def main():
(args, params) = parseArguments()
opts = {'verbose': args.verbose,
'console': True,
}
log.init_logging(opts)
ecs_api = api.EcsApi(args.access_key, args.access_key_secret)
if args.info and args.api:
api_info = ecs_api.get_api_info(args.api)
print_api_info(api_info)
return 0
if args.info:
api_list = ecs_api.get_available_apis()
print_api_list(api_list)
return 0
method = getattr(ecs_api, args.api)
if not method:
log.error("API %s not implemented", args.api)
raise exp.InvalidParameterValue(opt_name="api", opt_value=method)
resp = method(**params)
print "Aliyun ECS API %s results:\n%s" % (args.api, json.dumps(resp,
indent=2,
sort_keys=True))
return 0
def main():
try:
# prepare the working environment
args = prepare()
# load PMT/BMT table
xmt_mgr = xmt.XmtManager()
xmt_mgr.load(xmt.XMT_TYPE_PMT, args.pmt)
hostname = socket.gethostname()
pmt_entry = xmt_mgr.get_entry(xmt.XMT_TYPE_PMT, hostname)
log.debug("Get the PMT entry: %s", pmt_entry)
# modify puppet config with proper environment value
cmd = "sed -i \'s/environment=production$/environment=" + \
pmt_entry['environment'] + "/g\' /etc/puppet/puppet.conf"
utils.run_command(cmd)
# start deployment with the specified build
pbt = load_yaml(args.pbt)
deploy_build(args, pmt_entry, pbt)
except Exception as e:
log.error("Error happens during post boot: %s", e)
raise
return 0
def parseArguments():
params = {}
parser = argparse.ArgumentParser(description="Shucaibao aliyun ecs api tool")
parser.add_argument('--verbose', action="store_true", required=False,
help="print the debug information")
parser.add_argument('--access_key', required=False, type=str,
help="Aliyun ECS access key")
parser.add_argument('--access_key_secret', required=False, type=str,
help="Aliyun ECS access key secret")
parser.add_argument('--api', required=False, type=str,
help="Aliyun ECS api name")
parser.add_argument('--params', required=False, nargs='+',
type=key_equal_value,
help="ECS API request parameters in key=value")
parser.add_argument('--info', required=False, action='store_true',
help="Print out the ECS API details")
args = parser.parse_args()
if args.api and args.params:
if args.access_key is None or args.access_key_secret is None:
log.error("Please specify the access key and secret")
raise exp.InvalidParameterValue(opt_name="access_key/secret",
opt_value="missing")
if args.params is not None:
for param in args.params:
(k, v) = param.split('=')
params[k] = v
return (args, params)
def main():
try:
# prepare the working environment
args = prepare()
# load PMT/BMT table
xmt_mgr = xmt.XmtManager()
xmt_mgr.load(xmt.XMT_TYPE_PMT, args.pmt)
hostname = socket.gethostname()
pmt_entry = xmt_mgr.get_entry(xmt.XMT_TYPE_PMT, hostname)
log.debug("Get the PMT entry: %s", pmt_entry)
# modify puppet config with proper environment value
cmd = "sed -i \'s/environment=production$/environment=" + \
pmt_entry['environment'] + "/g\' /etc/puppet/puppet.conf"
utils.run_command(cmd)
# start deployment with the specified build
pbt = load_yaml(args.pbt)
deploy_build(args, pmt_entry, pbt)
except Exception as e:
log.error("Error happens during post boot: %s", e)
raise
return 0
def _request(self, method, url, extra_headers=None, params=None):
"""Fire a HTTPS request to github
"""
common_headers = {
'Content-type': 'applicatin/json',
'Cache-Control': 'no-cache',
'Connection': 'Keep-Alive',
}
if extra_headers is not None:
headers = dict(common_headers.items() + extra_headers.items())
else:
headers = common_headers
try:
request_matrix = {
'get':
(lambda: requests.get(url, headers=headers, params=params)),
'put':
(lambda: requests.put(url, headers=headers, params=params)),
'post':
(lambda: requests.post(url, headers=headers, params=params)),
'delete':
(lambda: requests.delete(url, headers=headers, params=params)),
}
r = request_matrix[method]()
if r.status_code != requests.codes.ok:
out = r.json()
e = exp.AliyunEcsException(status_code=r.status_code,
code=out['Code'],
host=out['HostId'],
message=out['Message'],
requestId=out['RequestId'])
raise e
return r
except requests.exceptions.ConnectionError as e:
log.error(
"Github request failed due to connection network problem: %s",
e)
raise e
except requests.exceptions.HTTPError as e:
log.error("Github request failed due to invalid HTTP response: %s",
e)
raise e
except requests.exceptions.Timeout as e:
log.error("Github request failed due to request times out")
raise e
except requests.exceptions.TooManyRedirects as e:
log.error(
"Github request failed because the request exceeds the configured "
+ "number of maximum redirections")
raise e
except requests.exceptions.RequestException as e:
log.error("Github request failed due to error %s", e)
raise e
def prepare():
args = parseArguments()
log.init_logging({'verbose': args.verbose,
'console': args.console,
'logfile': args.logfile,
})
if not os.path.isfile(args.pmt):
log.error("Error: file %s not exists", args.pmt)
raise exp.InvalidConfigurationOption(opt_name='pmt', opt_value=args.pmt)
return args
def load_yaml(yaml_path):
"""
Load yaml file into python dict (key is phase number)
"""
try:
with open(yaml_path, 'r') as yamlfile:
yaml_data = dict(map(lambda x: (x['phase'], x), yaml.load_all(yamlfile)))
except yaml.YAMLError as ex:
if hasattr(ex, 'problem_mark'):
mark = ex.problem_mark
log.error("YAML load error at position (%s:%s)",
mark.line + 1, mark.column + 1)
return yaml_data
def prepare():
args = parseArguments()
log.init_logging({
'verbose': args.verbose,
'console': args.console,
'logfile': args.logfile,
})
if not os.path.isfile(args.pmt):
log.error("Error: file %s not exists", args.pmt)
raise exp.InvalidConfigurationOption(opt_name='pmt',
opt_value=args.pmt)
return args
def validate_api_params(self, method, **params):
if method.lower() not in self.api_dict:
log.error("Invalid ECS API method: %s", method)
raise exp.InvalidParameterValue(opt_name='method',
opt_value=method)
valid_params = self.api_dict[method.lower()]['parameters']
for (k, v) in valid_params.iteritems():
if k == 'Action':
continue
if valid_params[k] is True and k not in params:
log.error("Invalid ECS API parameter: missing parameter %s", k)
raise exp.InvalidParameterValue(opt_name=k, opt_value="missing")
params['Action'] = self.get_api_action(method)
return params
def load_yaml(yaml_path):
"""
Load yaml file into python dict (key is phase number)
"""
try:
with open(yaml_path, 'r') as yamlfile:
yaml_data = dict(
map(lambda x: (x['phase'], x), yaml.load_all(yamlfile)))
except yaml.YAMLError as ex:
if hasattr(ex, 'problem_mark'):
mark = ex.problem_mark
log.error("YAML load error at position (%s:%s)", mark.line + 1,
mark.column + 1)
return yaml_data
def run_command(cmd):
"""
This method execute the command in another shell process in blocking mode
raise exception if the command exits with failure
"""
output = None
log.debug("Executing command [%s]", cmd)
try:
output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as ex:
errmsg = "<errorcode>: %d \n<output>: %s\n" % (ex.returncode, ex.output)
log.error("Executing command [%s] failed: %s", cmd, errmsg)
raise exp.CommandFailure(cmd=cmd, error=errmsg)
return output
def main():
pidfile = '/opt/shoowo/run/stats.pid'
try:
(args, metric_def) = prepare()
if args.pid is not None:
pidfile = args.pid
send_stats_data(args, metric_def['name'],
metric_def['value'], metric_def['type'])
except exp.ShoowoException as e:
log.error("Error happens during stats aggregation: %s", e.message)
raise e
finally:
if os.path.exists(pidfile):
os.unlink(pidfile)
return 0
def main():
try:
args = prepare()
xmt_mgr = xmt.XmtManager(pmt_path=args.pmt)
xmt_mgr.load(xmt.XMT_TYPE_PMT)
if args.boot is True:
set_hostname(args, xmt_mgr)
set_hosts(args, xmt_mgr)
set_host_meta(args, xmt_mgr)
except exp.ShucaibaoException as e:
log.error("Error happens during hosts refreshing: %s", e.message)
raise e
return 0
def set_hostname(args, xmt_mgr):
"""setup the hostname in boot mode
"""
try:
pmt_entries = xmt_mgr.get_entries(xmt.XMT_TYPE_PMT)
pmt_entry = [x for x in pmt_entries if args.ecs_name.find(x['ecs_name']) != -1][0]
# pmt_entry = xmt_mgr.get_entries(xmt.XMT_TYPE_PMT, filter={'ecs_name': args.ecs_name})[0]
except ValueError:
log.error("Didn't find the host give ecs_name [%s]!", args.ecs_name)
raise
hostname = pmt_entry['host']
cmd = "echo " + hostname + " > /etc/hostname"
utils.run_command(cmd)
utils.run_command('hostname ' + hostname)
log.info("set hostname to [%s]", hostname)
def validate_api_params(self, method, **params):
if method.lower() not in self.api_dict:
log.error("Invalid ECS API method: %s", method)
raise exp.InvalidParameterValue(opt_name='method',
opt_value=method)
valid_params = self.api_dict[method.lower()]['parameters']
for (k, v) in valid_params.iteritems():
if k == 'Action':
continue
if valid_params[k] is True and k not in params:
log.error("Invalid ECS API parameter: missing parameter %s", k)
raise exp.InvalidParameterValue(opt_name=k,
opt_value="missing")
params['Action'] = self.get_api_action(method)
return params
def main():
try:
args = prepare()
xmt_mgr = xmt.XmtManager(pmt_path=args.pmt)
xmt_mgr.load(xmt.XMT_TYPE_PMT)
if args.boot is True:
set_hostname(args, xmt_mgr)
set_hosts(args, xmt_mgr)
set_host_meta(args, xmt_mgr)
except exp.ShucaibaoException as e:
log.error("Error happens during hosts refreshing: %s", e.message)
raise e
return 0
def _request(self, method, url, extra_headers=None, params=None):
"""Fire a HTTPS request to github
"""
common_headers = {'Content-type': 'applicatin/json',
'Cache-Control': 'no-cache',
'Connection': 'Keep-Alive',
}
if extra_headers is not None:
headers = dict(common_headers.items() + extra_headers.items())
else:
headers = common_headers
try:
request_matrix = {
'get': (lambda: requests.get(url, headers=headers, params=params)),
'put': (lambda: requests.put(url, headers=headers, params=params)),
'post': (lambda: requests.post(url, headers=headers, params=params)),
'delete': (lambda: requests.delete(url, headers=headers, params=params)),
}
r = request_matrix[method]()
if r.status_code != requests.codes.ok:
out = r.json()
e = exp.AliyunEcsException(status_code=r.status_code, code=out['Code'],
host=out['HostId'], message=out['Message'],
requestId=out['RequestId'])
raise e
return r
except requests.exceptions.ConnectionError as e:
log.error("Github request failed due to connection network problem: %s", e)
raise e
except requests.exceptions.HTTPError as e:
log.error("Github request failed due to invalid HTTP response: %s", e)
raise e
except requests.exceptions.Timeout as e:
log.error("Github request failed due to request times out")
raise e
except requests.exceptions.TooManyRedirects as e:
log.error("Github request failed because the request exceeds the configured " +
"number of maximum redirections")
raise e
except requests.exceptions.RequestException as e:
log.error("Github request failed due to error %s", e)
raise e
def set_hostname(args, xmt_mgr):
"""setup the hostname in boot mode
"""
try:
pmt_entries = xmt_mgr.get_entries(xmt.XMT_TYPE_PMT)
pmt_entry = [
x for x in pmt_entries if args.ecs_name.find(x['ecs_name']) != -1
][0]
# pmt_entry = xmt_mgr.get_entries(xmt.XMT_TYPE_PMT, filter={'ecs_name': args.ecs_name})[0]
except ValueError:
log.error("Didn't find the host give ecs_name [%s]!", args.ecs_name)
raise
hostname = pmt_entry['host']
cmd = "echo " + hostname + " > /etc/hostname"
utils.run_command(cmd)
utils.run_command('hostname ' + hostname)
log.info("set hostname to [%s]", hostname)
def _request(self, method, url, extra_headers=None, params=None):
"""Fire a HTTPS request to github
"""
common_headers = {
'content-type': 'applicatin/json',
'Authorization': 'token ' + self.token,
}
if extra_headers is not None:
headers = dict(common_headers.items() + extra_headers.items())
else:
headers = common_headers
try:
request_matrix = {
'get': (lambda: requests.get(url, headers=headers, params=params)),
'put': (lambda: requests.put(url, headers=headers, params=params)),
'post': (lambda: requests.post(url, headers=headers, params=params)),
'delete': (lambda: requests.delete(url, headers=headers, params=params)),
}
r = request_matrix[method]()
if r.status_code != requests.codes.ok:
r.raise_for_status()
return r
except requests.exceptions.ConnectionError as e:
log.error("Github request failed due to connection network problem: %s", e)
raise e
except requests.exceptions.HTTPError as e:
log.error("Github request failed due to invalid HTTP response: %s", e)
raise e
except requests.exceptions.Timeout as e:
log.error("Github request failed due to request times out")
raise e
except requests.exceptions.TooManyRedirects as e:
log.error("Github request failed because the request exceeds the configured " +
"number of maximum redirections")
raise e
except requests.exceptions.RequestException as e:
log.error("Github request failed due to error %s", e)
raise e
def prepare():
args = parseArguments()
pid = str(os.getpid())
pidfile = args.pid
log.init_logging({'verbose': args.verbose,
'console': args.console,
'logfile': args.logfile})
if utils.check_pid(pidfile) is True:
log.error("Pid %s already exists, exiting.", pidfile)
sys.exit()
else:
file(pidfile, 'w').write(pid)
metric_def = validate_metric(args.metric, args.value)
# initialize statsdclient
global statsd_client
statsd_client = StatsClient(host=args.server, port=args.port,
prefix=(args.prefix + "." + args.source
if args.prefix is not None else args.source))
return (args, metric_def)
def prepare():
args = parseArguments()
log.init_logging({'verbose': args.verbose,
'console': args.console,
'logfile': args.logfile})
if args.boot is True and args.ecs_name is None:
log.error("ecs_name must be given in boot mode")
raise exp.InvalidConfigurationOption(opt_name="ecs_name",
opt_val=None)
if not os.path.isfile(args.pmt):
log.error("file %s not exists", args.pmt)
raise exp.InvalidConfigurationOption(opt_name='pmt',
opt_value=args.pmt)
if not os.path.isfile(args.template):
log.error("template file %s not exists", args.template)
raise exp.InvalidConfigurationOption(opt_name='template',
opt_value=args.template)
return args
def prepare():
args = parseArguments()
log.init_logging({
'verbose': args.verbose,
'console': args.console,
'logfile': args.logfile
})
if args.boot is True and args.ecs_name is None:
log.error("ecs_name must be given in boot mode")
raise exp.InvalidConfigurationOption(opt_name="ecs_name", opt_val=None)
if not os.path.isfile(args.pmt):
log.error("file %s not exists", args.pmt)
raise exp.InvalidConfigurationOption(opt_name='pmt',
opt_value=args.pmt)
if not os.path.isfile(args.template):
log.error("template file %s not exists", args.template)
raise exp.InvalidConfigurationOption(opt_name='template',
opt_value=args.template)
return args
def get_api_action(self, method):
if method.lower() not in self.api_dict:
log.error("Invalid ECS API method: %s", method)
raise exp.InvalidParameterValue(opt_name='method',
opt_value=method)
return self.api_dict[method.lower()]['parameters']['Action']
def get_api_action(self, method):
if method.lower() not in self.api_dict:
log.error("Invalid ECS API method: %s", method)
raise exp.InvalidParameterValue(opt_name='method',
opt_value=method)
return self.api_dict[method.lower()]['parameters']['Action']