def download_pkg(self, filename):
yield self.reporter.log_ok('Begin to download package '
'{} ...'.format(filename))
down_url = self.download_url + '?filename=' + filename
try:
response = yield self.client.fetch(
down_url,
connect_timeout=config.get('file_service_connect_timeout',
3600.0),
request_timeout=config.get('file_service_request_timeout',
3600.0),
validate_cert=False)
if response.code == 200:
if not nfs.exists(PKG_CACHE_DIR):
os.makedirs(PKG_CACHE_DIR)
nfs.copy(response.body, nfs.join(PKG_CACHE_DIR, filename))
yield self.reporter.log_ok(
'Download package {} success'.format(filename))
else:
raise MessageError(
'Download package {} failed, reason: {}!'.format(
filename, response.body))
except HTTPError as e:
raise MessageError(
'Download package {} failed, reason: {}, {}!'.format(
filename, e, e.message))
def _install(self, agent_platform, compress_name):
if not compress_name:
raise MessageError("compress_name can't be empty")
yield self.reporter.log_ok('Prepare environment for agent')
if 'aix' in agent_platform:
new_dst_name = re.sub('.gz$', '', compress_name)
uncompress_cmd = 'gunzip {compress_name} && ' \
'tar xf {new_dst_name} && rm -rf {new_dst_name}'\
.format(compress_name=compress_name, new_dst_name=new_dst_name)
else:
uncompress_cmd = \
'tar mzxf {compress_name} && rm -f {compress_name}'\
.format(compress_name=compress_name)
cmd = 'umask 0027 && cd {dst} && {uncompress_cmd} '\
.format(dst=self.dst, uncompress_cmd=uncompress_cmd)
try:
yield self.reporter.log_ok('Execute install cmd.')
result = yield self.do_ssh_cmd(cmd)
if result:
yield self.reporter.log_ok(result)
except Exception as e:
logger.error(traceback.format_exc())
yield self._rollback()
raise MessageError('Execute install error on agent. {}'.format(e))
def _deliver_to_agent(self, compress_name):
try:
file_url = urlparse.urljoin(self.upstream, UPSTREAM_SUFFIX)
file_url = urlparse.urljoin(file_url, 'file/')
file_url = urlparse.urljoin(file_url, compress_name)
if self.system == 'debian':
curl_cmd = \
'wget "{}" -c -P "{}" --no-check-certificate'\
.format(file_url, self.dst)
yield self.do_ssh_cmd(curl_cmd)
else:
curl_cmd = 'curl -s -w "%{{http_code}}" "{}" -s -m 3600 -o ' \
'"{}" -k --create-dirs'\
.format(file_url, self.dst_name)
http_code = yield self.do_ssh_cmd(curl_cmd)
logger.info('Download pkg {}, the http_code is {}'.format(
compress_name, http_code))
if http_code == '404':
raise MessageError(
'The package {} is not exists'.format(compress_name))
if http_code != '200':
raise MessageError('Http code is {}!'.format(http_code))
yield self.reporter.log_ok('Download agent pkg successfully')
except Exception as e:
raise MessageError('Download agent pkg failed. {}'.format(e))
def validate(self):
if not isinstance(self.task_message, list):
raise MessageError('Infos should be list')
for info in self.task_message:
if not isinstance(info, dict):
raise MessageError('Info should be dict')
for key in self.require:
if key not in info:
raise MessageError('{!r} is missing in info'.format(key))
def check_info(cls, module_name, info):
# Check required package fields
if set(REQUIRED_PKG_FIELDS) - set(info.keys()):
raise MessageError('{} requires {}'.format(info,
REQUIRED_PKG_FIELDS))
# Check fields type
for field in REQUIRED_PKG_FIELDS:
if not isinstance(info[field], basestring):
raise MessageError(be_string_msg.format(field, module_name))
# Check name and version
# we should remove DISABLE_POSTFIX from version if have
version = info['version'].strip()
if not semver.validate(version):
raise MessageError(version_invalid_msg.format(
version, module_name))
# dict fields
for field in ('platforms', 'scripts', 'dependencies'):
if field not in info:
continue
if not isinstance(info[field], (type(None), dict)):
raise MessageError(be_dict_msg.format(field, module_name))
if info[field] is None:
info[field] = {}
if field == 'scripts':
for value in info[field].values():
if not isinstance(value,
(type(None), basestring, list, dict)):
raise MessageError(
be_string_list_empty_msg.format(
field, module_name))
elif field == 'platforms':
for key in info[field].keys():
if FULLNAME_SEP not in key and key not in PLATFORM_LIST:
raise MessageError(
be_platform_err_msg.format(field, module_name))
else:
for value in info[field].values():
if not isinstance(value, list):
raise MessageError(
be_string_msg.format(field, module_name))
# 'main' field
if 'run' in info and not isinstance(info['run'],
(basestring, list, dict)):
raise MessageError(be_string_list_empty_msg.format('run'))
# 'priority' field
if 'priority' in info and not isinstance(info['priority'], int):
raise MessageError(be_string_list_empty_msg.format('priority'))
def _validate(self):
if not isinstance(self.task_message, list):
raise MessageError('Infos should be list')
for info in self.task_message:
if not isinstance(info, dict):
raise MessageError('Info should be dict')
for key in ('host', 'port', 'user', 'passwd', 'network_domain'):
if key not in info:
raise MessageError('{!r} is missing in info'.format(key))
task_length = len(self.task_message)
if task_length > 20:
self._executor = ThreadPoolExecutor(max_workers=20)
else:
self._executor = ThreadPoolExecutor(max_workers=task_length)
def disable(cls):
if nfs.exists(cls.enable_conf):
nfs.rename(cls.enable_conf, cls.disable_conf)
yield cls.rm_openresty()
if not nfs.exists(cls.disable_conf):
raise MessageError('Disable openresty failed !')
def reload(cls):
props = {'waiting': True, 'path': cls.enable_conf}
ret = yield cls.client.send_message('reloadconfig', **props)
if isinstance(ret, dict):
if ret.get('status') == 'error':
raise MessageError(ret.get('reason'))
raise gen.Return('Finish reload openresty!')
def rm_openresty(cls):
props = {"name": "openresty", "nostop": False, "waiting": False}
ret = yield cls.client.send_message('rm', **props)
if isinstance(ret, dict):
if ret.get('status') == 'error':
raise MessageError(ret.get('reason'))
raise gen.Return('Finish rm openresty!')
def exec_lifecycle_script(self, cmd_name, cwd=None, wait=True, env=None):
if not cwd:
cwd = os.path.dirname(self.config_path)
scripts = self.content.get('scripts')
if not scripts\
or cmd_name not in scripts \
or not scripts.get(cmd_name):
return
cmds = scripts[cmd_name]
cmds = deal_cmd(cmds)
if not cmds:
raise MessageError('{} is not support for this platform !'.format(
cmd_name))
if isinstance(cmds, str) and cmds.strip():
cmds = [cmds.strip()]
elif not isinstance(cmds, list):
raise MessageError('scirpts:{} must be str or list'.format(
cmd_name))
cmd = ''
try:
for cmd in cmds:
executable = cmd.split()[0]
sys_executable = '"{}"'.format(sys.executable) \
if ' ' in sys.executable else sys.executable
if executable.lower().strip() == 'python':
cmd = cmd.replace(executable, sys_executable, 1)
module_logger.info('Executing lifecycle {}'.format(cmd_name))
module_logger.info('Lifecycle cmd: {}, cwd: {}'.format(cmd,
cwd))
status, output = execute(cmd, cwd, wait=wait, env=env)
module_logger.info(
"Command '%s' returned exit status %d, the output is: %s" %
(cmd, status, output))
if status != 0:
raise MessageError(
"Command '%s' returned non-zero exit status %d, "
"the output is: %s" % (cmd, status, output))
except CalledProcessError as e:
raise MessageError(e)
except Exception as e:
raise MessageError('When executing "{0}": "{1}", meet an error:\n'
'{2}'.format(cmd_name, cmd, str(e)))
def _check_started(self):
cmd = 'ps -ef| grep -Ew "circled|upgrade" | ' \
'grep "{}/embedded/bin/python"| grep -v grep| wc -l'\
.format(AGENT_NAME)
result = yield self.do_ssh_cmd(cmd)
if result and result.strip() != '0':
raise MessageError('Ant agent on {} has been installed and '
'is running, {}'.format(self.host, result))
def check_module(cls, module_name, info):
if not module_name:
return
if module_name.strip() != info['name'].strip():
raise MessageError(
"The name of module {!r} is different from it's "
"{} info !!!".format(module_name, PKG_YAML_NAME))
cls.check_platforms(info['platforms'])
def start_circled(su_cmd):
logger.info('Starting Agent ...')
if IS_WINDOWS:
check_call([BIN_START], shell=True)
else:
status, _ = execute('{} "{}"'.format(su_cmd, BIN_START))
if status != 0:
raise MessageError('Start Agent failed')
logger.info('Start Agent done')
def get_agent_platform_detail(self):
cmd = 'cd {} && ./embedded/bin/python -m framework.actions.info' \
''.format(self.project_dir)
try:
yield self.reporter.log_ok('Execute info cmd')
result = yield self.do_ssh_cmd(cmd)
yield self.reporter.log_ok(result, True)
except Exception as e:
yield self._rollback()
raise MessageError('Execute info error on agent. {}'.format(e))
def _get_upstream(self):
if self.upstream:
raise gen.Return(True)
result = yield self.ssh_client.ssh('env |grep SSH_CLIENT')
upstream_ip = get_upstream_ip(result.strip())
if not upstream_ip:
raise MessageError('Get upstream error!')
self.upstream = 'http://{}:{}'.format(upstream_ip, NGINX_PORT)
def circle_cmd(self, cmd, module_name=None):
props = {'waiting': True, 'name': module_name, 'match': 'regex'} \
if module_name else {}
ret = yield self.circle_client.send_message(cmd, **props)
if isinstance(ret, dict):
if ret.get('status') == 'error':
raise MessageError(ret.get('reason'))
if module_name:
raise gen.Return('Finish {} {}!'.format(cmd, module_name))
raise gen.Return('Finish {}!'.format(cmd))
def get_hooks(self, scripts):
if not scripts:
return None
cmds_names = ['post_stop', 'post_start']
result = {}
for cmd_name in cmds_names:
hooks = scripts.get(cmd_name)
if not hooks:
continue
hooks = deal_cmd(hooks)
if not hooks:
raise MessageError(
'{} is not support for this platform !'.format(cmd_name))
if isinstance(hooks, list):
raise MessageError('scirpts:{} must be str'.format(cmd_name))
result[cmd_name] = hooks
return result
def post_upgrade(self, body):
response = yield self._http_client.fetch(
self._url,
method='POST',
headers={'Accept': 'application/json'},
body=body,
validate_cert=False,
raise_error=True)
if response.code != 200:
raise MessageError('The response code：{}，body：{}'.format(
response.code, response.body))
def do_ssh_cmd(ssh_client, cmd):
chan = ssh_client.get_transport().open_session()
chan.get_pty()
chan.exec_command(cmd)
stdout = chan.makefile('r', -1)
stderr = chan.makefile_stderr('r', -1)
output = stdout.read()
err = stderr.read()
if chan.recv_exit_status() != 0:
raise MessageError(err)
else:
return output
def _deliver_to_aix(self, compress_name):
try:
file_path = nfs.join(REPO_DIR, compress_name)
if not nfs.exists(file_path):
file_path = nfs.join(REPO_DIR, REPO_ANT_SPACENAME,
compress_name)
if not nfs.exists(file_path):
down_url = 'http://127.0.0.1:16600/file?filename={}'\
.format(compress_name)
client = AsyncHTTPClient(io_loop=ioloop.IOLoop.current())
response = yield client.fetch(down_url,
connect_timeout=3600.0,
request_timeout=3600.0,
validate_cert=False)
if response.code != 200:
raise MessageError("Can't download pkg by http")
yield self.do_ssh_cmd('umask 0027 && mkdir -p "{}"'.format(
self.dst))
yield self.ssh_client.scp(os.path.realpath(file_path),
self.dst_name)
except Exception as e:
raise MessageError('Download agent pkg failed. {}'.format(e))
def get_system_version(self, distribution):
version_cmd = \
"cat /etc/*-release 2>/dev/null |sed 's/^#//g' |" \
"grep -E \"^VERSION|^{distribution}\"|" \
"grep -v \"VERSION_ID\" || " \
"grep -E \"{distribution}\" /etc/issue 2>/dev/null" \
.format(distribution=distribution)
version = yield self.do_ssh_cmd(version_cmd)
search = re.search('\d+(\.\d+)?', version)
if search:
version = search.group(0)
raise gen.Return(version)
else:
raise MessageError('Get system version error!')
def _check_sudo(self, runas=None):
if self.user == 'root':
self.runner = 'root'
self.cmd_prefix = ''
raise gen.Return(True)
self.runner = runas if runas else self.user
self.cmd_prefix = 'echo \'{}\' | sudo -p "" -S su - {} -c'\
.format(self.passwd, self.runner)
try:
yield self.do_ssh_cmd('LANG=C && env > /dev/null')
except Exception as e:
raise MessageError('Need sudo to install agent, {}'.format(e))
raise gen.Return(True)
def generate_file(self, main_cmds, env, scripts):
if not main_cmds:
return
if not os.path.exists(CIRCLE_CONF_DIR):
os.makedirs(CIRCLE_CONF_DIR)
main_cmds = deal_cmd(main_cmds)
if not main_cmds:
raise MessageError('Run cmd is not support for this platform !')
main_cmds = [cmd for cmd in main_cmds if cmd.strip()] \
if isinstance(main_cmds, list) else [main_cmds]
watcher_names = [
'{}{}{}'.format(self.fullname, FULLNAME_SEP, i + 1)
for i in range(len(main_cmds))
]
module_logger.debug('Write config to {}'.format(self.able_path))
hooks = self.get_hooks(scripts)
with open(self.able_path, 'w') as config_file:
for i, cmd in enumerate(main_cmds):
if re.match(r'^python', cmd):
cmd = re.sub(r'^python',
sys.executable.replace('\\', '\\\\'), cmd)
config_file.write('[watcher:{}]\n'
'cmd={}\n'
'numprocess=1\n'
'stop_children=True\n\n'
.format(watcher_names[i], cmd))
config_file.write('[env:{}]\n'.format(watcher_names[i]))
if not env:
return
for key, value in env.iteritems():
config_file.write('{}={}\n'.format(key, value))
config_file.write('\n')
if hooks:
config_file.write('[hooks:{}]\n'.format(watcher_names[i]))
for key, value in hooks.iteritems():
config_file.write('{}={}\n'.format(key, value))
config_file.write('\n')
def get_info(cls, pkg_yml_path, name):
"""
If tgz is True, uncompress pkgs to PKG_SERVER_CACHE_DIR,
and allow uncompress names like "base_1.0.0_win", "base_1.0.0_linux".
Else, uncompress pkgs to PKG_DIR,
only allow uncompress names like "base_1.0.0".
"""
# Check if package.yml exists
if not os.path.exists(pkg_yml_path):
raise NotExistsError('The {} of {}'.format(PKG_YAML_NAME, name))
try:
with open(pkg_yml_path) as pkg_info:
info = yaml.load(pkg_info.read())
cls.check_info(info['name'], info)
return info
except yaml.scanner.ScannerError:
raise MessageError('The {} of {} is invalid yaml format'.format(
PKG_YAML_NAME, name))
def _get_agent_platform(self):
"""Detect agent platform (AIX, Linux) and cpu arch (x86, x64)"""
yield self.reporter.log_ok('Detecting agent platform')
try:
system_uname = yield self.do_ssh_cmd('uname')
system_uname = system_uname.lower()
if 'aix' in system_uname:
system = 'aix'
output = yield self.do_ssh_cmd(
'[[ `getconf HARDWARE_BITMODE | grep 64|wc -l` -gt 0 ]] '
'&& echo 64 || echo 32')
output = output.strip()
version = yield self.do_ssh_cmd('oslevel')
version = '.'.join(version.split('.')[0:2])
else:
find_str = 'grep -Eo "{}" '.format(KNOWN_DISTRIBUTION)
cmd = 'lsb_release -d 2>/dev/null | {find_str} ' \
'|| cat /etc/*-release 2>/dev/null | {find_str}' \
'|| {find_str} /etc/issue 2>/dev/null' \
'|| uname -s'.format(find_str=find_str)
distribution = yield self.do_ssh_cmd(cmd)
distribution = de_duplication(distribution)
system = distribution.lower()
if not system:
raise MessageError('Get system distribution error!')
output = yield self.do_ssh_cmd(
'[[ `uname -m|grep 64|wc -l` -gt 0 ]] '
'&& echo 64 || echo 32')
output = output.strip()
version = yield self.get_system_version(distribution)
logger.info('Get system version: {}'.format(version))
self.system = system
agent_platform = self.deal_agent_platform(system, version, output)
yield self.reporter.log_ok(
'Agent platform: {}'.format(agent_platform))
raise gen.Return(agent_platform)
except SSHRunError:
raise NotMatchError('Unsupport system')
def _do_bootstrap(self):
cmd = 'cd {project_dir} && ./embedded/bin/python -m ' \
'framework.actions.bootstrap --tenant {tenant} ' \
'--network-domain {network_domain} --upstream {upstream} ' \
'--ip {ip} --user {user}'.format(
project_dir=self.project_dir,
tenant=self.tenant,
network_domain=self.network_domain,
upstream=self.upstream,
ip=self.host,
user=self.runner)
try:
yield self.reporter.log_ok('Execute bootstrap cmd!')
if self.user == 'root':
result = yield self.do_ssh_cmd(cmd)
else:
result = yield self.do_ssh_cmd(
cmd,
'echo \'{}\' | sudo -p "" -S su -c '.format(self.passwd))
yield self.reporter.log_ok(result)
except Exception as e:
yield self._rollback()
raise MessageError(
'Execute bootstrap error on agent. {}'.format(e))
def register_upgrade_service(su_cmd):
if not IS_WINDOWS:
logger.info('Register upgrade service')
if 'ubuntu' in PLATFORM or 'debian' in PLATFORM:
format_upgrade_template(DESCRIPTION['like_debian'], su_cmd)
register_status, _ = execute(
'update-rc.d {} defaults'.format(UPGRADE_SERVICE_NAME))
if register_status != 0:
raise RegisterServiceError(
'Register {} Service Error'.format(UPGRADE_SERVICE_NAME))
elif any(_p in PLATFORM
for _p in ('centos', 'fedora', 'suse', 'redhat')):
format_upgrade_template(DESCRIPTION['like_redhat'], su_cmd)
chk_status, _ = execute(
'chkconfig {} on'.format(UPGRADE_SERVICE_NAME))
service_path = nfs.join(SERVICE_PATH, UPGRADE_SERVICE_NAME)
echo_status, _ = execute('echo "{service_path} start" '
'| tee --append {path} >/dev/null'.format(
service_path=service_path,
path=BOOT_SCRIPT))
chmod_status, _ = execute('chmod +x {}'.format(BOOT_SCRIPT))
if chk_status != 0 or echo_status != 0 or chmod_status != 0:
raise RegisterServiceError(
'Register {} Service Error'.format(UPGRADE_SERVICE_NAME))
else:
logger.warn('Unsupported platform')
status, output = execute('{} "{}"'.format(su_cmd, UPGRADE_BIN_START))
if status != 0:
raise MessageError(
'Start Upgrade failed: reason: {}'.format(output))
logger.info('Start Upgrade done')
else:
UpgradeWinService.install()
UpgradeWinService.start()
def enable(cls):
if nfs.exists(cls.disable_conf):
nfs.rename(cls.disable_conf, cls.enable_conf)
if not nfs.exists(cls.enable_conf):
raise MessageError('Enable openresty failed !')
yield cls.reload()
def handle_cli():
try:
cli_args = docopt(__doc__)
if IS_WINDOWS and not check_win_agent():
return
if not nfs.exists(UPGRADE_PYTHON_DIR):
nfs.copy(nfs.join(PYTHON_DIR, '*'), UPGRADE_PYTHON_DIR)
# Get upstream message
if cli_args['--upstream']:
baseurl = cli_args['--upstream']
upstream = urlparse.urljoin(cli_args['--upstream'],
UPSTREAM_SUFFIX)
upstream_mes = upstream_validate(cli_args['--upstream'])
if not upstream_mes:
logger.error('The upstream: {} is wrong!'
''.format(cli_args['--upstream']))
return
else:
upstream_ip = os.environ['SSH_CLIENT'].split()[0]
baseurl = 'http://{}:{}/'.format(upstream_ip, NGINX_PORT)
upstream = '{}{}'.format(baseurl, UPSTREAM_SUFFIX)
upstream_mes = [upstream_ip, NGINX_PORT]
if cli_args['--ip']:
if not ip_validate(cli_args['--ip']):
raise ValueError('Ant agent ip: {} is invalid'
''.format(cli_args['--ip']))
else:
agent_ip = cli_args['--ip']
else:
agent_ip = get_agent_ip(upstream_mes[0], int(upstream_mes[1]))
runner = cli_args['--user'] if cli_args['--user'] else os.environ.get(
'USER')
su_cmd = '' if runner == 'root' else 'su - {} -c '.format(runner)
conf_dict = {
'tenant': cli_args['--tenant'],
'ip': agent_ip,
'upstream': upstream,
'network_domain': cli_args['--network-domain']
}
init_conf(conf_dict)
init_bootstrap()
init_openresty(baseurl, upstream, runner)
register_service(su_cmd)
if runner and not IS_WINDOWS and runner != 'root':
status, result = execute('chown -R {user}:{user} {path}'.format(
user=runner, path=ROOT_DIR))
if status != 0:
raise MessageError(
'Change log path owen failed! Error[{}]: {}'.format(
status, result))
register_upgrade_service(su_cmd)
start_circled(su_cmd)
except Exception as e:
logger.error(e)
sys.exit(1)