def copy_to(self, dst, plugin_parent='plugins'):
if self.is_plugin:
plugin_yml_path = fs.join(self.path, 'module.yml')
if not fs.exists(plugin_yml_path):
plugin_yml_path = fs.join(self.path, 'plugin.yml')
if fs.exists(plugin_yml_path):
import yaml
info = yaml.load(open(plugin_yml_path))
fullname = '{}@{}'.format(info['name'], info['version'])
dst = fs.join(dst, plugin_parent, fullname)
fs.makedirs(dst)
else:
logger.error('module.yml or plugin.yml not exists')
sys.exit(1)
logger.info('Copy project: {!r} from {!r} to {!r}'.format(
self.name, self.path, dst))
for dirname in fs.listdir(self.path):
dirpath = fs.join(self.path, dirname)
if dirname in (EXCLUDE_DIRS + EXCLUDE_FILES) \
or dirname.startswith('.'):
continue
fs.copy(dirpath,
dst,
exclude_dirs=EXCLUDE_DIRS,
exclude_files=['*.exe', '*.bat']
if not IS_WINDOWS else ['*.sh'])
return dst
def __init__(self, new_agent_dir, http_handler):
self.new_agent_dir = new_agent_dir
self.http_handler = http_handler
self.bin_start = nfs.join(ROOT_DIR, 'bin',
'start' + SYSTEM_SCRIPT_TYPE)
self.bin_stop = nfs.join(ROOT_DIR, 'bin', 'stop' + SYSTEM_SCRIPT_TYPE)
self.copy = copy_files if IS_WINDOWS else nfs.copy
def backup_files(self):
if nfs.exists(AGENT_BACK_DIR):
nfs.remove(nfs.join(AGENT_BACK_DIR, '*'))
else:
nfs.makedirs(AGENT_BACK_DIR)
# Copy
self.http_handler.log_ok('Backup files')
for dir_name in nfs.listdir(ROOT_DIR):
if dir_name in EXCLUDE_BACK_DIRS:
continue
nfs.copy(nfs.join(ROOT_DIR, dir_name), AGENT_BACK_DIR)
self.http_handler.log_ok('Backup done')
def execute(self):
if IS_WINDOWS:
remove_script = fs.join(ROOT_DIR, 'bin', 'remove.bat')
subprocess.Popen([remove_script, '-f'],
close_fds=True,
shell=True,
creationflags=subprocess.CREATE_NEW_CONSOLE)
else:
remove_script = fs.join(ROOT_DIR, 'bin', 'remove.sh')
subprocess.Popen('/usr/bin/env bash {} -f &'.format(remove_script),
shell=True)
yield self.reporter.log_ok('Remove agent success!', done=True)
self.io_loop.stop()
def execute(self):
try:
for module in self.modules:
module_name = module['module_name']
module_env = self.deal_env(module.get('env'))
if not module.get('filename'):
yield self.reporter.log_error('Filename for {} is None'
''.format(module_name))
continue
if not nfs.exists(nfs.join(PKG_DIR, module_name)):
yield self.reporter.log_error('{} is not installed！'
''.format(module_name))
continue
yield self.reporter.log_ok('Begin to Upgrade {}'
''.format(module_name))
pkg_path = yield self.do_pre(module['filename'], module_env)
uninstall = Uninstall([module], self.io_loop, False)
uninstall.reporter = self.reporter
yield uninstall.remove(module_name)
yield self.do_install(module)
nfs.remove(pkg_path)
yield self.reporter.log_ok('Finish upgrade modules!', True)
yield self.circle_cmd('reloadconfig')
except Exception as e:
yield self.reporter.log_error(str(e), True)
sys.exit(1)
def download_pkg(self, filename):
yield self.reporter.log_ok('Begin to download package '
'{} ...'.format(filename))
down_url = self.download_url + '?filename=' + filename
try:
response = yield self.client.fetch(
down_url,
connect_timeout=config.get('file_service_connect_timeout',
3600.0),
request_timeout=config.get('file_service_request_timeout',
3600.0),
validate_cert=False)
if response.code == 200:
if not nfs.exists(PKG_CACHE_DIR):
os.makedirs(PKG_CACHE_DIR)
nfs.copy(response.body, nfs.join(PKG_CACHE_DIR, filename))
yield self.reporter.log_ok(
'Download package {} success'.format(filename))
else:
raise MessageError(
'Download package {} failed, reason: {}!'.format(
filename, response.body))
except HTTPError as e:
raise MessageError(
'Download package {} failed, reason: {}, {}!'.format(
filename, e, e.message))
def register_service(su_cmd):
if not IS_WINDOWS:
logger.info('Register service')
if 'ubuntu' in PLATFORM or 'debian' in PLATFORM:
format_template(DESCRIPTION['like_debian'], su_cmd)
register_status, _ = execute(
'update-rc.d {} defaults'.format(SERVICE_NAME))
if register_status != 0:
raise RegisterServiceError(
'Register {} Service Error'.format(SERVICE_NAME))
elif any(_p in PLATFORM
for _p in ('centos', 'fedora', 'suse', 'redhat')):
format_template(DESCRIPTION['like_redhat'], su_cmd)
chk_status, _ = execute('chkconfig {} on'.format(SERVICE_NAME))
service_path = nfs.join(SERVICE_PATH, SERVICE_NAME)
echo_status, _ = execute('echo "{service_path} start" '
'| tee --append {path} >/dev/null'.format(
service_path=service_path,
path=BOOT_SCRIPT))
chmod_status, _ = execute('chmod +x {}'.format(BOOT_SCRIPT))
if chk_status != 0 or echo_status != 0 or chmod_status != 0:
raise RegisterServiceError(
'Register {} Service Error'.format(SERVICE_NAME))
else:
logger.warn('Unsupported platform')
def deploy_new_agent(self):
self.http_handler.log_ok('Deploying agent')
dst = self.new_agent_dir if IS_WINDOWS \
else nfs.join(self.new_agent_dir, '*')
self.copy(dst, ROOT_DIR)
nfs.remove(self.new_agent_dir)
self.http_handler.log_ok('Deploy done')
def do_pre(self, filename, module_env):
yield self.download_pkg(filename)
pkg_path, pkg_info = PkgHelper.uncompress(filename, PKG_UPGRADE_DIR)
config = Config(nfs.join(pkg_path, PKG_YAML_NAME))
config.exec_lifecycle_script('pre_upgrade',
cwd=pkg_path,
env=module_env)
yield self.reporter.log_ok('Finish exec pre_upgrade!')
raise gen.Return(pkg_path)
def checkout_branch(self, dst=PROJECT_ROOT, branch='develop'):
fs.chdir(fs.join(dst, self.name))
try:
check_call(['git', 'checkout', branch])
except CalledProcessError as e:
if e.returncode == 1:
check_call(
['git', 'checkout', '-b', 'develop', 'origin/develop'])
fs.chdir('../../')
def get_agent_version():
from constants import ROOT_DIR
version = ''
module_yaml = nfs.join(ROOT_DIR, 'manifest.yaml')
with open(module_yaml) as temp:
module_mes = yaml.load(temp.read())
if 'version' in module_mes:
version = module_mes['version']
return version
def encrypt_py_project(target_root):
for dirname in fs.listdir(target_root):
if dirname in ('conf', 'embedded'):
continue
path = fs.join(target_root, dirname)
if not fs.isdir(path):
continue
compile_dir(path)
fs.remove(path, filter_files='*.py')
class Openresty(object):
enable_conf = nfs.join(CIRCLE_CONF_DIR, 'openresty.ini')
disable_conf = nfs.join(CIRCLE_CONF_DIR, 'openresty.ini.disable')
client = AsyncCircleClient()
@classmethod
@gen.coroutine
def reload(cls):
props = {'waiting': True, 'path': cls.enable_conf}
ret = yield cls.client.send_message('reloadconfig', **props)
if isinstance(ret, dict):
if ret.get('status') == 'error':
raise MessageError(ret.get('reason'))
raise gen.Return('Finish reload openresty!')
@classmethod
@gen.coroutine
def rm_openresty(cls):
props = {"name": "openresty", "nostop": False, "waiting": False}
ret = yield cls.client.send_message('rm', **props)
if isinstance(ret, dict):
if ret.get('status') == 'error':
raise MessageError(ret.get('reason'))
raise gen.Return('Finish rm openresty!')
@classmethod
@gen.coroutine
def enable(cls):
if nfs.exists(cls.disable_conf):
nfs.rename(cls.disable_conf, cls.enable_conf)
if not nfs.exists(cls.enable_conf):
raise MessageError('Enable openresty failed !')
yield cls.reload()
@classmethod
@gen.coroutine
def disable(cls):
if nfs.exists(cls.enable_conf):
nfs.rename(cls.enable_conf, cls.disable_conf)
yield cls.rm_openresty()
if not nfs.exists(cls.disable_conf):
raise MessageError('Disable openresty failed !')
def _load_command_and_env(self, action):
if action.startswith('core.'):
self.module_name = 'core'
command = settings.CORE_ACTIONS.get(action)
else:
self.module_name, _ = action.split('.')
module_path = nfs.join(settings.MODULES_DIR, self.module_name)
self.cwd = module_path
yaml_file = nfs.join(module_path, 'manifest.yaml')
if not nfs.exists(yaml_file):
return None
with open(yaml_file) as f:
data = yaml.load(f)
command = data['actions'].get(action)
env = utils.normalize_env(data.get('env', {}),
relpath_prefix=module_path)
logger.debug('Action task {!r} env: {}'.format(self.action, env))
# 处理PATH开头的环境变量，加入到PATH环境变量中
paths = [self.env.get('PATH', '')]
for name, value in env.iteritems():
if isinstance(value, int):
value = str(value)
else:
value = value.encode('utf-8')
if name.startswith('PATH'):
paths.append(value)
else:
self.env[name] = value
self.env['PATH'] = ';'.join(paths) if IS_WINDOWS else \
':'.join(paths)
logger.debug('Action task {!r} env PATH: {}'.format(
self.action, self.env['PATH']))
command = utils.normalize_cmdline(command)
return command
def maybe_download_python():
system = platform.system.lower()
cwd = os.getcwd()
if WINDOWS:
python = fs.join(cwd, 'embedded/python.exe')
pip = fs.join(cwd, 'embedded/Scripts/pip.exe')
else:
python = fs.join(cwd, 'embedded/bin/python')
pip = fs.join(cwd, 'embedded/bin/pip')
if not fs.exists(fs.join(cwd, 'embedded')):
python_name = 'python-{}-{}.{}'.format(system, platform.cpu,
POSTFIX.get(system, 'tgz'))
python_url = PYTHON_TEMPLATE_URL.format(python_name)
download(python_url, python_name)
fs.uncompress(python_name)
fs.rename('python-{}-{}'.format(system, platform.cpu), 'embedded')
return python, pip
def download(self, dst=PROJECT_ROOT):
logger.info('Download from {!r} to {!r}'.format(self.url, dst))
filename = self.url.split('/')[-1]
dst = fs.join(dst, filename)
req = requests.get(self.url, stream=True)
with open(dst, 'wb') as f:
for chunk in req.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
return dst
def get(self):
self.file_name = self.get_argument('filename') # type: str
self.space_dir = nfs.join(settings.REPO_DIR,
settings.REPO_ANT_SPACENAME)
if not nfs.exists(self.space_dir):
nfs.makedirs(self.space_dir)
self.file_path = nfs.join(self.space_dir, self.file_name)
lock_file_name = nfs.extsep + self.file_name + nfs.extsep + 'lock'
self.lock_file = nfs.join(self.space_dir, lock_file_name)
logger.info('#%d Request file: %s', id(self.request), self.file_name)
if nfs.exists(self.lock_file):
yield self.wait_for_file_complete()
else:
is_cache_hit = yield self.try_to_return_file_cache()
if is_cache_hit:
return
logger.info('#%d File cache missed: %s', id(self.request),
self.file_path)
nfs.touch(self.lock_file)
yield self.request_file_from_upstream()
def copy_basic_conf(self):
self.http_handler.log_ok('Coping basic conf')
try:
new_conf_path = nfs.join(self.new_agent_dir, CONFIG_NAME)
with open(CONFIG_PATH) as temp:
yaml.dump(yaml.load(temp.read()),
open(new_conf_path, 'a+'),
default_flow_style=False)
self.http_handler.log_ok('Copy conf done')
except IOError as e:
self.http_handler.log_error(str(e))
logger.error(str(e), exc_info=True)
self.http_handler.log_ok('Skip copy basic conf')
def remove(self, module_name):
try:
ret = yield self.circle_cmd('stop', module_name)
yield self.reporter.log_ok(ret)
except Exception as e:
if not re.match(r'program[\s\S]+not found', str(e)):
yield self.reporter.log_error(str(e))
raise gen.Return(str(e))
self.pre_stop(module_name)
config = Config(nfs.join(PKG_DIR, module_name, PKG_YAML_NAME))
# Execute pre_remove
config.exec_lifecycle_script('pre_remove')
# Remove
nfs.remove(nfs.join(PKG_DIR, module_name))
# Execute post_remove
config.exec_lifecycle_script('post_remove')
# Remove conf file for circled
circled_config = CircledConfig(module_name)
circled_config.remove_file()
# Result
yield self.reporter.log_ok('Remove {!r} successfully'
''.format(module_name))
def _deliver_to_aix(self, compress_name):
try:
file_path = nfs.join(REPO_DIR, compress_name)
if not nfs.exists(file_path):
file_path = nfs.join(REPO_DIR, REPO_ANT_SPACENAME,
compress_name)
if not nfs.exists(file_path):
down_url = 'http://127.0.0.1:16600/file?filename={}'\
.format(compress_name)
client = AsyncHTTPClient(io_loop=ioloop.IOLoop.current())
response = yield client.fetch(down_url,
connect_timeout=3600.0,
request_timeout=3600.0,
validate_cert=False)
if response.code != 200:
raise MessageError("Can't download pkg by http")
yield self.do_ssh_cmd('umask 0027 && mkdir -p "{}"'.format(
self.dst))
yield self.ssh_client.scp(os.path.realpath(file_path),
self.dst_name)
except Exception as e:
raise MessageError('Download agent pkg failed. {}'.format(e))
def run_hooks(self):
hooks_dir = nfs.join(self.new_agent_dir, 'hooks')
if not nfs.exists(hooks_dir):
return True
for dir_name in nfs.listdir(hooks_dir):
hooks_file = nfs.join(hooks_dir, dir_name)
if nfs.isfile(hooks_file) and hooks_file.endswith('.py'):
p = Popen([sys.executable, hooks_file],
stdout=PIPE,
stderr=STDOUT,
cwd=ROOT_DIR,
shell=True)
while p.poll() is None:
time.sleep(1)
if p.stdout:
logger.info(p.stdout.read())
if p.poll() != 0:
self.http_handler.log_ok('Run hooks {} failed!'
''.format(hooks_file))
return False
return True
def normalize_env(env, relpath_prefix=None):
u"""
将环境变量中的路径标准化，python开头的内容会被替换为sys.executable
凡是盘符（如C:/, C:\）或./或.\开头的字符串均会通过nfs.normpath进行标准化处理
如果提供relpath_prefix，比如relpath_prefix='c:/'，
环境变量中的相对路径均会改成绝对路径。
"""
for name, value in env.items():
value = value.strip()
if _is_path(value):
if relpath_prefix:
env[name] = nfs.normpath(nfs.join(relpath_prefix, value))
else:
env[name] = nfs.normpath(value)
else:
env[name] = re.sub(
'^ *python',
'"{}"'.format(sys.executable.replace('\\', '\\\\')),
value,
flags=re.IGNORECASE)
return env
def init_openresty(baseurl, upstream, runner):
if not nfs.exists(NGINX_CONF):
return
with open(NGINX_CONF) as temp:
content = temp.read()
content = content.replace('UPSTREAM', upstream)
content = content.replace('BASEURL', baseurl)
if not IS_WINDOWS and runner:
content = content.replace('RUNNER', runner)
if IS_WINDOWS:
BASEDIR = os.path.normpath(ROOT_DIR).replace('\\', '\\\\')
else:
BASEDIR = ROOT_DIR
content = content.replace('BASEDIR', BASEDIR)
with open(NGINX_CONF, 'w') as conf:
conf.write(content)
log_dir = nfs.join(ROOT_DIR, 'openresty', 'logs')
if not nfs.exists(log_dir):
nfs.makedirs(log_dir, 0750)
def register_upgrade_service(su_cmd):
if not IS_WINDOWS:
logger.info('Register upgrade service')
if 'ubuntu' in PLATFORM or 'debian' in PLATFORM:
format_upgrade_template(DESCRIPTION['like_debian'], su_cmd)
register_status, _ = execute(
'update-rc.d {} defaults'.format(UPGRADE_SERVICE_NAME))
if register_status != 0:
raise RegisterServiceError(
'Register {} Service Error'.format(UPGRADE_SERVICE_NAME))
elif any(_p in PLATFORM
for _p in ('centos', 'fedora', 'suse', 'redhat')):
format_upgrade_template(DESCRIPTION['like_redhat'], su_cmd)
chk_status, _ = execute(
'chkconfig {} on'.format(UPGRADE_SERVICE_NAME))
service_path = nfs.join(SERVICE_PATH, UPGRADE_SERVICE_NAME)
echo_status, _ = execute('echo "{service_path} start" '
'| tee --append {path} >/dev/null'.format(
service_path=service_path,
path=BOOT_SCRIPT))
chmod_status, _ = execute('chmod +x {}'.format(BOOT_SCRIPT))
if chk_status != 0 or echo_status != 0 or chmod_status != 0:
raise RegisterServiceError(
'Register {} Service Error'.format(UPGRADE_SERVICE_NAME))
else:
logger.warn('Unsupported platform')
status, output = execute('{} "{}"'.format(su_cmd, UPGRADE_BIN_START))
if status != 0:
raise MessageError(
'Start Upgrade failed: reason: {}'.format(output))
logger.info('Start Upgrade done')
else:
UpgradeWinService.install()
UpgradeWinService.start()
def pull(self, dst=PROJECT_ROOT):
fs.chdir(fs.join(dst, self.name))
check_call(['git', 'pull'])
def __init__(self, name, url):
self.name = name
self.url = url
self.path = fs.join(PROJECT_ROOT, name)
def __init__(self, task_message):
self.task_message = task_message
self.bin_start = nfs.join(ROOT_DIR, 'bin',
'start' + SYSTEM_SCRIPT_TYPE)
self.bin_stop = nfs.join(ROOT_DIR, 'bin', 'stop' + SYSTEM_SCRIPT_TYPE)
def handle_cli():
try:
cli_args = docopt(__doc__)
if IS_WINDOWS and not check_win_agent():
return
if not nfs.exists(UPGRADE_PYTHON_DIR):
nfs.copy(nfs.join(PYTHON_DIR, '*'), UPGRADE_PYTHON_DIR)
# Get upstream message
if cli_args['--upstream']:
baseurl = cli_args['--upstream']
upstream = urlparse.urljoin(cli_args['--upstream'],
UPSTREAM_SUFFIX)
upstream_mes = upstream_validate(cli_args['--upstream'])
if not upstream_mes:
logger.error('The upstream: {} is wrong!'
''.format(cli_args['--upstream']))
return
else:
upstream_ip = os.environ['SSH_CLIENT'].split()[0]
baseurl = 'http://{}:{}/'.format(upstream_ip, NGINX_PORT)
upstream = '{}{}'.format(baseurl, UPSTREAM_SUFFIX)
upstream_mes = [upstream_ip, NGINX_PORT]
if cli_args['--ip']:
if not ip_validate(cli_args['--ip']):
raise ValueError('Ant agent ip: {} is invalid'
''.format(cli_args['--ip']))
else:
agent_ip = cli_args['--ip']
else:
agent_ip = get_agent_ip(upstream_mes[0], int(upstream_mes[1]))
runner = cli_args['--user'] if cli_args['--user'] else os.environ.get(
'USER')
su_cmd = '' if runner == 'root' else 'su - {} -c '.format(runner)
conf_dict = {
'tenant': cli_args['--tenant'],
'ip': agent_ip,
'upstream': upstream,
'network_domain': cli_args['--network-domain']
}
init_conf(conf_dict)
init_bootstrap()
init_openresty(baseurl, upstream, runner)
register_service(su_cmd)
if runner and not IS_WINDOWS and runner != 'root':
status, result = execute('chown -R {user}:{user} {path}'.format(
user=runner, path=ROOT_DIR))
if status != 0:
raise MessageError(
'Change log path owen failed! Error[{}]: {}'.format(
status, result))
register_upgrade_service(su_cmd)
start_circled(su_cmd)
except Exception as e:
logger.error(e)
sys.exit(1)
def init_bootstrap():
bootstrap_file = nfs.join(ROOT_DIR, 'templates', 'bootstrap.py')
nfs.copy(bootstrap_file, ROOT_DIR)
