def copy_to(self, dst, plugin_parent='plugins'): if self.is_plugin: plugin_yml_path = fs.join(self.path, 'module.yml') if not fs.exists(plugin_yml_path): plugin_yml_path = fs.join(self.path, 'plugin.yml') if fs.exists(plugin_yml_path): import yaml info = yaml.load(open(plugin_yml_path)) fullname = '{}@{}'.format(info['name'], info['version']) dst = fs.join(dst, plugin_parent, fullname) fs.makedirs(dst) else: logger.error('module.yml or plugin.yml not exists') sys.exit(1) logger.info('Copy project: {!r} from {!r} to {!r}'.format( self.name, self.path, dst)) for dirname in fs.listdir(self.path): dirpath = fs.join(self.path, dirname) if dirname in (EXCLUDE_DIRS + EXCLUDE_FILES) \ or dirname.startswith('.'): continue fs.copy(dirpath, dst, exclude_dirs=EXCLUDE_DIRS, exclude_files=['*.exe', '*.bat'] if not IS_WINDOWS else ['*.sh']) return dst
def __init__(self, new_agent_dir, http_handler): self.new_agent_dir = new_agent_dir self.http_handler = http_handler self.bin_start = nfs.join(ROOT_DIR, 'bin', 'start' + SYSTEM_SCRIPT_TYPE) self.bin_stop = nfs.join(ROOT_DIR, 'bin', 'stop' + SYSTEM_SCRIPT_TYPE) self.copy = copy_files if IS_WINDOWS else nfs.copy
def backup_files(self): if nfs.exists(AGENT_BACK_DIR): nfs.remove(nfs.join(AGENT_BACK_DIR, '*')) else: nfs.makedirs(AGENT_BACK_DIR) # Copy self.http_handler.log_ok('Backup files') for dir_name in nfs.listdir(ROOT_DIR): if dir_name in EXCLUDE_BACK_DIRS: continue nfs.copy(nfs.join(ROOT_DIR, dir_name), AGENT_BACK_DIR) self.http_handler.log_ok('Backup done')
def execute(self): if IS_WINDOWS: remove_script = fs.join(ROOT_DIR, 'bin', 'remove.bat') subprocess.Popen([remove_script, '-f'], close_fds=True, shell=True, creationflags=subprocess.CREATE_NEW_CONSOLE) else: remove_script = fs.join(ROOT_DIR, 'bin', 'remove.sh') subprocess.Popen('/usr/bin/env bash {} -f &'.format(remove_script), shell=True) yield self.reporter.log_ok('Remove agent success!', done=True) self.io_loop.stop()
def execute(self): try: for module in self.modules: module_name = module['module_name'] module_env = self.deal_env(module.get('env')) if not module.get('filename'): yield self.reporter.log_error('Filename for {} is None' ''.format(module_name)) continue if not nfs.exists(nfs.join(PKG_DIR, module_name)): yield self.reporter.log_error('{} is not installed!' ''.format(module_name)) continue yield self.reporter.log_ok('Begin to Upgrade {}' ''.format(module_name)) pkg_path = yield self.do_pre(module['filename'], module_env) uninstall = Uninstall([module], self.io_loop, False) uninstall.reporter = self.reporter yield uninstall.remove(module_name) yield self.do_install(module) nfs.remove(pkg_path) yield self.reporter.log_ok('Finish upgrade modules!', True) yield self.circle_cmd('reloadconfig') except Exception as e: yield self.reporter.log_error(str(e), True) sys.exit(1)
def download_pkg(self, filename): yield self.reporter.log_ok('Begin to download package ' '{} ...'.format(filename)) down_url = self.download_url + '?filename=' + filename try: response = yield self.client.fetch( down_url, connect_timeout=config.get('file_service_connect_timeout', 3600.0), request_timeout=config.get('file_service_request_timeout', 3600.0), validate_cert=False) if response.code == 200: if not nfs.exists(PKG_CACHE_DIR): os.makedirs(PKG_CACHE_DIR) nfs.copy(response.body, nfs.join(PKG_CACHE_DIR, filename)) yield self.reporter.log_ok( 'Download package {} success'.format(filename)) else: raise MessageError( 'Download package {} failed, reason: {}!'.format( filename, response.body)) except HTTPError as e: raise MessageError( 'Download package {} failed, reason: {}, {}!'.format( filename, e, e.message))
def register_service(su_cmd): if not IS_WINDOWS: logger.info('Register service') if 'ubuntu' in PLATFORM or 'debian' in PLATFORM: format_template(DESCRIPTION['like_debian'], su_cmd) register_status, _ = execute( 'update-rc.d {} defaults'.format(SERVICE_NAME)) if register_status != 0: raise RegisterServiceError( 'Register {} Service Error'.format(SERVICE_NAME)) elif any(_p in PLATFORM for _p in ('centos', 'fedora', 'suse', 'redhat')): format_template(DESCRIPTION['like_redhat'], su_cmd) chk_status, _ = execute('chkconfig {} on'.format(SERVICE_NAME)) service_path = nfs.join(SERVICE_PATH, SERVICE_NAME) echo_status, _ = execute('echo "{service_path} start" ' '| tee --append {path} >/dev/null'.format( service_path=service_path, path=BOOT_SCRIPT)) chmod_status, _ = execute('chmod +x {}'.format(BOOT_SCRIPT)) if chk_status != 0 or echo_status != 0 or chmod_status != 0: raise RegisterServiceError( 'Register {} Service Error'.format(SERVICE_NAME)) else: logger.warn('Unsupported platform')
def deploy_new_agent(self): self.http_handler.log_ok('Deploying agent') dst = self.new_agent_dir if IS_WINDOWS \ else nfs.join(self.new_agent_dir, '*') self.copy(dst, ROOT_DIR) nfs.remove(self.new_agent_dir) self.http_handler.log_ok('Deploy done')
def do_pre(self, filename, module_env): yield self.download_pkg(filename) pkg_path, pkg_info = PkgHelper.uncompress(filename, PKG_UPGRADE_DIR) config = Config(nfs.join(pkg_path, PKG_YAML_NAME)) config.exec_lifecycle_script('pre_upgrade', cwd=pkg_path, env=module_env) yield self.reporter.log_ok('Finish exec pre_upgrade!') raise gen.Return(pkg_path)
def checkout_branch(self, dst=PROJECT_ROOT, branch='develop'): fs.chdir(fs.join(dst, self.name)) try: check_call(['git', 'checkout', branch]) except CalledProcessError as e: if e.returncode == 1: check_call( ['git', 'checkout', '-b', 'develop', 'origin/develop']) fs.chdir('../../')
def get_agent_version(): from constants import ROOT_DIR version = '' module_yaml = nfs.join(ROOT_DIR, 'manifest.yaml') with open(module_yaml) as temp: module_mes = yaml.load(temp.read()) if 'version' in module_mes: version = module_mes['version'] return version
def encrypt_py_project(target_root): for dirname in fs.listdir(target_root): if dirname in ('conf', 'embedded'): continue path = fs.join(target_root, dirname) if not fs.isdir(path): continue compile_dir(path) fs.remove(path, filter_files='*.py')
class Openresty(object): enable_conf = nfs.join(CIRCLE_CONF_DIR, 'openresty.ini') disable_conf = nfs.join(CIRCLE_CONF_DIR, 'openresty.ini.disable') client = AsyncCircleClient() @classmethod @gen.coroutine def reload(cls): props = {'waiting': True, 'path': cls.enable_conf} ret = yield cls.client.send_message('reloadconfig', **props) if isinstance(ret, dict): if ret.get('status') == 'error': raise MessageError(ret.get('reason')) raise gen.Return('Finish reload openresty!') @classmethod @gen.coroutine def rm_openresty(cls): props = {"name": "openresty", "nostop": False, "waiting": False} ret = yield cls.client.send_message('rm', **props) if isinstance(ret, dict): if ret.get('status') == 'error': raise MessageError(ret.get('reason')) raise gen.Return('Finish rm openresty!') @classmethod @gen.coroutine def enable(cls): if nfs.exists(cls.disable_conf): nfs.rename(cls.disable_conf, cls.enable_conf) if not nfs.exists(cls.enable_conf): raise MessageError('Enable openresty failed !') yield cls.reload() @classmethod @gen.coroutine def disable(cls): if nfs.exists(cls.enable_conf): nfs.rename(cls.enable_conf, cls.disable_conf) yield cls.rm_openresty() if not nfs.exists(cls.disable_conf): raise MessageError('Disable openresty failed !')
def _load_command_and_env(self, action): if action.startswith('core.'): self.module_name = 'core' command = settings.CORE_ACTIONS.get(action) else: self.module_name, _ = action.split('.') module_path = nfs.join(settings.MODULES_DIR, self.module_name) self.cwd = module_path yaml_file = nfs.join(module_path, 'manifest.yaml') if not nfs.exists(yaml_file): return None with open(yaml_file) as f: data = yaml.load(f) command = data['actions'].get(action) env = utils.normalize_env(data.get('env', {}), relpath_prefix=module_path) logger.debug('Action task {!r} env: {}'.format(self.action, env)) # 处理PATH开头的环境变量,加入到PATH环境变量中 paths = [self.env.get('PATH', '')] for name, value in env.iteritems(): if isinstance(value, int): value = str(value) else: value = value.encode('utf-8') if name.startswith('PATH'): paths.append(value) else: self.env[name] = value self.env['PATH'] = ';'.join(paths) if IS_WINDOWS else \ ':'.join(paths) logger.debug('Action task {!r} env PATH: {}'.format( self.action, self.env['PATH'])) command = utils.normalize_cmdline(command) return command
def maybe_download_python(): system = platform.system.lower() cwd = os.getcwd() if WINDOWS: python = fs.join(cwd, 'embedded/python.exe') pip = fs.join(cwd, 'embedded/Scripts/pip.exe') else: python = fs.join(cwd, 'embedded/bin/python') pip = fs.join(cwd, 'embedded/bin/pip') if not fs.exists(fs.join(cwd, 'embedded')): python_name = 'python-{}-{}.{}'.format(system, platform.cpu, POSTFIX.get(system, 'tgz')) python_url = PYTHON_TEMPLATE_URL.format(python_name) download(python_url, python_name) fs.uncompress(python_name) fs.rename('python-{}-{}'.format(system, platform.cpu), 'embedded') return python, pip
def download(self, dst=PROJECT_ROOT): logger.info('Download from {!r} to {!r}'.format(self.url, dst)) filename = self.url.split('/')[-1] dst = fs.join(dst, filename) req = requests.get(self.url, stream=True) with open(dst, 'wb') as f: for chunk in req.iter_content(chunk_size=1024): if chunk: f.write(chunk) return dst
def get(self): self.file_name = self.get_argument('filename') # type: str self.space_dir = nfs.join(settings.REPO_DIR, settings.REPO_ANT_SPACENAME) if not nfs.exists(self.space_dir): nfs.makedirs(self.space_dir) self.file_path = nfs.join(self.space_dir, self.file_name) lock_file_name = nfs.extsep + self.file_name + nfs.extsep + 'lock' self.lock_file = nfs.join(self.space_dir, lock_file_name) logger.info('#%d Request file: %s', id(self.request), self.file_name) if nfs.exists(self.lock_file): yield self.wait_for_file_complete() else: is_cache_hit = yield self.try_to_return_file_cache() if is_cache_hit: return logger.info('#%d File cache missed: %s', id(self.request), self.file_path) nfs.touch(self.lock_file) yield self.request_file_from_upstream()
def copy_basic_conf(self): self.http_handler.log_ok('Coping basic conf') try: new_conf_path = nfs.join(self.new_agent_dir, CONFIG_NAME) with open(CONFIG_PATH) as temp: yaml.dump(yaml.load(temp.read()), open(new_conf_path, 'a+'), default_flow_style=False) self.http_handler.log_ok('Copy conf done') except IOError as e: self.http_handler.log_error(str(e)) logger.error(str(e), exc_info=True) self.http_handler.log_ok('Skip copy basic conf')
def remove(self, module_name): try: ret = yield self.circle_cmd('stop', module_name) yield self.reporter.log_ok(ret) except Exception as e: if not re.match(r'program[\s\S]+not found', str(e)): yield self.reporter.log_error(str(e)) raise gen.Return(str(e)) self.pre_stop(module_name) config = Config(nfs.join(PKG_DIR, module_name, PKG_YAML_NAME)) # Execute pre_remove config.exec_lifecycle_script('pre_remove') # Remove nfs.remove(nfs.join(PKG_DIR, module_name)) # Execute post_remove config.exec_lifecycle_script('post_remove') # Remove conf file for circled circled_config = CircledConfig(module_name) circled_config.remove_file() # Result yield self.reporter.log_ok('Remove {!r} successfully' ''.format(module_name))
def _deliver_to_aix(self, compress_name): try: file_path = nfs.join(REPO_DIR, compress_name) if not nfs.exists(file_path): file_path = nfs.join(REPO_DIR, REPO_ANT_SPACENAME, compress_name) if not nfs.exists(file_path): down_url = 'http://127.0.0.1:16600/file?filename={}'\ .format(compress_name) client = AsyncHTTPClient(io_loop=ioloop.IOLoop.current()) response = yield client.fetch(down_url, connect_timeout=3600.0, request_timeout=3600.0, validate_cert=False) if response.code != 200: raise MessageError("Can't download pkg by http") yield self.do_ssh_cmd('umask 0027 && mkdir -p "{}"'.format( self.dst)) yield self.ssh_client.scp(os.path.realpath(file_path), self.dst_name) except Exception as e: raise MessageError('Download agent pkg failed. {}'.format(e))
def run_hooks(self): hooks_dir = nfs.join(self.new_agent_dir, 'hooks') if not nfs.exists(hooks_dir): return True for dir_name in nfs.listdir(hooks_dir): hooks_file = nfs.join(hooks_dir, dir_name) if nfs.isfile(hooks_file) and hooks_file.endswith('.py'): p = Popen([sys.executable, hooks_file], stdout=PIPE, stderr=STDOUT, cwd=ROOT_DIR, shell=True) while p.poll() is None: time.sleep(1) if p.stdout: logger.info(p.stdout.read()) if p.poll() != 0: self.http_handler.log_ok('Run hooks {} failed!' ''.format(hooks_file)) return False return True
def normalize_env(env, relpath_prefix=None): u""" 将环境变量中的路径标准化,python开头的内容会被替换为sys.executable 凡是盘符(如C:/, C:\)或./或.\开头的字符串均会通过nfs.normpath进行标准化处理 如果提供relpath_prefix,比如relpath_prefix='c:/', 环境变量中的相对路径均会改成绝对路径。 """ for name, value in env.items(): value = value.strip() if _is_path(value): if relpath_prefix: env[name] = nfs.normpath(nfs.join(relpath_prefix, value)) else: env[name] = nfs.normpath(value) else: env[name] = re.sub( '^ *python', '"{}"'.format(sys.executable.replace('\\', '\\\\')), value, flags=re.IGNORECASE) return env
def init_openresty(baseurl, upstream, runner): if not nfs.exists(NGINX_CONF): return with open(NGINX_CONF) as temp: content = temp.read() content = content.replace('UPSTREAM', upstream) content = content.replace('BASEURL', baseurl) if not IS_WINDOWS and runner: content = content.replace('RUNNER', runner) if IS_WINDOWS: BASEDIR = os.path.normpath(ROOT_DIR).replace('\\', '\\\\') else: BASEDIR = ROOT_DIR content = content.replace('BASEDIR', BASEDIR) with open(NGINX_CONF, 'w') as conf: conf.write(content) log_dir = nfs.join(ROOT_DIR, 'openresty', 'logs') if not nfs.exists(log_dir): nfs.makedirs(log_dir, 0750)
def register_upgrade_service(su_cmd): if not IS_WINDOWS: logger.info('Register upgrade service') if 'ubuntu' in PLATFORM or 'debian' in PLATFORM: format_upgrade_template(DESCRIPTION['like_debian'], su_cmd) register_status, _ = execute( 'update-rc.d {} defaults'.format(UPGRADE_SERVICE_NAME)) if register_status != 0: raise RegisterServiceError( 'Register {} Service Error'.format(UPGRADE_SERVICE_NAME)) elif any(_p in PLATFORM for _p in ('centos', 'fedora', 'suse', 'redhat')): format_upgrade_template(DESCRIPTION['like_redhat'], su_cmd) chk_status, _ = execute( 'chkconfig {} on'.format(UPGRADE_SERVICE_NAME)) service_path = nfs.join(SERVICE_PATH, UPGRADE_SERVICE_NAME) echo_status, _ = execute('echo "{service_path} start" ' '| tee --append {path} >/dev/null'.format( service_path=service_path, path=BOOT_SCRIPT)) chmod_status, _ = execute('chmod +x {}'.format(BOOT_SCRIPT)) if chk_status != 0 or echo_status != 0 or chmod_status != 0: raise RegisterServiceError( 'Register {} Service Error'.format(UPGRADE_SERVICE_NAME)) else: logger.warn('Unsupported platform') status, output = execute('{} "{}"'.format(su_cmd, UPGRADE_BIN_START)) if status != 0: raise MessageError( 'Start Upgrade failed: reason: {}'.format(output)) logger.info('Start Upgrade done') else: UpgradeWinService.install() UpgradeWinService.start()
def pull(self, dst=PROJECT_ROOT): fs.chdir(fs.join(dst, self.name)) check_call(['git', 'pull'])
def __init__(self, name, url): self.name = name self.url = url self.path = fs.join(PROJECT_ROOT, name)
def __init__(self, task_message): self.task_message = task_message self.bin_start = nfs.join(ROOT_DIR, 'bin', 'start' + SYSTEM_SCRIPT_TYPE) self.bin_stop = nfs.join(ROOT_DIR, 'bin', 'stop' + SYSTEM_SCRIPT_TYPE)
def handle_cli(): try: cli_args = docopt(__doc__) if IS_WINDOWS and not check_win_agent(): return if not nfs.exists(UPGRADE_PYTHON_DIR): nfs.copy(nfs.join(PYTHON_DIR, '*'), UPGRADE_PYTHON_DIR) # Get upstream message if cli_args['--upstream']: baseurl = cli_args['--upstream'] upstream = urlparse.urljoin(cli_args['--upstream'], UPSTREAM_SUFFIX) upstream_mes = upstream_validate(cli_args['--upstream']) if not upstream_mes: logger.error('The upstream: {} is wrong!' ''.format(cli_args['--upstream'])) return else: upstream_ip = os.environ['SSH_CLIENT'].split()[0] baseurl = 'http://{}:{}/'.format(upstream_ip, NGINX_PORT) upstream = '{}{}'.format(baseurl, UPSTREAM_SUFFIX) upstream_mes = [upstream_ip, NGINX_PORT] if cli_args['--ip']: if not ip_validate(cli_args['--ip']): raise ValueError('Ant agent ip: {} is invalid' ''.format(cli_args['--ip'])) else: agent_ip = cli_args['--ip'] else: agent_ip = get_agent_ip(upstream_mes[0], int(upstream_mes[1])) runner = cli_args['--user'] if cli_args['--user'] else os.environ.get( 'USER') su_cmd = '' if runner == 'root' else 'su - {} -c '.format(runner) conf_dict = { 'tenant': cli_args['--tenant'], 'ip': agent_ip, 'upstream': upstream, 'network_domain': cli_args['--network-domain'] } init_conf(conf_dict) init_bootstrap() init_openresty(baseurl, upstream, runner) register_service(su_cmd) if runner and not IS_WINDOWS and runner != 'root': status, result = execute('chown -R {user}:{user} {path}'.format( user=runner, path=ROOT_DIR)) if status != 0: raise MessageError( 'Change log path owen failed! Error[{}]: {}'.format( status, result)) register_upgrade_service(su_cmd) start_circled(su_cmd) except Exception as e: logger.error(e) sys.exit(1)
def init_bootstrap(): bootstrap_file = nfs.join(ROOT_DIR, 'templates', 'bootstrap.py') nfs.copy(bootstrap_file, ROOT_DIR)