def uncompress(cls, fullname, parent_dir, overwrite=True): parent_dir = parent_dir if parent_dir else PKG_DIR src = os.path.join(PKG_CACHE_DIR, fullname) if not os.path.exists(src): raise NotExistsError('The cache of '.format(fullname)) # ensure make target_dir for dir_name in [parent_dir, PKG_UNCOMPRESS_DIR]: if not nfs.exists(dir_name): nfs.makedirs(dir_name, 0750) # clear PKG_UNCOMPRESS_DIR for name in os.listdir(PKG_UNCOMPRESS_DIR): nfs.remove(os.path.join(PKG_UNCOMPRESS_DIR, name)) # extract file with tarfile.open(src) as tar: tar.extractall(PKG_UNCOMPRESS_DIR) extract_path = os.path.join(PKG_UNCOMPRESS_DIR, os.listdir(PKG_UNCOMPRESS_DIR)[0]) pkg_yml_path = os.path.join(extract_path, PKG_YAML_NAME) pkg_info = cls.get_info(pkg_yml_path, fullname) dst = os.path.join(parent_dir, pkg_info['name']) if not overwrite and os.path.exists(dst): return dst nfs.rename(extract_path, dst, overwrite) return dst, pkg_info
def execute(self): try: for module in self.modules: module_name = module['module_name'] module_env = self.deal_env(module.get('env')) if not module.get('filename'): yield self.reporter.log_error('Filename for {} is None' ''.format(module_name)) continue if not nfs.exists(nfs.join(PKG_DIR, module_name)): yield self.reporter.log_error('{} is not installed!' ''.format(module_name)) continue yield self.reporter.log_ok('Begin to Upgrade {}' ''.format(module_name)) pkg_path = yield self.do_pre(module['filename'], module_env) uninstall = Uninstall([module], self.io_loop, False) uninstall.reporter = self.reporter yield uninstall.remove(module_name) yield self.do_install(module) nfs.remove(pkg_path) yield self.reporter.log_ok('Finish upgrade modules!', True) yield self.circle_cmd('reloadconfig') except Exception as e: yield self.reporter.log_error(str(e), True) sys.exit(1)
def do_upgrade(self): try: try: self.http_handler.log_ok('Start upgrader') self.copy_basic_conf() self.stop_agent() self.deploy_new_agent() self.start_agent() if not self.run_hooks(): self.roll_back() self.http_handler.log_error( 'Upgrade failed. Roll back successfully', done=True) else: nfs.remove(AGENT_BACK_DIR) self.http_handler.log_ok('Upgrade successfully', done=True) except Exception as e: self.http_handler.log_error(str(e)) logger.error(str(e), exc_info=True) self.roll_back() self.http_handler.log_error( 'Upgrade failed. Roll back successfully', done=True) except Exception: self.start_agent() self.http_handler.log_error( 'Upgrade failed. Roll back successfully', done=True)
def deploy_new_agent(self): self.http_handler.log_ok('Deploying agent') dst = self.new_agent_dir if IS_WINDOWS \ else nfs.join(self.new_agent_dir, '*') self.copy(dst, ROOT_DIR) nfs.remove(self.new_agent_dir) self.http_handler.log_ok('Deploy done')
def encrypt_py_project(target_root): for dirname in fs.listdir(target_root): if dirname in ('conf', 'embedded'): continue path = fs.join(target_root, dirname) if not fs.isdir(path): continue compile_dir(path) fs.remove(path, filter_files='*.py')
def init_bootstrap(): bootstrap = os.path.join(os.getcwd(), 'bootstrap.py') if os.path.exists(bootstrap): logger.info('Detect bootstrap script, ready to perform') exit_code = subprocess.call([sys.executable, bootstrap]) if exit_code != 0: logger.info('Abnormal exit code: %d', exit_code) sys.exit(exit_code) else: logger.info('Perform bootstrap script successfully') nfs.remove(bootstrap)
def backup_files(self): if nfs.exists(AGENT_BACK_DIR): nfs.remove(nfs.join(AGENT_BACK_DIR, '*')) else: nfs.makedirs(AGENT_BACK_DIR) # Copy self.http_handler.log_ok('Backup files') for dir_name in nfs.listdir(ROOT_DIR): if dir_name in EXCLUDE_BACK_DIRS: continue nfs.copy(nfs.join(ROOT_DIR, dir_name), AGENT_BACK_DIR) self.http_handler.log_ok('Backup done')
def close_file_resource(self): try: if self.temp_file and not self.temp_file.closed: self.temp_file.close() if nfs.exists(self.lock_file): nfs.remove(self.lock_file) except Exception as exc: logger.error('#%d Error while closing resource (%s): %s', id(self.request), self.file_path, exc, exc_info=True) self.send_error(500, message=exc) # FIXME: 有可能是请求结束后调用
def try_to_return_file_cache(self): is_cache_hit = False if nfs.exists(self.file_path): flag = yield self.check_file_mtime() if flag: logger.info('#%d File cache hit: %s', id(self.request), self.file_path) self.write(self.file_path) # 直接返回本地缓存文件的路径 is_cache_hit = True else: logger.info('#{} The cache file is too old and need to ' 'download the new file'.format(id(self.request))) nfs.remove(self.file_path) raise gen.Return(is_cache_hit)
def umcompress(self, compress_agent_path): self.http_handler.log_ok('Removing {!r}...' ''.format(AGENT_UNCOMPRESS_DIRNAME)) if nfs.exists(AGENT_UNCOMPRESS_DIR): nfs.remove(AGENT_UNCOMPRESS_DIR) self.http_handler.log_ok('Remove {!r} done' ''.format(AGENT_UNCOMPRESS_DIRNAME)) self.http_handler.log_ok('Uncompressing {}...' ''.format(self.task_message['filename'])) os.makedirs(AGENT_UNCOMPRESS_DIR) uncompress_agent_path = nfs.uncompress(compress_agent_path, AGENT_UNCOMPRESS_DIR, temp_dir=AGENT_UNCOMPRESS_DIR) self.http_handler.log_ok('Uncompress done') return uncompress_agent_path
def deploy_dispatcher(self): with cd(self.project_name): execute(['../node/bin/node', '../node/bin/yarn']) execute(['../node/bin/node', '../node/lib/node_modules/npm/bin/npm-cli.js', 'run', 'build']) if nfs.exists('dispatcher'): nfs.remove('dispatcher') nfs.rename('dist', 'dispatcher') cmd = 'su uyun -c "pm2 delete all"' excutor_cmd(cmd) for file in ['node_modules', 'bin', 'scripts', 'install.sh', 'uninstall.sh', 'check_status.sh', 'dispatcher']: local_path = os.path.join(self.project_name, file) server_path = INSTALL_DIR[self.project_name] scp_upload_file(local_path, server_path) cmd = 'cd {} && su uyun -c "pm2 start process.json"'.format( INSTALL_DIR[self.project_name]) excutor_cmd(cmd)
def pip_install(python_package, requeriment_txt, exclude_pkgs): with open(requeriment_txt) as f: to_install = f.readlines() for pkg in to_install: if '-r' in pkg: print pkg to_install.remove(pkg) txt = pkg.split()[-1] with open(txt) as file: new_install = file.readlines() else: new_install = [] to_install = [x.strip() for x in to_install + new_install] if exclude_pkgs: for pkg in exclude_pkgs.strip().split(','): if pkg in to_install: to_install.remove(pkg) if 'win' in python_package: python_exe = '' pip_exe = os.path.join(python_package, 'Scripts\pip.exe') python_url = PYTHON_URL.format(python_package, 'zip') else: python_exe = os.path.join(python_package, 'bin/python') pip_exe = os.path.join(python_package, 'bin/pip') python_url = PYTHON_URL.format(python_package, 'tgz') target_dir = os.path.join(ROOT_DIR, os.path.basename(python_url)) if not os.path.exists(python_package): Step.download(python_url, target_dir) fs.uncompress(target_dir, ROOT_DIR) fs.remove(target_dir) already_install = Step.get_output('{} {} freeze'.format( python_exe, pip_exe)) already_install = [x.strip() for x in already_install] if set(to_install) & set(already_install) != set(to_install): for requ in list(set(to_install) - set(already_install)): Step.get_output('{} {} install -U {}'.format( python_exe, pip_exe, requ))
def remove(self, module_name): try: ret = yield self.circle_cmd('stop', module_name) yield self.reporter.log_ok(ret) except Exception as e: if not re.match(r'program[\s\S]+not found', str(e)): yield self.reporter.log_error(str(e)) raise gen.Return(str(e)) self.pre_stop(module_name) config = Config(nfs.join(PKG_DIR, module_name, PKG_YAML_NAME)) # Execute pre_remove config.exec_lifecycle_script('pre_remove') # Remove nfs.remove(nfs.join(PKG_DIR, module_name)) # Execute post_remove config.exec_lifecycle_script('post_remove') # Remove conf file for circled circled_config = CircledConfig(module_name) circled_config.remove_file() # Result yield self.reporter.log_ok('Remove {!r} successfully' ''.format(module_name))
def delete(module_name): glob_path = os.path.join(PKG_DIR, module_name) for pkg_path in glob(glob_path): nfs.remove(pkg_path)
def remove_file(self): if os.path.exists(self.able_path): nfs.remove(self.able_path)
def remove(self): fs.remove(self.path)
def clear_old_dir(dst): if fs.exists(dst): logger.info('Clear {!r}'.format(dst)) fs.remove(dst)