def uncompress(cls, fullname, parent_dir, overwrite=True):
parent_dir = parent_dir if parent_dir else PKG_DIR
src = os.path.join(PKG_CACHE_DIR, fullname)
if not os.path.exists(src):
raise NotExistsError('The cache of '.format(fullname))
# ensure make target_dir
for dir_name in [parent_dir, PKG_UNCOMPRESS_DIR]:
if not nfs.exists(dir_name):
nfs.makedirs(dir_name, 0750)
# clear PKG_UNCOMPRESS_DIR
for name in os.listdir(PKG_UNCOMPRESS_DIR):
nfs.remove(os.path.join(PKG_UNCOMPRESS_DIR, name))
# extract file
with tarfile.open(src) as tar:
tar.extractall(PKG_UNCOMPRESS_DIR)
extract_path = os.path.join(PKG_UNCOMPRESS_DIR,
os.listdir(PKG_UNCOMPRESS_DIR)[0])
pkg_yml_path = os.path.join(extract_path, PKG_YAML_NAME)
pkg_info = cls.get_info(pkg_yml_path, fullname)
dst = os.path.join(parent_dir, pkg_info['name'])
if not overwrite and os.path.exists(dst):
return dst
nfs.rename(extract_path, dst, overwrite)
return dst, pkg_info
def execute(self):
try:
for module in self.modules:
module_name = module['module_name']
module_env = self.deal_env(module.get('env'))
if not module.get('filename'):
yield self.reporter.log_error('Filename for {} is None'
''.format(module_name))
continue
if not nfs.exists(nfs.join(PKG_DIR, module_name)):
yield self.reporter.log_error('{} is not installed！'
''.format(module_name))
continue
yield self.reporter.log_ok('Begin to Upgrade {}'
''.format(module_name))
pkg_path = yield self.do_pre(module['filename'], module_env)
uninstall = Uninstall([module], self.io_loop, False)
uninstall.reporter = self.reporter
yield uninstall.remove(module_name)
yield self.do_install(module)
nfs.remove(pkg_path)
yield self.reporter.log_ok('Finish upgrade modules!', True)
yield self.circle_cmd('reloadconfig')
except Exception as e:
yield self.reporter.log_error(str(e), True)
sys.exit(1)
def do_upgrade(self):
try:
try:
self.http_handler.log_ok('Start upgrader')
self.copy_basic_conf()
self.stop_agent()
self.deploy_new_agent()
self.start_agent()
if not self.run_hooks():
self.roll_back()
self.http_handler.log_error(
'Upgrade failed. Roll back successfully', done=True)
else:
nfs.remove(AGENT_BACK_DIR)
self.http_handler.log_ok('Upgrade successfully', done=True)
except Exception as e:
self.http_handler.log_error(str(e))
logger.error(str(e), exc_info=True)
self.roll_back()
self.http_handler.log_error(
'Upgrade failed. Roll back successfully', done=True)
except Exception:
self.start_agent()
self.http_handler.log_error(
'Upgrade failed. Roll back successfully', done=True)
def deploy_new_agent(self):
self.http_handler.log_ok('Deploying agent')
dst = self.new_agent_dir if IS_WINDOWS \
else nfs.join(self.new_agent_dir, '*')
self.copy(dst, ROOT_DIR)
nfs.remove(self.new_agent_dir)
self.http_handler.log_ok('Deploy done')
def encrypt_py_project(target_root):
for dirname in fs.listdir(target_root):
if dirname in ('conf', 'embedded'):
continue
path = fs.join(target_root, dirname)
if not fs.isdir(path):
continue
compile_dir(path)
fs.remove(path, filter_files='*.py')
def init_bootstrap():
bootstrap = os.path.join(os.getcwd(), 'bootstrap.py')
if os.path.exists(bootstrap):
logger.info('Detect bootstrap script, ready to perform')
exit_code = subprocess.call([sys.executable, bootstrap])
if exit_code != 0:
logger.info('Abnormal exit code: %d', exit_code)
sys.exit(exit_code)
else:
logger.info('Perform bootstrap script successfully')
nfs.remove(bootstrap)
def backup_files(self):
if nfs.exists(AGENT_BACK_DIR):
nfs.remove(nfs.join(AGENT_BACK_DIR, '*'))
else:
nfs.makedirs(AGENT_BACK_DIR)
# Copy
self.http_handler.log_ok('Backup files')
for dir_name in nfs.listdir(ROOT_DIR):
if dir_name in EXCLUDE_BACK_DIRS:
continue
nfs.copy(nfs.join(ROOT_DIR, dir_name), AGENT_BACK_DIR)
self.http_handler.log_ok('Backup done')
def close_file_resource(self):
try:
if self.temp_file and not self.temp_file.closed:
self.temp_file.close()
if nfs.exists(self.lock_file):
nfs.remove(self.lock_file)
except Exception as exc:
logger.error('#%d Error while closing resource (%s): %s',
id(self.request),
self.file_path,
exc,
exc_info=True)
self.send_error(500, message=exc) # FIXME: 有可能是请求结束后调用
def try_to_return_file_cache(self):
is_cache_hit = False
if nfs.exists(self.file_path):
flag = yield self.check_file_mtime()
if flag:
logger.info('#%d File cache hit: %s', id(self.request),
self.file_path)
self.write(self.file_path) # 直接返回本地缓存文件的路径
is_cache_hit = True
else:
logger.info('#{} The cache file is too old and need to '
'download the new file'.format(id(self.request)))
nfs.remove(self.file_path)
raise gen.Return(is_cache_hit)
def umcompress(self, compress_agent_path):
self.http_handler.log_ok('Removing {!r}...'
''.format(AGENT_UNCOMPRESS_DIRNAME))
if nfs.exists(AGENT_UNCOMPRESS_DIR):
nfs.remove(AGENT_UNCOMPRESS_DIR)
self.http_handler.log_ok('Remove {!r} done'
''.format(AGENT_UNCOMPRESS_DIRNAME))
self.http_handler.log_ok('Uncompressing {}...'
''.format(self.task_message['filename']))
os.makedirs(AGENT_UNCOMPRESS_DIR)
uncompress_agent_path = nfs.uncompress(compress_agent_path,
AGENT_UNCOMPRESS_DIR,
temp_dir=AGENT_UNCOMPRESS_DIR)
self.http_handler.log_ok('Uncompress done')
return uncompress_agent_path
def deploy_dispatcher(self):
with cd(self.project_name):
execute(['../node/bin/node', '../node/bin/yarn'])
execute(['../node/bin/node',
'../node/lib/node_modules/npm/bin/npm-cli.js', 'run',
'build'])
if nfs.exists('dispatcher'):
nfs.remove('dispatcher')
nfs.rename('dist', 'dispatcher')
cmd = 'su uyun -c "pm2 delete all"'
excutor_cmd(cmd)
for file in ['node_modules', 'bin', 'scripts', 'install.sh',
'uninstall.sh', 'check_status.sh', 'dispatcher']:
local_path = os.path.join(self.project_name, file)
server_path = INSTALL_DIR[self.project_name]
scp_upload_file(local_path, server_path)
cmd = 'cd {} && su uyun -c "pm2 start process.json"'.format(
INSTALL_DIR[self.project_name])
excutor_cmd(cmd)
def pip_install(python_package, requeriment_txt, exclude_pkgs):
with open(requeriment_txt) as f:
to_install = f.readlines()
for pkg in to_install:
if '-r' in pkg:
print pkg
to_install.remove(pkg)
txt = pkg.split()[-1]
with open(txt) as file:
new_install = file.readlines()
else:
new_install = []
to_install = [x.strip() for x in to_install + new_install]
if exclude_pkgs:
for pkg in exclude_pkgs.strip().split(','):
if pkg in to_install:
to_install.remove(pkg)
if 'win' in python_package:
python_exe = ''
pip_exe = os.path.join(python_package, 'Scripts\pip.exe')
python_url = PYTHON_URL.format(python_package, 'zip')
else:
python_exe = os.path.join(python_package, 'bin/python')
pip_exe = os.path.join(python_package, 'bin/pip')
python_url = PYTHON_URL.format(python_package, 'tgz')
target_dir = os.path.join(ROOT_DIR, os.path.basename(python_url))
if not os.path.exists(python_package):
Step.download(python_url, target_dir)
fs.uncompress(target_dir, ROOT_DIR)
fs.remove(target_dir)
already_install = Step.get_output('{} {} freeze'.format(
python_exe, pip_exe))
already_install = [x.strip() for x in already_install]
if set(to_install) & set(already_install) != set(to_install):
for requ in list(set(to_install) - set(already_install)):
Step.get_output('{} {} install -U {}'.format(
python_exe, pip_exe, requ))
def remove(self, module_name):
try:
ret = yield self.circle_cmd('stop', module_name)
yield self.reporter.log_ok(ret)
except Exception as e:
if not re.match(r'program[\s\S]+not found', str(e)):
yield self.reporter.log_error(str(e))
raise gen.Return(str(e))
self.pre_stop(module_name)
config = Config(nfs.join(PKG_DIR, module_name, PKG_YAML_NAME))
# Execute pre_remove
config.exec_lifecycle_script('pre_remove')
# Remove
nfs.remove(nfs.join(PKG_DIR, module_name))
# Execute post_remove
config.exec_lifecycle_script('post_remove')
# Remove conf file for circled
circled_config = CircledConfig(module_name)
circled_config.remove_file()
# Result
yield self.reporter.log_ok('Remove {!r} successfully'
''.format(module_name))
def delete(module_name):
glob_path = os.path.join(PKG_DIR, module_name)
for pkg_path in glob(glob_path):
nfs.remove(pkg_path)
def remove_file(self):
if os.path.exists(self.able_path):
nfs.remove(self.able_path)
def remove(self):
fs.remove(self.path)
def clear_old_dir(dst):
if fs.exists(dst):
logger.info('Clear {!r}'.format(dst))
fs.remove(dst)
