def run(param):
version = param['version']
tar_filename = param['config_package']['source']['file'].format(
version=version)
tar_file = os.path.join(param['package_path']['misc_dir'], 'download',
tar_filename)
main_dir = param['config_package']['source'].get(
'main', '').format(version=version)
dst_dir = param['config_package'].get('path', {}).get('source')
if not dst_dir:
return {'success': False, 'message': 'Path "source" is not specified'}
safe_rmdir(dst_dir)
safe_mkdir(dst_dir)
if main_dir:
strip_number = main_dir.strip(os.sep).count(os.sep) + 1
cmd = [
'tar', '--strip-components',
str(strip_number), '-xvf', tar_file, main_dir
]
else:
cmd = ['tar', '-xvf', tar_file]
with open(param['log_file'], 'w') as f:
ret = call_and_log(cmd, log=f, cwd=dst_dir)
return {'success': ret == 0, 'message': 'Tar exit code: {0}'.format(ret)}
def execute(self,
package,
category,
subdir,
version,
category_origin,
subdir_origin,
version_origin,
from_install=False):
config_package_name = 'package_install' if from_install else 'package_runtime'
pkg_cfg_origin = copy.deepcopy(
self._config[config_package_name].package_config(
category_origin, subdir_origin, package,
version_origin)['config_origin'])
pkg_cfg_origin['version'] = version
pkg_path = package_path(self._config['app'], self._config['category'],
category, subdir, package, version)
ctg_cf, sd_cf, pkg_cf, ver_cf = check_conflict_package(
pkg_path['main_dir'], self._config['package_runtime'])
if ctg_cf:
raise InstallPackageConfigError(
'Package path conflicts with package "{0}", category "{1}", subdir "{2}", version "{3}"'
.format(pkg_cf, ctg_cf, sd_cf, ver_cf))
safe_mkdir(pkg_path['config_dir'])
dump_config(pkg_cfg_origin, pkg_path['config_file'])
self._config.reset()
def run(param):
build_dir = param['config_package'].get('path', {}).get('build')
if not build_dir:
return {'success': False, 'message': 'Path "build" is not specified'}
install_dir = param['config_package'].get('path', {}).get('install')
if not install_dir:
return {'success': False, 'message': 'Path "install" is not specified'}
safe_mkdir(install_dir)
install_args = param['config_package'].get('make_install',
{}).get('args', ['install'])
install_args = ensure_list(install_args)
env = param.get('env')
env_install = env.copy()
for k, v in param['config_package'].get('make_install',
{}).get('env', {}).items():
env_install[k] = v.format(**param['config_package'].get('path', {}))
with open(param['log_file'], 'w') as f:
cmd = ['make'] + install_args
ret = call_and_log(cmd, log=f, cwd=build_dir, env=env_install)
return {
'success': ret == 0,
'message': 'Make install exit code: {0}'.format(ret)
}
def run(param):
release_version = param['config_scenario']['version']
if len(param['command']) > 1:
destination = param['command'][1]
else:
destination = os.getcwd()
destination = expand_path(destination)
pack_dir = os.path.join(destination, 'pack_' + release_version)
for category in param['config_package_install']:
for subdir in param['config_package_install'][category]:
for package in param['config_package_install'][category][subdir]:
for version, value in param['config_package_install'][
category][subdir][package].items():
pkg_cfg = value['config_origin']
download = pkg_cfg.get('install', {}).get('download')
if not download:
continue
url = pkg_cfg.get('source', {}).get('url')
if not url:
continue
pkg_dir = os.path.join(pack_dir, package, version)
safe_mkdir(pkg_dir)
_logger.info('Packing {0}...'.format(package))
if download == 'http':
url = url.format(version=version)
pkg_file = _download_http(url, pkg_dir)
elif download == 'svn':
url = url.format(version=version)
pkg_file = _download_svn(url, pkg_dir, package,
version)
elif download == 'git':
tag = pkg_cfg.get('source', {}).get('tag', 'master')
tag = tag.format(version=version)
pkg_file = _download_git(url, tag, pkg_dir, package,
version)
else:
_logger.warn('Package {0} not packed'.format(package))
continue
with open(os.path.join(pkg_dir, 'md5sum.txt'), 'w') as f:
call(['md5sum', pkg_file], cwd=pkg_dir, stdout=f)
with open(os.path.join(pkg_dir, 'sha1sum.txt'), 'w') as f:
call(['sha1sum', pkg_file], cwd=pkg_dir, stdout=f)
with open(os.path.join(pkg_dir, 'url.txt'), 'w') as f:
f.write(url + '\n')
_logger.info('Package {0} packed'.format(package))
_logger.info('All packages in version {0} packed in {1}'.format(
release_version, pack_dir))
def execute(self, param):
pkg = param['package']
step = param['step']
sub_step = param['sub_step']
if sub_step == 0:
step_full_name = '{0} - {1}'.format(pkg, step)
else:
step_full_name = '{0} - {1} - {2}'.format(pkg, step, sub_step)
if not param['action_install']:
_logger.debug('Skip step: {0}'.format(step_full_name))
return {'success': True, 'skip': True}
result = {}
result['start'] = datetime.datetime.utcnow()
safe_mkdir(os.path.dirname(param['log_file']))
try:
with Handler() as h:
result_action = h.run('install', param)
except HandlerNotFoundError as e:
_logger.error('Install handler "{0}" not found for "{1}"'.format(
param['action'], step_full_name))
raise
except Exception as e:
_logger.error('Install handler "{0}" error for {1}: {2}'.format(
param['action'], step_full_name, e))
if param['config_output']['verbose']:
_logger.error('\n{0}'.format(traceback.format_exc()))
raise
if isinstance(result_action, bool):
result['success'] = result_action
elif isinstance(result_action, dict):
result['success'] = result_action.get('success', False)
else:
result['success'] = False
if not result['success']:
if isinstance(result_action, dict) and 'message' in result_action:
_logger.error('"{0}" execution error: {1}'.format(
step_full_name, result_action['message']))
_logger.error('"{0}" execution error. Find log in "{1}"'.format(
step_full_name, param['log_file']))
raise InstallExecutorError(
'"{0}" execution error'.format(step_full_name))
result['action'] = result_action
result['end'] = datetime.datetime.utcnow()
return result
def execute(self, config_user, config_version, destination):
release_version = config_version.get('version')
try:
config_release = ConfigRelease(config_version)
except ConfigReleaseError:
_logger.debug(
'Install release definition: {0}'.format(release_version))
install = Install(config_user, config_version)
config_release = install.config_release()
self.__pkg_mgr = PackageManager(config_version, config_release)
if not destination:
destination = os.getcwd()
destination = expand_path(destination)
pack_dir = os.path.join(destination, 'pack_' + release_version)
for pkg, pkg_info in self.__pkg_mgr.package_all().items():
version = pkg_info.get('package', {}).get('version')
if not version:
continue
download = pkg_info.get('install', {}).get('download')
if not download:
continue
url = download.get('param', {}).get('url')
if not url:
continue
url = url.format(version=version)
pkg_dir = os.path.join(pack_dir, pkg, version)
safe_mkdir(pkg_dir)
_logger.info('Packing {0}...'.format(pkg))
if download.get('handler') == 'http':
pkg_file = _download_http(url, pkg_dir)
elif download.get('handler') == 'svn':
pkg_file = _download_svn(url, pkg_dir, pkg, version)
with open(os.path.join(pkg_dir, 'md5sum.txt'), 'w') as f:
call(['md5sum', pkg_file], cwd=pkg_dir, stdout=f)
with open(os.path.join(pkg_dir, 'sha1sum.txt'), 'w') as f:
call(['sha1sum', pkg_file], cwd=pkg_dir, stdout=f)
with open(os.path.join(pkg_dir, 'url.txt'), 'w') as f:
f.write(url + '\n')
_logger.info('Package {0} packed'.format(pkg))
_logger.info('All packages in version {0} packed in {1}'.format(
release_version, pack_dir))
def execute(self, param):
pkg = param['package']
action = param['action']
sub_action = param['sub_action']
if sub_action == 0:
action_full_name = '{0} - {1}'.format(pkg, action)
else:
action_full_name = '{0} - {1} - {2}'.format(
pkg, action, sub_action)
result = {}
result['start'] = datetime.datetime.utcnow()
safe_mkdir(param['pkg_info']['dir']['log'])
try:
result_action = run_handler('', 'install', param['action_name'],
param)
except Exception as e:
_logger.critical('"{0}" install handler error: {1}'.format(
action_full_name, e))
if param['config_user']['verbose']:
_logger.critical('\n{0}'.format(traceback.format_exc()))
raise
result['success'] = False
if isinstance(result_action, bool) and result_action:
result['success'] = True
if isinstance(
result_action, dict
) and 'success' in result_action and result_action['success']:
result['success'] = True
if not result['success']:
if isinstance(result_action, dict) and 'message' in result_action:
_logger.error('"{0}" execution error: {1}'.format(
action_full_name, result_action['message']))
_logger.critical('"{0}" execution error. Find log in "{1}"'.format(
action_full_name, param['log_file']))
raise InstallExecutorError(
'"{0}" execution error'.format(action_full_name))
result['action'] = result_action
result['end'] = datetime.datetime.utcnow()
return result
def save_release_status(self, end_time):
if not self['category'].get('install'):
_logger.warn('Will not save release status for: {0}'.format(
self['name']))
return
release_status = self.__load_status(self.__release_status_file())
if 'version' not in release_status:
release_status['version'] = []
release_version = self.__config_release.get('version')
if release_version and release_version not in release_status['version']:
release_status['version'].append(release_version)
safe_mkdir(self['dir']['status'])
dump_config(release_status, self.__release_status_file())
def run(param):
version = param['version']
url = param['config_package']['source']['url'].format(version=version)
source_dir = param['config_package'].get('path', {}).get('source')
if not source_dir:
return {'success': False, 'message': 'Path "source" is not specified'}
safe_rmdir(source_dir)
safe_mkdir(source_dir)
cmd = ['svn', 'export', '--force', url, source_dir]
with open(param['log_file'], 'w') as f:
ret = call_and_log(cmd, log=f)
return {'success': ret == 0, 'message': 'Svn exit code: {0}'.format(ret)}
def execute(self, package, category, subdir, version):
pkg_path = package_path(self._config['app'], self._config['category'],
category, subdir, package, version)
ctg_cf, sd_cf, pkg_cf, ver_cf = check_conflict_package(
pkg_path['main_dir'], self._config['package_runtime'])
if ctg_cf:
raise CreatePackageConfigError(
'Package path conflicts with package "{0}", category "{1}", subdir "{2}", version "{3}"'
.format(pkg_cf, ctg_cf, sd_cf, ver_cf))
safe_mkdir(pkg_path['config_dir'])
if not os.path.exists(pkg_path['config_file']):
open(pkg_path['config_file'], 'w').close()
self._config.reset()
def save_release_status(self, pkg, end_time):
if pkg not in self.__pkgs:
raise PackageNotFoundError('Package {0} not found'.format(pkg))
if not self.__pkgs[pkg]['config_category'].get('install'):
_logger.warn('Will not save release status for: {0}'.format(pkg))
return
release_status = self.__load_status(self.__release_status_file(pkg))
if 'version' not in release_status:
release_status['version'] = []
release_version = self.__config_release.get('version')
if release_version and release_version not in release_status['version']:
release_status['version'].append(release_version)
safe_mkdir(self.__pkgs[pkg]['dir']['status'])
dump_config(release_status, self.__release_status_file(pkg))
def run(param):
version = param['version']
url = param['config_package']['source']['url'].format(version=version)
filename = param['config_package']['source']['file'].format(
version=version)
md5url = param['config_package']['source'].get('md5url',
'').format(version=version)
dst_dir = os.path.join(param['package_path']['misc_dir'], 'download')
safe_rmdir(dst_dir)
safe_mkdir(dst_dir)
with open(param['log_file'], 'w') as f:
cmd = ['curl', '-v', '-f', '-L', '-s', '-S', '-R', '-O', url]
ret = call_and_log(cmd, log=f, cwd=dst_dir)
if ret != 0:
return {
'success': False,
'message':
'File download error, CURL exit code: {0}'.format(ret)
}
if md5url:
cmd = [
'curl', '-v', '-f', '-L', '-s', '-S', '-R', '-o', 'md5sum.txt',
md5url
]
ret = call_and_log(cmd, log=f, cwd=dst_dir)
if ret != 0:
return {
'success':
False,
'message':
'md5sum download error, CURL exit code: {0}'.format(ret)
}
cmd = ['md5sum', '-c', 'md5sum.txt']
ret = call_and_log(cmd, log=f, cwd=dst_dir)
if ret != 0:
return {
'success': False,
'message': 'md5 checksum error: {0}'.format(ret)
}
return {'success': ret == 0, 'message': 'CURL exit code: {0}'.format(ret)}
def save_action_status(self, action, start, end):
if not self['category'].get('install'):
_logger.warn('Will not save install status for: {0}.{1}'.format(
self['name'], action))
return
install_status = self.__load_status(self.__install_status_file())
if 'steps' not in install_status:
install_status['steps'] = {}
if action not in install_status['steps']:
install_status['steps'][action] = {}
install_status['steps'][action]['finished'] = True
install_status['steps'][action]['start'] = start
install_status['steps'][action]['end'] = end
safe_mkdir(self['dir']['status'])
dump_config(install_status, self.__install_status_file())
def save_action_status(self, pkg, action, start, end):
if pkg not in self.__pkgs:
raise PackageNotFoundError('Package {0} not found'.format(pkg))
if not self.__pkgs[pkg]['config_category'].get('install'):
_logger.warn('Will not save install status for: {0}.{1}'.format(
pkg, action))
return
install_status = self.__load_status(self.__install_status_file(pkg))
if 'steps' not in install_status:
install_status['steps'] = {}
if action not in install_status['steps']:
install_status['steps'][action] = {}
install_status['steps'][action]['finished'] = True
install_status['steps'][action]['start'] = start
install_status['steps'][action]['end'] = end
safe_mkdir(self.__pkgs[pkg]['dir']['status'])
dump_config(install_status, self.__install_status_file(pkg))
def run(param):
source_dir = param['config_package'].get('path', {}).get('source')
if not source_dir:
return {'success': False, 'message': 'Path "source" is not specified'}
build_dir = param['config_package'].get('path', {}).get('build')
if not build_dir:
return {'success': False, 'message': 'Path "build" is not specified'}
install_dir = param['config_package'].get('path', {}).get('install')
if not install_dir:
return {'success': False, 'message': 'Path "install" is not specified'}
if source_dir != build_dir:
safe_rmdir(build_dir)
safe_mkdir(build_dir)
cmake_args = param['config_package'].get('cmake', {}).get('args', [])
cmake_args = ensure_list(cmake_args)
cmake_args = [p.format(**param['config_package_install_path']) for p in cmake_args]
if not param['config_package'].get('cmake', {}).get('ignore_install_prefix', False):
cmake_args.insert(0, '-DCMAKE_INSTALL_PREFIX='+install_dir)
cmake_var = param['config_package'].get('cmake', {}).get('var', {})
for k, v in cmake_var.items():
full_value = v.format(**param['config_package_install_path'])
full_arg = '-D{0}={1}'.format(k, full_value)
cmake_args.append(full_arg)
env = param.get('env')
with open(param['log_file'], 'w') as f:
cmd = ['cmake', source_dir] + cmake_args
ret = call_and_log(cmd, log=f, cwd=build_dir, env=env)
return {'success': ret==0, 'message': 'CMake exit code: {0}'.format(ret)}
def run(param):
version = param['version']
url = param['config_package']['source']['url']
tag = param['config_package']['source'].get('tag',
'').format(version=version)
keep_dotgit = param['config_package']['source'].get('keep_dotgit', False)
branch = param['config_package']['source'].get('branch', 'develop')
if not tag:
return {'success': False, 'message': 'Git tag is not specified'}
source_dir = param['config_package'].get('path', {}).get('source')
if not source_dir:
return {'success': False, 'message': 'Path "source" is not specified'}
safe_rmdir(source_dir)
safe_mkdir(source_dir)
with open(param['log_file'], 'w') as f:
cmd = ['git', 'clone', url, source_dir]
ret = call_and_log(cmd, log=f)
if ret != 0:
return {
'success': False,
'message': 'Git clone exit code: {0}'.format(ret)
}
cmd = ['git', 'checkout', tag, '-b', branch]
ret = call_and_log(cmd, log=f, cwd=source_dir)
if ret != 0:
return {
'success': False,
'message': 'Git checkout tag exit code: {0}'.format(ret)
}
if not keep_dotgit:
safe_rmdir(os.path.join(source_dir, '.git'))
return {'success': ret == 0, 'message': 'Git OK'}
def run(param):
source_dir = param['config_package'].get('path', {}).get('source')
if not source_dir:
return {'success': False, 'message': 'Path "source" is not specified'}
build_dir = param['config_package'].get('path', {}).get('build')
if not build_dir:
return {'success': False, 'message': 'Path "build" is not specified'}
install_dir = param['config_package'].get('path', {}).get('install')
if not install_dir:
return {'success': False, 'message': 'Path "install" is not specified'}
if source_dir != build_dir:
safe_rmdir(build_dir)
safe_mkdir(build_dir)
configure_args = param['config_package'].get('configure', {}).get('args', [])
configure_args = ensure_list(configure_args)
configure_args = [p.format(**param['config_package_install_path']) for p in configure_args]
if not param['config_package'].get('configure', {}).get('ignore_install_prefix', False):
configure_args.insert(0, '--prefix='+install_dir)
env = param.get('env')
env_configure = env.copy()
for k, v in param['config_package'].get('configure', {}).get('env', {}).items():
env_configure[k] = v.format(**param['config_package_install_path'])
configure_path = os.path.join(source_dir, 'configure')
with open(param['log_file'], 'w') as f:
cmd = [configure_path] + configure_args
ret = call_and_log(cmd, log=f, cwd=build_dir, env=env_configure)
return {'success': ret==0, 'message': 'Configure exit code: {0}'.format(ret)}
def __dag_run(self):
# Get clean environment
env = Env(initial_env=self._env.env_final(),
env_prefix=self._config['app']['env_prefix'])
env.unload_packages()
env.unload_release()
env.unload_app()
env.load_app(self._config['app'])
env.load_release(self._config['scenario'], self._config['option'],
self._config['release'])
selector = InstallSelector(self._config)
processor = MultiThreadProcessor()
executor = InstallExecutor(self._config, env.env_final())
start_time = datetime.datetime.utcnow()
with Handler(self._config['release_path']['handler_python_dir']):
dag_run(self.__dag,
selector=selector,
processor=processor,
executor=executor)
end_time = datetime.datetime.utcnow()
self._config['release_status']['install'] = {}
self._config['release_status']['install']['start'] = start_time
self._config['release_status']['install']['end'] = end_time
self._config['release_status']['install']['finished'] = True
self._config['release_status']['option'] = {}
for opt in self._config['release_install']['options_to_save']:
if opt in self._config['option']:
self._config['release_status']['option'][opt] = self._config[
'option'][opt]
safe_mkdir(self._config['release_path']['status_dir'])
self._config['release_status'].save_to_file(
self._config['release_path']['status_file'])
def save_package_config(self, category, subdir, name, version):
pkg_install_cfg = self.package_config(category, subdir, name, version)
safe_mkdir(pkg_install_cfg['package_path']['config_dir'])
dump_config(pkg_install_cfg['config_origin'],
pkg_install_cfg['package_path']['config_file'])
def save_install_status(self, category, subdir, name, version):
pkg_install_cfg = self.package_config(category, subdir, name, version)
safe_mkdir(pkg_install_cfg['package_path']['status_dir'])
dump_config(pkg_install_cfg['install_status'],
pkg_install_cfg['package_path']['status_install_file'])