def __transform_package(self, category, name, subdir, version, pkg_cfg,
config_app, config_scenario, config_release_path,
config_attribute, config_release, config_category):
param = {}
param['operation'] = 'install'
param['name'] = name
param['category'] = category
param['subdir'] = subdir
param['version'] = version
param['config_package'] = copy.deepcopy(pkg_cfg)
param['config_app'] = config_app.data_copy
param['config_scenario'] = config_scenario.data_copy
param['config_release_path'] = config_release_path.data_copy
param['config_attribute'] = config_attribute.data_copy
param['config_release'] = config_release.data_copy
param['config_category'] = config_category.data_copy
try:
with Handler() as h:
result = h.run('transform_package', param)
if isinstance(result, dict):
return result
except HandlerNotFoundError as e:
_logger.debug('Transformer for package not found: {0}'.format(e))
return copy.deepcopy(pkg_cfg)
def execute(self, command):
param = {}
param['command'] = command.copy()
param['config_app'] = self._config['app'].data_copy()
param['config_output'] = self._config['output'].data_copy()
param['config_scenario'] = self._config['scenario'].data_copy()
param['config_option'] = self._config['option'].data_copy()
param['config_release_path'] = self._config['release_path'].data_copy()
param['config_attribute'] = self._config['attribute'].data_copy()
param['config_release'] = self._config['release'].data_copy()
param['config_category'] = self._config['category'].data_copy()
param['config_category_priority'] = self._config['category_priority'].data_copy()
param['config_package_install'] = self._config['package_install'].data_copy()
param['config_package_install_path'] = self._config['package_install_path'].data_copy()
param['config_package_runtime'] = self._config['package_runtime'].data_copy()
param['config_package_runtime_path'] = self._config['package_runtime_path'].data_copy()
param['config_package_check'] = self._config['package_check'].data_copy()
with Handler(self._config['release_path']['handler_python_dir']) as h:
try:
return h.run('command', param)
except HandlerNotFoundError as e:
_logger.error('Could not find out how to run command: {0}'.format(command))
raise
def select(self, running, idle):
param = {}
param['config_app'] = self.__config['app'].data_copy()
param['config_output'] = self.__config['output'].data_copy()
param['config_scenario'] = self.__config['scenario'].data_copy()
param['config_option'] = self.__config['option'].data_copy()
param['config_release_path'] = self.__config['release_path'].data_copy()
param['config_attribute'] = self.__config['attribute'].data_copy()
param['config_release'] = self.__config['release'].data_copy()
param['config_category'] = self.__config['category'].data_copy()
param['config_category_priority'] = self.__config['category_priority'].data_copy()
param['running'] = running
param['idle'] = idle
try:
with Handler() as h:
return h.run('select', param)
except HandlerNotFoundError as e:
_logger.debug('Select handler not found, will randomly select one step')
return [next(iter(idle))]
except Exception as e:
_logger.error('Select handler run error: {0}'.format(e))
raise
def __load_option_list(self):
self.__config['option_list'] = ConfigCommon()
if 'handler_python_dir' not in self['release_path']:
return
try:
with Handler(self['release_path']['handler_python_dir']) as h:
self['option_list'].update(h.run('option'))
except HandlerNotFoundError:
_logger.debug('Handler for option not found')
def execute(self):
self._env.unload_packages()
category_auto_env = [ctg for ctg in self._config['category_priority'] if self._config['category'].get(ctg, {}).get('auto_env')]
with Handler(self._config['release_path']['handler_python_dir']) as h:
for package, value in load_packages(h, category_auto_env, self._config['package_runtime']).items():
category, subdir, version = value
pkg_cfg = self._config['package_runtime'].package_config(category, subdir, package, version)
self._env.load_package(pkg_cfg['config'])
def execute(self,
package,
category=None,
subdir=None,
version=None,
category_origin=None,
subdir_origin=None,
version_origin=None):
category_priority = self._config['category_priority']
from_install = False
with Handler(self._config['release_path']['handler_python_dir']) as h:
ctg, sd, ver = find_package(h, category_priority,
self._config['package_runtime'],
package, category, subdir, version)
if ctg is not None:
_logger.debug(
'Package destination found: {0}.{1}.{2}.{3}'.format(
ctg, sd, package, ver))
return ctg, sd, ver, None, None, None, False
ctg_org, sd_org, ver_org = find_package(
h, category_priority, self._config['package_runtime'], package,
category_origin, subdir_origin, version_origin)
if ctg_org is None:
_logger.debug(
'Try to find reference in config_package_install')
ctg_org, sd_org, ver_org = find_package(
h, category_priority, self._config['package_install'],
package, category_origin, subdir_origin, version_origin)
if ctg_org is None:
_logger.error(
'Could not find matching reference package to install "{0}"'
.format(package))
return None, None, None, None, None, None, False
from_install = True
_logger.debug(
'Package reference found in config_package_install: {0}.{1}.{2}.{3}'
.format(ctg_org, sd_org, package, ver_org))
else:
_logger.debug(
'Package reference found in config_package_runtime: {0}.{1}.{2}.{3}'
.format(ctg_org, sd_org, package, ver_org))
if category is None:
category = ctg_org
if subdir is None:
subdir = sd_org
if version is None:
version = ver_org
return category, subdir, version, ctg_org, sd_org, ver_org, from_install
def execute(self, param):
pkg = param['package']
step = param['step']
sub_step = param['sub_step']
if sub_step == 0:
step_full_name = '{0} - {1}'.format(pkg, step)
else:
step_full_name = '{0} - {1} - {2}'.format(pkg, step, sub_step)
if not param['action_install']:
_logger.debug('Skip step: {0}'.format(step_full_name))
return {'success': True, 'skip': True}
result = {}
result['start'] = datetime.datetime.utcnow()
safe_mkdir(os.path.dirname(param['log_file']))
try:
with Handler() as h:
result_action = h.run('install', param)
except HandlerNotFoundError as e:
_logger.error('Install handler "{0}" not found for "{1}"'.format(
param['action'], step_full_name))
raise
except Exception as e:
_logger.error('Install handler "{0}" error for {1}: {2}'.format(
param['action'], step_full_name, e))
if param['config_output']['verbose']:
_logger.error('\n{0}'.format(traceback.format_exc()))
raise
if isinstance(result_action, bool):
result['success'] = result_action
elif isinstance(result_action, dict):
result['success'] = result_action.get('success', False)
else:
result['success'] = False
if not result['success']:
if isinstance(result_action, dict) and 'message' in result_action:
_logger.error('"{0}" execution error: {1}'.format(
step_full_name, result_action['message']))
_logger.error('"{0}" execution error. Find log in "{1}"'.format(
step_full_name, param['log_file']))
raise InstallExecutorError(
'"{0}" execution error'.format(step_full_name))
result['action'] = result_action
result['end'] = datetime.datetime.utcnow()
return result
def __init__(self, config_release_path, config_category_priority,
config_package):
super(PackagePath, self).__init__()
with Handler(config_release_path['handler_python_dir']) as h:
for package, value in load_packages(h, config_category_priority,
config_package).items():
category, subdir, version = value
pkg_cfg = config_package.package_config(
category, subdir, package, version)
self[package] = pkg_cfg['config'].get('path', {}).copy()
def execute(self):
handler_dir = None
if 'handler_dir' in self._config['release_path']:
handler_dir = self._config['release_path']['handler_dir']
try:
with Handler(handler_dir) as h:
return h.run('option')
except HandlerNotFoundError as e:
_logger.debug('No option loaded: {0}'.format(e))
return ''
def __init__(self, config_entry, config_app, config_output,
config_scenario, config_option, config_release_path,
config_attribute, config_release, config_release_install,
config_category, config_category_priority):
super(PackageBase, self).__init__()
if not ('version' in config_scenario and config_scenario['version']):
_logger.debug('"version" not specified in config package')
return
with Handler(config_release_path['handler_python_dir']) as h:
self._init_package(h, config_entry, config_app, config_output,
config_scenario, config_option,
config_release_path, config_attribute,
config_release, config_release_install,
config_category, config_category_priority)
def __load_option_list(self):
cfg = ConfigCommonDict()
if 'handler_python_dir' not in self['release_path']:
_logger.debug('handler_python_dir not in release_path')
return cfg
try:
with Handler(self['release_path']['handler_python_dir']) as h:
cfg.update(h.run('option'))
except HandlerNotFoundError:
_logger.debug('Handler for option list not found')
except Exception as e:
_logger.error('Handler for option list run error: {0}'.format(e))
raise
return cfg
def __init__(self, config_app, config_output, config_scenario,
config_option, config_release_path, config_attribute,
config_release, config_release_install, config_category,
config_category_priority):
super(PackageCheck, self).__init__()
category_check = [
ctg for ctg, ctg_cfg in config_category.items()
if ctg_cfg.get('pre_check')
]
_logger.debug('Category for check: {0}'.format(category_check))
with Handler(config_release_path['handler_python_dir']) as h:
for identifier, pkg_cfg in config_release.get('package',
{}).items():
try:
category_name, _, pkg_name = package_param_from_identifier(
identifier, pkg_cfg)
except ConfigPackageParamError:
continue
if category_name not in category_check:
_logger.debug(
'Package "{0}" could not be checked with category "{1}"'
.format(pkg_name, category_name))
continue
self.setdefault(category_name, {})
if pkg_name in self[category_name]:
_logger.warning(
'Duplicated package found: category({0}), package({1})'
.format(category_name, pkg_name))
self[category_name].setdefault(pkg_name, {})
final_config = self[category_name][pkg_name]
final_config['config_origin'] = copy.deepcopy(pkg_cfg)
final_config['config'] = self.__transform_package(
h, category_name, pkg_name, pkg_cfg, config_app,
config_output, config_scenario, config_option,
config_release_path, config_attribute, config_release,
config_category, config_category_priority)
final_config['config']['name'] = pkg_name
final_config['config']['category'] = category_name
def __transform(self, config_app, config_scenario, config_release_path,
config_attribute):
param = {}
param['config_app'] = config_app.data_copy
param['config_scenario'] = config_scenario.data_copy
param['config_release_path'] = config_release_path.data_copy
param['config_release'] = self.data_copy
param['config_attribute'] = config_attribute.data_copy
try:
with Handler(config_release_path['handler_python_dir']) as h:
result = h.run('transform_release', param)
if isinstance(result, dict):
self.clear()
self.update(result)
except HandlerNotFoundError as e:
_logger.debug('Transformer for release not found: {0}'.format(e))
except Exception as e:
_logger.error('Transformer for release run error: {0}'.format(e))
raise
def __dag_run(self):
# Get clean environment
env = Env(initial_env=self._env.env_final(),
env_prefix=self._config['app']['env_prefix'])
env.unload_packages()
env.unload_release()
env.unload_app()
env.load_app(self._config['app'])
env.load_release(self._config['scenario'], self._config['option'],
self._config['release'])
selector = InstallSelector(self._config)
processor = MultiThreadProcessor()
executor = InstallExecutor(self._config, env.env_final())
start_time = datetime.datetime.utcnow()
with Handler(self._config['release_path']['handler_python_dir']):
dag_run(self.__dag,
selector=selector,
processor=processor,
executor=executor)
end_time = datetime.datetime.utcnow()
self._config['release_status']['install'] = {}
self._config['release_status']['install']['start'] = start_time
self._config['release_status']['install']['end'] = end_time
self._config['release_status']['install']['finished'] = True
self._config['release_status']['option'] = {}
for opt in self._config['release_install']['options_to_save']:
if opt in self._config['option']:
self._config['release_status']['option'][opt] = self._config[
'option'][opt]
safe_mkdir(self._config['release_path']['status_dir'])
self._config['release_status'].save_to_file(
self._config['release_path']['status_file'])
def execute(self, package, category, subdir, version, rebuild=False):
pkg_cfg = self._config['package_runtime'].package_config(
category, subdir, package, version)
param = {}
param['type'] = 'rebuild' if rebuild else 'build'
param['name'] = package
param['category'] = category
param['subdir'] = subdir
param['version'] = version
param['config_package'] = copy.deepcopy(pkg_cfg['config'])
param['package_path'] = copy.deepcopy(pkg_cfg['package_path'])
param['config_app'] = self._config['app'].data_copy()
param['config_output'] = self._config['output'].data_copy()
param['config_scenario'] = self._config['scenario'].data_copy()
param['config_option'] = self._config['option'].data_copy()
param['config_release_path'] = self._config['release_path'].data_copy()
param['config_attribute'] = self._config['attribute'].data_copy()
param['config_release'] = self._config['release'].data_copy()
param['config_category'] = self._config['category'].data_copy()
param['config_category_priority'] = self._config[
'category_priority'].data_copy()
param['config_package_runtime'] = self._config[
'package_runtime'].data_copy()
param['config_package_runtime_path'] = self._config[
'package_runtime_path'].data_copy()
with Handler(self._config['release_path']['handler_python_dir']) as h:
try:
return h.run('build_package', param)
except HandlerNotFoundError as e:
_logger.error(
'Could not find out how to build package "{0}", category "{1}", subdir "{2}", version "{3}"'
.format(package, category, subdir, version))
raise
def execute(self, check_type):
env = Env(initial_env=self._env.env_final(),
env_prefix=self._config['app']['env_prefix'])
env.unload_packages()
env.unload_release()
env.unload_app()
env.load_app(self._config['app'])
env.load_release(self._config['scenario'], self._config['option'],
self._config['release'])
missing = {}
with Handler(self._config['release_path']['handler_python_dir']) as h:
for category in self._config['package_check']:
for package, value in self._config['package_check'][
category].items():
if not self.__check_package(h, env.env_final(),
value['config'], check_type):
missing.setdefault(package, [])
missing[package].append(category)
self.__check_summary(h, missing, check_type)
return missing
def __load_attribute(self):
cfg = ConfigCommonDict()
if 'handler_python_dir' not in self['release_path']:
_logger.debug('handler_python_dir not in release_path')
return cfg
param = {}
param['config_app'] = self['app'].data_copy()
param['config_output'] = self['output'].data_copy()
param['config_scenario'] = self['scenario'].data_copy()
param['config_option'] = self['option'].data_copy()
param['config_release_path'] = self['release_path'].data_copy()
param['config_release_origin'] = self['release_origin'].data_copy()
try:
with Handler(self['release_path']['handler_python_dir']) as h:
cfg.update(h.run('attribute', param))
except HandlerNotFoundError:
_logger.debug('Handler for attribute not found')
except Exception as e:
_logger.error('Handler for attribute run error: {0}'.format(e))
raise
return cfg
def execute(self, package, category=None, subdir=None, version=None, from_install=False):
config_package_name = 'package_install' if from_install else 'package_runtime'
with Handler(self._config['release_path']['handler_python_dir']) as h:
return find_package(h, self._config['category_priority'], self._config[config_package_name], package, category, subdir, version)