def __expand_env(self, config_scenario, config_attribute):
format_dict = {}
format_dict.update(config_attribute)
format_dict.update(config_scenario)
release_env = self.get('setting', {}).get('env', {})
env_prepend_path = release_env.get('prepend_path', {})
for k, v in env_prepend_path.items():
result = []
for i in ensure_list(v):
result.append(i.format(**format_dict))
env_prepend_path[k] = result
env_append_path = release_env.get('append_path', {})
for k, v in env_append_path.items():
result = []
for i in ensure_list(v):
result.append(i.format(**format_dict))
env_append_path[k] = result
env_set_env = release_env.get('set_env', {})
for k, v in env_set_env.items():
env_set_env[k] = v.format(**format_dict)
env_alias = release_env.get('alias', {})
for k, v in env_alias.items():
env_alias[k] = v.format(**format_dict)
def expand_package_env(pkg_cfg):
format_dict = {}
format_dict['name'] = pkg_cfg['name']
format_dict['category'] = pkg_cfg['category']
format_dict['subdir'] = pkg_cfg['subdir']
format_dict['version'] = pkg_cfg['version']
format_dict.update(pkg_cfg.get('path', {}))
env_prepend_path = pkg_cfg.get('env', {}).get('prepend_path', {})
for k, v in env_prepend_path.items():
result = []
for i in ensure_list(v):
result.append(i.format(**format_dict))
env_prepend_path[k] = result
env_append_path = pkg_cfg.get('env', {}).get('append_path', {})
for k, v in env_append_path.items():
result = []
for i in ensure_list(v):
result.append(i.format(**format_dict))
env_append_path[k] = result
env_set_env = pkg_cfg.get('env', {}).get('set_env', {})
for k, v in env_set_env.items():
env_set_env[k] = v.format(**format_dict)
env_alias = pkg_cfg.get('env', {}).get('alias', {})
for k, v in env_alias.items():
env_alias[k] = v.format(**format_dict)
def __init__(self, config_release):
super(ReleaseInstall, self).__init__()
install_cfg = config_release.get('setting', {}).get('install', {})
self['steps'] = ensure_list(install_cfg.get('steps', []))
if len(self['steps']) != len(set(self['steps'])):
raise ConfigReleaseInstallError(
'Duplicated steps found in: {0}'.format(self['steps']))
if not self['steps']:
_logger.warning('No install steps specified')
self['atomic_start'] = install_cfg.get('atomic_start')
self['atomic_end'] = install_cfg.get('atomic_end')
if self['atomic_start'] not in self['steps'] or self[
'atomic_end'] not in self['steps']:
raise ConfigReleaseInstallError(
'Can not find atomic start/end: {0} ... {1}'.format(
self['atomic_start'], self['atomic_end']))
if self['steps'].index(self['atomic_start']) > self['steps'].index(
self['atomic_end']):
raise ConfigReleaseInstallError(
'atomic_start should not be after atomic_end')
self['options_to_save'] = ensure_list(
install_cfg.get('options_to_save', []))
def install_step(config_release_install, pkg_cfg, install_status, reinstall):
result = {}
for step in config_release_install['steps']:
finished = install_status.get('steps',
{}).get(step, {}).get('finished', False)
install = reinstall or not finished
config_action = ensure_list(pkg_cfg.get('install', {}).get(step, []))
sub_index = 0
for cfg_action in config_action:
handler, param = _step_param(cfg_action)
if handler:
result.setdefault(step, [])
result[step].append({
'handler': handler,
'param': param,
'install': install
})
sub_index += 1
if sub_index == 0:
result.setdefault(step, [])
result[step].append({'handler': '', 'param': {}, 'install': False})
sub_index += 1
return result
def __load_steps(self):
self.__steps = {}
pkg_mgr = PackageManager(self.__config_version, self.__config_release)
for pkg, pkg_info in pkg_mgr.package_all().items():
if not pkg_info.get('config_category', {}).get('install'):
continue
self.__steps[pkg] = []
for action in self.__all_steps:
if not pkg_info.get('config', {}).get('install',
{}).get(action):
continue
if action not in self.__no_skip and pkg_mgr.is_finished(
pkg, action):
continue
config_action = ensure_list(
pkg_info['config']['install'][action])
sub_index = 0
for cfg_action in config_action:
handler, param = _step_param(cfg_action)
if handler:
self.__steps[pkg].append({
'action': action,
'sub_action': sub_index,
'handler': handler,
'param': param
})
sub_index += 1
def run(param):
missing_package = param['missing_package']
if not missing_package:
_logger.debug('There is no missing system package')
return
_logger.warn('Missing package(s): {0}'.format(', '.join(missing_package.keys())))
mgr_type = _detect_manager_type()
if not mgr_type:
_logger.warn('Could not detect current package manager')
return
check_type = param['type']
pkg_install_name = []
for package, categories in param['missing_package'].items():
for category in categories:
pkg_cfg = param['config_package_check'][category][package]['config']
install_name = pkg_cfg.get(check_type, {}).get('package_manager', {}).get(mgr_type, [])
pkg_install_name += ensure_list(install_name)
install_cmd = PACKAGE_MANAGER[mgr_type]
_logger.info('The missing package(s) could be installed with the following command:\n' + ' '.join(install_cmd+pkg_install_name))
haha('OS not supported')
def __install_step(self, all_steps, pkg_cfg):
result = []
for action in all_steps:
config_action = ensure_list(
pkg_cfg.get('install', {}).get(action, []))
sub_index = 0
for cfg_action in config_action:
handler, param = _step_param(cfg_action)
if handler:
result.append({
'action': action,
'sub_action': sub_index,
'handler': handler,
'param': param
})
sub_index += 1
if sub_index == 0:
result.append({
'action': action,
'sub_action': sub_index,
'handler': '',
'param': {}
})
return result
def run(param):
build_dir = param['config_package'].get('path', {}).get('build')
if not build_dir:
return {'success': False, 'message': 'Path "build" is not specified'}
install_dir = param['config_package'].get('path', {}).get('install')
if not install_dir:
return {'success': False, 'message': 'Path "install" is not specified'}
safe_mkdir(install_dir)
install_args = param['config_package'].get('make_install',
{}).get('args', ['install'])
install_args = ensure_list(install_args)
env = param.get('env')
env_install = env.copy()
for k, v in param['config_package'].get('make_install',
{}).get('env', {}).items():
env_install[k] = v.format(**param['config_package'].get('path', {}))
with open(param['log_file'], 'w') as f:
cmd = ['make'] + install_args
ret = call_and_log(cmd, log=f, cwd=build_dir, env=env_install)
return {
'success': ret == 0,
'message': 'Make install exit code: {0}'.format(ret)
}
def run(param, cfg):
name = param['name']
category = param['category']
cfg.setdefault('env', {})
cfg['env'].setdefault('set_env', {})
cfg['env']['set_env'].setdefault(name+'_HOME', '{install}')
if category != 'data' and 'source' not in cfg.get('clean', []):
cfg['env']['set_env'].setdefault(name+'_SOURCE', '{source}')
if 'bin' in cfg.get('path', {}):
cfg['env']['set_env'].setdefault(name+'_BIN', '{bin}')
if 'lib' in cfg.get('path', {}):
cfg['env']['set_env'].setdefault(name+'_LIB', '{lib}')
if 'inc' in cfg.get('path', {}):
cfg['env']['set_env'].setdefault(name+'_INCLUDE', '{inc}')
cfg['env'].setdefault('prepend_path', {})
for k, v in PATH_ENV.items():
if k in cfg.get('path', {}):
cfg['env']['prepend_path'][v] = ensure_list(cfg['env']['prepend_path'].get(v, []))
cfg['env']['prepend_path'][v].append('{{{0}}}'.format(k))
def __load_env(self, config_env):
env_info = {}
env_info['set_env'] = []
env_info['path'] = {}
env_info['alias'] = []
if 'unset_env' in config_env:
unset_env = ensure_list(config_env['unset_env'])
for e in unset_env:
if e in self.__env:
del self.__env[e]
if 'set_env' in config_env:
for e, v in config_env['set_env'].items():
self.__env[e] = v
env_info['set_env'].append(e)
if 'prepend_path' in config_env:
for e, v in config_env['prepend_path'].items():
path_list = ensure_list(v)
self.__merge_path(e, path_list)
env_info['path'].setdefault(e, [])
env_info['path'][e] += path_list
if 'append_path' in config_env:
for e, v in config_env['append_path'].items():
path_list = ensure_list(v)
self.__merge_path(e, path_list, True)
env_info['path'].setdefault(e, [])
env_info['path'][e] += path_list
if 'unalias' in config_env:
unalias_list = ensure_list(config_env['unalias'])
for e in unalias_list:
if e in self.__alias:
del self.__alias[e]
else:
self.__unalias.append(e)
if 'alias' in config_env:
self.__alias.update(config_env['alias'])
env_info['alias'] += list(config_env['alias'].keys())
return env_info
def __update_priority(self, config_container, config_category):
priority = ensure_list(config_container.get('category_priority', []))
priority = [ctg for ctg in priority if ctg in config_category]
priority += [
ctg for ctg in config_container.get('category', {})
if ctg not in priority
]
# prepend the new priority to the beginning
self[:0] = priority
def run(param, cfg):
option = param['config_option']
if not option.get('keep_log'):
return
if 'clean' in cfg:
clean_list = ensure_list(cfg['clean'])
clean_list.remove('log')
cfg['clean'] = clean_list
def _build_steps(cfg):
cfg['build'] = ensure_list(cfg['build'])
try:
make_index = cfg['build'].index('make')
except ValueError:
return
build_step_number = len(cfg['build'])
if make_index > 0 and 'configure' not in cfg['install']:
cfg['install']['configure'] = cfg['build'][make_index - 1]
if 'compile' not in cfg['install']:
cfg['install']['compile'] = 'make'
if make_index + 1 < build_step_number and 'install' not in cfg['install']:
cfg['install']['install'] = cfg['build'][make_index + 1]
def __build_dag(self):
self.__dag = Dag()
for pkg, pkg_info in self.__pkg_mgr.package_all().items():
if not pkg_info.get('config_category', {}).get('auto_env'):
continue
self.__dag.add_vertex(pkg)
for pkg, pkg_info in self.__pkg_mgr.package_all().items():
if not pkg_info.get('config_category', {}).get('auto_env'):
continue
pkgs_dep = ensure_list(pkg_info.get('config', {}).get('dep', []))
for pkg_dep in pkgs_dep:
if not self.__pkg_mgr.package_info(pkg_dep).get(
'config_category', {}).get('auto_env'):
continue
self.__dag.add_edge(pkg_dep, pkg)
def __build_dag(self):
self.__dag = Dag()
all_steps = []
for category in self._config['package_install']:
for subdir in self._config['package_install'][category]:
for package in self._config['package_install'][category][subdir]:
if self._config['category'][category]['version_dir']:
for version in self._config['package_install'][category][subdir][package]:
all_steps +=
# For a single package
for pkg, pkg_steps in self.__step_info.package_steps_all().items():
previous_vertex = None
for step in pkg_steps:
vertex_pkg = (pkg, step['action'], step['sub_action'])
self.__dag.add_vertex(vertex_pkg)
if previous_vertex is not None:
_logger.debug('DAG add edge: {0} -> {1}'.format(previous_vertex, vertex_pkg))
self.__dag.add_edge(previous_vertex, vertex_pkg)
previous_vertex = vertex_pkg
# For the package dependencies
for pkg, pkg_steps in self.__step_info.package_steps_all().items():
start_step = self.__step_info.find_atomic_start(pkg)
_logger.debug('start_step: {0} {1}'.format(pkg, start_step))
if not start_step:
continue
start_vertex = (pkg, start_step['action'], start_step['sub_action'])
pkg_deps = ensure_list(self.__pkg_mgr.package_info(pkg).get('config', {}).get('dep', []))
for pkg_dep in pkg_deps:
if pkg_dep not in self.__step_info.package_steps_all():
continue
end_step = self.__step_info.find_atomic_end(pkg_dep)
_logger.debug('end_step: {0} {1}'.format(pkg_dep, end_step))
if not end_step:
continue
end_vertex = (pkg_dep, end_step['action'], end_step['sub_action'])
_logger.debug('DAG add edge: {0} -> {1}'.format(end_vertex, start_vertex))
self.__dag.add_edge(end_vertex, start_vertex)
def run(param, config_release_setting):
option = param['config_option']
category = config_release_setting.get('category', {})
category_priority = ensure_list(
config_release_setting.get('category_priority', []))
if 'work' in category:
if 'work_root' in option:
category['work']['root'] = option['work_root']
else:
del category['work']
category_priority.remove('work')
if 'workver' in category:
if 'workver_root' in option:
category['workver']['root'] = option['workver_root']
else:
del category['workver']
category_priority.remove('workver')
def run(param):
source_dir = param['config_package'].get('path', {}).get('source')
if not source_dir:
return {'success': False, 'message': 'Path "source" is not specified'}
build_dir = param['config_package'].get('path', {}).get('build')
if not build_dir:
return {'success': False, 'message': 'Path "build" is not specified'}
install_dir = param['config_package'].get('path', {}).get('install')
if not install_dir:
return {'success': False, 'message': 'Path "install" is not specified'}
if source_dir != build_dir:
safe_rmdir(build_dir)
safe_mkdir(build_dir)
cmake_args = param['config_package'].get('cmake', {}).get('args', [])
cmake_args = ensure_list(cmake_args)
cmake_args = [p.format(**param['config_package_install_path']) for p in cmake_args]
if not param['config_package'].get('cmake', {}).get('ignore_install_prefix', False):
cmake_args.insert(0, '-DCMAKE_INSTALL_PREFIX='+install_dir)
cmake_var = param['config_package'].get('cmake', {}).get('var', {})
for k, v in cmake_var.items():
full_value = v.format(**param['config_package_install_path'])
full_arg = '-D{0}={1}'.format(k, full_value)
cmake_args.append(full_arg)
env = param.get('env')
with open(param['log_file'], 'w') as f:
cmd = ['cmake', source_dir] + cmake_args
ret = call_and_log(cmd, log=f, cwd=build_dir, env=env)
return {'success': ret==0, 'message': 'CMake exit code: {0}'.format(ret)}
def __load_setting(self):
setting_install = self.__config_release.get('setting',
{}).get('install', {})
self.__all_steps = setting_install.get('steps', [])
self.__atomic_start = setting_install.get('atomic_start')
self.__atomic_end = setting_install.get('atomic_end')
self.__no_skip = ensure_list(setting_install.get('no_skip', []))
if len(self.__all_steps) != len(set(self.__all_steps)):
raise InstallStepError('Duplicated steps found: {0}'.format(
self.__all_steps))
if self.__atomic_start not in self.__all_steps or self.__atomic_end not in self.__all_steps:
raise InstallStepError(
'Can not find atomic start/end: {0}/{1}'.format(
self.__atomic_start, self.__atomic_end))
if self.__all_steps.index(
self.__atomic_start) > self.__all_steps.index(
self.__atomic_end):
raise InstallStepError(
'atomic_start should not be after atomic_end')
def run(param):
source_dir = param['config_package'].get('path', {}).get('source')
if not source_dir:
return {'success': False, 'message': 'Path "source" is not specified'}
build_dir = param['config_package'].get('path', {}).get('build')
if not build_dir:
return {'success': False, 'message': 'Path "build" is not specified'}
install_dir = param['config_package'].get('path', {}).get('install')
if not install_dir:
return {'success': False, 'message': 'Path "install" is not specified'}
if source_dir != build_dir:
safe_rmdir(build_dir)
safe_mkdir(build_dir)
configure_args = param['config_package'].get('configure', {}).get('args', [])
configure_args = ensure_list(configure_args)
configure_args = [p.format(**param['config_package_install_path']) for p in configure_args]
if not param['config_package'].get('configure', {}).get('ignore_install_prefix', False):
configure_args.insert(0, '--prefix='+install_dir)
env = param.get('env')
env_configure = env.copy()
for k, v in param['config_package'].get('configure', {}).get('env', {}).items():
env_configure[k] = v.format(**param['config_package_install_path'])
configure_path = os.path.join(source_dir, 'configure')
with open(param['log_file'], 'w') as f:
cmd = [configure_path] + configure_args
ret = call_and_log(cmd, log=f, cwd=build_dir, env=env_configure)
return {'success': ret==0, 'message': 'Configure exit code: {0}'.format(ret)}
def _init_package(self, handler, config_entry, config_app, config_output,
config_scenario, config_option, config_release_path,
config_attribute, config_release, config_release_install,
config_category, config_category_priority):
reinstall = config_entry.get('reinstall', False)
category_install = [
ctg for ctg, ctg_cfg in config_category.items()
if ctg_cfg['install']
]
_logger.debug('Category for install: {0}'.format(category_install))
for identifier, pkg_cfg in config_release.get('package', {}).items():
try:
category_name, subdir, pkg_name = package_param_from_identifier(
identifier, pkg_cfg)
except ConfigPackageParamError:
continue
if category_name not in category_install:
_logger.debug(
'Package "{0}" could not be installed with category "{1}"'.
format(pkg_name, category_name))
continue
version = ensure_list(pkg_cfg.get('version', []))
version_dir = config_category[category_name]['version_dir']
if (not version_dir) and len(version) > 1:
_logger.warning(
'Only one version could be installed when category version_dir is false'
)
version = version[:1]
if not version:
_logger.warning(
'No version is specified for category({0}), package({1})'.
format(category_name, pkg_name))
for ver in version:
self.setdefault(category_name, {})
self[category_name].setdefault(subdir, {})
self[category_name][subdir].setdefault(pkg_name, {})
if ver in self[category_name][subdir][pkg_name]:
_logger.warning(
'Duplicated package found: category({0}), subdir({1}), package({2}), version({3})'
.format(category_name, subdir, pkg_name, ver))
self[category_name][subdir][pkg_name].setdefault(ver, {})
final_config = self[category_name][subdir][pkg_name][ver]
final_config['config_origin'] = copy.deepcopy(pkg_cfg)
final_config['config_origin']['version'] = ver
pkg_path = package_path(config_app, config_category,
category_name, subdir, pkg_name, ver)
final_config['config'] = transform_package(
handler, 'install', category_name, subdir, pkg_name, ver,
pkg_cfg, pkg_path, config_app, config_output,
config_scenario, config_option, config_release_path,
config_attribute, config_release, config_release_install,
config_category, config_category_priority)
final_config['config']['name'] = pkg_name
final_config['config']['category'] = category_name
final_config['config']['subdir'] = subdir
if 'version' not in final_config['config']:
final_config['config']['version'] = ver
final_config['package_path'] = package_path(
config_app, config_category, category_name, subdir,
pkg_name, final_config['config']['version'])
expand_package_path(final_config['package_path']['main_dir'],
final_config['config'])
expand_package_env(final_config['config'])
final_config['install_status'] = install_status(
final_config['package_path']['status_install_file'])
final_config['step'] = install_step(
config_release_install, final_config['config'],
final_config['install_status'], reinstall)
def load(self, config_app, config_scenario, config_release_path,
config_attribute, config_release, config_category):
if not ('version' in config_scenario and config_scenario['version']):
_logger.debug('"version" not specified in config install')
return
category_install = [
ctg for ctg, ctg_cfg in config_category.items()
if ctg_cfg['install']
]
all_steps = config_release.get('setting', {}).get('install',
{}).get('steps', [])
if len(all_steps) != len(set(all_steps)):
raise ConfigInstallStepError(
'Duplicated steps found in: {0}'.format(all_steps))
if len(all_steps) == 0:
_logger.warn('No install steps specified')
sys.path.insert(0, config_release_path['handler_python_dir'])
config_release_package = config_release.get('package', {})
for identifier, pkg_cfg in config_release_package.items():
frag = identifier.split(os.sep)
# Use the first part as default category name
if 'category' in pkg_cfg:
category_name = pkg_cfg['category']
elif len(frag) > 1:
category_name = frag[0]
else:
_logger.warn(
'Category not specified for {0}'.format(identifier))
continue
if category_name not in category_install:
continue
# Use the last part as default package name
if 'name' in pkg_cfg:
pkg_name = pkg_cfg['name']
elif frag[-1]:
pkg_name = frag[-1]
else:
_logger.error(
'Package name not found for {0}'.format(identifier))
continue
# Use the middle part as default subdir
if 'subdir' in pkg_cfg:
subdir = pkg_cfg['subdir']
elif frag[1:-1]:
subdir = os.path.join(*frag[1:-1])
else:
subdir = ''
version = ensure_list(pkg_cfg.get('version', []))
version_dir = config_category[category_name]['version_dir']
if (not version_dir) and len(version) > 1:
_logger.warn(
'Only one version could be installed when category version_dir is false'
)
version = version[:1]
if not version:
_logger.warn(
'No version is specified for category({0}), package({1})'.
format(category_name, pkg_name))
for ver in version:
self.setdefault(category_name, {})
self[category_name].setdefault(subdir, {})
self[category_name][subdir].setdefault(pkg_name, {})
if version_dir:
self[category_name][subdir][pkg_name].setdefault(ver, {})
if ver in self[category_name][subdir][pkg_name]:
_logger.warn(
'Duplicated package found: category({0}), subdir({1}), package({2}), version({3})'
.format(category_name, subdir, pkg_name, ver))
final_config = self[category_name][subdir][pkg_name][ver]
else:
if pkg_name in self[category_name][subdir]:
_logger.warn(
'Duplicated package found: category({0}), subdir({1}), package({2})'
.format(category_name, subdir, pkg_name))
final_config = self[category_name][subdir][pkg_name]
final_config['config_origin'] = copy.deepcopy(pkg_cfg)
final_config['config_origin']['version'] = ver
final_config['config'] = self.__transform_package(
category_name, pkg_name, subdir, ver, pkg_cfg, config_app,
config_scenario, config_release_path, config_attribute,
config_release, config_category)
final_config['config']['name'] = pkg_name
final_config['config']['category'] = category_name
final_config['config']['subdir'] = subdir
if 'version' not in final_config['config']:
final_config['config']['version'] = ver
final_config['common_path'] = self.__common_path(
config_category, final_config['config'])
self.__expand_package_path(
final_config['common_path']['main_dir'],
final_config['config'])
self.__expand_env(final_config['config'])
final_config['step'] = self.__install_step(
all_steps, final_config['config'])
sys.path.remove(config_release_path['handler_python_dir'])
def __init__(self, output='', commands=[], script_types=[]):
self.__output = output
self.__commands = ensure_list(commands)
self.__script_types = ensure_list(script_types)
def __build_dag(self):
self.__dag = Dag()
steps = self._config['release_install']['steps']
atomic_start = self._config['release_install']['atomic_start']
atomic_end = self._config['release_install']['atomic_end']
package_steps = {}
# For a single package
for category in self._config['package_install']:
for subdir in self._config['package_install'][category]:
for package in self._config['package_install'][category][
subdir]:
for version, value in self._config['package_install'][
category][subdir][package].items():
previous_vertex = None
for step in steps:
for sub_step in range(len(value['step'][step])):
vertex_pkg = (category, subdir, package,
version, step, sub_step)
self.__dag.add_vertex(vertex_pkg)
if previous_vertex is not None:
_logger.debug(
'DAG add edge for single package: {0} -> {1}'
.format(previous_vertex, vertex_pkg))
self.__dag.add_edge(
previous_vertex, vertex_pkg)
previous_vertex = vertex_pkg
package_steps.setdefault(package, [])
package_steps[package].append(
(category, subdir, package, version))
# For the package dependencies
for category in self._config['package_install']:
for subdir in self._config['package_install'][category]:
for package in self._config['package_install'][category][
subdir]:
for version, value in self._config['package_install'][
category][subdir][package].items():
start_vertex = (category, subdir, package, version,
atomic_start, 0)
pkg_deps = ensure_list(value['config'].get('dep', []))
for pkg_dep in pkg_deps:
if pkg_dep not in package_steps:
_logger.debug(
'Dependency for package "{0}" not available: {1}'
.format(package, pkg_dep))
continue
for dep in package_steps[pkg_dep]:
dep_end_index = len(
self._config['package_install'][dep[0]][
dep[1]][dep[2]][
dep[3]]['step'][atomic_end]) - 1
end_vertex = (dep[0], dep[1], dep[2], dep[3],
atomic_end, dep_end_index)
_logger.debug(
'DAG add edge for package dependency: {0} -> {1}'
.format(end_vertex, start_vertex))
self.__dag.add_edge(end_vertex, start_vertex)
def __init__(self, output='', script_types=[]):
self.__output = output
self.__script_types = ensure_list(script_types)
