def __expand_env(self, config_scenario, config_attribute): format_dict = {} format_dict.update(config_attribute) format_dict.update(config_scenario) release_env = self.get('setting', {}).get('env', {}) env_prepend_path = release_env.get('prepend_path', {}) for k, v in env_prepend_path.items(): result = [] for i in ensure_list(v): result.append(i.format(**format_dict)) env_prepend_path[k] = result env_append_path = release_env.get('append_path', {}) for k, v in env_append_path.items(): result = [] for i in ensure_list(v): result.append(i.format(**format_dict)) env_append_path[k] = result env_set_env = release_env.get('set_env', {}) for k, v in env_set_env.items(): env_set_env[k] = v.format(**format_dict) env_alias = release_env.get('alias', {}) for k, v in env_alias.items(): env_alias[k] = v.format(**format_dict)
def expand_package_env(pkg_cfg): format_dict = {} format_dict['name'] = pkg_cfg['name'] format_dict['category'] = pkg_cfg['category'] format_dict['subdir'] = pkg_cfg['subdir'] format_dict['version'] = pkg_cfg['version'] format_dict.update(pkg_cfg.get('path', {})) env_prepend_path = pkg_cfg.get('env', {}).get('prepend_path', {}) for k, v in env_prepend_path.items(): result = [] for i in ensure_list(v): result.append(i.format(**format_dict)) env_prepend_path[k] = result env_append_path = pkg_cfg.get('env', {}).get('append_path', {}) for k, v in env_append_path.items(): result = [] for i in ensure_list(v): result.append(i.format(**format_dict)) env_append_path[k] = result env_set_env = pkg_cfg.get('env', {}).get('set_env', {}) for k, v in env_set_env.items(): env_set_env[k] = v.format(**format_dict) env_alias = pkg_cfg.get('env', {}).get('alias', {}) for k, v in env_alias.items(): env_alias[k] = v.format(**format_dict)
def __init__(self, config_release): super(ReleaseInstall, self).__init__() install_cfg = config_release.get('setting', {}).get('install', {}) self['steps'] = ensure_list(install_cfg.get('steps', [])) if len(self['steps']) != len(set(self['steps'])): raise ConfigReleaseInstallError( 'Duplicated steps found in: {0}'.format(self['steps'])) if not self['steps']: _logger.warning('No install steps specified') self['atomic_start'] = install_cfg.get('atomic_start') self['atomic_end'] = install_cfg.get('atomic_end') if self['atomic_start'] not in self['steps'] or self[ 'atomic_end'] not in self['steps']: raise ConfigReleaseInstallError( 'Can not find atomic start/end: {0} ... {1}'.format( self['atomic_start'], self['atomic_end'])) if self['steps'].index(self['atomic_start']) > self['steps'].index( self['atomic_end']): raise ConfigReleaseInstallError( 'atomic_start should not be after atomic_end') self['options_to_save'] = ensure_list( install_cfg.get('options_to_save', []))
def install_step(config_release_install, pkg_cfg, install_status, reinstall): result = {} for step in config_release_install['steps']: finished = install_status.get('steps', {}).get(step, {}).get('finished', False) install = reinstall or not finished config_action = ensure_list(pkg_cfg.get('install', {}).get(step, [])) sub_index = 0 for cfg_action in config_action: handler, param = _step_param(cfg_action) if handler: result.setdefault(step, []) result[step].append({ 'handler': handler, 'param': param, 'install': install }) sub_index += 1 if sub_index == 0: result.setdefault(step, []) result[step].append({'handler': '', 'param': {}, 'install': False}) sub_index += 1 return result
def __load_steps(self): self.__steps = {} pkg_mgr = PackageManager(self.__config_version, self.__config_release) for pkg, pkg_info in pkg_mgr.package_all().items(): if not pkg_info.get('config_category', {}).get('install'): continue self.__steps[pkg] = [] for action in self.__all_steps: if not pkg_info.get('config', {}).get('install', {}).get(action): continue if action not in self.__no_skip and pkg_mgr.is_finished( pkg, action): continue config_action = ensure_list( pkg_info['config']['install'][action]) sub_index = 0 for cfg_action in config_action: handler, param = _step_param(cfg_action) if handler: self.__steps[pkg].append({ 'action': action, 'sub_action': sub_index, 'handler': handler, 'param': param }) sub_index += 1
def run(param): missing_package = param['missing_package'] if not missing_package: _logger.debug('There is no missing system package') return _logger.warn('Missing package(s): {0}'.format(', '.join(missing_package.keys()))) mgr_type = _detect_manager_type() if not mgr_type: _logger.warn('Could not detect current package manager') return check_type = param['type'] pkg_install_name = [] for package, categories in param['missing_package'].items(): for category in categories: pkg_cfg = param['config_package_check'][category][package]['config'] install_name = pkg_cfg.get(check_type, {}).get('package_manager', {}).get(mgr_type, []) pkg_install_name += ensure_list(install_name) install_cmd = PACKAGE_MANAGER[mgr_type] _logger.info('The missing package(s) could be installed with the following command:\n' + ' '.join(install_cmd+pkg_install_name)) haha('OS not supported')
def __install_step(self, all_steps, pkg_cfg): result = [] for action in all_steps: config_action = ensure_list( pkg_cfg.get('install', {}).get(action, [])) sub_index = 0 for cfg_action in config_action: handler, param = _step_param(cfg_action) if handler: result.append({ 'action': action, 'sub_action': sub_index, 'handler': handler, 'param': param }) sub_index += 1 if sub_index == 0: result.append({ 'action': action, 'sub_action': sub_index, 'handler': '', 'param': {} }) return result
def run(param): build_dir = param['config_package'].get('path', {}).get('build') if not build_dir: return {'success': False, 'message': 'Path "build" is not specified'} install_dir = param['config_package'].get('path', {}).get('install') if not install_dir: return {'success': False, 'message': 'Path "install" is not specified'} safe_mkdir(install_dir) install_args = param['config_package'].get('make_install', {}).get('args', ['install']) install_args = ensure_list(install_args) env = param.get('env') env_install = env.copy() for k, v in param['config_package'].get('make_install', {}).get('env', {}).items(): env_install[k] = v.format(**param['config_package'].get('path', {})) with open(param['log_file'], 'w') as f: cmd = ['make'] + install_args ret = call_and_log(cmd, log=f, cwd=build_dir, env=env_install) return { 'success': ret == 0, 'message': 'Make install exit code: {0}'.format(ret) }
def run(param, cfg): name = param['name'] category = param['category'] cfg.setdefault('env', {}) cfg['env'].setdefault('set_env', {}) cfg['env']['set_env'].setdefault(name+'_HOME', '{install}') if category != 'data' and 'source' not in cfg.get('clean', []): cfg['env']['set_env'].setdefault(name+'_SOURCE', '{source}') if 'bin' in cfg.get('path', {}): cfg['env']['set_env'].setdefault(name+'_BIN', '{bin}') if 'lib' in cfg.get('path', {}): cfg['env']['set_env'].setdefault(name+'_LIB', '{lib}') if 'inc' in cfg.get('path', {}): cfg['env']['set_env'].setdefault(name+'_INCLUDE', '{inc}') cfg['env'].setdefault('prepend_path', {}) for k, v in PATH_ENV.items(): if k in cfg.get('path', {}): cfg['env']['prepend_path'][v] = ensure_list(cfg['env']['prepend_path'].get(v, [])) cfg['env']['prepend_path'][v].append('{{{0}}}'.format(k))
def __load_env(self, config_env): env_info = {} env_info['set_env'] = [] env_info['path'] = {} env_info['alias'] = [] if 'unset_env' in config_env: unset_env = ensure_list(config_env['unset_env']) for e in unset_env: if e in self.__env: del self.__env[e] if 'set_env' in config_env: for e, v in config_env['set_env'].items(): self.__env[e] = v env_info['set_env'].append(e) if 'prepend_path' in config_env: for e, v in config_env['prepend_path'].items(): path_list = ensure_list(v) self.__merge_path(e, path_list) env_info['path'].setdefault(e, []) env_info['path'][e] += path_list if 'append_path' in config_env: for e, v in config_env['append_path'].items(): path_list = ensure_list(v) self.__merge_path(e, path_list, True) env_info['path'].setdefault(e, []) env_info['path'][e] += path_list if 'unalias' in config_env: unalias_list = ensure_list(config_env['unalias']) for e in unalias_list: if e in self.__alias: del self.__alias[e] else: self.__unalias.append(e) if 'alias' in config_env: self.__alias.update(config_env['alias']) env_info['alias'] += list(config_env['alias'].keys()) return env_info
def __update_priority(self, config_container, config_category): priority = ensure_list(config_container.get('category_priority', [])) priority = [ctg for ctg in priority if ctg in config_category] priority += [ ctg for ctg in config_container.get('category', {}) if ctg not in priority ] # prepend the new priority to the beginning self[:0] = priority
def run(param, cfg): option = param['config_option'] if not option.get('keep_log'): return if 'clean' in cfg: clean_list = ensure_list(cfg['clean']) clean_list.remove('log') cfg['clean'] = clean_list
def _build_steps(cfg): cfg['build'] = ensure_list(cfg['build']) try: make_index = cfg['build'].index('make') except ValueError: return build_step_number = len(cfg['build']) if make_index > 0 and 'configure' not in cfg['install']: cfg['install']['configure'] = cfg['build'][make_index - 1] if 'compile' not in cfg['install']: cfg['install']['compile'] = 'make' if make_index + 1 < build_step_number and 'install' not in cfg['install']: cfg['install']['install'] = cfg['build'][make_index + 1]
def __build_dag(self): self.__dag = Dag() for pkg, pkg_info in self.__pkg_mgr.package_all().items(): if not pkg_info.get('config_category', {}).get('auto_env'): continue self.__dag.add_vertex(pkg) for pkg, pkg_info in self.__pkg_mgr.package_all().items(): if not pkg_info.get('config_category', {}).get('auto_env'): continue pkgs_dep = ensure_list(pkg_info.get('config', {}).get('dep', [])) for pkg_dep in pkgs_dep: if not self.__pkg_mgr.package_info(pkg_dep).get( 'config_category', {}).get('auto_env'): continue self.__dag.add_edge(pkg_dep, pkg)
def __build_dag(self): self.__dag = Dag() all_steps = [] for category in self._config['package_install']: for subdir in self._config['package_install'][category]: for package in self._config['package_install'][category][subdir]: if self._config['category'][category]['version_dir']: for version in self._config['package_install'][category][subdir][package]: all_steps += # For a single package for pkg, pkg_steps in self.__step_info.package_steps_all().items(): previous_vertex = None for step in pkg_steps: vertex_pkg = (pkg, step['action'], step['sub_action']) self.__dag.add_vertex(vertex_pkg) if previous_vertex is not None: _logger.debug('DAG add edge: {0} -> {1}'.format(previous_vertex, vertex_pkg)) self.__dag.add_edge(previous_vertex, vertex_pkg) previous_vertex = vertex_pkg # For the package dependencies for pkg, pkg_steps in self.__step_info.package_steps_all().items(): start_step = self.__step_info.find_atomic_start(pkg) _logger.debug('start_step: {0} {1}'.format(pkg, start_step)) if not start_step: continue start_vertex = (pkg, start_step['action'], start_step['sub_action']) pkg_deps = ensure_list(self.__pkg_mgr.package_info(pkg).get('config', {}).get('dep', [])) for pkg_dep in pkg_deps: if pkg_dep not in self.__step_info.package_steps_all(): continue end_step = self.__step_info.find_atomic_end(pkg_dep) _logger.debug('end_step: {0} {1}'.format(pkg_dep, end_step)) if not end_step: continue end_vertex = (pkg_dep, end_step['action'], end_step['sub_action']) _logger.debug('DAG add edge: {0} -> {1}'.format(end_vertex, start_vertex)) self.__dag.add_edge(end_vertex, start_vertex)
def run(param, config_release_setting): option = param['config_option'] category = config_release_setting.get('category', {}) category_priority = ensure_list( config_release_setting.get('category_priority', [])) if 'work' in category: if 'work_root' in option: category['work']['root'] = option['work_root'] else: del category['work'] category_priority.remove('work') if 'workver' in category: if 'workver_root' in option: category['workver']['root'] = option['workver_root'] else: del category['workver'] category_priority.remove('workver')
def run(param): source_dir = param['config_package'].get('path', {}).get('source') if not source_dir: return {'success': False, 'message': 'Path "source" is not specified'} build_dir = param['config_package'].get('path', {}).get('build') if not build_dir: return {'success': False, 'message': 'Path "build" is not specified'} install_dir = param['config_package'].get('path', {}).get('install') if not install_dir: return {'success': False, 'message': 'Path "install" is not specified'} if source_dir != build_dir: safe_rmdir(build_dir) safe_mkdir(build_dir) cmake_args = param['config_package'].get('cmake', {}).get('args', []) cmake_args = ensure_list(cmake_args) cmake_args = [p.format(**param['config_package_install_path']) for p in cmake_args] if not param['config_package'].get('cmake', {}).get('ignore_install_prefix', False): cmake_args.insert(0, '-DCMAKE_INSTALL_PREFIX='+install_dir) cmake_var = param['config_package'].get('cmake', {}).get('var', {}) for k, v in cmake_var.items(): full_value = v.format(**param['config_package_install_path']) full_arg = '-D{0}={1}'.format(k, full_value) cmake_args.append(full_arg) env = param.get('env') with open(param['log_file'], 'w') as f: cmd = ['cmake', source_dir] + cmake_args ret = call_and_log(cmd, log=f, cwd=build_dir, env=env) return {'success': ret==0, 'message': 'CMake exit code: {0}'.format(ret)}
def __load_setting(self): setting_install = self.__config_release.get('setting', {}).get('install', {}) self.__all_steps = setting_install.get('steps', []) self.__atomic_start = setting_install.get('atomic_start') self.__atomic_end = setting_install.get('atomic_end') self.__no_skip = ensure_list(setting_install.get('no_skip', [])) if len(self.__all_steps) != len(set(self.__all_steps)): raise InstallStepError('Duplicated steps found: {0}'.format( self.__all_steps)) if self.__atomic_start not in self.__all_steps or self.__atomic_end not in self.__all_steps: raise InstallStepError( 'Can not find atomic start/end: {0}/{1}'.format( self.__atomic_start, self.__atomic_end)) if self.__all_steps.index( self.__atomic_start) > self.__all_steps.index( self.__atomic_end): raise InstallStepError( 'atomic_start should not be after atomic_end')
def run(param): source_dir = param['config_package'].get('path', {}).get('source') if not source_dir: return {'success': False, 'message': 'Path "source" is not specified'} build_dir = param['config_package'].get('path', {}).get('build') if not build_dir: return {'success': False, 'message': 'Path "build" is not specified'} install_dir = param['config_package'].get('path', {}).get('install') if not install_dir: return {'success': False, 'message': 'Path "install" is not specified'} if source_dir != build_dir: safe_rmdir(build_dir) safe_mkdir(build_dir) configure_args = param['config_package'].get('configure', {}).get('args', []) configure_args = ensure_list(configure_args) configure_args = [p.format(**param['config_package_install_path']) for p in configure_args] if not param['config_package'].get('configure', {}).get('ignore_install_prefix', False): configure_args.insert(0, '--prefix='+install_dir) env = param.get('env') env_configure = env.copy() for k, v in param['config_package'].get('configure', {}).get('env', {}).items(): env_configure[k] = v.format(**param['config_package_install_path']) configure_path = os.path.join(source_dir, 'configure') with open(param['log_file'], 'w') as f: cmd = [configure_path] + configure_args ret = call_and_log(cmd, log=f, cwd=build_dir, env=env_configure) return {'success': ret==0, 'message': 'Configure exit code: {0}'.format(ret)}
def _init_package(self, handler, config_entry, config_app, config_output, config_scenario, config_option, config_release_path, config_attribute, config_release, config_release_install, config_category, config_category_priority): reinstall = config_entry.get('reinstall', False) category_install = [ ctg for ctg, ctg_cfg in config_category.items() if ctg_cfg['install'] ] _logger.debug('Category for install: {0}'.format(category_install)) for identifier, pkg_cfg in config_release.get('package', {}).items(): try: category_name, subdir, pkg_name = package_param_from_identifier( identifier, pkg_cfg) except ConfigPackageParamError: continue if category_name not in category_install: _logger.debug( 'Package "{0}" could not be installed with category "{1}"'. format(pkg_name, category_name)) continue version = ensure_list(pkg_cfg.get('version', [])) version_dir = config_category[category_name]['version_dir'] if (not version_dir) and len(version) > 1: _logger.warning( 'Only one version could be installed when category version_dir is false' ) version = version[:1] if not version: _logger.warning( 'No version is specified for category({0}), package({1})'. format(category_name, pkg_name)) for ver in version: self.setdefault(category_name, {}) self[category_name].setdefault(subdir, {}) self[category_name][subdir].setdefault(pkg_name, {}) if ver in self[category_name][subdir][pkg_name]: _logger.warning( 'Duplicated package found: category({0}), subdir({1}), package({2}), version({3})' .format(category_name, subdir, pkg_name, ver)) self[category_name][subdir][pkg_name].setdefault(ver, {}) final_config = self[category_name][subdir][pkg_name][ver] final_config['config_origin'] = copy.deepcopy(pkg_cfg) final_config['config_origin']['version'] = ver pkg_path = package_path(config_app, config_category, category_name, subdir, pkg_name, ver) final_config['config'] = transform_package( handler, 'install', category_name, subdir, pkg_name, ver, pkg_cfg, pkg_path, config_app, config_output, config_scenario, config_option, config_release_path, config_attribute, config_release, config_release_install, config_category, config_category_priority) final_config['config']['name'] = pkg_name final_config['config']['category'] = category_name final_config['config']['subdir'] = subdir if 'version' not in final_config['config']: final_config['config']['version'] = ver final_config['package_path'] = package_path( config_app, config_category, category_name, subdir, pkg_name, final_config['config']['version']) expand_package_path(final_config['package_path']['main_dir'], final_config['config']) expand_package_env(final_config['config']) final_config['install_status'] = install_status( final_config['package_path']['status_install_file']) final_config['step'] = install_step( config_release_install, final_config['config'], final_config['install_status'], reinstall)
def load(self, config_app, config_scenario, config_release_path, config_attribute, config_release, config_category): if not ('version' in config_scenario and config_scenario['version']): _logger.debug('"version" not specified in config install') return category_install = [ ctg for ctg, ctg_cfg in config_category.items() if ctg_cfg['install'] ] all_steps = config_release.get('setting', {}).get('install', {}).get('steps', []) if len(all_steps) != len(set(all_steps)): raise ConfigInstallStepError( 'Duplicated steps found in: {0}'.format(all_steps)) if len(all_steps) == 0: _logger.warn('No install steps specified') sys.path.insert(0, config_release_path['handler_python_dir']) config_release_package = config_release.get('package', {}) for identifier, pkg_cfg in config_release_package.items(): frag = identifier.split(os.sep) # Use the first part as default category name if 'category' in pkg_cfg: category_name = pkg_cfg['category'] elif len(frag) > 1: category_name = frag[0] else: _logger.warn( 'Category not specified for {0}'.format(identifier)) continue if category_name not in category_install: continue # Use the last part as default package name if 'name' in pkg_cfg: pkg_name = pkg_cfg['name'] elif frag[-1]: pkg_name = frag[-1] else: _logger.error( 'Package name not found for {0}'.format(identifier)) continue # Use the middle part as default subdir if 'subdir' in pkg_cfg: subdir = pkg_cfg['subdir'] elif frag[1:-1]: subdir = os.path.join(*frag[1:-1]) else: subdir = '' version = ensure_list(pkg_cfg.get('version', [])) version_dir = config_category[category_name]['version_dir'] if (not version_dir) and len(version) > 1: _logger.warn( 'Only one version could be installed when category version_dir is false' ) version = version[:1] if not version: _logger.warn( 'No version is specified for category({0}), package({1})'. format(category_name, pkg_name)) for ver in version: self.setdefault(category_name, {}) self[category_name].setdefault(subdir, {}) self[category_name][subdir].setdefault(pkg_name, {}) if version_dir: self[category_name][subdir][pkg_name].setdefault(ver, {}) if ver in self[category_name][subdir][pkg_name]: _logger.warn( 'Duplicated package found: category({0}), subdir({1}), package({2}), version({3})' .format(category_name, subdir, pkg_name, ver)) final_config = self[category_name][subdir][pkg_name][ver] else: if pkg_name in self[category_name][subdir]: _logger.warn( 'Duplicated package found: category({0}), subdir({1}), package({2})' .format(category_name, subdir, pkg_name)) final_config = self[category_name][subdir][pkg_name] final_config['config_origin'] = copy.deepcopy(pkg_cfg) final_config['config_origin']['version'] = ver final_config['config'] = self.__transform_package( category_name, pkg_name, subdir, ver, pkg_cfg, config_app, config_scenario, config_release_path, config_attribute, config_release, config_category) final_config['config']['name'] = pkg_name final_config['config']['category'] = category_name final_config['config']['subdir'] = subdir if 'version' not in final_config['config']: final_config['config']['version'] = ver final_config['common_path'] = self.__common_path( config_category, final_config['config']) self.__expand_package_path( final_config['common_path']['main_dir'], final_config['config']) self.__expand_env(final_config['config']) final_config['step'] = self.__install_step( all_steps, final_config['config']) sys.path.remove(config_release_path['handler_python_dir'])
def __init__(self, output='', commands=[], script_types=[]): self.__output = output self.__commands = ensure_list(commands) self.__script_types = ensure_list(script_types)
def __build_dag(self): self.__dag = Dag() steps = self._config['release_install']['steps'] atomic_start = self._config['release_install']['atomic_start'] atomic_end = self._config['release_install']['atomic_end'] package_steps = {} # For a single package for category in self._config['package_install']: for subdir in self._config['package_install'][category]: for package in self._config['package_install'][category][ subdir]: for version, value in self._config['package_install'][ category][subdir][package].items(): previous_vertex = None for step in steps: for sub_step in range(len(value['step'][step])): vertex_pkg = (category, subdir, package, version, step, sub_step) self.__dag.add_vertex(vertex_pkg) if previous_vertex is not None: _logger.debug( 'DAG add edge for single package: {0} -> {1}' .format(previous_vertex, vertex_pkg)) self.__dag.add_edge( previous_vertex, vertex_pkg) previous_vertex = vertex_pkg package_steps.setdefault(package, []) package_steps[package].append( (category, subdir, package, version)) # For the package dependencies for category in self._config['package_install']: for subdir in self._config['package_install'][category]: for package in self._config['package_install'][category][ subdir]: for version, value in self._config['package_install'][ category][subdir][package].items(): start_vertex = (category, subdir, package, version, atomic_start, 0) pkg_deps = ensure_list(value['config'].get('dep', [])) for pkg_dep in pkg_deps: if pkg_dep not in package_steps: _logger.debug( 'Dependency for package "{0}" not available: {1}' .format(package, pkg_dep)) continue for dep in package_steps[pkg_dep]: dep_end_index = len( self._config['package_install'][dep[0]][ dep[1]][dep[2]][ dep[3]]['step'][atomic_end]) - 1 end_vertex = (dep[0], dep[1], dep[2], dep[3], atomic_end, dep_end_index) _logger.debug( 'DAG add edge for package dependency: {0} -> {1}' .format(end_vertex, start_vertex)) self.__dag.add_edge(end_vertex, start_vertex)
def __init__(self, output='', script_types=[]): self.__output = output self.__script_types = ensure_list(script_types)