def __build_dag(self):
self.__dag = Dag()
for pkg, pkg_info in self.__pkg_mgr.package_all().items():
if not pkg_info.get('config_category', {}).get('auto_env'):
continue
self.__dag.add_vertex(pkg)
for pkg, pkg_info in self.__pkg_mgr.package_all().items():
if not pkg_info.get('config_category', {}).get('auto_env'):
continue
pkgs_dep = ensure_list(pkg_info.get('config', {}).get('dep', []))
for pkg_dep in pkgs_dep:
if not self.__pkg_mgr.package_info(pkg_dep).get(
'config_category', {}).get('auto_env'):
continue
self.__dag.add_edge(pkg_dep, pkg)
def __build_dag(self):
self.__dag = Dag()
all_steps = []
for category in self._config['package_install']:
for subdir in self._config['package_install'][category]:
for package in self._config['package_install'][category][subdir]:
if self._config['category'][category]['version_dir']:
for version in self._config['package_install'][category][subdir][package]:
all_steps +=
# For a single package
for pkg, pkg_steps in self.__step_info.package_steps_all().items():
previous_vertex = None
for step in pkg_steps:
vertex_pkg = (pkg, step['action'], step['sub_action'])
self.__dag.add_vertex(vertex_pkg)
if previous_vertex is not None:
_logger.debug('DAG add edge: {0} -> {1}'.format(previous_vertex, vertex_pkg))
self.__dag.add_edge(previous_vertex, vertex_pkg)
previous_vertex = vertex_pkg
# For the package dependencies
for pkg, pkg_steps in self.__step_info.package_steps_all().items():
start_step = self.__step_info.find_atomic_start(pkg)
_logger.debug('start_step: {0} {1}'.format(pkg, start_step))
if not start_step:
continue
start_vertex = (pkg, start_step['action'], start_step['sub_action'])
pkg_deps = ensure_list(self.__pkg_mgr.package_info(pkg).get('config', {}).get('dep', []))
for pkg_dep in pkg_deps:
if pkg_dep not in self.__step_info.package_steps_all():
continue
end_step = self.__step_info.find_atomic_end(pkg_dep)
_logger.debug('end_step: {0} {1}'.format(pkg_dep, end_step))
if not end_step:
continue
end_vertex = (pkg_dep, end_step['action'], end_step['sub_action'])
_logger.debug('DAG add edge: {0} -> {1}'.format(end_vertex, start_vertex))
self.__dag.add_edge(end_vertex, start_vertex)
class Install(Base):
def execute(self):
self.__build_dag()
sys.path.insert(0, self._config['release_path']['handler_python_dir'])
self.__dag_run()
sys.path.remove(self._config['release_path']['handler_python_dir'])
def __build_dag(self):
self.__dag = Dag()
all_steps = []
for category in self._config['package_install']:
for subdir in self._config['package_install'][category]:
for package in self._config['package_install'][category][subdir]:
if self._config['category'][category]['version_dir']:
for version in self._config['package_install'][category][subdir][package]:
all_steps +=
# For a single package
for pkg, pkg_steps in self.__step_info.package_steps_all().items():
previous_vertex = None
for step in pkg_steps:
vertex_pkg = (pkg, step['action'], step['sub_action'])
self.__dag.add_vertex(vertex_pkg)
if previous_vertex is not None:
_logger.debug('DAG add edge: {0} -> {1}'.format(previous_vertex, vertex_pkg))
self.__dag.add_edge(previous_vertex, vertex_pkg)
previous_vertex = vertex_pkg
# For the package dependencies
for pkg, pkg_steps in self.__step_info.package_steps_all().items():
start_step = self.__step_info.find_atomic_start(pkg)
_logger.debug('start_step: {0} {1}'.format(pkg, start_step))
if not start_step:
continue
start_vertex = (pkg, start_step['action'], start_step['sub_action'])
pkg_deps = ensure_list(self.__pkg_mgr.package_info(pkg).get('config', {}).get('dep', []))
for pkg_dep in pkg_deps:
if pkg_dep not in self.__step_info.package_steps_all():
continue
end_step = self.__step_info.find_atomic_end(pkg_dep)
_logger.debug('end_step: {0} {1}'.format(pkg_dep, end_step))
if not end_step:
continue
end_vertex = (pkg_dep, end_step['action'], end_step['sub_action'])
_logger.debug('DAG add edge: {0} -> {1}'.format(end_vertex, start_vertex))
self.__dag.add_edge(end_vertex, start_vertex)
def __dag_run(self):
selector = InstallSelector(self.__config_user, self.__config_version, self.__config_release)
processor = MultiThreadProcessor()
# processor = SequentialProcessor()
executor = InstallExecutor(self.__config_user, self.__config_version, self.__config_release, self.__step_info)
dag_run(self.__dag, selector=selector, processor=processor, executor=executor)
class InstallOld(object):
def __init__(self, config_user, config_version, config_release):
self.__config_user = config_user
self.__config_version = config_version
self.__config_release = config_release
self.__pkg_mgr = PackageManager(self.__config_version, self.__config_release)
def check(self):
check = Check(self.__config_release, 'install')
missing_pkg, pkg_install_name = check.check()
return missing_pkg, check.install_cmd, pkg_install_name
def package_list(self):
pkg_list = []
for pkg, pkg_info in self.__pkg_mgr.package_all().items():
if not pkg_info.get('config_category', {}).get('install'):
continue
pkg_version = pkg_info['config'].get('version', 'unknown')
pkg_list.append((pkg, pkg_version, pkg_info['dir']['root']))
return pkg_list
def install_packages(self):
self.__step_info = Step(self.__config_version, self.__config_release)
self.__build_dag()
sys.path.insert(0, self.__config_version.handler_dir)
self.__dag_run()
sys.path.remove(self.__config_version.handler_dir)
def __find_dest_action(self, pkg, steps, dest_start, dest_end):
start_found = False
for step in steps:
if not start_found and step == atomic_start:
start_found = True
if start_found and (pkg, step) in self.__dag.vertice():
return step
if step == dest_end:
break
return ''
def __build_dag(self):
self.__dag = Dag()
all_steps = self.__step_info.package_steps_all()
# For a single package
for pkg, pkg_steps in self.__step_info.package_steps_all().items():
previous_vertex = None
for step in pkg_steps:
vertex_pkg = (pkg, step['action'], step['sub_action'])
self.__dag.add_vertex(vertex_pkg)
if previous_vertex is not None:
_logger.debug('DAG add edge: {0} -> {1}'.format(previous_vertex, vertex_pkg))
self.__dag.add_edge(previous_vertex, vertex_pkg)
previous_vertex = vertex_pkg
# For the package dependencies
for pkg, pkg_steps in self.__step_info.package_steps_all().items():
start_step = self.__step_info.find_atomic_start(pkg)
_logger.debug('start_step: {0} {1}'.format(pkg, start_step))
if not start_step:
continue
start_vertex = (pkg, start_step['action'], start_step['sub_action'])
pkg_deps = ensure_list(self.__pkg_mgr.package_info(pkg).get('config', {}).get('dep', []))
for pkg_dep in pkg_deps:
if pkg_dep not in self.__step_info.package_steps_all():
continue
end_step = self.__step_info.find_atomic_end(pkg_dep)
_logger.debug('end_step: {0} {1}'.format(pkg_dep, end_step))
if not end_step:
continue
end_vertex = (pkg_dep, end_step['action'], end_step['sub_action'])
_logger.debug('DAG add edge: {0} -> {1}'.format(end_vertex, start_vertex))
self.__dag.add_edge(end_vertex, start_vertex)
def __dag_run(self):
selector = InstallSelector(self.__config_user, self.__config_version, self.__config_release)
processor = MultiThreadProcessor()
# processor = SequentialProcessor()
executor = InstallExecutor(self.__config_user, self.__config_version, self.__config_release, self.__step_info)
dag_run(self.__dag, selector=selector, processor=processor, executor=executor)
class Use(object):
def __init__(self, config_user, config_version, config_release):
self.__config_user = config_user
self.__config_version = config_version
self.__config_release = config_release
self.__pkg_mgr = PackageManager(config_version, config_release)
self.__build_dag()
def __build_dag(self):
self.__dag = Dag()
for pkg, pkg_info in self.__pkg_mgr.package_all().items():
if not pkg_info.get('config_category', {}).get('auto_env'):
continue
self.__dag.add_vertex(pkg)
for pkg, pkg_info in self.__pkg_mgr.package_all().items():
if not pkg_info.get('config_category', {}).get('auto_env'):
continue
pkgs_dep = ensure_list(pkg_info.get('config', {}).get('dep', []))
for pkg_dep in pkgs_dep:
if not self.__pkg_mgr.package_info(pkg_dep).get(
'config_category', {}).get('auto_env'):
continue
self.__dag.add_edge(pkg_dep, pkg)
def check(self):
check = Check(self.__config_release, 'runtime')
missing_pkg, pkg_install_name = check.check()
return missing_pkg, check.install_cmd, pkg_install_name
def run(self):
sorted_pkgs = dag_run(self.__dag)
software_root = self.__config_version.config['software_root']
release_version = self.__config_version.config['version']
env = Env()
env.clean()
env.set_release(software_root, release_version)
global_env = self.__config_release.get('setting',
{}).get('global_env', {})
new_global_env = {}
for k, v in global_env.items():
new_global_env[k] = v.format(**self.__config_version.config)
env.set_global(new_global_env)
path_def = self.__config_release.get('setting', {}).get('path_def', {})
for pkg in sorted_pkgs:
pkg_info = self.__pkg_mgr.package_info(pkg)
env.set_package(path_def, pkg_info)
env_change = env.env_change()
_logger.info('From software root: {0}'.format(software_root))
_logger.info('Using version: {0}'.format(release_version))
return env_change
def __build_dag(self):
self.__dag = Dag()
steps = self._config['release_install']['steps']
atomic_start = self._config['release_install']['atomic_start']
atomic_end = self._config['release_install']['atomic_end']
package_steps = {}
# For a single package
for category in self._config['package_install']:
for subdir in self._config['package_install'][category]:
for package in self._config['package_install'][category][
subdir]:
for version, value in self._config['package_install'][
category][subdir][package].items():
previous_vertex = None
for step in steps:
for sub_step in range(len(value['step'][step])):
vertex_pkg = (category, subdir, package,
version, step, sub_step)
self.__dag.add_vertex(vertex_pkg)
if previous_vertex is not None:
_logger.debug(
'DAG add edge for single package: {0} -> {1}'
.format(previous_vertex, vertex_pkg))
self.__dag.add_edge(
previous_vertex, vertex_pkg)
previous_vertex = vertex_pkg
package_steps.setdefault(package, [])
package_steps[package].append(
(category, subdir, package, version))
# For the package dependencies
for category in self._config['package_install']:
for subdir in self._config['package_install'][category]:
for package in self._config['package_install'][category][
subdir]:
for version, value in self._config['package_install'][
category][subdir][package].items():
start_vertex = (category, subdir, package, version,
atomic_start, 0)
pkg_deps = ensure_list(value['config'].get('dep', []))
for pkg_dep in pkg_deps:
if pkg_dep not in package_steps:
_logger.debug(
'Dependency for package "{0}" not available: {1}'
.format(package, pkg_dep))
continue
for dep in package_steps[pkg_dep]:
dep_end_index = len(
self._config['package_install'][dep[0]][
dep[1]][dep[2]][
dep[3]]['step'][atomic_end]) - 1
end_vertex = (dep[0], dep[1], dep[2], dep[3],
atomic_end, dep_end_index)
_logger.debug(
'DAG add edge for package dependency: {0} -> {1}'
.format(end_vertex, start_vertex))
self.__dag.add_edge(end_vertex, start_vertex)
class InstallReleasePackages(Base):
def execute(self):
self.__build_dag()
self.__dag_run()
def __build_dag(self):
self.__dag = Dag()
steps = self._config['release_install']['steps']
atomic_start = self._config['release_install']['atomic_start']
atomic_end = self._config['release_install']['atomic_end']
package_steps = {}
# For a single package
for category in self._config['package_install']:
for subdir in self._config['package_install'][category]:
for package in self._config['package_install'][category][
subdir]:
for version, value in self._config['package_install'][
category][subdir][package].items():
previous_vertex = None
for step in steps:
for sub_step in range(len(value['step'][step])):
vertex_pkg = (category, subdir, package,
version, step, sub_step)
self.__dag.add_vertex(vertex_pkg)
if previous_vertex is not None:
_logger.debug(
'DAG add edge for single package: {0} -> {1}'
.format(previous_vertex, vertex_pkg))
self.__dag.add_edge(
previous_vertex, vertex_pkg)
previous_vertex = vertex_pkg
package_steps.setdefault(package, [])
package_steps[package].append(
(category, subdir, package, version))
# For the package dependencies
for category in self._config['package_install']:
for subdir in self._config['package_install'][category]:
for package in self._config['package_install'][category][
subdir]:
for version, value in self._config['package_install'][
category][subdir][package].items():
start_vertex = (category, subdir, package, version,
atomic_start, 0)
pkg_deps = ensure_list(value['config'].get('dep', []))
for pkg_dep in pkg_deps:
if pkg_dep not in package_steps:
_logger.debug(
'Dependency for package "{0}" not available: {1}'
.format(package, pkg_dep))
continue
for dep in package_steps[pkg_dep]:
dep_end_index = len(
self._config['package_install'][dep[0]][
dep[1]][dep[2]][
dep[3]]['step'][atomic_end]) - 1
end_vertex = (dep[0], dep[1], dep[2], dep[3],
atomic_end, dep_end_index)
_logger.debug(
'DAG add edge for package dependency: {0} -> {1}'
.format(end_vertex, start_vertex))
self.__dag.add_edge(end_vertex, start_vertex)
def __dag_run(self):
# Get clean environment
env = Env(initial_env=self._env.env_final(),
env_prefix=self._config['app']['env_prefix'])
env.unload_packages()
env.unload_release()
env.unload_app()
env.load_app(self._config['app'])
env.load_release(self._config['scenario'], self._config['option'],
self._config['release'])
selector = InstallSelector(self._config)
processor = MultiThreadProcessor()
executor = InstallExecutor(self._config, env.env_final())
start_time = datetime.datetime.utcnow()
with Handler(self._config['release_path']['handler_python_dir']):
dag_run(self.__dag,
selector=selector,
processor=processor,
executor=executor)
end_time = datetime.datetime.utcnow()
self._config['release_status']['install'] = {}
self._config['release_status']['install']['start'] = start_time
self._config['release_status']['install']['end'] = end_time
self._config['release_status']['install']['finished'] = True
self._config['release_status']['option'] = {}
for opt in self._config['release_install']['options_to_save']:
if opt in self._config['option']:
self._config['release_status']['option'][opt] = self._config[
'option'][opt]
safe_mkdir(self._config['release_path']['status_dir'])
self._config['release_status'].save_to_file(
self._config['release_path']['status_file'])