def __load_steps(self):
self.__steps = {}
pkg_mgr = PackageManager(self.__config_version, self.__config_release)
for pkg, pkg_info in pkg_mgr.package_all().items():
if not pkg_info.get('config_category', {}).get('install'):
continue
self.__steps[pkg] = []
for action in self.__all_steps:
if not pkg_info.get('config', {}).get('install',
{}).get(action):
continue
if action not in self.__no_skip and pkg_mgr.is_finished(
pkg, action):
continue
config_action = ensure_list(
pkg_info['config']['install'][action])
sub_index = 0
for cfg_action in config_action:
handler, param = _step_param(cfg_action)
if handler:
self.__steps[pkg].append({
'action': action,
'sub_action': sub_index,
'handler': handler,
'param': param
})
sub_index += 1
def __init__(self, config_user, config_version, config_release):
self.__config_user = config_user
self.__config_version = config_version
self.__config_release = config_release
self.__pkg_mgr = PackageManager(config_version, config_release)
self.__build_dag()
def __init__(self, config_user, config_version, config_release, step_info):
self.__config_user = config_user
self.__config_version = config_version
self.__config_release = config_release
self.__step_info = step_info
# Create independent env for installation
self.__env = Env()
self.__env.clean()
self.__pkg_mgr = PackageManager(config_version, config_release)
def execute(self, config_user, config_version, destination):
release_version = config_version.get('version')
try:
config_release = ConfigRelease(config_version)
except ConfigReleaseError:
_logger.debug(
'Install release definition: {0}'.format(release_version))
install = Install(config_user, config_version)
config_release = install.config_release()
self.__pkg_mgr = PackageManager(config_version, config_release)
if not destination:
destination = os.getcwd()
destination = expand_path(destination)
pack_dir = os.path.join(destination, 'pack_' + release_version)
for pkg, pkg_info in self.__pkg_mgr.package_all().items():
version = pkg_info.get('package', {}).get('version')
if not version:
continue
download = pkg_info.get('install', {}).get('download')
if not download:
continue
url = download.get('param', {}).get('url')
if not url:
continue
url = url.format(version=version)
pkg_dir = os.path.join(pack_dir, pkg, version)
safe_mkdir(pkg_dir)
_logger.info('Packing {0}...'.format(pkg))
if download.get('handler') == 'http':
pkg_file = _download_http(url, pkg_dir)
elif download.get('handler') == 'svn':
pkg_file = _download_svn(url, pkg_dir, pkg, version)
with open(os.path.join(pkg_dir, 'md5sum.txt'), 'w') as f:
call(['md5sum', pkg_file], cwd=pkg_dir, stdout=f)
with open(os.path.join(pkg_dir, 'sha1sum.txt'), 'w') as f:
call(['sha1sum', pkg_file], cwd=pkg_dir, stdout=f)
with open(os.path.join(pkg_dir, 'url.txt'), 'w') as f:
f.write(url + '\n')
_logger.info('Package {0} packed'.format(pkg))
_logger.info('All packages in version {0} packed in {1}'.format(
release_version, pack_dir))
class InstallOld(object):
def __init__(self, config_user, config_version, config_release):
self.__config_user = config_user
self.__config_version = config_version
self.__config_release = config_release
self.__pkg_mgr = PackageManager(self.__config_version, self.__config_release)
def check(self):
check = Check(self.__config_release, 'install')
missing_pkg, pkg_install_name = check.check()
return missing_pkg, check.install_cmd, pkg_install_name
def package_list(self):
pkg_list = []
for pkg, pkg_info in self.__pkg_mgr.package_all().items():
if not pkg_info.get('config_category', {}).get('install'):
continue
pkg_version = pkg_info['config'].get('version', 'unknown')
pkg_list.append((pkg, pkg_version, pkg_info['dir']['root']))
return pkg_list
def install_packages(self):
self.__step_info = Step(self.__config_version, self.__config_release)
self.__build_dag()
sys.path.insert(0, self.__config_version.handler_dir)
self.__dag_run()
sys.path.remove(self.__config_version.handler_dir)
def __find_dest_action(self, pkg, steps, dest_start, dest_end):
start_found = False
for step in steps:
if not start_found and step == atomic_start:
start_found = True
if start_found and (pkg, step) in self.__dag.vertice():
return step
if step == dest_end:
break
return ''
def __build_dag(self):
self.__dag = Dag()
all_steps = self.__step_info.package_steps_all()
# For a single package
for pkg, pkg_steps in self.__step_info.package_steps_all().items():
previous_vertex = None
for step in pkg_steps:
vertex_pkg = (pkg, step['action'], step['sub_action'])
self.__dag.add_vertex(vertex_pkg)
if previous_vertex is not None:
_logger.debug('DAG add edge: {0} -> {1}'.format(previous_vertex, vertex_pkg))
self.__dag.add_edge(previous_vertex, vertex_pkg)
previous_vertex = vertex_pkg
# For the package dependencies
for pkg, pkg_steps in self.__step_info.package_steps_all().items():
start_step = self.__step_info.find_atomic_start(pkg)
_logger.debug('start_step: {0} {1}'.format(pkg, start_step))
if not start_step:
continue
start_vertex = (pkg, start_step['action'], start_step['sub_action'])
pkg_deps = ensure_list(self.__pkg_mgr.package_info(pkg).get('config', {}).get('dep', []))
for pkg_dep in pkg_deps:
if pkg_dep not in self.__step_info.package_steps_all():
continue
end_step = self.__step_info.find_atomic_end(pkg_dep)
_logger.debug('end_step: {0} {1}'.format(pkg_dep, end_step))
if not end_step:
continue
end_vertex = (pkg_dep, end_step['action'], end_step['sub_action'])
_logger.debug('DAG add edge: {0} -> {1}'.format(end_vertex, start_vertex))
self.__dag.add_edge(end_vertex, start_vertex)
def __dag_run(self):
selector = InstallSelector(self.__config_user, self.__config_version, self.__config_release)
processor = MultiThreadProcessor()
# processor = SequentialProcessor()
executor = InstallExecutor(self.__config_user, self.__config_version, self.__config_release, self.__step_info)
dag_run(self.__dag, selector=selector, processor=processor, executor=executor)
class Use(object):
def __init__(self, config_user, config_version, config_release):
self.__config_user = config_user
self.__config_version = config_version
self.__config_release = config_release
self.__pkg_mgr = PackageManager(config_version, config_release)
self.__build_dag()
def __build_dag(self):
self.__dag = Dag()
for pkg, pkg_info in self.__pkg_mgr.package_all().items():
if not pkg_info.get('config_category', {}).get('auto_env'):
continue
self.__dag.add_vertex(pkg)
for pkg, pkg_info in self.__pkg_mgr.package_all().items():
if not pkg_info.get('config_category', {}).get('auto_env'):
continue
pkgs_dep = ensure_list(pkg_info.get('config', {}).get('dep', []))
for pkg_dep in pkgs_dep:
if not self.__pkg_mgr.package_info(pkg_dep).get(
'config_category', {}).get('auto_env'):
continue
self.__dag.add_edge(pkg_dep, pkg)
def check(self):
check = Check(self.__config_release, 'runtime')
missing_pkg, pkg_install_name = check.check()
return missing_pkg, check.install_cmd, pkg_install_name
def run(self):
sorted_pkgs = dag_run(self.__dag)
software_root = self.__config_version.config['software_root']
release_version = self.__config_version.config['version']
env = Env()
env.clean()
env.set_release(software_root, release_version)
global_env = self.__config_release.get('setting',
{}).get('global_env', {})
new_global_env = {}
for k, v in global_env.items():
new_global_env[k] = v.format(**self.__config_version.config)
env.set_global(new_global_env)
path_def = self.__config_release.get('setting', {}).get('path_def', {})
for pkg in sorted_pkgs:
pkg_info = self.__pkg_mgr.package_info(pkg)
env.set_package(path_def, pkg_info)
env_change = env.env_change()
_logger.info('From software root: {0}'.format(software_root))
_logger.info('Using version: {0}'.format(release_version))
return env_change
class Executor(object):
def __init__(self, config_user, config_version, config_release, step_info):
self.__config_user = config_user
self.__config_version = config_version
self.__config_release = config_release
self.__step_info = step_info
# Create independent env for installation
self.__env = Env()
self.__env.clean()
self.__pkg_mgr = PackageManager(config_version, config_release)
def param(self, vertex):
pkg, action, sub_action = vertex
par = {}
par['package'] = pkg
par['action'] = action
par['sub_action'] = sub_action
step = self.__step_info.package_step(pkg, action, sub_action)
par['action_name'] = step.get('action')
par['action_param'] = step.get('param')
par['log_file'] = os.path.join(
self.__pkg_mgr.package_info(pkg)['dir']['log'],
'{0}_{1}_{2}.log'.format(pkg, action, sub_action))
par['env'] = copy.deepcopy(self.__env.env_final())
par['config_user'] = copy.deepcopy(self.__config_user)
par['def_dir'] = self.__config_version.def_dir
par['pkg_info'] = copy.deepcopy(self.__pkg_mgr.package_info(pkg))
par['pkg_dir_list'] = copy.deepcopy(self.__pkg_mgr.package_dir_list())
return par
# Do NOT access or modify any variables outside this function (global and member variables)
def execute(self, param):
pkg = param['package']
action = param['action']
sub_action = param['sub_action']
if sub_action == 0:
action_full_name = '{0} - {1}'.format(pkg, action)
else:
action_full_name = '{0} - {1} - {2}'.format(
pkg, action, sub_action)
result = {}
result['start'] = datetime.datetime.utcnow()
safe_mkdir(param['pkg_info']['dir']['log'])
try:
result_action = run_handler('', 'install', param['action_name'],
param)
except Exception as e:
_logger.critical('"{0}" install handler error: {1}'.format(
action_full_name, e))
if param['config_user']['verbose']:
_logger.critical('\n{0}'.format(traceback.format_exc()))
raise
result['success'] = False
if isinstance(result_action, bool) and result_action:
result['success'] = True
if isinstance(
result_action, dict
) and 'success' in result_action and result_action['success']:
result['success'] = True
if not result['success']:
if isinstance(result_action, dict) and 'message' in result_action:
_logger.error('"{0}" execution error: {1}'.format(
action_full_name, result_action['message']))
_logger.critical('"{0}" execution error. Find log in "{1}"'.format(
action_full_name, param['log_file']))
raise InstallExecutorError(
'"{0}" execution error'.format(action_full_name))
result['action'] = result_action
result['end'] = datetime.datetime.utcnow()
return result
def report_start(self, vertice):
pass
def report_finish(self, vertice_result):
for vertex, result in vertice_result:
pkg, action, sub_action = vertex
if isinstance(result['action'], dict) and 'env_package' in result[
'action'] and result['action']['env_package']:
path_def = self.__config_release.config['setting'].get(
'path_def', {})
pkg_info = self.__pkg_mgr.package_info(pkg)
self.__env.set_package(path_def, pkg_info)
if isinstance(
result['action'],
dict) and 'save_release_status' in result[
'action'] and result['action']['save_release_status']:
self.__pkg_mgr.save_release_status(pkg, result['end'])
if result['success']:
_logger.info(' > {0} {1} {2} finished'.format(
pkg, action, sub_action))
if self.__step_info.is_last_sub_action(pkg, action,
sub_action):
self.__pkg_mgr.save_action_status(pkg, action,
result['start'],
result['end'])
def report_running(self, vertice):
if not vertice:
return
running_vertice = []
for v in vertice:
if v[2] == 0:
action_full_name = '{0}({1})'
else:
action_full_name = '{0}({1}.{2})'
running_vertice.append(action_full_name.format(*v))
_logger.info('Running: ' + ', '.join(running_vertice))
def deliver(self, vertex, result):
pass
def abort(self, vertice):
pass