def __init__(self, config, config_version, config_release):
self.__config = config
self.__config_version = config_version
self.__config_release = config_release
self.__pkg_mgr = PackageManager(config_version, config_release)
self.__build_dag()
def __init__(self, config, config_version, config_release):
self.__config = config
self.__config_version = config_version
self.__config_release = config_release
self.__env = Env()
self.__env.clean()
self.__pkg_mgr = PackageManager(config_version, config_release)
def execute(self, config, config_version, destination):
release_version = config_version.get('version')
try:
config_release = ConfigRelease(config_version)
except ConfigReleaseError:
_logger.debug('Install release definition: {0}'.format(release_version))
install = Install(config, config_version)
config_release = install.config_release()
self.__pkg_mgr = PackageManager(config_version, config_release)
if not destination:
destination = os.getcwd()
destination = expand_path(destination)
pack_dir = os.path.join(destination, 'pack_'+release_version)
for pkg, pkg_info in self.__pkg_mgr.package_all().items():
version = pkg_info.get('package', {}).get('version')
if not version:
continue
download = pkg_info.get('install', {}).get('download')
if not download:
continue
url = download.get('param', {}).get('url')
if not url:
continue
url = url.format(version=version)
pkg_dir = os.path.join(pack_dir, pkg, version)
safe_mkdir(pkg_dir)
_logger.info('Packing {0}...'.format(pkg))
if download.get('handler') == 'http':
pkg_file = _download_http(url, pkg_dir)
elif download.get('handler') == 'svn':
pkg_file = _download_svn(url, pkg_dir, pkg, version)
with open(os.path.join(pkg_dir, 'md5sum.txt'), 'w') as f:
call(['md5sum', pkg_file], cwd=pkg_dir, stdout=f)
with open(os.path.join(pkg_dir, 'sha1sum.txt'), 'w') as f:
call(['sha1sum', pkg_file], cwd=pkg_dir, stdout=f)
with open(os.path.join(pkg_dir, 'url.txt'), 'w') as f:
f.write(url+'\n')
_logger.info('Package {0} packed'.format(pkg))
_logger.info('All packages in version {0} packed in {1}'.format(release_version, pack_dir))
def __init__(self, config, config_version, config_release=None, transformers=[]):
self.__config = config
self.__config_version = config_version
self.__config_release = config_release
if self.__config_release is None:
install_definition(self.__config_version)
install_handler(self.__config_version)
# Must initialize ConfigRelease after install_definition
self.__config_release = ConfigRelease(self.__config_version)
for transformer in transformers:
self.__config_release.transform(transformer)
self.__pkg_mgr = PackageManager(self.__config_version, self.__config_release)
class Install(object):
def __init__(self, config, config_version, config_release=None, transformers=[]):
self.__config = config
self.__config_version = config_version
self.__config_release = config_release
if self.__config_release is None:
install_definition(self.__config_version)
install_handler(self.__config_version)
# Must initialize ConfigRelease after install_definition
self.__config_release = ConfigRelease(self.__config_version)
for transformer in transformers:
self.__config_release.transform(transformer)
self.__pkg_mgr = PackageManager(self.__config_version, self.__config_release)
def config_release(self):
return self.__config_release
def check(self):
check = Check(self.__config_release, 'install')
missing_pkg, pkg_install_name = check.check()
return missing_pkg, check.install_cmd, pkg_install_name
def package_list(self):
pkg_list = []
for pkg, pkg_info in self.__pkg_mgr.package_all().items():
if not pkg_info.get('category', {}).get('install'):
continue
pkg_version = pkg_info['package'].get('version', 'unknown')
pkg_list.append((pkg, pkg_version, pkg_info['dir']['root']))
return pkg_list
def install_packages(self):
self.__build_dag()
sys.path.insert(0, self.__config_version.handler_dir)
self.__dag_run()
sys.path.remove(self.__config_version.handler_dir)
def __build_dag(self):
self.__dag = Dag()
for pkg, pkg_info in self.__pkg_mgr.package_all().items():
if not pkg_info.get('category', {}).get('install'):
continue
self.__dag.add_vertex((pkg, 'download'))
self.__dag.add_vertex((pkg, 'extract'))
self.__dag.add_vertex((pkg, 'pre_compile'))
self.__dag.add_vertex((pkg, 'compile'))
self.__dag.add_vertex((pkg, 'post_compile'))
self.__dag.add_vertex((pkg, 'clean'))
self.__dag.add_edge((pkg, 'download'), (pkg, 'extract'))
self.__dag.add_edge((pkg, 'extract'), (pkg, 'pre_compile'))
self.__dag.add_edge((pkg, 'pre_compile'), (pkg, 'compile'))
self.__dag.add_edge((pkg, 'compile'), (pkg, 'post_compile'))
self.__dag.add_edge((pkg, 'post_compile'), (pkg, 'clean'))
for pkg, pkg_info in self.__pkg_mgr.package_all().items():
if not pkg_info.get('category', {}).get('install'):
continue
pkgs_dep = ensure_list(pkg_info.get('attribute', {}).get('dep', []))
for pkg_dep in pkgs_dep:
if not self.__pkg_mgr.package_info(pkg_dep).get('category', {}).get('install'):
continue
self.__dag.add_edge((pkg_dep, 'post_compile'), (pkg, 'pre_compile'))
def __dag_run(self):
selector = InstallSelector(self.__config, self.__config_release)
processor = MultiThreadProcessor()
# processor = SequentialProcessor()
executor = InstallExecutor(self.__config, self.__config_version, self.__config_release)
dag_run(self.__dag, selector=selector, processor=processor, executor=executor)
class Use(object):
def __init__(self, config, config_version, config_release):
self.__config = config
self.__config_version = config_version
self.__config_release = config_release
self.__pkg_mgr = PackageManager(config_version, config_release)
self.__build_dag()
def __build_dag(self):
self.__dag = Dag()
for pkg, pkg_info in self.__pkg_mgr.package_all().items():
if not pkg_info.get('category', {}).get('auto_env'):
continue
self.__dag.add_vertex(pkg)
for pkg, pkg_info in self.__pkg_mgr.package_all().items():
if not pkg_info.get('category', {}).get('auto_env'):
continue
pkgs_dep = ensure_list(
pkg_info.get('attribute', {}).get('dep', []))
for pkg_dep in pkgs_dep:
if not self.__pkg_mgr.package_info(pkg_dep).get(
'category', {}).get('auto_env'):
continue
self.__dag.add_edge(pkg_dep, pkg)
def check(self):
check = Check(self.__config_release, 'runtime')
missing_pkg, pkg_install_name = check.check()
return missing_pkg, check.install_cmd, pkg_install_name
def run(self):
sorted_pkgs = dag_run(self.__dag)
software_root = self.__config_version.config['software_root']
release_version = self.__config_version.config['version']
env = Env()
env.clean()
env.set_release(software_root, release_version)
global_env = self.__config_release.get('setting',
{}).get('global_env', {})
new_global_env = {}
for k, v in global_env.items():
new_global_env[k] = v.format(**self.__config_version.config)
env.set_global(new_global_env)
path_usage = self.__config_release.get('setting',
{}).get('path_usage', {})
for pkg in sorted_pkgs:
pkg_info = self.__pkg_mgr.package_info(pkg)
env.set_package(path_usage, pkg_info)
env_change = env.env_change()
_logger.info('From software root: {0}'.format(software_root))
_logger.info('Using version: {0}'.format(release_version))
_logger.info('Current platform: {0}'.format(
self.__config_version.config['platform']))
return env_change
class Executor(object):
def __init__(self, config, config_version, config_release):
self.__config = config
self.__config_version = config_version
self.__config_release = config_release
self.__env = Env()
self.__env.clean()
self.__pkg_mgr = PackageManager(config_version, config_release)
def param(self, vertex):
pkg, action = vertex
par = {}
par['package'] = pkg
par['action'] = action
handler = self.__pkg_mgr.package_info(pkg)['install'].get(
action, {}).get('handler')
if handler:
par['action_handler'] = handler
par['action_param'] = self.__pkg_mgr.package_info(
pkg)['install'][action].get('param', {})
par['config'] = self.__config
par['def_dir'] = self.__config_version.def_dir
par['pkg_info'] = self.__pkg_mgr.package_info(pkg)
par['pkg_dir_list'] = self.__pkg_mgr.package_dir_list()
par['log_file'] = os.path.join(
self.__pkg_mgr.package_info(pkg)['dir']['log'],
'{0}_{1}.log'.format(action, pkg))
par['env'] = self.__env.env_final()
par['finished'] = self.__pkg_mgr.is_ready(pkg, action)
return par
# Do NOT access or modify any variables outside this function (global and member variables)
def execute(self, param):
pkg = param['package']
action = param['action']
if 'action_handler' not in param:
_logger.debug('No handler: {0} - {1}'.format(pkg, action))
return None
if param['finished']:
_logger.debug('Skip doing: {0} - {1}'.format(pkg, action))
return None
result = {}
result['start'] = datetime.datetime.utcnow()
safe_mkdir(param['pkg_info']['dir']['log'])
module_name = HANDLER_MODULE_NAME + '.install.' + param[
'action_handler']
f = load_func(module_name, 'run')
result_action = f(param)
if result_action is not None and not result_action:
_logger.critical(
'"{0} - {1}" execution error. Find log in "{2}"'.format(
pkg, action, param['log_file']))
raise InstallExecutorError('"{0} - {1}" execution error'.format(
pkg, action))
result['log_file'] = param['log_file']
result['end'] = datetime.datetime.utcnow()
return result
def report_start(self, vertice):
pass
def report_finish(self, vertice_result):
for vertex, result in vertice_result:
pkg, action = vertex
# TODO: post_compile may be absent, find out a secure way
if action == 'post_compile':
path_usage = self.__config_release.config['setting'].get(
'path_usage', {})
pkg_info = self.__pkg_mgr.package_info(pkg)
self.__env.set_package(path_usage, pkg_info)
if result:
_logger.info(' > {0} {1} finished'.format(pkg, action))
safe_mkdir(self.__pkg_mgr.package_info(pkg)['dir']['status'])
self.__pkg_mgr.save_action_status(pkg, action, result['start'],
result['end'])
def report_running(self, vertice):
if not vertice:
return
running_vertice = ', '.join(['{0}({1})'.format(*v) for v in vertice])
_logger.info('Running: ' + running_vertice)
def deliver(self, vertex, result):
pass
def abort(self, vertice):
pass