def __dag_run(self):
# Get clean environment
env = Env(initial_env=self._env.env_final(),
env_prefix=self._config['app']['env_prefix'])
env.unload_packages()
env.unload_release()
env.unload_app()
env.load_app(self._config['app'])
env.load_release(self._config['scenario'], self._config['option'],
self._config['release'])
selector = InstallSelector(self._config)
processor = MultiThreadProcessor()
executor = InstallExecutor(self._config, env.env_final())
start_time = datetime.datetime.utcnow()
with Handler(self._config['release_path']['handler_python_dir']):
dag_run(self.__dag,
selector=selector,
processor=processor,
executor=executor)
end_time = datetime.datetime.utcnow()
self._config['release_status']['install'] = {}
self._config['release_status']['install']['start'] = start_time
self._config['release_status']['install']['end'] = end_time
self._config['release_status']['install']['finished'] = True
self._config['release_status']['option'] = {}
for opt in self._config['release_install']['options_to_save']:
if opt in self._config['option']:
self._config['release_status']['option'][opt] = self._config[
'option'][opt]
safe_mkdir(self._config['release_path']['status_dir'])
self._config['release_status'].save_to_file(
self._config['release_path']['status_file'])
def execute(self, check_type):
env = Env(initial_env=self._env.env_final(),
env_prefix=self._config['app']['env_prefix'])
env.unload_packages()
env.unload_release()
env.unload_app()
env.load_app(self._config['app'])
env.load_release(self._config['scenario'], self._config['option'],
self._config['release'])
missing = {}
with Handler(self._config['release_path']['handler_python_dir']) as h:
for category in self._config['package_check']:
for package, value in self._config['package_check'][
category].items():
if not self.__check_package(h, env.env_final(),
value['config'], check_type):
missing.setdefault(package, [])
missing[package].append(category)
self.__check_summary(h, missing, check_type)
return missing
class Executor(object):
def __init__(self, config, initial_env, run_type='install'):
self.__config = config
self.__env = Env(initial_env=initial_env,
env_prefix=self.__config['app']['env_prefix'])
self.__current_running = set()
self.__run_type = run_type
if run_type == 'runtime':
self.__config_package_name = 'package_runtime'
else:
self.__config_package_name = 'package_install'
def param(self, vertex):
ctg, subdir, pkg, version, step, sub_step = vertex
par = {}
par['category'] = ctg
par['subdir'] = subdir
par['package'] = pkg
par['version'] = version
par['step'] = step
par['sub_step'] = sub_step
pkg_cfg = self.__config[self.__config_package_name].package_config(
ctg, subdir, pkg, version)
step_info = pkg_cfg['step'][step][sub_step]
par['action'] = step_info.get('handler')
par['action_param'] = step_info.get('param')
par['action_install'] = step_info.get('install', False)
pkg_path = pkg_cfg['package_path']
par['package_path'] = copy.deepcopy(pkg_path)
par['log_file'] = os.path.join(
pkg_path['log_dir'], '{0}_{1}_{2}.log'.format(pkg, step, sub_step))
par['env'] = copy.deepcopy(self.__env.env_final())
par['config_package'] = copy.deepcopy(pkg_cfg['config'])
par['config_app'] = self.__config['app'].data_copy()
par['config_output'] = self.__config['output'].data_copy()
par['config_scenario'] = self.__config['scenario'].data_copy()
par['config_option'] = self.__config['option'].data_copy()
par['config_release_path'] = self.__config['release_path'].data_copy()
par['config_attribute'] = self.__config['attribute'].data_copy()
par['config_release'] = self.__config['release'].data_copy()
par['config_category'] = self.__config['category'].data_copy()
par['config_category_priority'] = self.__config[
'category_priority'].data_copy()
par['config_package_all'] = self.__config[
self.__config_package_name].data_copy()
par['config_package_all_path'] = self.__config[
self.__config_package_name + '_path'].data_copy()
return par
# This method must be thread safe
# Do NOT access or modify any variables outside this function (global and member variables)
# All parameters are passed by "param" argument
def execute(self, param):
pkg = param['package']
step = param['step']
sub_step = param['sub_step']
if sub_step == 0:
step_full_name = '{0} - {1}'.format(pkg, step)
else:
step_full_name = '{0} - {1} - {2}'.format(pkg, step, sub_step)
if not param['action_install']:
_logger.debug('Skip step: {0}'.format(step_full_name))
return {'success': True, 'skip': True}
result = {}
result['start'] = datetime.datetime.utcnow()
safe_mkdir(os.path.dirname(param['log_file']))
try:
with Handler() as h:
result_action = h.run('install', param)
except HandlerNotFoundError as e:
_logger.error('Install handler "{0}" not found for "{1}"'.format(
param['action'], step_full_name))
raise
except Exception as e:
_logger.error('Install handler "{0}" error for {1}: {2}'.format(
param['action'], step_full_name, e))
if param['config_output']['verbose']:
_logger.error('\n{0}'.format(traceback.format_exc()))
raise
if isinstance(result_action, bool):
result['success'] = result_action
elif isinstance(result_action, dict):
result['success'] = result_action.get('success', False)
else:
result['success'] = False
if not result['success']:
if isinstance(result_action, dict) and 'message' in result_action:
_logger.error('"{0}" execution error: {1}'.format(
step_full_name, result_action['message']))
_logger.error('"{0}" execution error. Find log in "{1}"'.format(
step_full_name, param['log_file']))
raise InstallExecutorError(
'"{0}" execution error'.format(step_full_name))
result['action'] = result_action
result['end'] = datetime.datetime.utcnow()
return result
def report_start(self, vertice):
pass
def report_finish(self, vertice_result):
steps = self.__config['release_install']['steps']
atomic_start = self.__config['release_install']['atomic_start']
atomic_end = self.__config['release_install']['atomic_end']
for vertex, result in vertice_result:
ctg, subdir, pkg, version, step, sub_step = vertex
pkg_cfg = self.__config[self.__config_package_name].package_config(
ctg, subdir, pkg, version)
if step == atomic_end and sub_step == (len(pkg_cfg['step'][step]) -
1):
_logger.debug('Load package env for {0}'.format(pkg))
self.__env.load_package(pkg_cfg['config'])
if result['success']:
if not result.get('skip'):
_logger.info(' > {0} {1} {2} finished'.format(
pkg, step, sub_step))
if sub_step == (len(pkg_cfg['step'][step]) - 1):
_logger.debug(
'Save install status for {0} - {1}'.format(
pkg, step))
pkg_cfg['install_status'].setdefault('steps', {})
pkg_cfg['install_status']['steps'][step] = {}
pkg_cfg['install_status']['steps'][step][
'finished'] = True
pkg_cfg['install_status']['steps'][step][
'start'] = result['start']
pkg_cfg['install_status']['steps'][step][
'end'] = result['end']
self.__config[
self.__config_package_name].save_install_status(
ctg, subdir, pkg, version)
if step == steps[-1] and sub_step == (
len(pkg_cfg['step'][step]) - 1):
if not pkg_cfg['install_status'].get('finished'):
pkg_cfg['install_status']['finished'] = True
self.__config[
self.__config_package_name].save_install_status(
ctg, subdir, pkg, version)
if self.__run_type == 'install':
_logger.debug(
'Save package config for {0}'.format(pkg))
self.__config[
self.
__config_package_name].save_package_config(
ctg, subdir, pkg, version)
def report_running(self, vertice):
if not vertice:
return
new_running = set()
for v in vertice:
ctg, subdir, pkg, version, step, sub_step = v
pkg_cfg = self.__config[self.__config_package_name].package_config(
ctg, subdir, pkg, version)
if not pkg_cfg['step'][step][sub_step].get('install'):
continue
new_running.add(v)
if new_running == self.__current_running or not new_running:
return
self.__current_running = new_running
running_vertice = []
for v in self.__current_running:
ctg, subdir, pkg, version, step, sub_step = v
if sub_step == 0:
step_full_name = '{2}({4})'
else:
step_full_name = '{2}({4}.{5})'
running_vertice.append(step_full_name.format(*v))
_logger.info('Running: ' + ', '.join(running_vertice))
def deliver(self, vertex, result):
pass
def abort(self, vertice):
pass
class Bsm(object):
def __init__(self, config_entry={}, initial_env=None):
if initial_env is None:
initial_env = os.environ
self.__config_entry_input = config_entry
self.__config_entry = copy.deepcopy(self.__config_entry_input)
self.__config = Config(self.__config_entry, initial_env)
add_stream_logger(self.__config['output']['verbose'],
self.__config['output']['quiet'])
self.__env = Env(initial_env=initial_env,
env_prefix=self.__config['app']['env_prefix'])
self.__operation = Operation(self.__config, self.__env)
def version(self):
return BSM_VERSION
def home(self):
return BSM_HOME
def __auto_reload(method):
def inner(self, *args, **kargs):
self.__watch_config()
return method(self, *args, **kargs)
return inner
def __watch_config(self):
if self.__config_entry == self.__config_entry_input:
return
self.reload_config()
def reload_config(self):
self.__config_entry = copy.deepcopy(self.__config_entry_input)
self.__config.reset(self.__config_entry)
@property
def entry(self):
return self.__config_entry_input
def switch(self, scenario):
self.__config_entry_input['scenario'] = scenario
@__auto_reload
def app(self):
return self.__config['app']['id']
@__auto_reload
def config_all(self):
return self.__config.data
@__auto_reload
def config(self, config_type):
return self.__config.config(config_type)
@__auto_reload
def option(self):
return self.__operation.execute('option')
@__auto_reload
def ls_remote(self, list_all=False):
return self.__operation.execute('ls_remote', list_all)
@__auto_reload
def check_build(self):
return self.__operation.execute('check', 'build')
@__auto_reload
def check_runtime(self):
return self.__operation.execute('check', 'runtime')
@__auto_reload
def install_release(self):
return self.__operation.execute('install-release')
@__auto_reload
def install_package(self, category=None, subdir=None, version=None):
return self.__operation.execute('install-package', category, subdir,
version)
@__auto_reload
def install_software(self):
return self.__operation.execute('install-software')
@__auto_reload
def ls(self):
return self.__operation.execute('ls')
@__auto_reload
def use(self):
self.__operation.execute('use')
@__auto_reload
def ls_package(self):
pass
@__auto_reload
def run_release_command(self, command, args):
# run customized commands defined in release
# like bsm run pack (only in current version)
pass
def apply_env_changes(self):
return self.__env.apply_changes()
def env_final(self):
return self.__env.env_final()
def default_version(self, shell=None):
try:
info = Info()
default_version = info.default_version
if default_version:
config_version = ConfigVersion(config_user, default_version)
config_release = ConfigRelease(config_version)
obj = BsmUse(config_user, config_version, config_release)
set_env, unset_env = obj.run()
else:
env = Env()
env.clean()
set_env, unset_env = env.env_change()
for e in unset_env:
shell.unset_env(e)
for k, v in set_env.items():
shell.set_env(k, v)
except Exception as e:
_logger.warn('Cat not load default version: {0}'.format(e))
class Bsm(object):
def __init__(self, config_entry={}, initial_env=None):
self.reload(config_entry=config_entry, initial_env=initial_env)
def reload(self, **kwargs):
if 'config_entry' in kwargs:
self.__config_entry = kwargs['config_entry']
if 'initial_env' in kwargs:
if kwargs['initial_env'] is None:
initial_env = os.environ
else:
initial_env = kwargs['initial_env']
else:
initial_env = self.__env.env_final()
self.__config = Config(self.__config_entry, initial_env)
create_stream_logger(self.__config['output']['verbose'],
self.__config['output']['quiet'])
self.__env = Env(initial_env=initial_env,
env_prefix=self.__config['app']['env_prefix'])
self.__env.load_app(self.__config['app'])
self.__operation = Operation(self.__config, self.__env)
def version(self):
return BSM_VERSION
def home(self):
return BSM_HOME
def app(self):
return self.__config['app']['id']
def config_all(self):
return self.__config.data()
def config(self, config_type):
return self.__config.config(config_type)
def apply_env_changes(self):
return self.__env.apply_changes()
def env_final(self):
return self.__env.env_final()
def default(self):
return self.__config['info'].get('default', {})
def ls_remote(self, list_all=False):
return self.__operation.execute('ls-remote', list_all)
def check_missing_install(self):
return self.__operation.execute('check-missing', 'install')
def check_missing_runtime(self):
return self.__operation.execute('check-missing', 'runtime')
def install_release(self):
return self.__operation.execute('install-release')
def install_release_packages(self):
return self.__operation.execute('install-release-packages')
def ls_release_version(self):
return self.__operation.execute('ls-release-version')
def load_release(self):
return self.__operation.execute('load-release')
def load_release_packages(self):
return self.__operation.execute('load-release-packages')
def clean(self):
return self.__operation.execute('clean')
def exit(self):
return self.__operation.execute('exit')
def save_as_default(self):
return self.__operation.execute('save-as-default')
def current(self):
return self.__env.current_release()
def detect_category(self, directory):
return self.__operation.execute('detect-category', directory)
def run_release_command(self, command):
return self.__operation.execute('run-release-command', command)
def find_package(self,
package,
category=None,
subdir=None,
version=None,
from_install=False):
return self.__operation.execute('find-package', package, category,
subdir, version, from_install)
def match_install_package(self,
package,
category=None,
subdir=None,
version=None,
category_origin=None,
subdir_origin=None,
version_origin=None):
return self.__operation.execute('match-install-package', package,
category, subdir, version,
category_origin, subdir_origin,
version_origin)
def package_path(self, package, category, subdir, version):
return self.__operation.execute('package-path', package, category,
subdir, version)
def package_config(self, package, category, subdir, version):
return self.__operation.execute('package-config', package, category,
subdir, version)
def package_exist(self, package, category, subdir, version):
return self.__operation.execute('package-exist', package, category,
subdir, version)
def install_package_config(self,
package,
category,
subdir,
version,
category_origin,
subdir_origin,
version_origin,
from_install=False):
return self.__operation.execute('install-package-config', package,
category, subdir, version,
category_origin, subdir_origin,
version_origin, from_install)
def install_package(self, package, category, subdir, version):
return self.__operation.execute('install-package', package, category,
subdir, version)
def remove_package(self, package, category, subdir, version):
return self.__operation.execute('remove-package', package, category,
subdir, version)
def detect_package_param(self, package_dir):
return self.__operation.execute('detect-package-param', package_dir)
def detect_package(self, directory):
return self.__operation.execute('detect-package', directory)
def check_conflict_package(self, directory):
return self.__operation.execute('check-conflict-package', directory)
def create_package_config(self, package, category, subdir, version):
return self.__operation.execute('create-package-config', package,
category, subdir, version)
def build_package(self, package, category, subdir, version, rebuild=False):
return self.__operation.execute('build-package', package, category,
subdir, version, rebuild)
def clean_package(self, package):
return self.__operation.execute('clean-package', package)
def load_package(self, package, category, subdir, version):
return self.__operation.execute('load-package', package, category,
subdir, version)
def ls_all_package(self):
return self.__operation.execute('ls-all-package')
def ls_active_package(self):
return self.__operation.execute('ls-active-package')
class Executor(object):
def __init__(self, config_user, config_version, config_release, step_info):
self.__config_user = config_user
self.__config_version = config_version
self.__config_release = config_release
self.__step_info = step_info
# Create independent env for installation
self.__env = Env()
self.__env.clean()
self.__pkg_mgr = PackageManager(config_version, config_release)
def param(self, vertex):
pkg, action, sub_action = vertex
par = {}
par['package'] = pkg
par['action'] = action
par['sub_action'] = sub_action
step = self.__step_info.package_step(pkg, action, sub_action)
par['action_name'] = step.get('action')
par['action_param'] = step.get('param')
par['log_file'] = os.path.join(
self.__pkg_mgr.package_info(pkg)['dir']['log'],
'{0}_{1}_{2}.log'.format(pkg, action, sub_action))
par['env'] = copy.deepcopy(self.__env.env_final())
par['config_user'] = copy.deepcopy(self.__config_user)
par['def_dir'] = self.__config_version.def_dir
par['pkg_info'] = copy.deepcopy(self.__pkg_mgr.package_info(pkg))
par['pkg_dir_list'] = copy.deepcopy(self.__pkg_mgr.package_dir_list())
return par
# Do NOT access or modify any variables outside this function (global and member variables)
def execute(self, param):
pkg = param['package']
action = param['action']
sub_action = param['sub_action']
if sub_action == 0:
action_full_name = '{0} - {1}'.format(pkg, action)
else:
action_full_name = '{0} - {1} - {2}'.format(
pkg, action, sub_action)
result = {}
result['start'] = datetime.datetime.utcnow()
safe_mkdir(param['pkg_info']['dir']['log'])
try:
result_action = run_handler('', 'install', param['action_name'],
param)
except Exception as e:
_logger.critical('"{0}" install handler error: {1}'.format(
action_full_name, e))
if param['config_user']['verbose']:
_logger.critical('\n{0}'.format(traceback.format_exc()))
raise
result['success'] = False
if isinstance(result_action, bool) and result_action:
result['success'] = True
if isinstance(
result_action, dict
) and 'success' in result_action and result_action['success']:
result['success'] = True
if not result['success']:
if isinstance(result_action, dict) and 'message' in result_action:
_logger.error('"{0}" execution error: {1}'.format(
action_full_name, result_action['message']))
_logger.critical('"{0}" execution error. Find log in "{1}"'.format(
action_full_name, param['log_file']))
raise InstallExecutorError(
'"{0}" execution error'.format(action_full_name))
result['action'] = result_action
result['end'] = datetime.datetime.utcnow()
return result
def report_start(self, vertice):
pass
def report_finish(self, vertice_result):
for vertex, result in vertice_result:
pkg, action, sub_action = vertex
if isinstance(result['action'], dict) and 'env_package' in result[
'action'] and result['action']['env_package']:
path_def = self.__config_release.config['setting'].get(
'path_def', {})
pkg_info = self.__pkg_mgr.package_info(pkg)
self.__env.set_package(path_def, pkg_info)
if isinstance(
result['action'],
dict) and 'save_release_status' in result[
'action'] and result['action']['save_release_status']:
self.__pkg_mgr.save_release_status(pkg, result['end'])
if result['success']:
_logger.info(' > {0} {1} {2} finished'.format(
pkg, action, sub_action))
if self.__step_info.is_last_sub_action(pkg, action,
sub_action):
self.__pkg_mgr.save_action_status(pkg, action,
result['start'],
result['end'])
def report_running(self, vertice):
if not vertice:
return
running_vertice = []
for v in vertice:
if v[2] == 0:
action_full_name = '{0}({1})'
else:
action_full_name = '{0}({1}.{2})'
running_vertice.append(action_full_name.format(*v))
_logger.info('Running: ' + ', '.join(running_vertice))
def deliver(self, vertex, result):
pass
def abort(self, vertice):
pass