def _generate_files(self, format_args, tmp_sh, tmp_cont):
project = self.project
step = self.stage
vars = self.variables
with tmp_sh:
tmp_sh.write_shebang()
if vars:
try:
vars_json = '\n'.join([f'{k}: {v}' for k,v in vars.items() if not str(k).startswith('!')])
except Exception as e:
vars_json = str(vars)
tmp_sh.write_section('CONFIGURATION', '\n'.join(['# ' + x for x in vars_json.splitlines()]))
if project.init_shell:
configured = configure_string(project.init_shell, format_args)
tmp_sh.write_section('INIT SHELL', configured)
configured = configure_string(step.shell, format_args)
tmp_sh.write_section('STEP SHELL', configured)
args = ['bash', tmp_sh.path]
# logger.warning(tmp_sh.content)
if step.container:
with tmp_cont:
tmp_cont.write_shebang()
tmp_cont.write(configure_string(step.container.exec % tmp_sh.path, format_args))
args = ['bash', tmp_cont.path]
return args
def __init__(self, step_cache, step_git, global_args=None, cwd='.'):
"""
Function will clone given git
and will checkout to a given branch and commit
:type step_cache: cihpc.core.structures.project_step_cache.ProjectStepCache
:type step_git: list[cihpc.core.structures.project_step_git.ProjectStepGit]
:type global_args: dict
"""
self.global_args = global_args
self.step_cache = step_cache
self.step_git = step_git
self.value = None
self.cwd = os.path.abspath(cwd)
self.storage = configure_string(self.step_cache.storage,
self.global_args)
self.directories = configure_object(self.step_cache.directories,
self.global_args)
self.cache_folder_props = configure_object(self._cache_folder_format,
global_args)
cache_folder_name = '-'.join(self.cache_folder_props)
self.location = os.path.join(self.storage, cache_folder_name)
def test_config_special_variables(self):
from cihpc.core.structures.project import Project
from cihpc.cfg import cfgutil
project = Project(name='foobar')
self.assertEqual(
cfgutil.configure_string('My home is <os.HOME>',
project.global_args),
'My home is %s' % os.environ.get('HOME'))
self.assertEqual(
cfgutil.configure_string('My home is <$.HOME>',
project.global_args),
'My home is %s' % os.environ.get('HOME'))
kwargs = merge_dict(project.global_args,
foobar=dict(foo='foo', bar=dict(foo='foo.bar')))
self.assertEqual(
cfgutil.configure_string(
'foobar.bar.foo is <foobar.bar.foo> and I am <$.USER>',
kwargs),
'foobar.bar.foo is foo.bar and I am %s' % os.environ.get('USER'))
def get_script(cls, args, execute_path=None):
install_cmd = cls._generate_install_command(args)
execute_cmd = cls._generage_execute_command(args)
execute_script = cls._load_content(execute_path)
kwargs = {
'ci-hpc-exec' : ' '.join(global_configuration.exec_args) + ' ' + execute_cmd,
'ci-hpc-exec-no-interpret': global_configuration.main_py + ' ' + execute_cmd,
'ci-hpc-exec-only-args' : execute_cmd,
'ci-hpc-install' : install_cmd,
}
return cfgutil.configure_string(
execute_script,
kwargs
)
def __init__(self, name, global_args_extra, **kwargs):
# globals
self.name = name
self.init_shell = kwargs.get('init-shell', None)
self.use_database = not kwargs.get('no-database', False)
self._secure_name = strings.secure_filename(self.name)
self._counter = Counter()
self._os = EnvGetter()
self._global_args = dict(
__project__=dict(
start=DatePoint(),
current=DatePoint(),
counter=self._counter,
),
os=self._os,
)
self._global_args['$'] = self._global_args['os']
self._global_args.update(global_args_extra)
self.workdir = os.path.abspath(
configure_string(kwargs.get('workdir', '.'), self.global_args)
)
# sections
self.install = ProjectSection('install', kwargs.get('install', []))
self.test = ProjectSection('test', kwargs.get('test', []))
# stages
self.stages = ProjectStages(kwargs.get('stages'))
# git info
self.git = ProjectGit(
configure_object(kwargs.get('git'), self.global_args)
)
self._global_args['git'] = self.git.main_repo
self._global_args['deps'] = self.git.deps
def __init__(self, project, stage, variables):
"""
Parameters
----------
project: cihpc.core.structures.project.Project
stage which is this instance bound to
stage: cihpc.core.structures.project_stage.ProjectStage
stage which is this instance bound to
variables: dict
optional variables for the sub stage
"""
super(ProcessStage, self).__init__()
self.stage = stage
self.variables = variables
self.project = project
self.name = stage.name
self.name_prefix = ''
self._shell_result = None
self._cache = None
self.collect_result = None
self.current_index = None
if self.stage.index:
self.current_index = configure_object(self.stage.index, self.variables)
# here we try to determine how many cpus we want to use in this step
# based on a cpu-prop value
if self.stage.parallel and self.stage.parallel.prop:
cpu_prop_conf = configure_string(self.stage.parallel.prop, self.variables)
cpu_prop = parse_cpu_property(cpu_prop_conf)
self.variables.update(dict(__cpu__=cpu_prop))
def process_step_collect(project, step, process_result, format_args=None):
"""
Function will collect artifacts for the given step
:type step: cihpc.core.structures.project_stage.ProjectStage
:type project: structures.project.Project
:type process_result: proc.step.step_shell.ProcessStepResult
"""
logger.debug(f'collecting artifacts')
result = namedtuple('CollectResult', ['total', 'items'])(total=[],
items=[])
logger.info(f'loading module {step.collect.module}')
module = importlib.import_module(step.collect.module)
CollectModule = module.CollectModule
# obtain git information
artifacts_base.CIHPCReport.init(step.collect.repo)
# enrich result section
if step.collect.extra:
extra = configure_object(step.collect.extra, format_args)
artifacts_base.CIHPCReport.global_problem.update(extra)
# if ord is set
if step.index:
index = configure_object(step.index, format_args)
artifacts_base.CIHPCReport.global_index.update(index)
# create instance of the CollectModule
instance = CollectModule(
project.name) # type: artifacts_base.AbstractCollectModule
# get either yaml or json
conversion = convert_method(step.collect.type)
# --------------------------------------------------
results = list()
timers_info = []
timers_total = 0
if step.collect.parse:
reports = process_step_collect_parse(project, step, process_result,
format_args)
logger.debug(f'artifacts: found {len(reports)} reports to process')
for report, file in iter_reports(reports, conversion, is_file=False):
try:
collect_result = instance.process(report, file)
timers_total += len(collect_result.items)
timers_info.append(
(os.path.basename(file), len(collect_result.items)))
results.append(collect_result)
except Exception as e:
logger.exception(
f'artifact processing failed (parse method) \n'
f'module: {CollectModule}\n'
f'file: {file}\n')
logger.debug(str(report))
for file, timers in timers_info:
logger.debug(f'%20s: %5d timers found' % (file, timers))
logger.debug(
f'artifacts: found {timers_total} timer(s) in {len(reports)} file(s)'
)
# insert artifacts into db
if step.collect.save_to_db:
instance.save_to_db(results)
if timers_total:
result.total.append(timers_total)
if timers_info:
result.items.append(timers_info)
# --------------------------------------------------
results = list()
timers_info = []
timers_total = 0
if step.collect.files:
files_glob = configure_string(step.collect.files, format_args)
files = glob.glob(files_glob, recursive=True)
logger.debug(f'artifacts: found {len(files)} files to process')
for report, file in iter_reports(files, conversion, is_file=True):
try:
collect_result = instance.process(report, file)
timers_total += len(collect_result.items)
timers_info.append(
(os.path.basename(file), len(collect_result.items)))
results.append(collect_result)
except Exception as e:
logger.warning(f'artifact processing failed (files method) \n'
f'module: {CollectModule}\n'
f'file: {file}\n')
logger.debug(str(report))
for file, timers in timers_info:
logger.debug(f'%20s: %5d timers found' % (file, timers))
logger.debug(
f'artifacts: found {timers_total} timer(s) in {len(reports)} file(s)'
)
# insert artifacts into db
if step.collect.save_to_db:
instance.save_to_db(results)
# move results to they are not processed twice
if step.collect.move_to:
move_to = configure_string(step.collect.move_to, format_args)
logger.debug(f'artifacts: moving {len(files)} files to {move_to}')
for file in files:
old_filepath = os.path.abspath(file)
# customize location and prefix of the dir structure if set
if step.collect.cut_prefix:
new_rel_filepath = old_filepath.replace(
step.collect.cut_prefix, '').lstrip('/')
new_dirname = os.path.join(
move_to,
os.path.dirname(new_rel_filepath),
)
new_filepath = os.path.join(new_dirname,
os.path.basename(file))
os.makedirs(new_dirname, exist_ok=True)
os.rename(old_filepath, new_filepath)
else:
new_filepath = os.path.join(move_to,
os.path.basename(file))
os.makedirs(move_to, exist_ok=True)
os.rename(old_filepath, new_filepath)
if timers_total:
result.total.append(timers_total)
if timers_info:
result.items.append(timers_info)
return result
def test_configure_string(self):
from cihpc.cfg import cfgutil
variables = dict(foo='foo', bar=dict(
foo='123',
bar=True,
))
self.assertEqual(cfgutil.configure_string('<foo>', variables), 'foo')
self.assertEqual(cfgutil.configure_string('<bar.foo>', variables),
'123')
self.assertEqual(cfgutil.configure_string('<bar.foo|s>', variables),
'123')
self.assertEqual(cfgutil.configure_string('<bar.foo|i>', variables),
123)
self.assertEqual(cfgutil.configure_string('<bar.foo|f>', variables),
123.0)
self.assertEqual(cfgutil.configure_string('<bar.bar>', variables),
'True')
self.assertEqual(cfgutil.configure_string('<bar.bar|b>', variables),
True)
self.assertIsInstance(cfgutil.configure_string('<bar>', variables),
str)
self.assertIsInstance(cfgutil.configure_string('<bar.foo>', variables),
str)
self.assertIsInstance(
cfgutil.configure_string('<bar.foo|i>', variables), int)
self.assertIsInstance(
cfgutil.configure_string('<bar.bar|b>', variables), bool)
variables = dict(foo=random.random(), bar=dict(foo=random.random(), ))
config = list(cfgutil.configure_file(config_path, variables))[0]
self.assertEqual(config['stages'][0]['foo'], variables['foo'])
self.assertEqual(config['stages'][0]['bar-foo'],
'<bar.foo>') # is not supported
def configure_git(git, global_args):
if global_args:
git.branch = configure_string(git.branch, global_args)
git.commit = configure_string(git.commit, global_args)
return git