def __init__(self, yaml_config_file): self.yaml_config_file = yaml_config_file self.root = Paths.dirname(self.yaml_config_file) self.yamls = self._get_all_yamls() self.cases = list() self.common_config = None # create dummy case for every yaml file in folder if not Paths.exists(self.yaml_config_file): self.common_config = deepcopy(DEFAULTS) for y in self.yamls: dummy_case = deepcopy(DEFAULTS) dummy_case['file'] = [y] self.cases.append(dummy_case) else: # setup common config values self.yaml_config = self._read_yaml() self.common_config = self.merge(DEFAULTS, self.yaml_config.get('common_config', {})) # first process files which are specified in test_cases missing = [Paths.basename(y) for y in self.yamls] for case in self.yaml_config.get('test_cases', []): case_config = self.merge(self.common_config, case) self.cases.append(case_config) for f in case_config['file']: if f in missing: missing.remove(f) # process rest (dummy case) for y in missing: dummy_case = deepcopy(self.common_config) dummy_case['file'] = [y] self.cases.append(dummy_case)
def _run(self): if Paths.exists(self.dir): try: shutil.rmtree(self.dir) self.returncode = 0 except OSError as e: self.returncode = 4 self.error = str(e)
def get_pbs_module(hostname_hint=None): """ file host_table.yaml serves as lookup table when using python script in queue mode each key is hostname and each value names a module which should be loaded modules are located in /src/python/scripts/pbs/modules If no matching key for current machine exists try to use pbs_<hostname> where all dots(.) are replaced with underscores(_) if hostname_hint is not set node name will be used :rtype : scripts.pbs.modules.pbs_tarkil_cesnet_cz """ pbs_module_path = None host_file = Paths.join(Paths.flow123d_root(), 'config', 'host_table.yaml') host_file_exists = Paths.exists(host_file) hostname = hostname_hint or platform.node() from_host = False # try to get name from json file if host_file_exists: with open(host_file, 'r') as fp: hosts = yaml.load(fp) pbs_module_path = hosts.get(hostname, None) from_host = pbs_module_path is not None if not pbs_module_path: hostname = hostname.replace('.', '_') pbs_module_path = 'pbs_{}'.format(hostname) # construct full path for import full_module_path = 'scripts.pbs.modules.{module_name}'.format( module_name=pbs_module_path) # try to get pbs_module try: return importlib.import_module(full_module_path) except ImportError: Printer.all.err('Could not load module "{}" ({}) for hostname "{}"', pbs_module_path, full_module_path, hostname) with Printer.all.with_level(2): if host_file_exists: if from_host: Printer.all.err( 'Value specified in host_table.yaml "{}" points to non-existing module', pbs_module_path) else: Printer.all.err( 'Config file host_table.yaml does not have entry for hostname "{}"', hostname) else: Printer.all.err( 'Config file host_table.yaml does not exists ({}) and auto module detection failed', host_file) raise
def run(self): # wipe out dirs on demand if self.wipeout_dir: for d in self.wipeout_dir: if Paths.exists(d): Printer.all.out('Deleting directory {}', d) shutil.rmtree(d) total = 0 for p in self: total += 1 p.copy() Printer.all.out("Copied out {} files", total)
def get_pbs_module(hostname_hint=None): """ file host_table.yaml serves as lookup table when using python script in queue mode each key is hostname and each value names a module which should be loaded modules are located in /src/python/scripts/pbs/modules If no matching key for current machine exists try to use pbs_<hostname> where all dots(.) are replaced with underscores(_) if hostname_hint is not set node name will be used :rtype : scripts.pbs.modules.pbs_tarkil_cesnet_cz """ pbs_module_path = None host_file = Paths.join(Paths.flow123d_root(), 'config', 'host_table.yaml') host_file_exists = Paths.exists(host_file) hostname = hostname_hint or platform.node() from_host = False # try to get name from json file if host_file_exists: with open(host_file, 'r') as fp: hosts = yaml.load(fp) pbs_module_path = hosts.get(hostname, None) from_host = pbs_module_path is not None if not pbs_module_path: hostname = hostname.replace('.', '_') pbs_module_path = 'pbs_{}'.format(hostname) # construct full path for import full_module_path = 'scripts.pbs.modules.{module_name}'.format(module_name=pbs_module_path) # try to get pbs_module try: return importlib.import_module(full_module_path) except ImportError: Printer.all.err('Could not load module "{}" ({}) for hostname "{}"', pbs_module_path, full_module_path, hostname) with Printer.all.with_level(2): if host_file_exists: if from_host: Printer.all.err('Value specified in host_table.yaml "{}" points to non-existing module', pbs_module_path) else: Printer.all.err('Config file host_table.yaml does not have entry for hostname "{}"', hostname) else: Printer.all.err('Config file host_table.yaml does not exists ({}) and auto module detection failed', host_file) raise
def _run(self): """ Run method for this module """ if self.arg_options.random_output_dir: import scripts.yamlc as yamlc from core.base import System yamlc.TEST_RESULTS = 'test_results-{}'.format(System.rnd8) self.all_yamls = list() for path in self.others: if not Paths.exists(path): Printer.all.err('given path does not exists, ignoring path "{}"', path) sys.exit(3) # append files to all_yamls if Paths.is_dir(path): self.all_yamls.extend(Paths.walk(path, ConfigPool.yaml_filters)) else: self.all_yamls.append(path) Printer.all.out("Found {} yaml file/s", len(self.all_yamls)) if not self.all_yamls: Printer.all.wrn('No yaml files found in locations: \n {}', '\n '.join(self.others)) sys.exit(0) self.configs = self.read_configs(self.all_yamls) self.configs.update( proc=self.arg_options.cpu, time_limit=self.arg_options.time_limit, memory_limit=self.arg_options.memory_limit, ) # filter tags for includes and excludes self.configs.filter_tags( include=self.include, exclude=self.exclude ) if self.arg_options.queue: Printer.all.out('Running in PBS mode') return self.run_pbs_mode() else: Printer.all.out('Running in LOCAL mode') return self.run_local_mode()
def _run(self): """ Run method for this module """ if self.arg_options.random_output_dir: import scripts.yamlc as yamlc yamlc.TEST_RESULTS = 'test_results-{}'.format(self.arg_options.random_output_dir) self.all_yamls = list() for path in self.arg_options.args: if not Paths.exists(path): Printer.all.err('given path does not exists, path "{}"', path) sys.exit(3) # append files to all_yamls if Paths.is_dir(path): self.all_yamls.extend(Paths.walk(path, ConfigPool.yaml_filters)) else: self.all_yamls.append(path) Printer.all.out("Found {} yaml file/s", len(self.all_yamls)) if not self.all_yamls: Printer.all.wrn('No yaml files found in locations: \n {}', '\n '.join(self.arg_options.args)) sys.exit(0) self.configs = self.read_configs(self.all_yamls) self.configs.update( proc=self.arg_options.cpu, time_limit=self.arg_options.time_limit, memory_limit=self.arg_options.memory_limit, ) # filter tags for includes and excludes self.configs.filter_tags( include=self.include, exclude=self.exclude ) if self.arg_options.queue: Printer.all.out('Running in PBS mode') return self.run_pbs_mode() else: Printer.all.out('Running in LOCAL mode') return self.run_local_mode()
def __init__(self, yaml_config_file, missing_policy=MISSING_POLICY_CREATE_DEFAULT): self.yaml_config_file = yaml_config_file self.root = Paths.dirname(self.yaml_config_file) self.yamls = self._get_all_yamls() self.cases = list() self.common_config = None self.missing_policy = missing_policy # create dummy case for every yaml file in folder if not Paths.exists(self.yaml_config_file): self.common_config = deepcopy(yamlc.DEFAULTS) for y in self.yamls: dummy_case = deepcopy(yamlc.DEFAULTS) dummy_case['files'] = [y] self.cases.append(dummy_case) else: # setup common config values self.yaml_config = self._read_yaml() self.common_config = self.merge( yamlc.DEFAULTS, self.yaml_config.get('common_config', {})) # first process files which are specified in test_cases missing = [Paths.basename(y) for y in self.yamls] for case in self.yaml_config.get(yamlc.TAG_TEST_CASES, []): case_config = self.merge(self.common_config, case) # ensure that value are array case_config[yamlc.TAG_FILES] = ensure_iterable( case_config.get(yamlc.TAG_FILES, [])) # keep correct order self.cases.append(case_config) for f in case_config[yamlc.TAG_FILES]: if f in missing: missing.remove(f) # process rest (dummy case) if missing_policy == self.MISSING_POLICY_CREATE_DEFAULT: for y in missing: dummy_case = deepcopy(self.common_config) dummy_case[yamlc.TAG_FILES] = [y] self.cases.append(dummy_case)
def _walk_files(self): # switching processing logic self.dir_mode = False # in this loop we are processing all given files/folders all_yamls = list() for path in self.arg_options.args: if not Paths.exists(path): printf.error('given path does not exists, path "{}"', path) sys.exit(3) # append files to all_yamls if Paths.is_dir(path): self.dir_mode = True all_yamls.extend(Paths.walk(path, ConfigPool.yaml_filters)) else: all_yamls.append(path) return all_yamls
def get_pbs_module(hostname=None): """ :rtype : scripts.pbs.modules.pbs_tarkil_cesnet_cz """ pbs_module_path = None if not hostname: hostname = platform.node() # try to get name from json file host_file = Paths.join(Paths.source_dir(), 'host_table.json') if Paths.exists(host_file): with open(host_file, 'r') as fp: hosts = json.load(fp) pbs_module_path = hosts.get(hostname, None) if not pbs_module_path: hostname = hostname.replace('.', '_') pbs_module_path = 'pbs_{}'.format(hostname) Printer.wrn('Warning! no host specified assuming module {}', pbs_module_path) # try to get pbs_module return importlib.import_module('scripts.pbs.modules.{}'.format(pbs_module_path))
def __init__(self, yaml_config_file): self.yaml_config_file = yaml_config_file self.root = Paths.dirname(self.yaml_config_file) self.yamls = self._get_all_yamls() self.cases = list() self.common_config = None # create dummy case for every yaml file in folder if not Paths.exists(self.yaml_config_file): self.common_config = deepcopy(yamlc.DEFAULTS) for y in self.yamls: dummy_case = deepcopy(yamlc.DEFAULTS) dummy_case['files'] = [y] self.cases.append(dummy_case) else: # setup common config values self.yaml_config = self._read_yaml() self.common_config = self.merge( yamlc.DEFAULTS, self.yaml_config.get('common_config', {})) # first process files which are specified in test_cases missing = [Paths.basename(y) for y in self.yamls] for case in self.yaml_config.get(yamlc.TAG_TEST_CASES, []): case_config = self.merge(self.common_config, case) # ensure that value is array case_config[yamlc.TAG_FILES] = ensure_iterable( case_config.get(yamlc.TAG_FILES, [])) # keep correct order self.cases.append(case_config) for f in case_config[yamlc.TAG_FILES]: if f in missing: missing.remove(f) # process rest (dummy case) for y in missing: dummy_case = deepcopy(self.common_config) dummy_case[yamlc.TAG_FILES] = [y] self.cases.append(dummy_case)
def do_work(parser, args=None, debug=False): """ :type parser: utils.argparser.ArgParser """ # parse arguments global arg_options, arg_others, arg_rest arg_options, arg_others, arg_rest = parser.parse(args) Paths.format = PathFormat.ABSOLUTE Paths.base_dir('' if not arg_options.root else arg_options.root) # configure printer Printer.batch_output = arg_options.batch Printer.dynamic_output = not arg_options.batch # we need flow123d, mpiexec and ndiff to exists in LOCAL mode if not arg_options.queue and not Paths.test_paths('flow123d', 'mpiexec', 'ndiff'): Printer.err('Missing obligatory files! Exiting') GlobalResult.error = "missing obligatory files" sys.exit(1) # test yaml args if not arg_others: parser.exit_usage('Error: No yaml files or folder given') GlobalResult.error = "no yaml files or folder given" sys.exit(2) all_yamls = list() for path in arg_others: if not Paths.exists(path): Printer.err('Error! given path does not exists, ignoring path "{}"', path) GlobalResult.error = "path does not exist" sys.exit(3) if Paths.is_dir(path): all_yamls.extend(Paths.walk(path, filters=[ PathFilters.filter_type_is_file(), PathFilters.filter_ext('.yaml'), PathFilters.filter_not(PathFilters.filter_name('config.yaml')) ])) else: all_yamls.append(path) Printer.out("Found {} .yaml file/s", len(all_yamls)) if not all_yamls: Printer.wrn('Warning! No yaml files found in locations: \n {}', '\n '.join(arg_others)) GlobalResult.error = "no yaml files or folders given" sys.exit(3) configs = read_configs(all_yamls) configs.update( proc=arg_options.cpu, time_limit=arg_options.time_limit, memory_limit=arg_options.memory_limit, ) if arg_options.queue: Printer.out('Running in PBS mode') return run_pbs_mode(configs, debug) else: Printer.out('Running in LOCAL mode') return run_local_mode(configs, debug)