class ConfigPool(object): """ Class ConfigPool collects configs from multiple yaml files :type configs : dict[str, ConfigBase] :type files : dict[str, ConfigBase] """ # match only yaml files not in ref_out or test_results having config.yaml # in the same directory (excluding config.yaml itself) yaml_filters = [ PathFilters.filter_type_is_file(), PathFilters.filter_ignore_dirs( [yamlc.TEST_RESULTS, yamlc.TEST_RESULTS]), PathFilters.filter_not(PathFilters.filter_name('config.yaml')), PathFilters.filter_endswith('.yaml'), PathFilters.filter_dir_contains_file('config.yaml'), ] def __init__(self): self.configs = dict() self.files = dict() def add_config(self, yaml_config_file): self.configs[yaml_config_file] = None return self def add_case(self, yaml_case_file): config = Paths.join(Paths.dirname(yaml_case_file), yamlc.CONFIG_YAML) self.configs[config] = None self.files[yaml_case_file] = None return self def add(self, yaml_file): """ :type yaml_file: str """ if yaml_file.endswith(yamlc.CONFIG_YAML): return self.add_config(yaml_file) return self.add_case(yaml_file) def parse(self, missing_policy=ConfigBase.MISSING_POLICY_CREATE_DEFAULT): for k, v in list(self.configs.items()): self.configs[k] = ConfigBase(k, missing_policy) for k, v in list(self.files.items()): config = Paths.join(Paths.dirname(k), yamlc.CONFIG_YAML) self.files[k] = self.configs[config] __iadd__ = add def update(self, proc, time_limit, memory_limit, **kwargs): for config in list(self.configs.values()): config.update(proc, time_limit, memory_limit, **kwargs) def filter_tags(self, include, exclude): include = set(include) if include else None exclude = set(exclude) if exclude else None for config in list(self.configs.values()): config.filter_tags(include, exclude)
def list_tests(): test_dir = Paths.join(Paths.flow123d_root(), 'tests') tests = Paths.walk(test_dir, [ PathFilters.filter_type_is_file(), PathFilters.filter_endswith('.yaml'), PathFilters.filter_not(PathFilters.filter_name('config.yaml')), ]) result = dict() for r in tests: dirname = Paths.dirname(r) basename = Paths.basename(r) if Paths.dirname(dirname) != test_dir: continue if dirname not in result: result[dirname] = list() result[dirname].append(basename) keys = sorted(result.keys()) for dirname in keys: Printer.all.out(Paths.relpath(dirname, test_dir)) with Printer.all.with_level(1): for basename in result[dirname]: Printer.all.out('{: >4s} {: <40s} {}', '', basename, Paths.relpath(Paths.join(dirname, basename), test_dir)) Printer.all.newline()
def list_tests(): test_dir = Paths.join(Paths.flow123d_root(), 'tests') tests = Paths.walk(test_dir, [ PathFilters.filter_type_is_file(), PathFilters.filter_endswith('.yaml'), PathFilters.filter_not(PathFilters.filter_name('config.yaml')), ]) result = dict() for r in tests: dirname = Paths.dirname(r) basename = Paths.basename(r) if Paths.dirname(dirname) != test_dir: continue if dirname not in result: result[dirname] = list() result[dirname].append(basename) keys = sorted(result.keys()) for dirname in keys: printf.warning(Paths.relpath(dirname, test_dir)) with printf: paths = list() wrap = 2 for basename in result[dirname]: paths.append(basename) for i in range(0, len(paths), wrap): printf.out(' '.join( ['{:<40s}'.format(x) for x in paths[i:i + wrap]])) printf.sep()
def _get_ref_output_files(self, comp_data): """ :type comp_data: dict """ # parse filters filters = [ PathFilters.filter_wildcards(x) for x in comp_data.get('files', []) ] # browse files and make them relative to ref output so filters works properly files = Paths.walk(self.case.fs.ref_output, [PathFilters.filter_type_is_file()]) files = [Paths.relpath(f, self.case.fs.ref_output) for f in files]
def _get_ref_output_files(self, comp_data): """ :type comp_data: dict """ # parse filters filters = [PathFilters.filter_wildcards(x) for x in comp_data.get('files', [])] # browse files and make them relative to ref output so filters works properly files = Paths.walk(self.case.fs.ref_output, [PathFilters.filter_type_is_file()]) files = [Paths.relpath(f, self.case.fs.ref_output) for f in files] # filter files and make them absolute again files = Paths.match(files, filters) files = [Paths.join(self.case.fs.ref_output, f) for f in files] return zip(files, self._get_mirror_files(files))
def do_work(parser, args=None, debug=False): """ :type parser: utils.argparser.ArgParser """ # parse arguments global arg_options, arg_others, arg_rest arg_options, arg_others, arg_rest = parser.parse(args) Paths.format = PathFormat.ABSOLUTE Paths.base_dir('' if not arg_options.root else arg_options.root) # configure printer Printer.batch_output = arg_options.batch Printer.dynamic_output = not arg_options.batch # we need flow123d, mpiexec and ndiff to exists in LOCAL mode if not arg_options.queue and not Paths.test_paths('flow123d', 'mpiexec', 'ndiff'): Printer.err('Missing obligatory files! Exiting') GlobalResult.error = "missing obligatory files" sys.exit(1) # test yaml args if not arg_others: parser.exit_usage('Error: No yaml files or folder given') GlobalResult.error = "no yaml files or folder given" sys.exit(2) all_yamls = list() for path in arg_others: if not Paths.exists(path): Printer.err('Error! given path does not exists, ignoring path "{}"', path) GlobalResult.error = "path does not exist" sys.exit(3) if Paths.is_dir(path): all_yamls.extend(Paths.walk(path, filters=[ PathFilters.filter_type_is_file(), PathFilters.filter_ext('.yaml'), PathFilters.filter_not(PathFilters.filter_name('config.yaml')) ])) else: all_yamls.append(path) Printer.out("Found {} .yaml file/s", len(all_yamls)) if not all_yamls: Printer.wrn('Warning! No yaml files found in locations: \n {}', '\n '.join(arg_others)) GlobalResult.error = "no yaml files or folders given" sys.exit(3) configs = read_configs(all_yamls) configs.update( proc=arg_options.cpu, time_limit=arg_options.time_limit, memory_limit=arg_options.memory_limit, ) if arg_options.queue: Printer.out('Running in PBS mode') return run_pbs_mode(configs, debug) else: Printer.out('Running in LOCAL mode') return run_local_mode(configs, debug)