def list_tests():
test_dir = Paths.join(Paths.flow123d_root(), 'tests')
tests = Paths.walk(test_dir, [
PathFilters.filter_type_is_file(),
PathFilters.filter_endswith('.yaml'),
PathFilters.filter_not(PathFilters.filter_name('config.yaml')),
])
result = dict()
for r in tests:
dirname = Paths.dirname(r)
basename = Paths.basename(r)
if Paths.dirname(dirname) != test_dir:
continue
if dirname not in result:
result[dirname] = list()
result[dirname].append(basename)
keys = sorted(result.keys())
for dirname in keys:
Printer.all.out(Paths.relpath(dirname, test_dir))
with Printer.all.with_level(1):
for basename in result[dirname]:
Printer.all.out('{: >4s} {: <40s} {}', '', basename, Paths.relpath(Paths.join(dirname, basename), test_dir))
Printer.all.newline()
def list_tests():
test_dir = Paths.join(Paths.flow123d_root(), 'tests')
tests = Paths.walk(test_dir, [
PathFilters.filter_type_is_file(),
PathFilters.filter_endswith('.yaml'),
PathFilters.filter_not(PathFilters.filter_name('config.yaml')),
])
result = dict()
for r in tests:
dirname = Paths.dirname(r)
basename = Paths.basename(r)
if Paths.dirname(dirname) != test_dir:
continue
if dirname not in result:
result[dirname] = list()
result[dirname].append(basename)
keys = sorted(result.keys())
for dirname in keys:
printf.warning(Paths.relpath(dirname, test_dir))
with printf:
paths = list()
wrap = 2
for basename in result[dirname]:
paths.append(basename)
for i in range(0, len(paths), wrap):
printf.out(' '.join(
['{:<40s}'.format(x) for x in paths[i:i + wrap]]))
printf.sep()
def _get_all_yamls(self):
yamls = Paths.browse(
self.root,(
PathFilters.filter_endswith(YAML),
PathFilters.filter_not(PathFilters.filter_endswith(CONFIG_YAML))
))
return yamls
def list_tests():
test_dir = Paths.join(Paths.flow123d_root(), 'tests')
tests = Paths.walk(test_dir, [
PathFilters.filter_type_is_file(),
PathFilters.filter_endswith('.yaml'),
PathFilters.filter_not(PathFilters.filter_name('config.yaml')),
])
result = dict()
for r in tests:
dirname = Paths.dirname(r)
basename = Paths.basename(r)
if Paths.dirname(dirname) != test_dir:
continue
if dirname not in result:
result[dirname] = list()
result[dirname].append(basename)
keys = sorted(result.keys())
for dirname in keys:
Printer.all.out(Paths.relpath(dirname, test_dir))
with Printer.all.with_level(1):
for basename in result[dirname]:
Printer.all.out('{: >4s} {: <40s} {}', '', basename, Paths.relpath(Paths.join(dirname, basename), test_dir))
Printer.all.newline()
def _get_ref_output_files(self, comp_data):
"""
:type comp_data: dict
"""
# parse filters
filters = [
PathFilters.filter_wildcards(x)
for x in comp_data.get('files', [])
]
# browse files and make them relative to ref output so filters works properly
files = Paths.walk(self.case.fs.ref_output,
[PathFilters.filter_type_is_file()])
files = [Paths.relpath(f, self.case.fs.ref_output) for f in files]
def _get_ref_output_files(self, comp_data):
"""
:type comp_data: dict
"""
# parse filters
filters = [PathFilters.filter_wildcards(x) for x in comp_data.get('files', [])]
# browse files and make them relative to ref output so filters works properly
files = Paths.walk(self.case.fs.ref_output, [PathFilters.filter_type_is_file()])
files = [Paths.relpath(f, self.case.fs.ref_output) for f in files]
# filter files and make them absolute again
files = Paths.match(files, filters)
files = [Paths.join(self.case.fs.ref_output, f) for f in files]
return zip(files, self._get_mirror_files(files))
class ConfigPool(object):
"""
Class ConfigPool collects configs from multiple yaml files
:type configs : dict[str, ConfigBase]
:type files : dict[str, ConfigBase]
"""
# match only yaml files not in ref_out or test_results having config.yaml
# in the same directory (excluding config.yaml itself)
yaml_filters = [
PathFilters.filter_type_is_file(),
PathFilters.filter_ignore_dirs(
[yamlc.TEST_RESULTS, yamlc.TEST_RESULTS]),
PathFilters.filter_not(PathFilters.filter_name('config.yaml')),
PathFilters.filter_endswith('.yaml'),
PathFilters.filter_dir_contains_file('config.yaml'),
]
def __init__(self):
self.configs = dict()
self.files = dict()
def add_config(self, yaml_config_file):
self.configs[yaml_config_file] = None
return self
def add_case(self, yaml_case_file):
config = Paths.join(Paths.dirname(yaml_case_file), yamlc.CONFIG_YAML)
self.configs[config] = None
self.files[yaml_case_file] = None
return self
def add(self, yaml_file):
"""
:type yaml_file: str
"""
if yaml_file.endswith(yamlc.CONFIG_YAML):
return self.add_config(yaml_file)
return self.add_case(yaml_file)
def parse(self, missing_policy=ConfigBase.MISSING_POLICY_CREATE_DEFAULT):
for k, v in list(self.configs.items()):
self.configs[k] = ConfigBase(k, missing_policy)
for k, v in list(self.files.items()):
config = Paths.join(Paths.dirname(k), yamlc.CONFIG_YAML)
self.files[k] = self.configs[config]
__iadd__ = add
def update(self, proc, time_limit, memory_limit, **kwargs):
for config in list(self.configs.values()):
config.update(proc, time_limit, memory_limit, **kwargs)
def filter_tags(self, include, exclude):
include = set(include) if include else None
exclude = set(exclude) if exclude else None
for config in list(self.configs.values()):
config.filter_tags(include, exclude)
def generate_status_file(cls, target):
"""
Will generate status file if target has option turned on
:type target: PyPy
"""
if target.status_file:
IO.write(target.status_file, json.dumps(target.status(), indent=4))
output_dir = Paths.dirname(target.status_file)
files = Paths.browse(
output_dir,
[PathFilters.filter_wildcards('*/profiler_info_*.log.json')])
# profiler json is missing?
if not files:
IO.write(
Paths.join(output_dir, 'profiler_info_dummy.log.json'),
'{}')
def do_work(parser, args=None, debug=False):
"""
:type parser: utils.argparser.ArgParser
"""
# parse arguments
global arg_options, arg_others, arg_rest
arg_options, arg_others, arg_rest = parser.parse(args)
Paths.format = PathFormat.ABSOLUTE
Paths.base_dir('' if not arg_options.root else arg_options.root)
# configure printer
Printer.batch_output = arg_options.batch
Printer.dynamic_output = not arg_options.batch
# we need flow123d, mpiexec and ndiff to exists in LOCAL mode
if not arg_options.queue and not Paths.test_paths('flow123d', 'mpiexec', 'ndiff'):
Printer.err('Missing obligatory files! Exiting')
GlobalResult.error = "missing obligatory files"
sys.exit(1)
# test yaml args
if not arg_others:
parser.exit_usage('Error: No yaml files or folder given')
GlobalResult.error = "no yaml files or folder given"
sys.exit(2)
all_yamls = list()
for path in arg_others:
if not Paths.exists(path):
Printer.err('Error! given path does not exists, ignoring path "{}"', path)
GlobalResult.error = "path does not exist"
sys.exit(3)
if Paths.is_dir(path):
all_yamls.extend(Paths.walk(path, filters=[
PathFilters.filter_type_is_file(),
PathFilters.filter_ext('.yaml'),
PathFilters.filter_not(PathFilters.filter_name('config.yaml'))
]))
else:
all_yamls.append(path)
Printer.out("Found {} .yaml file/s", len(all_yamls))
if not all_yamls:
Printer.wrn('Warning! No yaml files found in locations: \n {}', '\n '.join(arg_others))
GlobalResult.error = "no yaml files or folders given"
sys.exit(3)
configs = read_configs(all_yamls)
configs.update(
proc=arg_options.cpu,
time_limit=arg_options.time_limit,
memory_limit=arg_options.memory_limit,
)
if arg_options.queue:
Printer.out('Running in PBS mode')
return run_pbs_mode(configs, debug)
else:
Printer.out('Running in LOCAL mode')
return run_local_mode(configs, debug)
def _get_all_yamls(self):
yamls = Paths.browse(
self.root, (PathFilters.filter_endswith(yamlc.YAML),
PathFilters.filter_not(
PathFilters.filter_endswith(yamlc.CONFIG_YAML))))
return yamls
