def browse_test_config_files(self, test_name, test_option):
# browse con and yamls files
cwd = io.join_path(self.tests_root, test_name)
cons = check_output_secure('ls *.con', cwd=cwd).strip().split()
yamls = check_output_secure('ls *.yaml', cwd=cwd).strip().split()
# union these lists and filter them
test_option['problem'] = lists.union(cons, yamls)
test_option['problem'] = lists.filter(test_option['problem'],
lambda x: bool(x))
test_option['problem'] = lists.filter(
test_option['problem'], lambda x: self.select_ini_rule.match(x))
def insert_process(self, dirname, env):
profilers = io.browse(dirname)
profilers = lists.filter(profilers,
lambda x: str(x).lower().find('info-') != -1)
for profiler in profilers:
json_data = read_json(profiler)
context = self._extract_process_context(json_data)
metrics = self._extract_process_metrics(json_data)
context.update(dict(env=env, process=True))
# metrics_id = self.insert_metrics(metrics.copy()).inserted_id
# context_id = self.insert_context(context.copy()).inserted_id
self._insert_pts_simple(',', metrics, context)
child_context = context.copy()
child_context.update(dict(process=False))
if 'children' in json_data:
whole_program = json_data['children'][0]
self.insert_time_frame(whole_program, [],
context=child_context)
return context, metrics
def insert_many(self, dirname, filters=[]):
dirs = io.listdir(dirname)
for f in filters:
dirs = lists.filter(dirs, f)
with timer.measured("Processing many, folder {dirname}".format(dirname=dirname), False):
for dir in dirs:
self.insert_one(dir)
def insert_many(self, dirname, filters=[]):
dirs = io.listdir(dirname)
for f in filters:
dirs = lists.filter(dirs, f)
with timer.measured(
'Processing many, folder {dirname}'.format(dirname=dirname),
False):
for dir in dirs:
self.insert_one(dir)
def insert_process(self, dirname, env):
profilers = io.browse(dirname)
profilers = lists.filter(profilers, lambda x: str(x).lower().find('info-') != -1)
for profiler in profilers:
json_data = read_json(profiler)
context = self._extract_process_context(json_data)
metrics = self._extract_process_metrics(json_data)
context.update(dict(env=env, process=True))
# metrics_id = self.insert_metrics(metrics.copy()).inserted_id
# context_id = self.insert_context(context.copy()).inserted_id
self._insert_pts_simple(',', metrics, context)
child_context = context.copy()
child_context.update(dict(process=False))
if 'children' in json_data:
whole_program = json_data['children'][0]
self.insert_time_frame(whole_program, [], context=child_context)
return context, metrics
def __init__(self, **kwargs):
if kwargs.get('flow_root'):
self.flow_root = os.path.abspath(kwargs.get('flow_root'))
if kwargs.get('test_root'):
self.tests_root = os.path.abspath(kwargs.get('test_root'))
else:
self.tests_root = io.join_path(self.flow_root, 'tests')
self.randomize_output = bool(kwargs.get('randomize_output', False))
self.nproc = kwargs.get('nproc', [1, 2, 3])
self.output_dir = os.path.abspath(
io.join_path(self.tests_root, kwargs.get('tests_output',
'_output')))
output_timestamp_dir = kwargs.get('output_timestamp_dir',
'%Y-%m-%d_%H-%M-%S')
if output_timestamp_dir:
self.output_dir = io.join_path(
self.output_dir,
datetime.datetime.now().strftime(output_timestamp_dir))
self.select_dir_rule = re.compile(
kwargs.get('select_dir_rule', r'\d+_.*'))
self.select_ini_rule = re.compile(kwargs.get('select_ini_rule', r'.*'))
self.select_artifact_rule = re.compile(
kwargs.get('select_artifact_rule', r'.*/profiler.*\.json$'))
self.compare_result = kwargs.get('compare-result', False)
self.save_stderr = kwargs.get('save-stderr', True)
self.save_stdout = kwargs.get('save-stdout', True)
f, m, n = kwargs.get("flow123d"), kwargs.get("mpiexec"), kwargs.get(
"ndiff")
self.bins = {
"flow123d":
f if f else io.join_path(self.flow_root, 'build_tree', 'bin',
'flow123d'),
"mpiexec":
m if m else io.join_path(self.flow_root, 'build_tree', 'bin',
'mpiexec'),
"ndiff":
n
if n else io.join_path(self.flow_root, 'bin', 'ndiff', 'ndiff.pl')
}
all_tests = sorted([test for test in os.listdir(self.tests_root)])
# filter folders
all_tests = lists.filter(
all_tests,
lambda x: os.path.isdir(io.join_path(self.tests_root, x)))
self.selected_tests = lists.filter(
all_tests, lambda x: self.select_dir_rule.match(x))
self.tests = {
test: self.test_template()
for test in self.selected_tests
}
for val in [
'flow_root',
'tests_root',
'selected_tests',
'nproc',
'output_dir',
'randomize_output',
]:
logger.debug("{name:20s}: {val:s}".format(name=val,
val=str(
getattr(self, val))))
def run(self):
logger.debug("Running tests...")
for test_name, test_option in self.tests.items():
self.browse_test_config_files(test_name, test_option)
logger.debug("{test_name}:{test_option[problem]}".format(
test_name=test_name, test_option=test_option))
self.setup_test_paths(test_name, test_option)
executors = self.prepare_test_executor(test_name, test_option)
for executor in executors:
logger.debug(
"{test_name}:{e.environment[problem_config]}: running".
format(test_name=test_name, e=executor))
environment = executor.environment
# purge output directory
if os.path.exists(environment['output_path']):
rmtree(environment['output_path'])
# run test
executor.run()
# save info about test
logger.debug(
"{test_name}:{e.environment[problem_config]}: generating report"
.format(test_name=test_name, e=executor))
json_report = self.generate_report(executor)
# get comparisons
if self.compare_result:
logger.debug(
"{test_name}:{e.environment[problem_config]}: comparing output result"
.format(test_name=test_name, e=executor))
comparisons = self.compare_results_files(environment)
if not comparisons or max(pluck(comparisons,
'exit_code')) == 0:
json_report['correct'] = True
else:
json_report['correct'] = False
json_report['comparisons'] = [
ex.environment['file'] for ex in comparisons
if ex.exit_code != 0
]
if self.save_stderr:
json_report['stderr'] = executor.stderr
if self.save_stdout:
json_report['stdout'] = executor.stdout
# create name for json file based on settings
info_json = executor.environment['info_json'].format(
**json_report)
info_json = io.join_path(self.output_dir, info_json)
mkdir(info_json, is_file=True)
# merge artifacts (so fat only one - profiler info)
profilers = browse(environment['output_path'])
profilers = lists.filter(
profilers, lambda x: self.select_artifact_rule.match(x))
# merge report with profiler
json_report = FlowJson.merge_json_info(json_report, profilers)
json_report = FlowJson.clean_json(json_report)
logger.debug(to_json(json_report, info_json))
