def compare_results_files(self, environment):
# locate ref output dir
problem_config = os.path.basename(environment['problem_config'])
ref_dir = io.join_path(environment['ref_output_path'], problem_config)
output_dir = environment['output_path']
# grab all files in dir
files_to_compare = browse(ref_dir)
files_to_compare = [
f.replace(ref_dir, '').lstrip('/\\') for f in files_to_compare
]
# compare them
comparisons = []
for filename in files_to_compare:
variables = {
'ndiff': self.bins['ndiff'],
'ref': io.join_path(ref_dir, filename),
'res': io.join_path(output_dir, filename),
'file': filename,
'name': "comparison {file}".format(file=filename)
}
comparison = exec_single(
command="""perl {ndiff} "{ref}" "{res}" """,
variables=variables,
plugins=[
PluginEnv(variables),
PluginBriefInfo(name=variables['name'])
])
comparisons.append(comparison)
return comparisons
def insert_one(self, dirname):
with timer.measured("Processing one, folder {dirname}".format(dirname=dirname), False):
env = self.mongo._extract_environment(io.join_path(dirname, "environment.json"))
cal = self.mongo._extract_calibration(io.join_path(dirname, "performance.json"))
env["cal"] = cal
self.mongo.insert_process(dirname, env)
def insert_one(self, dirname):
with timer.measured(
'Processing one, folder {dirname}'.format(dirname=dirname),
False):
env = self.mongo._extract_environment(
io.join_path(dirname, 'environment.json'))
cal = self.mongo._extract_calibration(
io.join_path(dirname, 'performance.json'))
env['cal'] = cal
self.mongo.insert_process(dirname, env)
def setup_test_paths(self, test_name, test_option):
test_option['input_path'] = io.join_path(self.tests_root, test_name,
test_option['input'])
test_option['output_path'] = io.join_path(self.tests_root, test_name,
test_option['output'])
test_option['ref_output_path'] = io.join_path(
self.tests_root, test_name, test_option['ref_output'])
test_option['problem_config'] = lists.prepend_path(
test_option['problem'], self.tests_root, test_name)
del test_option['input']
del test_option['output']
del test_option['ref_output']
del test_option['problem']
def browse_test_config_files(self, test_name, test_option):
# browse con and yamls files
cwd = io.join_path(self.tests_root, test_name)
cons = check_output_secure('ls *.con', cwd=cwd).strip().split()
yamls = check_output_secure('ls *.yaml', cwd=cwd).strip().split()
# union these lists and filter them
test_option['problem'] = lists.union(cons, yamls)
test_option['problem'] = lists.filter(test_option['problem'],
lambda x: bool(x))
test_option['problem'] = lists.filter(
test_option['problem'], lambda x: self.select_ini_rule.match(x))
io.join_path(dirname, 'performance.json'))
env['cal'] = cal
self.mongo.insert_process(dirname, env)
def insert_many(self, dirname, filters=[]):
dirs = io.listdir(dirname)
for f in filters:
dirs = lists.filter(dirs, f)
with timer.measured(
'Processing many, folder {dirname}'.format(dirname=dirname),
False):
for dir in dirs:
self.insert_one(dir)
mongo = MongoDB()
mongo.remove_all()
mongo.close()
mongo = MongoDB()
experiments = Experiments(MongoDB())
# experiments.insert_many('/home/jan-hybs/Dropbox/meta', [lambda x: str(x).startswith('test')])
meta_folder = r'c:\Users\Jan\Dropbox\meta'
experiments.insert_many(io.join_path(meta_folder, 'article', 'A'))
experiments.insert_many(io.join_path(meta_folder, 'article', 'B'))
experiments.insert_many(io.join_path(meta_folder, 'article', 'C'))
# experiments.insert_one('/home/jan-hybs/Dropbox/meta/test-13')
env = self.mongo._extract_environment(io.join_path(dirname, "environment.json"))
cal = self.mongo._extract_calibration(io.join_path(dirname, "performance.json"))
env["cal"] = cal
self.mongo.insert_process(dirname, env)
def insert_many(self, dirname, filters=[]):
dirs = io.listdir(dirname)
for f in filters:
dirs = lists.filter(dirs, f)
with timer.measured("Processing many, folder {dirname}".format(dirname=dirname), False):
for dir in dirs:
self.insert_one(dir)
mongo = MongoDB()
mongo.remove_all()
mongo.close()
mongo = MongoDB()
experiments = Experiments(MongoDB())
# experiments.insert_many('/home/jan-hybs/Dropbox/meta', [lambda x: str(x).startswith('test')])
meta_folder = r"c:\Users\Jan\Dropbox\meta"
experiments.insert_many(io.join_path(meta_folder, "article", "A"))
experiments.insert_many(io.join_path(meta_folder, "article", "B"))
experiments.insert_many(io.join_path(meta_folder, "article", "C"))
# experiments.insert_one('/home/jan-hybs/Dropbox/meta/test-13')
def __init__(self, **kwargs):
if kwargs.get('flow_root'):
self.flow_root = os.path.abspath(kwargs.get('flow_root'))
if kwargs.get('test_root'):
self.tests_root = os.path.abspath(kwargs.get('test_root'))
else:
self.tests_root = io.join_path(self.flow_root, 'tests')
self.randomize_output = bool(kwargs.get('randomize_output', False))
self.nproc = kwargs.get('nproc', [1, 2, 3])
self.output_dir = os.path.abspath(
io.join_path(self.tests_root, kwargs.get('tests_output',
'_output')))
output_timestamp_dir = kwargs.get('output_timestamp_dir',
'%Y-%m-%d_%H-%M-%S')
if output_timestamp_dir:
self.output_dir = io.join_path(
self.output_dir,
datetime.datetime.now().strftime(output_timestamp_dir))
self.select_dir_rule = re.compile(
kwargs.get('select_dir_rule', r'\d+_.*'))
self.select_ini_rule = re.compile(kwargs.get('select_ini_rule', r'.*'))
self.select_artifact_rule = re.compile(
kwargs.get('select_artifact_rule', r'.*/profiler.*\.json$'))
self.compare_result = kwargs.get('compare-result', False)
self.save_stderr = kwargs.get('save-stderr', True)
self.save_stdout = kwargs.get('save-stdout', True)
f, m, n = kwargs.get("flow123d"), kwargs.get("mpiexec"), kwargs.get(
"ndiff")
self.bins = {
"flow123d":
f if f else io.join_path(self.flow_root, 'build_tree', 'bin',
'flow123d'),
"mpiexec":
m if m else io.join_path(self.flow_root, 'build_tree', 'bin',
'mpiexec'),
"ndiff":
n
if n else io.join_path(self.flow_root, 'bin', 'ndiff', 'ndiff.pl')
}
all_tests = sorted([test for test in os.listdir(self.tests_root)])
# filter folders
all_tests = lists.filter(
all_tests,
lambda x: os.path.isdir(io.join_path(self.tests_root, x)))
self.selected_tests = lists.filter(
all_tests, lambda x: self.select_dir_rule.match(x))
self.tests = {
test: self.test_template()
for test in self.selected_tests
}
for val in [
'flow_root',
'tests_root',
'selected_tests',
'nproc',
'output_dir',
'randomize_output',
]:
logger.debug("{name:20s}: {val:s}".format(name=val,
val=str(
getattr(self, val))))
def run(self):
logger.debug("Running tests...")
for test_name, test_option in self.tests.items():
self.browse_test_config_files(test_name, test_option)
logger.debug("{test_name}:{test_option[problem]}".format(
test_name=test_name, test_option=test_option))
self.setup_test_paths(test_name, test_option)
executors = self.prepare_test_executor(test_name, test_option)
for executor in executors:
logger.debug(
"{test_name}:{e.environment[problem_config]}: running".
format(test_name=test_name, e=executor))
environment = executor.environment
# purge output directory
if os.path.exists(environment['output_path']):
rmtree(environment['output_path'])
# run test
executor.run()
# save info about test
logger.debug(
"{test_name}:{e.environment[problem_config]}: generating report"
.format(test_name=test_name, e=executor))
json_report = self.generate_report(executor)
# get comparisons
if self.compare_result:
logger.debug(
"{test_name}:{e.environment[problem_config]}: comparing output result"
.format(test_name=test_name, e=executor))
comparisons = self.compare_results_files(environment)
if not comparisons or max(pluck(comparisons,
'exit_code')) == 0:
json_report['correct'] = True
else:
json_report['correct'] = False
json_report['comparisons'] = [
ex.environment['file'] for ex in comparisons
if ex.exit_code != 0
]
if self.save_stderr:
json_report['stderr'] = executor.stderr
if self.save_stdout:
json_report['stdout'] = executor.stdout
# create name for json file based on settings
info_json = executor.environment['info_json'].format(
**json_report)
info_json = io.join_path(self.output_dir, info_json)
mkdir(info_json, is_file=True)
# merge artifacts (so fat only one - profiler info)
profilers = browse(environment['output_path'])
profilers = lists.filter(
profilers, lambda x: self.select_artifact_rule.match(x))
# merge report with profiler
json_report = FlowJson.merge_json_info(json_report, profilers)
json_report = FlowJson.clean_json(json_report)
logger.debug(to_json(json_report, info_json))
#!/usr/bin/python
# -*- coding: utf-8 -*-
# author: Jan Hybs
import logging
import traceback
import datetime
import os
from flowrunner.utils import io
logging.basicConfig(
filename=io.join_path(os.getcwd(), "python.log"),
level=logging.NOTSET,
format="%(asctime)s %(name)s %(levelname)-4s: %(message)s",
)
class Logger(object):
def __init__(self, name=__name__, debug=True):
self.logger = logging.getLogger(name)
self.level = 0
# add console log
if __debug__ or debug:
stream = logging.StreamHandler()
stream.setLevel(logging.NOTSET)
self.logger.addHandler(stream)
with open("python.log", "a+") as fp:
fp.write("-" * 110)
#!/usr/bin/python
# -*- coding: utf-8 -*-
# author: Jan Hybs
import logging
import traceback
import datetime
import os
from flowrunner.utils import io
logging.basicConfig(
filename=io.join_path(os.getcwd(), 'python.log'),
level=logging.NOTSET,
format='%(asctime)s %(name)s %(levelname)-4s: %(message)s'
)
class Logger(object):
def __init__(self, name=__name__, debug=True):
self.logger = logging.getLogger(name)
self.level = 0
# add console log
if __debug__ or debug:
stream = logging.StreamHandler()
stream.setLevel(logging.NOTSET)
self.logger.addHandler(stream)
with open('python.log', 'a+') as fp:
fp.write('-' * 110)
