def browse_test_config_files(self, test_name, test_option): # browse con and yamls files cwd = io.join_path(self.tests_root, test_name) cons = check_output_secure('ls *.con', cwd=cwd).strip().split() yamls = check_output_secure('ls *.yaml', cwd=cwd).strip().split() # union these lists and filter them test_option['problem'] = lists.union(cons, yamls) test_option['problem'] = lists.filter(test_option['problem'], lambda x: bool(x)) test_option['problem'] = lists.filter( test_option['problem'], lambda x: self.select_ini_rule.match(x))
def insert_process(self, dirname, env): profilers = io.browse(dirname) profilers = lists.filter(profilers, lambda x: str(x).lower().find('info-') != -1) for profiler in profilers: json_data = read_json(profiler) context = self._extract_process_context(json_data) metrics = self._extract_process_metrics(json_data) context.update(dict(env=env, process=True)) # metrics_id = self.insert_metrics(metrics.copy()).inserted_id # context_id = self.insert_context(context.copy()).inserted_id self._insert_pts_simple(',', metrics, context) child_context = context.copy() child_context.update(dict(process=False)) if 'children' in json_data: whole_program = json_data['children'][0] self.insert_time_frame(whole_program, [], context=child_context) return context, metrics
def insert_many(self, dirname, filters=[]): dirs = io.listdir(dirname) for f in filters: dirs = lists.filter(dirs, f) with timer.measured("Processing many, folder {dirname}".format(dirname=dirname), False): for dir in dirs: self.insert_one(dir)
def insert_many(self, dirname, filters=[]): dirs = io.listdir(dirname) for f in filters: dirs = lists.filter(dirs, f) with timer.measured( 'Processing many, folder {dirname}'.format(dirname=dirname), False): for dir in dirs: self.insert_one(dir)
def insert_process(self, dirname, env): profilers = io.browse(dirname) profilers = lists.filter(profilers, lambda x: str(x).lower().find('info-') != -1) for profiler in profilers: json_data = read_json(profiler) context = self._extract_process_context(json_data) metrics = self._extract_process_metrics(json_data) context.update(dict(env=env, process=True)) # metrics_id = self.insert_metrics(metrics.copy()).inserted_id # context_id = self.insert_context(context.copy()).inserted_id self._insert_pts_simple(',', metrics, context) child_context = context.copy() child_context.update(dict(process=False)) if 'children' in json_data: whole_program = json_data['children'][0] self.insert_time_frame(whole_program, [], context=child_context) return context, metrics
def __init__(self, **kwargs): if kwargs.get('flow_root'): self.flow_root = os.path.abspath(kwargs.get('flow_root')) if kwargs.get('test_root'): self.tests_root = os.path.abspath(kwargs.get('test_root')) else: self.tests_root = io.join_path(self.flow_root, 'tests') self.randomize_output = bool(kwargs.get('randomize_output', False)) self.nproc = kwargs.get('nproc', [1, 2, 3]) self.output_dir = os.path.abspath( io.join_path(self.tests_root, kwargs.get('tests_output', '_output'))) output_timestamp_dir = kwargs.get('output_timestamp_dir', '%Y-%m-%d_%H-%M-%S') if output_timestamp_dir: self.output_dir = io.join_path( self.output_dir, datetime.datetime.now().strftime(output_timestamp_dir)) self.select_dir_rule = re.compile( kwargs.get('select_dir_rule', r'\d+_.*')) self.select_ini_rule = re.compile(kwargs.get('select_ini_rule', r'.*')) self.select_artifact_rule = re.compile( kwargs.get('select_artifact_rule', r'.*/profiler.*\.json$')) self.compare_result = kwargs.get('compare-result', False) self.save_stderr = kwargs.get('save-stderr', True) self.save_stdout = kwargs.get('save-stdout', True) f, m, n = kwargs.get("flow123d"), kwargs.get("mpiexec"), kwargs.get( "ndiff") self.bins = { "flow123d": f if f else io.join_path(self.flow_root, 'build_tree', 'bin', 'flow123d'), "mpiexec": m if m else io.join_path(self.flow_root, 'build_tree', 'bin', 'mpiexec'), "ndiff": n if n else io.join_path(self.flow_root, 'bin', 'ndiff', 'ndiff.pl') } all_tests = sorted([test for test in os.listdir(self.tests_root)]) # filter folders all_tests = lists.filter( all_tests, lambda x: os.path.isdir(io.join_path(self.tests_root, x))) self.selected_tests = lists.filter( all_tests, lambda x: self.select_dir_rule.match(x)) self.tests = { test: self.test_template() for test in self.selected_tests } for val in [ 'flow_root', 'tests_root', 'selected_tests', 'nproc', 'output_dir', 'randomize_output', ]: logger.debug("{name:20s}: {val:s}".format(name=val, val=str( getattr(self, val))))
def run(self): logger.debug("Running tests...") for test_name, test_option in self.tests.items(): self.browse_test_config_files(test_name, test_option) logger.debug("{test_name}:{test_option[problem]}".format( test_name=test_name, test_option=test_option)) self.setup_test_paths(test_name, test_option) executors = self.prepare_test_executor(test_name, test_option) for executor in executors: logger.debug( "{test_name}:{e.environment[problem_config]}: running". format(test_name=test_name, e=executor)) environment = executor.environment # purge output directory if os.path.exists(environment['output_path']): rmtree(environment['output_path']) # run test executor.run() # save info about test logger.debug( "{test_name}:{e.environment[problem_config]}: generating report" .format(test_name=test_name, e=executor)) json_report = self.generate_report(executor) # get comparisons if self.compare_result: logger.debug( "{test_name}:{e.environment[problem_config]}: comparing output result" .format(test_name=test_name, e=executor)) comparisons = self.compare_results_files(environment) if not comparisons or max(pluck(comparisons, 'exit_code')) == 0: json_report['correct'] = True else: json_report['correct'] = False json_report['comparisons'] = [ ex.environment['file'] for ex in comparisons if ex.exit_code != 0 ] if self.save_stderr: json_report['stderr'] = executor.stderr if self.save_stdout: json_report['stdout'] = executor.stdout # create name for json file based on settings info_json = executor.environment['info_json'].format( **json_report) info_json = io.join_path(self.output_dir, info_json) mkdir(info_json, is_file=True) # merge artifacts (so fat only one - profiler info) profilers = browse(environment['output_path']) profilers = lists.filter( profilers, lambda x: self.select_artifact_rule.match(x)) # merge report with profiler json_report = FlowJson.merge_json_info(json_report, profilers) json_report = FlowJson.clean_json(json_report) logger.debug(to_json(json_report, info_json))