def run_synthesis_eval(cmd_args): benchmarks: Dict[str, Type[Benchmark]] = discover_benchmarks() path_matcher: Pattern = re.compile(cmd_args.path_regex) results = [] model_store: ModelStore = None if not cmd_args.load_models_on_demand: logger.info("Loading models ahead of time") path_map = {'function-model': cmd_args.function_model_dir} arg_model_paths = glob.glob(cmd_args.arg_model_dir + '/*/*/model_best.pickle') for path in arg_model_paths: func_name, arg_name = path.split('/')[-3:-1] path_map[func_name, arg_name] = os.path.dirname(path) model_store: ModelStore = ModelStore(path_map) logger.info("Loaded models") for qual_name, benchmark_cls in benchmarks.items(): if not path_matcher.match(qual_name): continue try: logger.info("Running benchmark {}".format(qual_name)) with SignalTimeout(seconds=cmd_args.timeout): evaluator = NeuralSynthesisEvaluator(benchmark_cls(), cmd_args, model_store=model_store) result = evaluator.run(qual_name) results.append(result) logger.info("Result for {} : {}".format(qual_name, results[-1])) except TimeoutError: logger.info("Timed out for {}".format(qual_name)) result = { 'benchmark': qual_name, 'num_seqs_explored': {}, 'num_candidates_generated': {}, 'solution_found': False, 'time': cmd_args.timeout } results.append(result) except Exception as e: logger.warn("Failed for {}".format(qual_name)) logging.exception(e) if not cmd_args.load_models_on_demand: model_store.close() results = pd.DataFrame(results) print(results) with open(cmd_args.outfile, 'w') as f: results.to_csv(f)
def run_synthesis_eval(cmd_args): benchmarks: Dict[str, Type[Benchmark]] = discover_benchmarks() path_matcher: Pattern = re.compile(cmd_args.path_regex) results = [] for qual_name, benchmark_cls in benchmarks.items(): if not path_matcher.match(qual_name): continue try: logger.info("Running benchmark {}".format(qual_name)) result = call_with_timeout(run_synthesis_for_benchmark, qual_name, cmd_args, timeout=cmd_args.timeout) results.append(result) logger.info("Result for {} : {}".format(qual_name, results[-1])) except TimeoutError: logger.info("Timed out for {}".format(qual_name)) result = { 'benchmark': qual_name, 'num_seqs_explored': {}, 'num_candidates_generated': {}, 'solution_found': False, 'time': cmd_args.timeout } results.append(result) except Exception as e: logger.warn("Failed for {}".format(qual_name)) logging.exception(e) results = pd.DataFrame(results) print(results) with open(cmd_args.outfile, 'w') as f: results.to_csv(f)
def run_generator_model_eval(cmd_args: ArgNamespace): benchmarks: Dict[str, Type[Benchmark]] = discover_benchmarks() path_matcher: Pattern = re.compile(cmd_args.path_regex) results = [] for qual_name, benchmark_cls in benchmarks.items(): if not path_matcher.match(qual_name): continue try: logger.info("Running benchmark {}".format(qual_name)) benchmark = benchmark_cls() evaluator = GeneratorModelEvaluator(benchmark, cmd_args) results.append(evaluator.run(qual_name)) logger.info("Result for {} : {}".format(qual_name, results[-1])) except Exception as e: logger.warn("Failed for {}".format(qual_name)) logging.exception(e) results = pd.DataFrame(results) print(results) with open(cmd_args.outfile, 'w') as f: results.to_csv(f)
def run_synthesis_for_benchmark(qual_name: str, cmd_args: ArgNamespace): benchmarks: Dict[str, Type[Benchmark]] = discover_benchmarks() benchmark_cls = benchmarks[qual_name] benchmark = benchmark_cls() evaluator = NeuralSynthesisEvaluator(benchmark, cmd_args) return evaluator.run(qual_name)