def process_output(cls, config, short_name, opt_list, stdout_filename, stderr_filename): with open(stdout_filename, "r") as stdout_file: output = stdout_file.read() results_token = "results: " result = "" if results_token not in output: result = "FAILED" else: result_line = filter(lambda x: results_token in x, output.split("\n"))[0] result_list = result_line.replace(results_token, "").split(",") err_msg = ("Expecting at least %s results " "but only found %s" % (config.IGNORED_TRIALS + 1, len(result_list))) assert len(result_list) > config.IGNORED_TRIALS, err_msg result_list = result_list[config.IGNORED_TRIALS:] result += "Runtime: %s, %.3f, %s, %s, %s\n" % \ stats_for_results([x.split(";")[0] for x in result_list]) result += "Train Set Metric: %s, %.3f, %s, %s, %s\n" %\ stats_for_results([x.split(";")[1] for x in result_list]) result += "Test Set Metric: %s, %.3f, %s, %s, %s" %\ stats_for_results([x.split(";")[2] for x in result_list]) result_string = "%s, %s\n%s" % (short_name, " ".join(opt_list), result) sys.stdout.flush() return result_string
def process_output(cls, config, short_name, opt_list, stdout_filename, stderr_filename): with open(stdout_filename, "r") as stdout_file: output = stdout_file.read() results_token = "results: " result_string = "" if results_token not in output: result_string = "FAILED" else: result_line = filter(lambda x: results_token in x, output.split("\n"))[-1] result_json = result_line.replace(results_token, "") try: result_dict = json.loads(result_json) except: print "Failed to parse JSON:\n", result_json raise num_results = len(result_dict['results']) err_msg = ("Expecting at least %s results " "but only found %s" % (config.IGNORED_TRIALS + 1, num_results)) assert num_results > config.IGNORED_TRIALS, err_msg # 2 modes: prediction problems (4 metrics) and others (time only) if 'trainingTime' in result_dict['results'][0]: # prediction problem trainingTimes = [ r['trainingTime'] for r in result_dict['results'] ] testTimes = [r['testTime'] for r in result_dict['results']] trainingMetrics = [ r['trainingMetric'] for r in result_dict['results'] ] testMetrics = [r['testMetric'] for r in result_dict['results']] trainingTimes = trainingTimes[config.IGNORED_TRIALS:] testTimes = testTimes[config.IGNORED_TRIALS:] trainingMetrics = trainingMetrics[config.IGNORED_TRIALS:] testMetrics = testMetrics[config.IGNORED_TRIALS:] result_string += "Training time: %s, %.3f, %s, %s, %s\n" % \ stats_for_results(trainingTimes) result_string += "Test time: %s, %.3f, %s, %s, %s\n" % \ stats_for_results(testTimes) result_string += "Training Set Metric: %s, %.3f, %s, %s, %s\n" % \ stats_for_results(trainingMetrics) result_string += "Test Set Metric: %s, %.3f, %s, %s, %s" % \ stats_for_results(testMetrics) else: # non-prediction problem times = [r['time'] for r in result_dict['results']] times = times[config.IGNORED_TRIALS:] result_string += "Time: %s, %.3f, %s, %s, %s\n" % \ stats_for_results(times) result_string = "%s, %s\n%s" % (short_name, " ".join(opt_list), result_string) sys.stdout.flush() return result_string
def process_output(cls, config, short_name, opt_list, stdout_filename, stderr_filename): with open(stdout_filename, "r") as stdout_file: output = stdout_file.read() results_token = "results: " result_string = "" if results_token not in output: result_string = "FAILED" else: result_line = filter(lambda x: results_token in x, output.split("\n"))[-1] result_json = result_line.replace(results_token, "") try: result_dict = json.loads(result_json) except: print "Failed to parse JSON:\n", result_json raise num_results = len(result_dict['results']) err_msg = ("Expecting at least %s results " "but only found %s" % (config.IGNORED_TRIALS + 1, num_results)) assert num_results > config.IGNORED_TRIALS, err_msg # 2 modes: prediction problems (4 metrics) and others (time only) if 'trainingTime' in result_dict['results'][0]: # prediction problem trainingTimes = [r['trainingTime'] for r in result_dict['results']] testTimes = [r['testTime'] for r in result_dict['results']] trainingMetrics = [r['trainingMetric'] for r in result_dict['results']] testMetrics = [r['testMetric'] for r in result_dict['results']] trainingTimes = trainingTimes[config.IGNORED_TRIALS:] testTimes = testTimes[config.IGNORED_TRIALS:] trainingMetrics = trainingMetrics[config.IGNORED_TRIALS:] testMetrics = testMetrics[config.IGNORED_TRIALS:] result_string += "Training time: %s, %.3f, %s, %s, %s\n" % \ stats_for_results(trainingTimes) result_string += "Test time: %s, %.3f, %s, %s, %s\n" % \ stats_for_results(testTimes) result_string += "Training Set Metric: %s, %.3f, %s, %s, %s\n" % \ stats_for_results(trainingMetrics) result_string += "Test Set Metric: %s, %.3f, %s, %s, %s" % \ stats_for_results(testMetrics) else: # non-prediction problem times = [r['time'] for r in result_dict['results']] times = times[config.IGNORED_TRIALS:] result_string += "Time: %s, %.3f, %s, %s, %s\n" % \ stats_for_results(times) result_string = "%s, %s\n%s" % (short_name, " ".join(opt_list), result_string) sys.stdout.flush() return result_string
def process_output(cls, config, short_name, opt_list, stdout_filename, stderr_filename): with open(stdout_filename, "r") as stdout_file: output = stdout_file.read() results_token = "results: " if results_token not in output: print("Test did not produce expected results. Output was:") print(output) sys.exit(1) result_line = filter(lambda x: results_token in x, output.split("\n"))[-1] result_list = result_line.replace(results_token, "").split(",") err_msg = ("Expecting at least %s results " "but only found %s" % (config.IGNORED_TRIALS + 1, len(result_list))) assert len(result_list) > config.IGNORED_TRIALS, err_msg result_list = result_list[config.IGNORED_TRIALS:] result_string = "%s, %s, " % (short_name, " ".join(opt_list)) result_string += "%s, %.3f, %s, %s, %s\n" % stats_for_results( result_list) sys.stdout.flush() return result_string
def process_output(cls, config, short_name, opt_list, stdout_filename, stderr_filename): with open(stdout_filename, "r") as stdout_file: output = stdout_file.read() results_token = "results: " if results_token not in output: print("Test did not produce expected results. Output was:") print(output) sys.exit(1) result_line = filter(lambda x: results_token in x, output.split("\n"))[0] result_list = result_line.replace(results_token, "").split(",") err_msg = ("Expecting at least %s results " "but only found %s" % (config.IGNORED_TRIALS + 1, len(result_list))) assert len(result_list) > config.IGNORED_TRIALS, err_msg result_list = result_list[config.IGNORED_TRIALS:] result_string = "%s, %s, " % (short_name, " ".join(opt_list)) result_string += "%s, %.3f, %s, %s, %s\n" % stats_for_results(result_list) sys.stdout.flush() return result_string