def write_scores_html(score_dir, auto_refresh=True, append=False): filename = 'detailed_results.html' image_paths = sorted(ls(os.path.join(score_dir, '*.png'))) if auto_refresh: html_head = '<html><head> <meta http-equiv="refresh" content="5"> ' +\ '</head><body><pre>' else: html_head = """<html><body><pre>""" html_end = '</pre></body></html>' if append: mode = 'a' else: mode = 'w' filepath = os.path.join(score_dir, filename) with open(filepath, mode) as html_file: html_file.write(html_head) for image_path in image_paths: with open(image_path, "rb") as image_file: encoded_string = base64.b64encode(image_file.read()) encoded_string = encoded_string.decode('utf-8') s = '<img src="data:image/png;charset=utf-8;base64,{}"/>'\ .format(encoded_string) html_file.write(s + '<br>') html_file.write(html_end) logger.debug("Wrote learning curve page to {}".format(filepath))
def get_prediction_files(prediction_dir, basename): """Return prediction files for the task <basename>. Examples of prediction file name: mini.predict_0, mini.predict_1 """ prediction_files = ls(os.path.join(prediction_dir, basename + '*.predict_*')) return prediction_files
def get_task_name(solution_dir): """Get the task name from solution directory.""" solution_names = sorted(ls(os.path.join(solution_dir, '*.solution'))) if len(solution_names) != 1: # Assert only one file is found logger.warning("{} solution files found: {}! "\ .format(len(solution_names), solution_names) + "Return `None` as task name.") return None solution_file = solution_names[0] task_name = solution_file.split(os.sep)[-1].split('.')[0] return task_name
def get_solution(solution_dir): """Get the solution array from solution directory.""" solution_names = sorted(ls(os.path.join(solution_dir, '*.solution'))) if len(solution_names) != 1: # Assert only one file is found logger.warning("{} solution files found: {}! "\ .format(len(solution_names), solution_names) + "Return `None` as solution.") return None solution_file = solution_names[0] solution = read_array(solution_file) return solution
def get_prediction_files(prediction_dir, basename, start): """Return prediction files for the task <basename>. Examples of prediction file name: mini.predict_0, mini.predict_1 """ prediction_files = ls( os.path.join(prediction_dir, basename + '*.predict_*')) # Exclude all files (if any) generated before start prediction_files = [ f for f in prediction_files if os.path.getmtime(f) > start ] return prediction_files
def get_prediction_files(prediction_dir): """Return prediction files in prediction directory. Examples of prediction file name: mini.predict_0, mini.predict_1 """ prediction_files = ls(os.path.join(prediction_dir, '*.predict_*')) task_names = set( [os.path.basename(f.split('.')[-2]) for f in prediction_files]) if len(task_names) > 1: raise ValueError("Predictions of multiple tasks are found: {}!" \ .format(prediction_files)) order_key = lambda filename: int(filename.split('_')[-1]) prediction_files = sorted(prediction_files, key=order_key) return prediction_files
def get_new_prediction_files(self): """Fetch new prediction file(name)s found in prediction directory and update corresponding attributes. Examples of prediction file name: mini.predict_0, mini.predict_1 Returns: List of new prediction files found. """ prediction_files = ls(os.path.join(self.prediction_dir, self.prediction_filename_pattern())) logger.debug("Prediction files: {}".format(prediction_files)) new_prediction_files = [p for p in prediction_files if p not in self.prediction_files_so_far] order_key = lambda filename: int(filename.split('_')[-1]) self.new_prediction_files = sorted(new_prediction_files, key=order_key) return self.new_prediction_files
def write_scores_html(score_dir, auto_refresh=True): filename = 'detailed_results.html' image_paths = sorted(ls(os.path.join(score_dir, '*.png'))) if auto_refresh: html_head = """<html><head> <meta http-equiv="refresh" content="5"> </head><body><pre>""" else: html_head = """<html><body><pre>""" html_end = '</pre></body></html>' with open(os.path.join(score_dir, filename), 'w') as html_file: # Automatic refreshing the page on file change using Live.js html_file.write(html_head) for image_path in image_paths: with open(image_path, "rb") as image_file: encoded_string = base64.b64encode(image_file.read()) encoded_string = encoded_string.decode('utf-8') s = '<img src="data:image/png;charset=utf-8;base64,%s"/>'%encoded_string html_file.write(s + '<br>') html_file.write(html_end)
) == 1: # Use the default input and output directories if no arguments are provided input_dir = default_input_dir output_dir = default_output_dir else: input_dir = argv[1] output_dir = argv[2] # Create the output directory, if it does not already exist and open output files mkdir(output_dir) score_file = open(os.path.join(output_dir, 'scores.txt'), 'wb') html_file = open(os.path.join(output_dir, 'scores.html'), 'wb') # Get the metric metric_name, scoring_function = _load_scoring_function() # Get all the solution files from the solution directory solution_names = sorted(ls(os.path.join(input_dir, 'ref', '*.solution'))) # Loop over files in solution directory and search for predictions with extension .predict having the same basename for i, solution_file in enumerate(solution_names): set_num = i + 1 # 1-indexed score_name = 'set%s_score' % set_num # Extract the dataset name from the file name basename = solution_file[-solution_file[::-1].index(filesep): -solution_file[::-1].index('.') - 1] try: # Get the last prediction from the res subdirectory (must end with '.predict') predict_file = ls( os.path.join(input_dir, 'res', basename + '*.predict'))[-1] if (predict_file == []):
# print_log("Scoring datetime:", the_date) # Create the output directory, if it does not already exist and open output files os.makedirs(score_dir, exist_ok=True) score_file = open(os.path.join(score_dir, 'scores.txt'), 'w') detailed_results_filepath = os.path.join(score_dir, 'detailed_results.html') # Initialize detailed_results.html init_scores_html(detailed_results_filepath) # Get the metric scoring_function = autodl_bac metric_name = "Area under Learning Curve" # Get all the solution files from the solution directory solution_names = sorted(ls(os.path.join(solution_dir, '*.solution'))) if not len(solution_names) == 1: # Assert only one file is found raise ValueError("Multiple solution file found!") solution_file = solution_names[0] # Extract the dataset name from the file name basename = get_basename(solution_file) nb_preds = {x:0 for x in solution_names} scores = {x:0 for x in solution_names} # Use 'duration.txt' file to detect if ingestion program exits early duration_filepath = os.path.join(prediction_dir, 'duration.txt') # Begin scoring process, along with ingestion program # Moniter training processes while time budget is not attained while(time.time() < start + TIME_BUDGET): time.sleep(0.5)