def process_run(args, run_file_name, annotation, description, thresh): ''' compute scores and generate output files for a single run :returns dict: max_scores for this one run ''' ## Generate confusion matrices from a run for each target_id ## and for each step of the confidence cutoff stats = build_confusion_matrix( os.path.join(args.run_dir, run_file_name) + '.gz', annotation, args.cutoff_step, args.unan_is_true, args.include_training, thresh=thresh, require_positives=args.require_positives, debug=args.debug) compile_and_average_performance_metrics(stats) max_scores = find_max_scores(stats) log(json.dumps(stats, indent=4, sort_keys=True)) base_output_filepath = os.path.join( args.run_dir, run_file_name + '-' + description) output_filepath = base_output_filepath + '.csv' write_performance_metrics(output_filepath, stats) ## Output a graph of the key performance statistics graph_filepath = base_output_filepath + '.png' write_graph(graph_filepath, stats) return max_scores
def process_run(args, run_file_name, annotation, description, thresh): ''' compute scores and generate output files for a single run :returns dict: max_scores for this one run ''' ## Generate confusion matrices from a run for each target_id ## and for each step of the confidence cutoff stats = build_confusion_matrix(os.path.join(args.run_dir, run_file_name) + '.gz', annotation, args.cutoff_step, args.unan_is_true, args.include_training, thresh=thresh, require_positives=args.require_positives, debug=args.debug) compile_and_average_performance_metrics(stats) max_scores = find_max_scores(stats) log(json.dumps(stats, indent=4, sort_keys=True)) base_output_filepath = os.path.join(args.run_dir, run_file_name + '-' + description) output_filepath = base_output_filepath + '.csv' write_performance_metrics(output_filepath, stats) ## Output a graph of the key performance statistics graph_filepath = base_output_filepath + '.png' write_graph(graph_filepath, stats) return max_scores
team_scores[mode][team_id][system_id] = max_scores ## Print the top F-Score log( ' %s: max(avg(F_1)): %.3f' % (mode, max_scores['average']['F'] )) log( ' %s: max(F_1(avg(P), avg(R))): %.3f' % (mode, max_scores['average']['F_recomputed'] )) log( ' %s: max(avg(SU)): %.3f' % (mode, max_scores['average']['SU'] )) ## Output the key performance statistics base_output_filepath = os.path.join( args.run_dir, run_file_name + '-' + description) output_filepath = base_output_filepath + '.csv' write_performance_metrics(output_filepath, CM[mode], Scores) print ' wrote metrics table to %s' % output_filepath if not plt: print ' not generating plot, because could not import matplotlib' else: ## Output a graph of the key performance statistics graph_filepath = base_output_filepath + '.png' write_graph(graph_filepath, Scores['average']) print ' wrote plot image to %s' % graph_filepath for mode in MODES: description = make_description(args, mode) ## When folder is finished running output a high level summary of the scores to overview.csv write_team_summary(description, team_scores[mode])
compile_and_average_performance_metrics(stats[mode]) max_scores = find_max_scores(stats[mode]) team_scores[mode][team_id][system_id] = max_scores ## Output the key performance statistics base_output_filepath = os.path.join( args.run_dir, run_file_name + '-' + description) output_filepath = base_output_filepath + '.csv' write_performance_metrics(output_filepath, stats[mode]) ## Output a graph of the key performance statistics graph_filepath = base_output_filepath + '.png' write_graph(graph_filepath, stats[mode]) log(json.dumps(stats, indent=4, sort_keys=True)) for mode in MODES: description = make_description(args, mode) ## When folder is finished running output a high level summary of the scores to overview.csv write_team_summary(description, team_scores[mode]) elapsed = time.time() - start_time log('finished after %d seconds at at %r' % (elapsed, datetime.utcnow()))
def score_all_runs(args, description, reject): ''' score all the runs in the specified runs dir using the various filters and configuration settings :param description: string used for file names :param reject: callable to rejects truth data ''' ## Load in the annotation data annotation = load_annotation(args.annotation, args.include_useful, args.include_neutral, args.min_len_clean_visible, reject) log( 'This assumes that all run file names end in .gz' ) team_scores = defaultdict(lambda: defaultdict(dict)) for run_file in os.listdir(args.run_dir): if not run_file.endswith('.gz'): continue ## take the name without the .gz run_file_name = '.'.join(run_file.split('.')[:-1]) log( 'processing: %s.gz' % run_file_name ) ## Generate the confusion matrix for a run CM = score_confusion_matrix( os.path.join(args.run_dir, run_file), annotation, args.cutoff_step, args.unan_is_true, args.include_training, debug=args.debug) ## Generate performance metrics for a run Scores = performance_metrics(CM) ## Generate the average metrics (CM['average'], Scores['average']) = full_run_metrics(CM, Scores, args.use_micro_averaging) max_scores = find_max_scores(Scores) ## split into team name and create stats file team_id, system_id = run_file_name.split('-') team_scores[team_id][system_id] = max_scores ## Print the top F-Score log( ' max(avg(F_1)): %.3f' % max_scores['average']['F'] ) log( ' max(F_1(avg(P), avg(R))): %.3f' % max_scores['average']['F_recomputed'] ) log( ' max(avg(SU)): %.3f' % max_scores['average']['SU'] ) base_output_filepath = os.path.join( args.run_dir, run_file_name + '-' + description) output_filepath = base_output_filepath + '.csv' write_performance_metrics(output_filepath, CM, Scores) log( ' wrote metrics table to %s' % output_filepath ) if not plt: log( ' not generating plot, because could not import matplotlib' ) else: ## Output a graph of the key performance statistics graph_filepath = base_output_filepath + '.png' write_graph(graph_filepath, Scores['average']) log( ' wrote plot image to %s' % graph_filepath ) ## When folder is finished running output a high level summary of the scores to overview.csv write_team_summary(description, team_scores)