Example #1
0
def process_run(args, run_file_name, annotation, description, thresh):
    '''
    compute scores and generate output files for a single run
    
    :returns dict: max_scores for this one run
    '''
    ## Generate confusion matrices from a run for each target_id
    ## and for each step of the confidence cutoff
    stats = build_confusion_matrix(
        os.path.join(args.run_dir, run_file_name) + '.gz',
        annotation, args.cutoff_step, args.unan_is_true, args.include_training,
        thresh=thresh,
        require_positives=args.require_positives,
        debug=args.debug)

    compile_and_average_performance_metrics(stats)

    max_scores = find_max_scores(stats)

    log(json.dumps(stats, indent=4, sort_keys=True))

    base_output_filepath = os.path.join(
        args.run_dir, 
        run_file_name + '-' + description)

    output_filepath = base_output_filepath + '.csv'
    write_performance_metrics(output_filepath, stats)

    ## Output a graph of the key performance statistics
    graph_filepath = base_output_filepath + '.png'
    write_graph(graph_filepath, stats)

    return max_scores
Example #2
0
            max_scores = find_max_scores(Scores)
            team_scores[mode][team_id][system_id] = max_scores

            ## Print the top F-Score
            log( '   %s: max(avg(F_1)): %.3f' % (mode, max_scores['average']['F'] ))
            log( '   %s: max(F_1(avg(P), avg(R))): %.3f' % (mode, max_scores['average']['F_recomputed'] ))
            log( '   %s: max(avg(SU)):  %.3f' % (mode, max_scores['average']['SU'] ))

            ## Output the key performance statistics
            base_output_filepath = os.path.join(
                args.run_dir, 
                run_file_name + '-' + description)

            output_filepath = base_output_filepath + '.csv'

            write_performance_metrics(output_filepath, CM[mode], Scores)
            print ' wrote metrics table to %s' % output_filepath

            if not plt:
                print ' not generating plot, because could not import matplotlib'
            else:
                ## Output a graph of the key performance statistics
                graph_filepath = base_output_filepath + '.png'
                write_graph(graph_filepath, Scores['average'])
                print ' wrote plot image to %s' % graph_filepath
    
    for mode in MODES:
        description = make_description(args, mode)

        ## When folder is finished running output a high level summary of the scores to overview.csv
        write_team_summary(description, team_scores[mode])
Example #3
0
            ## Generate performance metrics for a run
            compile_and_average_performance_metrics(stats[mode])

            max_scores = find_max_scores(stats[mode])

            team_scores[mode][team_id][system_id] = max_scores

            ## Output the key performance statistics
            base_output_filepath = os.path.join(
                args.run_dir, 
                run_file_name + '-' + description)

            output_filepath = base_output_filepath + '.csv'

            write_performance_metrics(output_filepath, stats[mode])

            ## Output a graph of the key performance statistics
            graph_filepath = base_output_filepath + '.png'
            write_graph(graph_filepath, stats[mode])

        log(json.dumps(stats, indent=4, sort_keys=True))

    for mode in MODES:
        description = make_description(args, mode)

        ## When folder is finished running output a high level summary of the scores to overview.csv
        write_team_summary(description, team_scores[mode])

    elapsed = time.time() - start_time
    log('finished after %d seconds at at %r'
Example #4
0
def score_all_runs(args, description, reject):
    '''
    score all the runs in the specified runs dir using the various
    filters and configuration settings

    :param description: string used for file names
    :param reject: callable to rejects truth data
    '''
    ## Load in the annotation data
    annotation = load_annotation(args.annotation, args.include_useful, args.include_neutral, 
                                 args.min_len_clean_visible, reject)
    log( 'This assumes that all run file names end in .gz' )

    team_scores = defaultdict(lambda: defaultdict(dict))
    for run_file in os.listdir(args.run_dir):
        if not run_file.endswith('.gz'):
            continue
        
        ## take the name without the .gz
        run_file_name = '.'.join(run_file.split('.')[:-1])
        log( 'processing: %s.gz' % run_file_name )
        
        ## Generate the confusion matrix for a run
        CM = score_confusion_matrix(
            os.path.join(args.run_dir, run_file), 
            annotation, args.cutoff_step, args.unan_is_true, args.include_training,
            debug=args.debug)

        ## Generate performance metrics for a run
        Scores = performance_metrics(CM)
        
        ## Generate the average metrics
        (CM['average'], Scores['average']) = full_run_metrics(CM, Scores, args.use_micro_averaging)

        max_scores = find_max_scores(Scores)

        ## split into team name and create stats file
        team_id, system_id = run_file_name.split('-')
        team_scores[team_id][system_id] = max_scores

        ## Print the top F-Score
        log( '   max(avg(F_1)): %.3f' % max_scores['average']['F'] )
        log( '   max(F_1(avg(P), avg(R))): %.3f' % max_scores['average']['F_recomputed'] )
        log( '   max(avg(SU)):  %.3f' % max_scores['average']['SU'] )
        
        base_output_filepath = os.path.join(
            args.run_dir, 
            run_file_name + '-' + description)

        output_filepath = base_output_filepath + '.csv'
        write_performance_metrics(output_filepath, CM, Scores)
        log( ' wrote metrics table to %s' % output_filepath )
        
        if not plt:
            log( ' not generating plot, because could not import matplotlib' )
        else:
            ## Output a graph of the key performance statistics
            graph_filepath = base_output_filepath + '.png'
            write_graph(graph_filepath, Scores['average'])
            log( ' wrote plot image to %s' % graph_filepath )

    ## When folder is finished running output a high level summary of the scores to overview.csv
    write_team_summary(description, team_scores)