Example #1
0
 def __init__(self, ref_dir, pred_dir, maxlen=100, log=sys.stderr):
   self.name == 'rouge'
   self.config_file = evaluate_duc.create_config(ref_dir, pred_dir)
   self.pred_dir = pred_dir
   self.ref_dir = ref_dir
   self.maxlen = maxlen
   self.output = log
Example #2
0
    
    options, task = parse_options()

    ## create SummaryProblem instances
    setup_start_time = time.time()
    if options.task == 'u08': framework.setup_TAC08(task)
    else: framework.setup_DUC_basic(task, skip_updates=False)

    ## only run the parser if compression is required (this is not known by the pickle stuff)
    parser = None
    if options.compress:
        parser = berkeleyparser.CommandLineParser(BERKELEY_PARSER_CMD)
    framework.setup_DUC_sentences(task, parser, reload=options.reload)

    setup_time = time.time() - setup_start_time

    ## go!
    run_standard(options)
    sys.stderr.write('Setup time [%1.2fs]\n' %setup_time)    

    ## evaluate
    if not options.manpath:
        sys.stderr.write('no manual path specified, skipping evaluation\n')
    else:
        import evaluate_duc
        sys.stderr.write('\nROUGE results\n')
        config_file = evaluate_duc.create_config(task.manual_path, options.output)
        evaluate_duc.run_rouge(ROUGE_SCORER, config_file, options.length, verbose=False)
        os.remove(config_file)

Example #3
0
                      type='float',
                      help='fudge factor',
                      default=1.0)
    parser.add_option('--shortened',
                      dest='shortened',
                      type='str',
                      help='location of .sent files for shortening experiment')
    (options, args) = get_options(parser)

    if options.experiment != 'eval':
        os.popen('mkdir -p %s' % options.outpath + '/summary/')
        ids = get_topic_ids(options.inputpath)
        for id in ids:
            if '-C' in id: continue
            make_summary_ie(options.inputpath, id, options.outpath,
                            options.outpath + '/summary/', options.length,
                            options)

    else:
        import evaluate_duc
        sys.stderr.write('\nROUGE results\n')
        config_file = evaluate_duc.create_config(options.manpath,
                                                 options.outpath + '/summary/',
                                                 chop_annotator_id=True)
        ROUGE_SCORER = 'scoring/ROUGE-1.5.5/ROUGE-1.5.5_faster.pl'
        evaluate_duc.run_rouge(ROUGE_SCORER,
                               config_file,
                               options.length,
                               verbose=True)
        os.remove(config_file)
Example #4
0
    os.popen('mkdir -p %s' %options.outpath + '/summary/')

    ## parameters
    length = 100
    
    ## run through all topics
    ids = get_topic_ids(options.inputpath)
    for id in ids:
        if '-C' in id: continue
        make_summary(options.inputpath, id, options.outpath, options.outpath + '/summary/', length, options)
        
    ## ROUGE evaluation
    ROUGE_SCORER = 'scoring/ROUGE-1.5.5/ROUGE-1.5.5_faster.pl'
    man_source = {}
    man_source['u09'] = '../09/UpdateSumm09_eval/ROUGE/models/'
    man_source['u08'] = '../08/tac08_results/ROUGE/models/'
    man_source['u07'] = '../07/duc07.results.data/results/updateEval/ROUGE/models/'
    man_source['m07'] = '../07/duc07.results.data/results/mainEval/ROUGE/models/'
    man_source['m06'] = '../06/results/NIST/NISTeval/ROUGE/models/'
    man_source['m05'] = '../05/results/NIST/results/ROUGE/models/'
    
    #sys.exit()
    manual_path = man_source[options.task]
    if options.manpath:
        manual_path = options.manpath
    import evaluate_duc
    sys.stderr.write('\nROUGE results\n')
    config_file = evaluate_duc.create_config(manual_path, options.outpath + '/summary/', chop_annotator_id=True)
    evaluate_duc.run_rouge(ROUGE_SCORER, config_file, length, verbose=False)
    os.remove(config_file)
Example #5
0
    data_path = "/u/dgillick/workspace/summ/bourbon/tac09_v4/"
    # data_path = '/u/dgillick/workspace/summ/bourbon/duc07_v3/'

    ## parameters
    length = 100

    ## run through all topics
    ids = get_topic_ids(data_path)
    for id in ids:
        if "-C" in id:
            continue
        make_summary(data_path, id, out_path, summ_path, length)

    ## ROUGE evaluation
    ROUGE_SCORER = "/u/favre/work/summarization/tools/ROUGE-1.5.5/ROUGE-1.5.5_faster.pl"
    man_source = {}
    man_source["u08"] = "/u/favre/work/summarization/tac08_results/ROUGE/models/"
    man_source["u07"] = "/u/drspeech/data/DUC/07/duc07.results.data/results/updateEval/ROUGE/models/"
    man_source["m07"] = "/u/drspeech/data/DUC/07/duc07.results.data/results/mainEval/ROUGE/models/"
    man_source["m06"] = "/u/drspeech/data/DUC/06/results/NIST/NISTeval/ROUGE/models/"
    man_source["m05"] = "/u/drspeech/data/DUC/05/results/NIST/results/ROUGE/models/"

    # sys.exit()
    manual_path = man_source["u08"]
    import evaluate_duc

    sys.stderr.write("\nROUGE results\n")
    config_file = evaluate_duc.create_config(manual_path, summ_path)
    evaluate_duc.run_rouge(ROUGE_SCORER, config_file, length, verbose=False)
    os.remove(config_file)
Example #6
0
    ## parameters
    length = 100

    ## run through all topics
    ids = get_topic_ids(data_path)
    for id in ids:
        if '-C' in id: continue
        make_summary(data_path, id, out_path, summ_path, length)

    ## ROUGE evaluation
    ROUGE_SCORER = '/u/favre/work/summarization/tools/ROUGE-1.5.5/ROUGE-1.5.5_faster.pl'
    man_source = {}
    man_source[
        'u08'] = '/u/favre/work/summarization/tac08_results/ROUGE/models/'
    man_source[
        'u07'] = '/u/drspeech/data/DUC/07/duc07.results.data/results/updateEval/ROUGE/models/'
    man_source[
        'm07'] = '/u/drspeech/data/DUC/07/duc07.results.data/results/mainEval/ROUGE/models/'
    man_source[
        'm06'] = '/u/drspeech/data/DUC/06/results/NIST/NISTeval/ROUGE/models/'
    man_source[
        'm05'] = '/u/drspeech/data/DUC/05/results/NIST/results/ROUGE/models/'

    #sys.exit()
    manual_path = man_source['u08']
    import evaluate_duc
    sys.stderr.write('\nROUGE results\n')
    config_file = evaluate_duc.create_config(manual_path, summ_path)
    evaluate_duc.run_rouge(ROUGE_SCORER, config_file, length, verbose=False)
    os.remove(config_file)