Ejemplo n.º 1
0
    def from_args(cls, args):
        c = cls()
        c.name = args.model_path if args.model_path else c.name
        c.param_path = args.param_path if args.param_path else c.param_path
        c.mode = args.mode
        c.train_set_path = args.train_path
        c.dev_set_path = args.dev_path
        c.test_set_path = args.test_path
        c.test_set_path2 = args.test_path2
        c.test_conllu_gold_path = args.test_conllu_gold_path
        c.wembpath = args.word_emb_path
        c.conll_format = const.CONLL06 if args.data_format is 'conll06' else \
            const.CONLL09 if args.data_format is 'conll09' else const.CONLLU
        c.tune_las = args.tune == 'las'

        c.use_bilstm_input = args.use_bilstm_input
        c.in_lstm_size = args.input_bilstm_size
        c.in_lstm_count = args.input_bilstm_layers
        c.hid_mlp_size = args.hidden_layer_size
        c.hid_mlp_count = args.hidden_layers
        c.hid_lstm_size = args.encoding_lstm_size
        c.hid_lstm_count = args.encoding_lstm_layers
        c.batch_size = args.batch_size

        c.stack_feats = args.stack_feats
        c.buffer_feats = args.buffer_feats
        c.max_epochs = args.epochs
        c.iter_per_test = args.iter_per_test
        c.min_iter_for_test = args.min_iter_for_test

        from networks import RRTNetwork
        c.nn = RRTNetwork.from_args(args)
        if c.conll_format is const.CONLLU:
            c.dev_ud = load_conllu_file(c.dev_set_path)
            c.gold_ud = load_conllu_file(c.test_conllu_gold_path)
        return c
Ejemplo n.º 2
0
    # Load the data
    root_factors = [ud_dataset.UDDataset.FORMS]
    train = ud_dataset.UDDataset("{}-ud-train.conllu".format(args.basename),
                                 args.lr_allow_copy, root_factors)
    dev = ud_dataset.UDDataset("{}-ud-dev.conllu".format(args.basename),
                               args.lr_allow_copy,
                               root_factors,
                               train=train,
                               shuffle_batches=False)
    dev_udpipe = ud_dataset.UDDataset("{}-ud-dev-udpipe.conllu".format(
        args.basename),
                                      args.lr_allow_copy,
                                      root_factors,
                                      train=train,
                                      shuffle_batches=False)
    dev_conllu = conll18_ud_eval.load_conllu_file("{}-ud-dev.conllu".format(
        args.basename))

    # Construct the network
    network = Network(threads=args.threads)
    network.construct(
        args, len(train.factors[train.FORMS].words),
        len(train.factors[train.FORMS].alphabet),
        dict((tag, len(train.factors[train.FACTORS_MAP[tag]].words))
             for tag in args.tags))

    if args.checkpoint:
        network.saver_train.restore(network.session, args.checkpoint)

    with open("{}/cmd".format(args.logdir), "w") as cmd_file:
        cmd_file.write(command_line)
    log_file = open("{}/log".format(args.logdir), "w")
Ejemplo n.º 3
0
def main():
    # Parse arguments
    parser = argparse.ArgumentParser()
    parser.add_argument("truth",
                        type=str,
                        help="Directory name of the truth dataset.")
    parser.add_argument("system",
                        type=str,
                        help="Directory name of system output.")
    parser.add_argument("output",
                        type=str,
                        help="Directory name of the output directory.")
    args = parser.parse_args()

    # Load input dataset metadata.json
    with open(args.truth + "/metadata.json", "r") as metadata_file:
        metadata = json.load(metadata_file)

    # Evaluate and compute sum of all treebanks
    metrics = [
        "Tokens", "Sentences", "Words", "UPOS", "XPOS", "UFeats", "AllTags",
        "Lemmas", "UAS", "LAS", "CLAS", "MLAS", "BLEX"
    ]
    treebanks = 0
    summation = {}
    results = []
    results_las, results_mlas, results_blex = {}, {}, {}
    for entry in metadata:
        treebanks += 1

        ltcode, goldfile, outfile = "_".join(
            (entry['lcode'],
             entry['tcode'])), entry['goldfile'], entry['outfile']

        # Load gold data
        try:
            gold = load_conllu_file(args.truth + "/" + goldfile)
        except:
            results.append(
                (ltcode + "-Status", "Error: Cannot load gold file"))
            continue

        # Load system data
        try:
            system = load_conllu_file(args.system + "/" + outfile)
        except UDError as e:
            if e.args[0].startswith("There is a cycle"):
                results.append(
                    (ltcode + "-Status",
                     "Error: There is a cycle in generated CoNLL-U file"))
                continue
            if e.args[0].startswith("There are multiple roots"):
                results.append((
                    ltcode + "-Status",
                    "Error: There are multiple roots in a sentence in generated CoNLL-U file"
                ))
                continue
            results.append((
                ltcode + "-Status",
                "Error: There is a format error (tabs, ID values, etc) in generated CoNLL-U file"
            ))
            continue
        except:
            results.append((ltcode + "-Status",
                            "Error: Cannot open generated CoNLL-U file"))
            continue

        # Check for correctness
        if not system.characters:
            results.append(
                (ltcode + "-Status", "Error: The system file is empty"))
            continue
        if system.characters != gold.characters:
            results.append((
                ltcode + "-Status",
                "Error: The concatenation of tokens in gold file and in system file differ, system file has {} nonspace characters, which is approximately {}% of the gold file"
                .format(
                    len(system.characters),
                    int(100 * len(system.characters) / len(gold.characters)))))
            continue

        # Evaluate
        try:
            evaluation = evaluate(gold, system)
        except:
            # Should not happen
            results.append((
                ltcode + "-Status",
                "Error: Cannot evaluate generated CoNLL-U file, internal error"
            ))
            continue

        # Generate output metrics and compute sum
        results.append((
            ltcode + "-Status",
            "OK: Result F1 scores rounded to 5% are LAS={:.0f}% MLAS={:.0f}% BLEX={:.0f}%"
            .format(100 * round_score(evaluation["LAS"].f1),
                    100 * round_score(evaluation["MLAS"].f1),
                    100 * round_score(evaluation["BLEX"].f1))))

        for metric in metrics:
            results.append((ltcode + "-" + metric + "-F1",
                            "{:.9f}".format(100 * evaluation[metric].f1)))
            summation[metric] = summation.get(metric,
                                              0) + evaluation[metric].f1
        results_las[ltcode] = evaluation["LAS"].f1
        results_mlas[ltcode] = evaluation["MLAS"].f1
        results_blex[ltcode] = evaluation["BLEX"].f1

    # Compute averages
    for metric in reversed(metrics):
        results.insert(0, ("total-" + metric + "-F1", "{:.9f}".format(
            100 * summation.get(metric, 0) / treebanks)))

    # Generate evaluation.prototext
    with open(args.output + "/evaluation.prototext", "w") as evaluation:
        for key, value in results:
            print('measure{{\n  key: "{}"\n  value: "{}"\n}}'.format(
                key, value),
                  file=evaluation)

    # Generate LAS-F1, MLAS-F1, BLEX-F1 + Status on stdout, Status on stderr
    for key, value in results:
        if not key.endswith("-Status"):
            continue

        ltcode = key[:-len("-Status")]
        print("{:13} LAS={:10.6f}% MLAS={:10.6f}% BLEX={:10.6f}% ({})".format(
            ltcode, 100 * results_las.get(ltcode, 0.),
            100 * results_mlas.get(ltcode, 0.),
            100 * results_blex.get(ltcode, 0.), value),
              file=sys.stdout)
        print("{:13} {}".format(ltcode, value), file=sys.stderr)
def conll_eval(system_file, gold_file):    
    gold_ud = load_conllu_file(gold_file)
    system_ud = load_conllu_file(system_file)
    return evaluate(gold_ud, system_ud)