def test_read_from_file(self): ptb_reader = PennTreeBankConstituencySpanDatasetReader() instances = ptb_reader.read( str(self.FIXTURES_ROOT / "data" / "example_ptb.trees")) assert len(instances) == 2 fields = instances[0].fields tokens = [x.text for x in fields["tokens"].tokens] pos_tags = fields["pos_tags"].labels spans = [(x.span_start, x.span_end) for x in fields["spans"].field_list] span_labels = fields["span_labels"].labels assert tokens == [ "Also", ",", "because", "UAL", "Chairman", "Stephen", "Wolf", "and", "other", "UAL", "executives", "have", "joined", "the", "pilots", "'", "bid", ",", "the", "board", "might", "be", "forced", "to", "exclude", "him", "from", "its", "deliberations", "in", "order", "to", "be", "fair", "to", "other", "bidders", ".", ] assert pos_tags == [ "RB", ",", "IN", "NNP", "NNP", "NNP", "NNP", "CC", "JJ", "NNP", "NNS", "VBP", "VBN", "DT", "NNS", "POS", "NN", ",", "DT", "NN", "MD", "VB", "VBN", "TO", "VB", "PRP", "IN", "PRP$", "NNS", "IN", "NN", "TO", "VB", "JJ", "TO", "JJ", "NNS", ".", ] assert spans == enumerate_spans(tokens) gold_tree = Tree.fromstring( "(S(ADVP(RB Also))(, ,)(SBAR(IN because)" "(S(NP(NP(NNP UAL)(NNP Chairman)(NNP Stephen)(NNP Wolf))" "(CC and)(NP(JJ other)(NNP UAL)(NNS executives)))(VP(VBP have)" "(VP(VBN joined)(NP(NP(DT the)(NNS pilots)(POS '))(NN bid))))))" "(, ,)(NP(DT the)(NN board))(VP(MD might)(VP(VB be)(VP(VBN " "forced)(S(VP(TO to)(VP(VB exclude)(NP(PRP him))(PP(IN from)" "(NP(PRP$ its)(NNS deliberations)))(SBAR(IN in)(NN order)(S(" "VP(TO to)(VP(VB be)(ADJP(JJ fair)(PP(TO to)(NP(JJ other)(NNS " "bidders))))))))))))))(. .))") assert fields["metadata"].metadata["gold_tree"] == gold_tree assert fields["metadata"].metadata["tokens"] == tokens correct_spans_and_labels = {} ptb_reader._get_gold_spans(gold_tree, 0, correct_spans_and_labels) for span, label in zip(spans, span_labels): if label != "NO-LABEL": assert correct_spans_and_labels[span] == label fields = instances[1].fields tokens = [x.text for x in fields["tokens"].tokens] pos_tags = fields["pos_tags"].labels spans = [(x.span_start, x.span_end) for x in fields["spans"].field_list] span_labels = fields["span_labels"].labels assert tokens == [ "That", "could", "cost", "him", "the", "chance", "to", "influence", "the", "outcome", "and", "perhaps", "join", "the", "winning", "bidder", ".", ] assert pos_tags == [ "DT", "MD", "VB", "PRP", "DT", "NN", "TO", "VB", "DT", "NN", "CC", "RB", "VB", "DT", "VBG", "NN", ".", ] assert spans == enumerate_spans(tokens) gold_tree = Tree.fromstring( "(S(NP(DT That))(VP(MD could)(VP(VB cost)(NP(PRP him))" "(NP(DT the)(NN chance)(S(VP(TO to)(VP(VP(VB influence)(NP(DT the)" "(NN outcome)))(CC and)(VP(ADVP(RB perhaps))(VB join)(NP(DT the)" "(VBG winning)(NN bidder)))))))))(. .))") assert fields["metadata"].metadata["gold_tree"] == gold_tree assert fields["metadata"].metadata["tokens"] == tokens correct_spans_and_labels = {} ptb_reader._get_gold_spans(gold_tree, 0, correct_spans_and_labels) for span, label in zip(spans, span_labels): if label != "NO-LABEL": assert correct_spans_and_labels[span] == label
def test_read_from_file(self): ptb_reader = PennTreeBankConstituencySpanDatasetReader() instances = ptb_reader.read('tests/fixtures/data/example_ptb.trees') assert len(instances) == 2 fields = instances[0].fields tokens = [x.text for x in fields["tokens"].tokens] pos_tags = fields["pos_tags"].labels spans = [(x.span_start, x.span_end) for x in fields["spans"].field_list] span_labels = fields["span_labels"].labels assert tokens == ['Also', ',', 'because', 'UAL', 'Chairman', 'Stephen', 'Wolf', 'and', 'other', 'UAL', 'executives', 'have', 'joined', 'the', 'pilots', "'", 'bid', ',', 'the', 'board', 'might', 'be', 'forced', 'to', 'exclude', 'him', 'from', 'its', 'deliberations', 'in', 'order', 'to', 'be', 'fair', 'to', 'other', 'bidders', '.'] assert pos_tags == ['RB', ',', 'IN', 'NNP', 'NNP', 'NNP', 'NNP', 'CC', 'JJ', 'NNP', 'NNS', 'VBP', 'VBN', 'DT', 'NNS', 'POS', 'NN', ',', 'DT', 'NN', 'MD', 'VB', 'VBN', 'TO', 'VB', 'PRP', 'IN', 'PRP$', 'NNS', 'IN', 'NN', 'TO', 'VB', 'JJ', 'TO', 'JJ', 'NNS', '.'] assert spans == enumerate_spans(tokens) gold_tree = Tree.fromstring("(VROOT(S(ADVP(RB Also))(, ,)(SBAR(IN because)" "(S(NP(NP(NNP UAL)(NNP Chairman)(NNP Stephen)(NNP Wolf))" "(CC and)(NP(JJ other)(NNP UAL)(NNS executives)))(VP(VBP have)" "(VP(VBN joined)(NP(NP(DT the)(NNS pilots)(POS '))(NN bid))))))" "(, ,)(NP(DT the)(NN board))(VP(MD might)(VP(VB be)(VP(VBN " "forced)(S(VP(TO to)(VP(VB exclude)(NP(PRP him))(PP(IN from)" "(NP(PRP$ its)(NNS deliberations)))(SBAR(IN in)(NN order)(S(" "VP(TO to)(VP(VB be)(ADJP(JJ fair)(PP(TO to)(NP(JJ other)(NNS " "bidders))))))))))))))(. .)))") assert fields["metadata"].metadata["gold_tree"] == gold_tree assert fields["metadata"].metadata["tokens"] == tokens correct_spans_and_labels = {} ptb_reader._get_gold_spans(gold_tree, 0, correct_spans_and_labels) for span, label in zip(spans, span_labels): if label != "NO-LABEL": assert correct_spans_and_labels[span] == label fields = instances[1].fields tokens = [x.text for x in fields["tokens"].tokens] pos_tags = fields["pos_tags"].labels spans = [(x.span_start, x.span_end) for x in fields["spans"].field_list] span_labels = fields["span_labels"].labels assert tokens == ['That', 'could', 'cost', 'him', 'the', 'chance', 'to', 'influence', 'the', 'outcome', 'and', 'perhaps', 'join', 'the', 'winning', 'bidder', '.'] assert pos_tags == ['DT', 'MD', 'VB', 'PRP', 'DT', 'NN', 'TO', 'VB', 'DT', 'NN', 'CC', 'RB', 'VB', 'DT', 'VBG', 'NN', '.'] assert spans == enumerate_spans(tokens) gold_tree = Tree.fromstring("(VROOT(S(NP(DT That))(VP(MD could)(VP(VB cost)(NP(PRP him))" "(NP(DT the)(NN chance)(S(VP(TO to)(VP(VP(VB influence)(NP(DT the)" "(NN outcome)))(CC and)(VP(ADVP(RB perhaps))(VB join)(NP(DT the)" "(VBG winning)(NN bidder)))))))))(. .)))") assert fields["metadata"].metadata["gold_tree"] == gold_tree assert fields["metadata"].metadata["tokens"] == tokens correct_spans_and_labels = {} ptb_reader._get_gold_spans(gold_tree, 0, correct_spans_and_labels) for span, label in zip(spans, span_labels): if label != "NO-LABEL": assert correct_spans_and_labels[span] == label
def test_read_from_file(self): ptb_reader = PennTreeBankConstituencySpanDatasetReader() instances = ptb_reader.read(str(self.FIXTURES_ROOT / 'data' / 'example_ptb.trees')) assert len(instances) == 2 fields = instances[0].fields tokens = [x.text for x in fields["tokens"].tokens] pos_tags = fields["pos_tags"].labels spans = [(x.span_start, x.span_end) for x in fields["spans"].field_list] span_labels = fields["span_labels"].labels assert tokens == ['Also', ',', 'because', 'UAL', 'Chairman', 'Stephen', 'Wolf', 'and', 'other', 'UAL', 'executives', 'have', 'joined', 'the', 'pilots', "'", 'bid', ',', 'the', 'board', 'might', 'be', 'forced', 'to', 'exclude', 'him', 'from', 'its', 'deliberations', 'in', 'order', 'to', 'be', 'fair', 'to', 'other', 'bidders', '.'] assert pos_tags == ['RB', ',', 'IN', 'NNP', 'NNP', 'NNP', 'NNP', 'CC', 'JJ', 'NNP', 'NNS', 'VBP', 'VBN', 'DT', 'NNS', 'POS', 'NN', ',', 'DT', 'NN', 'MD', 'VB', 'VBN', 'TO', 'VB', 'PRP', 'IN', 'PRP$', 'NNS', 'IN', 'NN', 'TO', 'VB', 'JJ', 'TO', 'JJ', 'NNS', '.'] assert spans == enumerate_spans(tokens) gold_tree = Tree.fromstring("(S(ADVP(RB Also))(, ,)(SBAR(IN because)" "(S(NP(NP(NNP UAL)(NNP Chairman)(NNP Stephen)(NNP Wolf))" "(CC and)(NP(JJ other)(NNP UAL)(NNS executives)))(VP(VBP have)" "(VP(VBN joined)(NP(NP(DT the)(NNS pilots)(POS '))(NN bid))))))" "(, ,)(NP(DT the)(NN board))(VP(MD might)(VP(VB be)(VP(VBN " "forced)(S(VP(TO to)(VP(VB exclude)(NP(PRP him))(PP(IN from)" "(NP(PRP$ its)(NNS deliberations)))(SBAR(IN in)(NN order)(S(" "VP(TO to)(VP(VB be)(ADJP(JJ fair)(PP(TO to)(NP(JJ other)(NNS " "bidders))))))))))))))(. .))") assert fields["metadata"].metadata["gold_tree"] == gold_tree assert fields["metadata"].metadata["tokens"] == tokens correct_spans_and_labels = {} ptb_reader._get_gold_spans(gold_tree, 0, correct_spans_and_labels) for span, label in zip(spans, span_labels): if label != "NO-LABEL": assert correct_spans_and_labels[span] == label fields = instances[1].fields tokens = [x.text for x in fields["tokens"].tokens] pos_tags = fields["pos_tags"].labels spans = [(x.span_start, x.span_end) for x in fields["spans"].field_list] span_labels = fields["span_labels"].labels assert tokens == ['That', 'could', 'cost', 'him', 'the', 'chance', 'to', 'influence', 'the', 'outcome', 'and', 'perhaps', 'join', 'the', 'winning', 'bidder', '.'] assert pos_tags == ['DT', 'MD', 'VB', 'PRP', 'DT', 'NN', 'TO', 'VB', 'DT', 'NN', 'CC', 'RB', 'VB', 'DT', 'VBG', 'NN', '.'] assert spans == enumerate_spans(tokens) gold_tree = Tree.fromstring("(S(NP(DT That))(VP(MD could)(VP(VB cost)(NP(PRP him))" "(NP(DT the)(NN chance)(S(VP(TO to)(VP(VP(VB influence)(NP(DT the)" "(NN outcome)))(CC and)(VP(ADVP(RB perhaps))(VB join)(NP(DT the)" "(VBG winning)(NN bidder)))))))))(. .))") assert fields["metadata"].metadata["gold_tree"] == gold_tree assert fields["metadata"].metadata["tokens"] == tokens correct_spans_and_labels = {} ptb_reader._get_gold_spans(gold_tree, 0, correct_spans_and_labels) for span, label in zip(spans, span_labels): if label != "NO-LABEL": assert correct_spans_and_labels[span] == label