def test_convert_binary_bracketing(self): given = "( 0 ( ( 1 2 ) 3 ) ) )" expected_tokens = [str(x) for x in range(4)] expected_transitions = [0, 0, 0, 1, 0, 1, 1] tokens, transitions = util.ConvertBinaryBracketedSeq(given.split(' ')) assert all(t == e for t, e in zip(tokens, expected_tokens)) assert all(t == e for t, e in zip(transitions, expected_transitions))
def load_data(path, lowercase=None): examples = [] with open(path) as f: for pairID, line in enumerate(f): line = line.strip() label, s1, s2 = line.split('\t') tokens1, transitions1 = util.ConvertBinaryBracketedSeq(s1.split(' ')) tokens2, transitions2 = util.ConvertBinaryBracketedSeq(s2.split(' ')) example = {} example["label"] = label example["premise"] = s1 example["premise_tokens"] = tokens1 example["premise_transitions"] = transitions1 example["hypothesis"] = s2 example["hypothesis_tokens"] = tokens2 example["hypothesis_transitions"] = transitions2 example["example_id"] = str(pairID) examples.append(example) return examples
def load_data(path, lowercase=None): examples = [] with open(path) as f: for example_id, line in enumerate(f): line = line.strip() label, seq = line.split('\t') tokens, transitions = util.ConvertBinaryBracketedSeq( seq.split(' ')) example = {} example["label"] = label example["sentence"] = seq example["tokens"] = tokens example["transitions"] = transitions example["example_id"] = str(example_id) examples.append(example) return examples