def test_constituency_parsing(self): predictor = pretrained.span_based_constituency_parsing_with_elmo_joshi_2018() sentence = """Pierre Vinken died aged 81; immortalised aged 61.""" result = predictor.predict_json({"sentence": sentence}) assert result["tokens"] == ["Pierre", "Vinken", "died", "aged", "81", ";", "immortalised", "aged", "61", "."] assert result["trees"] == "(S (NP (NNP Pierre) (NNP Vinken)) (VP (VP (VBD died) (NP (JJ aged) (CD 81))) (, ;) (VP (VBD immortalised) (S (ADJP (VBN aged) (NP (CD 61)))))) (. .))"
def __init__(self, config): super().__init__() self.config = config self.data_dir = config.data_dir self.save_data_dir = config.save_data_dir # Heuristic weather conditions self.weather_list = ['rainy', 'sunny', 'daytime', 'day', 'night'] self.difficult_pronouns = ["other", "it"] self.nlp = spacy.load('en_core_web_sm') # self.coref_model = pretrained.neural_coreference_resolution_lee_2017() # Constituency parser for getting ellipsis self.const_parser = pretrained.span_based_constituency_parsing_with_elmo_joshi_2018( ) self.heuristic_root_cp = ["S", "SQ", "SBARQ", "SINV"]
def __init__(self, config): super().__init__() self.config = config self.data_dir = config.data_dir self.save_data_dir = config.save_data_dir # Heuristic weather conditions self.weather_list = ['rainy', 'sunny', 'daytime', 'day', 'night'] self.difficult_pronouns = ["other", "it"] if config.create_stats_dic: self.nlp = spacy.load('en_core_web_sm') # self.coref_model = pretrained.neural_coreference_resolution_lee_2017() # Constituency parser for getting ellipsis self.const_parser = pretrained.span_based_constituency_parsing_with_elmo_joshi_2018() self.heuristic_root_cp = ["S", "SQ", "SBARQ", "SINV"] self.non_agreement_images = [128889, 525402, 270717, 562489, 204402, 431203, 566063, 391624, 34733, 357189, 194408, 290919, 179535, 321201, 434695, 422086, 389671, 219511, 288804, 135635, 506795, 565265, 76825, 113894, 569745, 454202, 648, 188270, 62886, 67435, 301586, 274049, 139477, 372849, 418086, 520026, 358062, 24025, 531916, 129793, 165015, 491809, 58794, 77166, 177397, 130307, 469621, 249431, 133036, 542075, 240603, 330121, 526149, 472987, 456541, 498784, 216234, 30786, 51931, 512332, 31706, 220643, 39771, 170070, 386874, 100528, 231110, 154609, 139962, 145204, 567307, 174821, 468028, 114981, 284112, 406280, 528160, 185561, 313049, 306269, 325335, 332510, 546153, 389417, 489182, 174023, 95844, 116883, 452505, 53772, 87395, 290313, 225029, 439314, 494256, 84393, 118025, 417343, 57931, 190947, 509900, 60776, 235054, 307886, 544849, 70689, 51184, 225737, 127730, 227148, 240892, 167060, 49132, 300446, 467899, 123729, 402930, 567184, 185894, 333125, 323557, 1872, 573045, 353753, 142963, 268723, 327133, 161055, 185565, 574189, 29737, 99643, 295627, 485732, 546554, 286929, 175345, 223379, 146821, 358981, 427711, 212259, 347890, 297528, 83797, 369360, 538790, 219444, 101666, 201002] self.hist_info_images = [257366, 425477, 191097, 552399, 12468, 458949, 109735, 311793, 437200, 355853, 98849, 57743, 83289, 488471, 446567, 196905, 308846, 328336, 289233, 52156, 366462, 511748, 457675, 518811, 413085, 432039, 531270, 430580, 293582, 544148, 80366, 179366, 150236, 400960, 10424, 451398, 498340, 268914, 384171, 172461, 387266, 214227, 555578, 181772, 149373, 251385, 407878, 574545, 544827, 120559, 19299, 73638, 496822, 204195, 97073, 209447, 53433, 403234, 524006, 178300, 376460, 570468, 292100, 227006, 170315, 456824, 525726, 179064, 98879, 558975, 193521, 377823, 449230, 44468, 573552, 288308, 237956, 69538, 250654, 439842, 146314, 458818, 122826, 33976, 322815, 239030, 209271, 560666, 361734, 225491, 27366, 29060, 191186, 394073, 120870, 580183, 111013]
'The other dataset', row[0], 'GEOQA') res.append(geoaq) line_count += 1 print(f'Processed {line_count} lines.') return res PRONOUN = dict({ 'where': '1', 'what': '2', 'which': '3', 'when': '4', 'who': '6', 'why': '7' }) model = pretrained.span_based_constituency_parsing_with_elmo_joshi_2018() fpt = 'data/place_type/type-set' factv = 'data/verb/action_verb.txt' fstav = 'data/verb/stative_verb.txt' pt_set, pt_dict = load_pt(fpt) actv = load_word(factv) stav = load_word(fstav) # loading ELMo pretrained word embedding model options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json" weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5" elmo = Elmo(options_file, weight_file, 2, dropout=0)
try: number = int(word) except ValueError: try: number = float(word) except ValueError: number = None return number tokenizer = BertDropTokenizer(pretrained_model="bert-base-uncased") number_tokenizer = WordTokenizer() words_splitter = WordTokenizer() sentences_splitter = SpacySentenceSplitter() ner_tagger = fine_grained_named_entity_recognition_with_elmo_peters_2018() pos_tagger = span_based_constituency_parsing_with_elmo_joshi_2018() def extract_letters_frequency(passage, sentence_idx=None): """ :param passage: :param sentence_idx: None for whole passage, else per sentence (index 0).. :return: """ if sentence_idx is None: return dict(filter(lambda k: k[0].isalpha(), Counter(passage).items())) else: sentences = extract_sentences(passage) sen = sentences[sentence_idx] return dict(filter(lambda k: k[0].isalpha(), Counter(sen).items()))