def test_train_test_tokenization_consistency(self): filepath = os.path.abspath( os.path.join(os.path.dirname(__file__), 'data', 'testdata.csv')) df = pd.read_csv(filepath) X = [] Y = [] for i, row in df.iterrows(): X.append(row["text"]) labels = json.loads(row["question_843"]) for label in labels: label['start'] = label['startOffset'] label['end'] = label['endOffset'] label['text'] = row["text"][label['start']:label['end']] Y.append(labels) for multilabel_setting in [True, False]: for base_model in [GPT, GPT2, BERT]: model = SequenceLabeler( chunk_long_sequences=True, base_model=base_model, multi_label_sequences=multilabel_setting) train_encoded = [ x for x in model.input_pipeline._text_to_ids( X, Y=Y, pad_token=model.config.pad_token) ] test_encoded = [ x for x in model.input_pipeline._text_to_ids(X) ] for chunk_id in range(len(train_encoded)): for train_token_ids, test_token_ids in zip( train_encoded[chunk_id].token_ids, test_encoded[chunk_id].token_ids): self.assertEqual(train_token_ids[0], test_token_ids[0])
def test_fit_predict_multi_model(self): """ Ensure model training does not error out Ensure model returns predictions """ self.model = SequenceLabeler(batch_size=2, max_length=256, lm_loss_coef=0.0, multi_label_sequences=True) raw_docs = ["".join(text) for text in self.texts] texts, annotations = finetune_to_indico_sequence( raw_docs, self.texts, self.labels, none_value=self.model.config.pad_token) train_texts, test_texts, train_annotations, _ = train_test_split( texts, annotations, test_size=0.1) self.model.fit(train_texts, train_annotations) self.model.predict(test_texts) probas = self.model.predict_proba(test_texts) self.assertIsInstance(probas, list) self.assertIsInstance(probas[0], list) self.assertIsInstance(probas[0][0], dict) self.assertIsInstance(probas[0][0]['confidence'], dict) self.model.save(self.save_file) model = SequenceLabeler.load(self.save_file) model.predict(test_texts)
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.model_config.update( dict( # general params that differ from finetune base_model=RoBERTa, batch_size=4, predict_batch_size=10, val_size=0.0, crf_sequence_labeling=False, low_memory_mode=True, class_weights="log", # auxiliary-specific params use_auxiliary_info=True, context_dim=4, default_context={ 'left': 0, 'right': 0, 'top': 0, 'bottom': 0, }, n_context_embed_per_channel=48, context_in_base_model=True, n_layers_with_aux=-1) ) self.model_config.update(kwargs) self.model = SequenceLabeler(**self.model_config)
def setUp(self): self.save_file = 'tests/saved-models/test-save-load' random.seed(42) np.random.seed(42) with open(self.processed_path, 'rt') as fp: self.texts, self.labels = json.load(fp) self.model = SequenceLabeler(**self.default_config())
def setUp(self): self.save_file = 'tests/saved-models/test-save-load' with open(self.processed_path, 'rt') as fp: self.texts, self.labels = json.load(fp) tf.reset_default_graph() self.model = SequenceLabeler(batch_size=2, max_length=256, verbose=False)
def test_sequence_labeler_no_auxiliary(self): """ Ensure model training does not error out Ensure model returns reasonable predictions """ model = SequenceLabeler(**self.default_config( use_auxiliary_info=False, val_set=(self.trainX, self.trainY))) model.fit(self.trainX, self.trainY_seq) preds = model.predict(self.trainX) self._evaluate_sequence_preds(preds, includes_context=False)
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.model_config = dict( use_auxiliary_info = False, n_layers_with_aux = 0, context_in_base_model = False, context_dim = 0 ) self.model_config.update(kwargs) self.model = SequenceLabeler(**self.model_config)
def test_sequence_labeler_auxiliary(self): """ Ensure model training does not error out Ensure model returns reasonable predictions """ # here we want to make sure we're actually using context model = SequenceLabeler(**self.default_config(n_epochs=1500)) model.fit(self.trainX, self.trainY_seq, context=self.train_context) preds = model.predict(self.trainX, context=self.train_context) self._evaluate_sequence_preds(preds, includes_context=True)
def setUpClass(cls): cls._download_data() #dataset preparation cls.classifier_dataset = pd.read_csv(cls.classifier_dataset_path, nrows=cls.n_sample * 10) path = os.path.join(os.path.dirname(__file__), "data", "testdata.json") with open(path, 'rt') as fp: cls.texts, cls.labels = json.load(fp) cls.animals = ["dog", "cat", "horse", "cow", "pig", "sheep", "goat", "chicken", "guinea pig", "donkey", "turkey", "duck", "camel", "goose", "llama", "rabbit", "fox"] cls.numbers = ["one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen"] #train and save sequence labeler for later use try: cls.s = SequenceLabeler.load(cls.sequence_labeler_path, **cls.default_seq_config(cls)) except FileNotFoundError: cls.s = SequenceLabeler(**cls.default_seq_config(cls)) cls.s.fit(cls.texts * 10, cls.labels * 10) cls.s.save(cls.sequence_labeler_path) #train and save classifier for later use train_sample = cls.classifier_dataset.sample(n=cls.n_sample*10) try: cls.cl = Classifier.load(cls.classifier_path) except FileNotFoundError: cls.cl = Classifier(**cls.default_config(cls)) cls.cl.fit(train_sample.Text, train_sample.Target) cls.cl.save(cls.classifier_path) if cls.do_comparison: #train and save comparison regressor for use cls.cr = ComparisonRegressor() n_per = 150 similar = [] different = [] for dataset in [cls.animals, cls.numbers]: for i in range(n_per // 2): similar.append([random.choice(dataset), random.choice(dataset)]) for i in range(n_per): different.append([random.choice(cls.animals), random.choice(cls.numbers)]) targets = np.asarray([1] * len(similar) + [0] * len(different)) data = similar + different cls.x_tr, cls.x_te, cls.t_tr, cls.t_te = train_test_split(data, targets, test_size=0.3, random_state=42) try: cls.cr = ComparisonRegressor.load(cls.comparison_regressor_path, **cls.default_config(cls)) except FileNotFoundError: cls.cr = ComparisonRegressor(**cls.default_config(cls)) cls.cr.fit(cls.x_tr, cls.t_tr) cls.cr.save(cls.comparison_regressor_path)
def test_fit_predict(self): """ Ensure model training does not error out Ensure model returns predictions Ensure class reweighting behaves as intended """ raw_docs = ["".join(text) for text in self.texts] texts, annotations = finetune_to_indico_sequence( raw_docs, self.texts, self.labels, none_value=self.model.config.pad_token ) train_texts, test_texts, train_annotations, test_annotations = train_test_split( texts, annotations, test_size=0.1 ) reweighted_model = SequenceLabeler( **self.default_config(class_weights={"Named Entity": 100.0}) ) reweighted_model.fit(train_texts, train_annotations) reweighted_predictions = reweighted_model.predict(test_texts) reweighted_token_recall = sequence_labeling_token_recall( test_annotations, reweighted_predictions ) self.model.fit(train_texts, train_annotations) predictions = self.model.predict(test_texts) probas = self.model.predict_proba(test_texts) self.assertIsInstance(probas, list) self.assertIsInstance(probas[0], list) self.assertIsInstance(probas[0][0], dict) self.assertIsInstance(probas[0][0]["confidence"], dict) token_precision = sequence_labeling_token_precision( test_annotations, predictions ) token_recall = sequence_labeling_token_recall(test_annotations, predictions) overlap_precision = sequence_labeling_overlap_precision( test_annotations, predictions ) overlap_recall = sequence_labeling_overlap_recall(test_annotations, predictions) self.assertIn("Named Entity", token_precision) self.assertIn("Named Entity", token_recall) self.assertIn("Named Entity", overlap_precision) self.assertIn("Named Entity", overlap_recall) self.model.save(self.save_file) self.assertGreater( reweighted_token_recall["Named Entity"], token_recall["Named Entity"] )
def test_auxiliary_sequence_labeler(self): """ Ensure model training does not error out Ensure model returns reasonable predictions """ (trainX, testX, trainY, testY) = self.dataset model = SequenceLabeler(**self.default_config()) model.fit(trainX, trainY) preds = model.predict(testX) token_precision = sequence_labeling_token_precision(preds, testY) token_recall = sequence_labeling_token_recall(preds, testY) self.assertIn("Named Entity", token_precision) self.assertIn("Named Entity", token_recall) token_precision = np.mean(list(token_precision.values())) token_recall = np.mean(list(token_recall.values())) self.assertGreater(token_precision, 0.6) self.assertGreater(token_recall, 0.6)
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.model_config.update(dict( pos_injection=True, n_layers_with_aux = 0, context_in_base_model = False )) self.model_config.update(kwargs) self.model = SequenceLabeler(**self.model_config)
def __init__(self, *args, **kwargs): """Initialize internal classifier.""" super().__init__(auto_resample=False, *args, **kwargs) self.model = SequenceLabeler(val_size=0)
texts.append(c.text) labels.append(label) docs.append(texts) docs_labels.append(labels) fd.close() os.remove(XML_PATH) raw_texts = ["".join(doc) for doc in docs] texts, annotations = finetune_to_indico_sequence(raw_texts, docs, docs_labels, none_value="<PAD>", subtoken_predictions=True) df = pd.DataFrame({'texts': texts, 'annotations': [json.dumps(annotation) for annotation in annotations]}) df.to_csv(DATA_PATH) if __name__ == "__main__": dataset = Reuters().dataframe dataset['annotations'] = [json.loads(annotation) for annotation in dataset['annotations']] trainX, testX, trainY, testY = train_test_split( dataset.texts.values, dataset.annotations.values, test_size=0.3, random_state=42 ) model = SequenceLabeler(base_model=GPT2, batch_size=2, val_size=0., chunk_long_sequences=True, subtoken_predictions=True) model.fit(trainX, trainY) predictions = model.predict(testX) print(predictions) print(annotation_report(testY, predictions))
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.model = SequenceLabeler(val_size=0)
fd.close() os.remove(XML_PATH) raw_texts = ["".join(doc) for doc in docs] texts, annotations = finetune_to_indico_sequence( raw_texts, docs, docs_labels) df = pd.DataFrame({ 'texts': texts, 'annotations': [json.dumps(annotation) for annotation in annotations] }) df.to_csv(DATA_PATH) if __name__ == "__main__": dataset = Reuters().dataframe dataset['annotations'] = [ json.loads(annotation) for annotation in dataset['annotations'] ] trainX, testX, trainY, testY = train_test_split(dataset.texts.values, dataset.annotations.values, test_size=0.3, random_state=42) model = SequenceLabeler(batch_size=2, val_size=0., chunk_long_sequences=True) model.fit(trainX, trainY) predictions = model.predict(testX) print(annotation_report(testY, predictions))
none_value="<PAD>", subtoken_predictions=True) df = pd.DataFrame({ 'texts': texts, 'annotations': [json.dumps(annotation) for annotation in annotations] }) df.to_csv(DATA_PATH) if __name__ == "__main__": dataset = Reuters().dataframe dataset['annotations'] = [ json.loads(annotation) for annotation in dataset['annotations'] ] trainX, testX, trainY, testY = train_test_split(dataset.texts.values, dataset.annotations.values, test_size=0.7, random_state=42) model = SequenceLabeler(base_model=RoBERTa, batch_size=1, val_size=0., max_length=16, chunk_long_sequences=True, subtoken_predictions=True) model.fit(trainX, trainY) predictions = model.predict(testX) print(predictions) print(annotation_report(testY, predictions))
fd.close() os.remove(XML_PATH) raw_texts = ["".join(doc) for doc in docs] texts, annotations = finetune_to_indico_sequence( raw_texts, docs, docs_labels) df = pd.DataFrame({ 'texts': texts, 'annotations': [json.dumps(annotation) for annotation in annotations] }) df.to_csv(DATA_PATH) if __name__ == "__main__": dataset = Reuters(nrows=1000).dataframe dataset['annotations'] = [ json.loads(annotation) for annotation in dataset['annotations'] ] trainX, testX, trainY, testY = train_test_split(dataset.texts, dataset.annotations, test_size=0.3, random_state=42) model = SequenceLabeler(verbose=False) model.fit(trainX, trainY) predictions = model.predict(testX) n_sample = 10 for i in range(n_sample): print(testX.values[i], predictions[i])
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.model_config = dict(val_size=0) self.model_config.update(kwargs) self.model = SequenceLabeler(**self.model_config)
subtoken_predictions=True) df = pd.DataFrame({ 'texts': texts, 'annotations': [json.dumps(annotation) for annotation in annotations] }) df.to_csv(DATA_PATH) if __name__ == "__main__": dataset = Reuters().dataframe dataset['annotations'] = [ json.loads(annotation) for annotation in dataset['annotations'] ] trainX, testX, trainY, testY = train_test_split(dataset.texts.values, dataset.annotations.values, test_size=0.3, random_state=42) model = SequenceLabeler(base_model=GPT2, batch_size=2, val_size=0., max_length=16, chunk_long_sequences=True, subtoken_predictions=True, filter_empty_examples=True) model.fit(trainX, trainY) predictions = model.predict(testX) print(predictions) print(annotation_report(testY, predictions))
df = pd.DataFrame({ 'texts': texts, 'annotations': [json.dumps(annotation) for annotation in annotations] }) df.to_csv(DATA_PATH) if __name__ == "__main__": dataset = Reuters().dataframe dataset['annotations'] = [ json.loads(annotation) for annotation in dataset['annotations'] ] trainX, testX, trainY, testY = train_test_split(dataset.texts.values, dataset.annotations.values, test_size=0.7, random_state=42) model = SequenceLabeler(base_model=RoBERTa, batch_size=1, n_epochs=3, val_size=0.0, max_length=16, chunk_long_sequences=True, subtoken_predictions=True, crf_sequence_labeling=True) model.fit(trainX, trainY) predictions = model.predict(testX) print(predictions) print(annotation_report(testY, predictions))
os.remove(XML_PATH) raw_texts = ["".join(doc) for doc in docs] texts, annotations = finetune_to_indico_sequence( raw_texts, docs, docs_labels) df = pd.DataFrame({ 'texts': texts, 'annotations': [json.dumps(annotation) for annotation in annotations] }) df.to_csv(DATA_PATH) if __name__ == "__main__": dataset = Reuters(nrows=1000).dataframe dataset['annotations'] = [ json.loads(annotation) for annotation in dataset['annotations'] ] trainX, testX, trainY, testY = train_test_split(dataset.texts.values, dataset.annotations.values, test_size=0.3, random_state=42) model = SequenceLabeler(verbose=False, max_length=64, chunk_long_sequences=True) model.fit(trainX, trainY) predictions = model.predict(testX) n_sample = 10 for i in range(n_sample): print(testX[i], predictions[i])