def create_flair_corpus(self, train_samples_path, test_samples_path, val_samples_path): """ Create a flair Corpus object and saive it to train, test, validation files. :param train_samples_path: Path to train samples :param test_samples_path: Path to test samples :param val_samples_path: Path to validation samples :return: """ if not path.exists("flair_train.txt"): train_samples = InputSample.read_dataset_json(train_samples_path) train_tagged = [ sample for sample in train_samples if len(sample.spans) > 0 ] print( f"Kept {len(train_tagged)} train samples after removal of non-tagged samples" ) train_data = InputSample.create_conll_dataset(train_tagged) self.to_flair(train_data, outfile="flair_train.txt") if not path.exists("flair_test.txt"): test_samples = InputSample.read_dataset_json(test_samples_path) test_data = InputSample.create_conll_dataset(test_samples) self.to_flair(test_data, outfile="flair_test.txt") if not path.exists("flair_val.txt"): val_samples = InputSample.read_dataset_json(val_samples_path) val_data = InputSample.create_conll_dataset(val_samples) self.to_flair(val_data, outfile="flair_val.txt")
def create_flair_corpus(self, train_samples_path, test_samples_path, val_samples_path): if not path.exists("flair_train.txt"): train_samples = read_synth_dataset(train_samples_path) train_tagged = [sample for sample in train_samples if len(sample.spans) > 0] print("Kept {} train samples after removal of non-tagged samples".format(len(train_tagged))) train_data = InputSample.create_conll_dataset(train_tagged) self.to_flair(train_data, outfile="flair_train.txt") if not path.exists("flair_test.txt"): test_samples = read_synth_dataset(test_samples_path) test_data = InputSample.create_conll_dataset(test_samples) self.to_flair(test_data, outfile="flair_test.txt") if not path.exists("flair_val.txt"): val_samples = read_synth_dataset(val_samples_path) val_data = InputSample.create_conll_dataset(val_samples) self.to_flair(val_data, outfile="flair_val.txt")
def test_to_conll(): import os dir_path = os.path.dirname(os.path.realpath(__file__)) input_samples = read_synth_dataset(os.path.join(dir_path, "data/generated_small.txt")) conll = InputSample.create_conll_dataset(input_samples) sentences = conll['sentence'].unique() assert len(sentences) == len(input_samples)
def _to_feature_set(self, dataset: List[InputSample]): samples_conll = InputSample.create_conll_dataset(dataset) sentences = samples_conll.groupby("sentence")[[ "text", "pos", "label" ]].apply(lambda x: x.values.tolist()) X_train = [self.sent2features(s) for s in sentences] y_train = [self.sent2labels(s) for s in sentences] return X_train, y_train