def transform(self, X, y=None, entity_labels=None):
""" Transforms the list of `Document` objects that are provided as
input to the BIO format and returns features, tokens, and labels per word.
Features of a word stored as a list of values.
"""
log.info("Generating features for {} documents...".format(len(X)))
tokens_per_doc, labels_per_doc = \
transform_annotated_documents_to_bio_format(X, entity_labels=entity_labels)
tokens_flat = [token for tokens in tokens_per_doc for token in tokens]
labels_flat = [label for labels in labels_per_doc for label in labels]
pos_tags_flat = [pos_tag for tokens in tokens_per_doc for pos_tag in tokens_to_pos_tags(tokens)]
features_flat = [self._word_to_context(tokens_flat, pos_tags_flat, idx)
for idx in range(len(tokens_flat))]
if not self.encoders:
# first time run
for idx in range(len(features_flat[0])):
if isinstance(features_flat[0][idx], str):
self.encoders[idx] = LabelEncoder()
column_vector = [features_flat[i][idx] for i in range(len(features_flat))]
column_vector.append(UNKNOWN_WORD)
self.encoders[idx].fit(column_vector)
for idx in self.encoders:
column_vector = [features_flat[i][idx] for i in range(len(features_flat))]
self._process_unknown_values(column_vector, self.encoders[idx].classes_.tolist(), UNKNOWN_WORD)
column_vector = self.encoders[idx].transform(column_vector).tolist()
for i in range(len(features_flat)):
features_flat[i][idx] = column_vector[i]
return features_flat, tokens_flat, labels_flat
def test_tokens_to_pos_tags():
sentence = "The cat (feline) eats a mouse."
split_tokens = sentence_to_tokens(sentence)
pos_tags = tokens_to_pos_tags(split_tokens)
assert_equal(['DT', 'NN', '(', 'NN', ')', 'VBZ', 'DT', 'NN', '.'],
pos_tags)
def transform(self, X, y=None):
""" Transforms the list of documents and returns tokens with their features.
Each document should represent a sentence.
"""
log.info("Generating features for {} documents...".format(len(X)))
features = []
for doc in X:
doc_features = []
for pos_tag in tokens_to_pos_tags(document_to_tokens(doc)):
if pos_tag in self.model.wv:
doc_features.append((pos_tag, self.model.wv[pos_tag]))
features.append(doc_features)
return features
def fit(self,
X,
y=None,
size=100,
min_count=5,
workers=1,
window=5,
sample=1e-3,
skipgram=False):
""" Trains a Word2vec model on given documents.
Each document should represent a sentence.
Args:
X: list(Document | AnnotatedDocument | list(str))
y: optional labels
size: Word vector dimensionality
min_count: Minimum word count
workers: Number of threads to run in parallel
window: Context window size
sample: Downsample setting for frequent words
skipgram: Use skip-gram if True and CBOW otherwise
"""
log.info("Checking parameters...")
self.config.set_parameters({
"size": size,
"min_count": min_count,
"workers": workers,
"window": window,
"sample": sample
})
# Get sentences as lists of tokens
log.info("Tokenizing {} documents...".format(len(X)))
sentences = []
for idx, doc in enumerate(X):
sentences.append(tokens_to_pos_tags(document_to_tokens(doc)))
log_progress(log, idx, len(X))
# Initialize and train the model (this will take some time)
log.info("Training Word2vec on {} sentences...".format(len(X)))
self.model = Word2Vec(sentences,
workers=self.config.get_parameter("workers"),
size=self.config.get_parameter("size"),
min_count=self.config.get_parameter("min_count"),
window=self.config.get_parameter("window"),
sample=self.config.get_parameter("sample"),
sg=1 if skipgram else 0)
# If you don't plan to train the model any further, calling
# init_sims() will make the model much more memory-efficient.
self.model.init_sims(replace=True)
return self
def _preprocessor(self, data):
""" Helper function for interconversions.
"""
tokens, labels = transform_annotated_documents_to_bio_format(data)
pos_tags = []
for t_i in tokens:
pos_tags.append(tokens_to_pos_tags(t_i))
sentences = []
for i in range(len(tokens)):
sentence = [
(text, pos, label)
for text, pos, label in zip(tokens[i], pos_tags[i], labels[i])
]
sentences.append(sentence)
features = [self._sent_to_features(s) for s in sentences]
labels = [self._sent_to_labels(s) for s in sentences]
return features, tokens, labels
def transform(self, X, y=None, entity_labels=None):
""" Transforms the list of `Document` objects that are provided as
input to the BIO format and returns features, tokens, and labels per word.
Features of a word stored as a dictionary of name-value pairs.
"""
tokens_per_doc, labels_per_doc = \
transform_annotated_documents_to_bio_format(X, entity_labels=entity_labels)
pos_tags_per_doc = []
for tokens in tokens_per_doc:
pos_tags_per_doc.append(tokens_to_pos_tags(tokens))
sentences = []
for i in range(len(tokens_per_doc)):
sentence = [(text, pos, label) for text, pos, label in zip(
tokens_per_doc[i], pos_tags_per_doc[i], labels_per_doc[i])]
sentences.append(sentence)
features_per_doc = [self._sent_to_features(s) for s in sentences]
labels_per_doc = [self._sent_to_labels(s) for s in sentences]
return features_per_doc, tokens_per_doc, labels_per_doc
def main():
parser = argparse.ArgumentParser(description='bioBERT_data_preprocessing.py')
parser.path_train('path_train', dest = 'path_train', deafult = None, type = str)
parser.path_test('path_test', dest = 'path_test', deafult = None, type = str)
parser.label('label', dest = 'label', default = None, type = str)
parser.output_dir_train('output_dir_train', dest = 'output_dir_train', deafult = None, type = str)
parser.output_dir_train_eval('output_dir_train_eval', dest = 'output_dir_train_eval', deafult = None, type = str)
parser.output_dir_eval('output_dir_eval', dest = 'output_dir_eval', deafult = None, type = str)
parser.output_dir_test('output_dir_test', dest = 'output_dir_test', deafult = None, type = str)
parser.path_google_bert('path_google_bert', dest = 'path_google_bert', deafult = None, type = str)
parser.output_dir_mapping_tokens('output_dir_mapping_tokens', dest = 'output_dir_mapping_tokens', deafult = None, type = str)
parser.output_dir_mapping_labels('output_dir_mapping_labels', dest = 'output_dir_mapping_labels', deafult = None, type = str)
args = parser.parse_args()
data_path_train = path_train
data_path_test = path_test
bi_train = BratInput(data_path_train).transform()
bi_test = BratInput(data_path_test).transform()
# Clean the data (train)
data_train = retain_annotations(bi_train, label)
clean_data_train = clean_annotated_documents(data_train)
non_overlap_data_train = resolve_overlaps(clean_data_train)
# Clean the data (test)
data_test = retain_annotations(bi_test, label)
clean_data_test = clean_annotated_documents(data_test)
non_overlap_data_test = resolve_overlaps(clean_data_test)
# Transform to BIO format
bio_train = transform_annotated_documents_to_bio_format(non_overlap_data_train, entity_labels=label)
bio_test = transform_annotated_documents_to_bio_format(non_overlap_data_test, entity_labels=label)
######################################
### Train, Eval and train-eval set ###
enum1 = []
enum2 = []
enum3 = []
for idx, inner in enumerate(bio_train[0], start=1):
enum2.append(idx)
for jdx, elt in enumerate(inner, start=1):
enum1.append(jdx)
enum3.append(elt)
# Number of tokens per sentence
nest_token_enum = []
for x in bio_train[0]:
nest_token_enum.append(len(x))
# Sentence ID
numbers_ar = np.arange(1, len(non_overlap_data_train)+1)
nest_token_enum_ar = np.asarray(nest_token_enum)
sent_id = np.repeat(numbers_ar, nest_token_enum_ar)
# BIO tags
list_bio2 = []
for x in bio_train[1]:
list_bio2.append(x)
flatten = lambda l: [item for sublist in l for item in sublist]
flatten_BIO = flatten(list_bio2)
# POS tags
pos = tokens_to_pos_tags(enum3)
# Sentence list with string
sentence = []
for i in sent_id:
sentence.append(f'Sentence: {i}')
# Create a dataframes (train(60/100), train_eval(80/100), eval(train_eval- train)=20/100)
d = {'Sentence_ID':sentence,'Token_ID':enum1, 'BIO':flatten_BIO, 'POS':pos, 'Token': enum3}
df = pd.DataFrame(d)
df['ID'] = df.index
l = df[df["Token_ID"]==1].index.tolist()
for c, i in enumerate(l):
dfs = np.split(df, [i+1+c])
df = pd.concat([dfs[0], pd.DataFrame([[np.NaN, np.NaN, np.NaN, np.NaN, np.NaN, np.NaN]], columns=df.columns), dfs[1]], ignore_index=True)
from itertools import repeat
numbers_ar = np.arange(1, len(non_overlap_data_test)+1)
nest_token_enum_ar = np.asarray(nest_token_enum)
sent_id = np.repeat(numbers_ar, nest_token_enum_ar)
# BIO tags
list_bio2 = []
for x in bio_test[1]:
list_bio2.append(x)
flatten = lambda l: [item for sublist in l for item in sublist]
flatten_BIO = flatten(list_bio2)
# Create POS tags
pos = tokens_to_pos_tags(enum3)
# Sentence list with string
sentence = []
for i in sent_id:
sentence.append(f'Sentence: {i}')
# Create a dataframe (test)
d = {'Sentence_ID':sentence,'Token_ID':enum1, 'BIO':flatten_BIO, 'POS':pos, 'Token': enum3}
df = pd.DataFrame(d)
df['ID'] = df.index
l = df[df["Token_ID"]==1].index.tolist()
for c, i in enumerate(l):
def fit(
self,
X,
y=None,
X_valid=None,
char_emb_size=32,
word_emb_size=128,
char_lstm_units=32,
word_lstm_units=128,
pos_emb_size=16,
dropout=0.5,
batch_size=8,
num_epochs=10,
use_crf=False,
use_char_emb=False,
shuffle=False,
use_pos_emb=False,
hparams_1={}, # specific to config
hparams_2={}): # other params
""" Trains the NER model. The input is a list of
`AnnotatedDocument` instances.
We should be careful with batch size:
it must satisfy len(X) % batch_size == 0.
Otherwise, it crashes with an error from time to time.
An example here is a token assigned a tag (the BIO scheme).
"""
log.info("Checking parameters...")
self.config.set_parameters({
"num_epochs": num_epochs,
"dropout": dropout,
"batch_size": batch_size,
"char_emb_size": char_emb_size,
"word_emb_size": word_emb_size,
"char_lstm_units": char_lstm_units,
"word_lstm_units": word_lstm_units,
"pos_emb_size": pos_emb_size,
"use_crf": use_crf,
"use_char_emb": use_char_emb,
"shuffle": shuffle,
"use_pos_emb": use_pos_emb
})
if hparams_1:
self.config.set_parameters(hparams_1)
self.config.validate()
if hparams_2:
self.hparams.update(hparams_2)
log.info("Transforming {} items to BIO format...".format(len(X)))
X_train, Y_train = self._transform_to_bio(X)
pos_tags = []
if use_pos_emb:
log.info("Getting POS tags for {} items...".format(len(X)))
for idx, x in enumerate(X_train):
pos_tags.append(tokens_to_pos_tags(x))
log_progress(log, idx, len(X))
self.p = IndexTransformer(
use_char=self.config.get_parameter("use_char_emb"))
self.p.fit(X_train, Y_train)
self.word_embeddings = filter_embeddings(
self.word_embeddings, self.p._word_vocab.vocab,
self.config.get_parameter("word_emb_size"))
# compile the model architecture
self._compile_model()
train_seq = NERSequence(X_train,
Y_train,
self.config.get_parameter("batch_size"),
preprocess=self.p.transform)
if X_valid:
X_valid, Y_valid = self._transform_to_bio(X_valid)
valid_seq = NERSequence(X_valid, Y_valid, batch_size,
self.p.transform)
f1 = F1score(valid_seq, preprocessor=self.p)
# train model
log.info("Training BiLSTM...")
self.model.fit_generator(
generator=train_seq,
epochs=self.config.get_parameter("num_epochs"),
validation_data=valid_seq if valid_seq else None,
verbose=1,
shuffle=self.config.get_parameter("shuffle"),
callbacks=self.hparams['callbacks'] +
[f1] if 'callbacks' in self.hparams else [f1])
return self
def transform(self, X, y=None):
""" Annotates the list of `Document` objects that are provided as
input and returns a list of `AnnotatedDocument` objects.
"""
log.info(
"Annotating named entities in {} documents with BiLSTM...".format(
len(X)))
annotated_documents = []
for idx, document in enumerate(X):
# get tokens
tokens, _ = transform_annotated_document_to_bio_format(document)
# encode tokens and pad the sequence
coded_tokens = [
self.encoder.encode_word(token) for token in tokens
]
x = pad_sequences(maxlen=self.encoder.max_len,
sequences=[coded_tokens],
padding="post",
value=self.encoder.encode_word(PAD_WORD))
inputs = [x]
# add encoded and padded char sequences if needed
if self.config.get_parameter("use_char_emb"):
c = [[self.encoder.encode_char(char) for char in token]
for token in tokens]
c = pad_sequences(
maxlen=self.encoder.max_len_char,
sequences=c,
padding="post",
value=self.encoder.encode_char(PAD_CHAR)).tolist()
# add padding chars for padding words
for i in range(len(tokens), self.encoder.max_len):
c.append([self.encoder.encode_char(PAD_CHAR)] *
self.encoder.max_len_char)
c = np.array([c], ndmin=3)
inputs.append(c)
# add encoded and padded POS tag sequences if needed
if self.config.get_parameter("use_pos_emb"):
pos_tags = tokens_to_pos_tags(tokens)
coded_pos_tags = [
self.encoder.encode_pos(pos) for pos in pos_tags
]
p = pad_sequences(maxlen=self.encoder.max_len,
sequences=[coded_pos_tags],
padding="post",
value=self.encoder.encode_pos(PAD_POS))
inputs.append(p)
# get predicted tags
output = self.model.predict(x=inputs)
coded_tags = np.argmax(output, axis=-1)[0]
tags = [self.encoder.decode_tag(idx) for idx in coded_tags]
tags = tags[:len(tokens)]
# annotate a document
annotated_documents.append(
transform_bio_tags_to_annotated_document(
tokens, tags, document))
# info
log_progress(log, idx, len(X))
return annotated_documents
def fit(self,
X,
y=None,
char_emb_size=32,
word_emb_size=128,
char_lstm_units=32,
word_lstm_units=128,
pos_emb_size=16,
dropout=0.5,
batch_size=8,
num_epochs=10,
use_crf=False,
use_char_emb=False,
shuffle=False,
use_pos_emb=False):
""" Trains the NER model. The input is a list of
`AnnotatedDocument` instances.
We should be careful with batch size:
it must satisfy len(X) % batch_size == 0.
Otherwise, it crushes with an error from time to time.
An example here is a token assigned a tag (the BIO scheme).
"""
log.info("Checking parameters...")
self.config.set_parameters({
"num_epochs":
num_epochs,
"dropout":
dropout,
"batch_size":
batch_size,
"char_emb_size":
char_emb_size,
"word_emb_size":
self.word_embeddings.vector_size
if self.word_embeddings else word_emb_size,
"char_lstm_units":
char_lstm_units,
"word_lstm_units":
word_lstm_units,
"pos_emb_size":
pos_emb_size,
"use_crf":
use_crf,
"use_char_emb":
use_char_emb,
"shuffle":
shuffle,
"use_pos_emb":
use_pos_emb
})
self.config.validate()
log.info("Transforming {} items to BIO format...".format(len(X)))
X_train, Y_train = self._transform_to_bio(X)
pos_tags = []
if use_pos_emb:
log.info("Getting POS tags for {} items...".format(len(X)))
for idx, x in enumerate(X_train):
pos_tags.append(tokens_to_pos_tags(x))
log_progress(log, idx, len(X))
# fit encoder
self.encoder.fit(X=X_train,
Y=Y_train,
use_chars=self.config.get_parameter("use_char_emb"),
pos_tags=pos_tags)
# compile the model architecture
self._compile_model()
# encode and pad word sequences
X = [[self.encoder.encode_word(word) for word in x] for x in X_train]
X = pad_sequences(maxlen=self.encoder.max_len,
sequences=X,
padding="post",
value=self.encoder.encode_word(PAD_WORD))
# add X to inputs
inputs = [X]
# encode and pad tag sequences
Y = [[self.encoder.encode_tag(tag) for tag in y] for y in Y_train]
Y = pad_sequences(maxlen=self.encoder.max_len,
sequences=Y,
padding="post",
value=self.encoder.encode_tag(PAD_TAG))
Y = np.array(
[to_categorical(y, num_classes=self.encoder.tag_count) for y in Y])
# encode and pad character sequences if needed
if self.config.get_parameter("use_char_emb"):
C = []
for x in X_train:
c = [[self.encoder.encode_char(char) for char in word]
for word in x]
c = pad_sequences(
maxlen=self.encoder.max_len_char,
sequences=c,
padding="post",
value=self.encoder.encode_char(PAD_CHAR)).tolist()
# add padding chars for padding words
for i in range(len(x), self.encoder.max_len):
c.append([self.encoder.encode_char(PAD_CHAR)] *
self.encoder.max_len_char)
C.append(c)
C = np.array(C, ndmin=3)
inputs.append(C)
# encode and pad POS tag sequences if needed
if self.config.get_parameter("use_pos_emb"):
P = [[self.encoder.encode_pos(pos) for pos in pos_seq]
for pos_seq in pos_tags]
P = pad_sequences(maxlen=self.encoder.max_len,
sequences=P,
padding="post",
value=self.encoder.encode_pos(PAD_POS))
inputs.append(P)
# train model
log.info("Training BiLSTM...")
self.model.fit(x=inputs,
y=Y,
epochs=self.config.get_parameter("num_epochs"),
batch_size=self.config.get_parameter("batch_size"),
verbose=1,
shuffle=self.config.get_parameter("shuffle"))
return self