def batch_generator(input_data, y, batch_size, max_sent_length, n_channels=1):
X = list(map(lambda s: numpy.stack(s[:max_sent_length]), input_data))
steps_per_epoch = math.ceil(len(X) / batch_size)
"""
Yields embedded sentences in matrices of shape (max_sent_length, embedding_size, 1)
"""
i = 0
# Loops indefinitely, as precised in https://keras.io/models/sequential/
while True:
sl = slice(i * batch_size, (i + 1) * batch_size)
mats = X[sl]
# max_sent_length = numpy.max([m.shape[0] for m in mats])
y_batch = y[sl]
# pad all the matrices (sentences) one by one.
e = get_embedding_dim()
for k, m in enumerate(mats):
mats[k] = numpy.vstack(
(mats[k], numpy.zeros(((max_sent_length - m.shape[0]), e))))
if n_channels > 0:
mats[k] = mats[k].reshape(max_sent_length, e, n_channels)
# Now stack em all in a 3D tensor of shape (batch_size, sent_length, embedding_size)
batch = numpy.array(mats)
# Avoid storing too large numbers by modulo.
i = (i + 1) % steps_per_epoch
# Reshape y
y_keras = to_categorical(y_batch, num_classes=51)
yield (batch, y_keras)
def VDCNN(embed_type, maxlen=250, filter_sizes={2, 3, 4, 5}):
embed_size = utils.get_embedding_dim(embed_type)
inp = Input(shape=(maxlen, embed_size))
x = inp
conv_ops = []
for filter_size in filter_sizes:
conv = Conv1D(256, filter_size, activation='relu')(x)
pool = MaxPool1D(5)(conv)
conv_ops.append(pool)
concat = Concatenate(axis=1)(conv_ops)
# concat = Dropout(0.1)(concat)
concat = BatchNormalization()(concat)
conv_2_main = Conv1D(256, 5, activation='relu', padding='same')(concat)
conv_2_main = BatchNormalization()(conv_2_main)
conv_2_main = Conv1D(256, 5, activation='relu',
padding='same')(conv_2_main)
conv_2_main = BatchNormalization()(conv_2_main)
conv_2 = Add()([concat, conv_2_main])
conv_2 = MaxPool1D(pool_size=2, strides=2)(conv_2)
# conv_2 = BatchNormalization()(conv_2)
# conv_2 = Dropout(0.1)(conv_2)
conv_3_main = Conv1D(256, 5, activation='relu', padding='same')(conv_2)
conv_3_main = BatchNormalization()(conv_3_main)
conv_3_main = Conv1D(256, 5, activation='relu',
padding='same')(conv_3_main)
conv_3_main = BatchNormalization()(conv_3_main)
conv_3 = Add()([conv_2, conv_3_main])
conv_3 = MaxPool1D(pool_size=2, strides=2)(conv_3)
# conv_3 = BatchNormalization()(conv_3)
# conv_3 = Dropout(0.1)(conv_3)
flat = Flatten()(conv_3)
op = Dense(256, activation="relu")(flat)
op = Dropout(0.5)(op)
op = BatchNormalization()(op)
op = Dense(128, activation="relu")(op)
op = Dropout(0.5)(op)
op = BatchNormalization()(op)
op = Dense(3, activation="softmax")(op)
model = Model(inputs=inp, outputs=op)
model.compile(loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=[
'sparse_categorical_accuracy',
km.sparse_categorical_f1_score()
])
return model, 'VDCNN_{}.hdf5'.format(embed_type)
def get_model(embed_type):
def count_parameters(model):
return sum(p.numel() for p in model.parameters() if p.requires_grad)
EMBEDDING_DIM = utils.get_embedding_dim(embed_type)
N_FILTERS = 512
FILTER_SIZES = [3, 4, 5, 6]
OUTPUT_DIM = 3
DROPOUT = 0.5
model = CNN1d(EMBEDDING_DIM, N_FILTERS, FILTER_SIZES, OUTPUT_DIM, DROPOUT)
print(model)
print('The model has {count_parameters(model):,} trainable parameters')
return model, 'cnn_{}.pt'.format(embed_type)
def LSTMCNN(embed_type, maxlen=250, filter_sizes={2, 3, 4, 5}):
embed_size = utils.get_embedding_dim(embed_type)
inp = Input(shape=(maxlen, embed_size))
x = inp
x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
conv_ops = []
for filter_size in filter_sizes:
conv = Conv1D(256, filter_size, activation='relu')(x)
pool = MaxPool1D(5)(conv)
conv_ops.append(pool)
concat = Concatenate(axis=1)(conv_ops)
concat = Dropout(0.5)(concat)
concat = BatchNormalization()(concat)
conv_2 = Conv1D(256, 5, activation='relu')(concat)
conv_2 = MaxPool1D(5)(conv_2)
conv_2 = BatchNormalization()(conv_2)
conv_2 = Dropout(0.5)(conv_2)
conv_3 = Conv1D(256, 5, activation='relu')(conv_2)
conv_3 = MaxPool1D(5)(conv_3)
conv_3 = BatchNormalization()(conv_3)
conv_3 = Dropout(0.1)(conv_3)
flat = Flatten()(conv_3)
op = Dense(256, activation="relu")(flat)
op = Dropout(0.5)(op)
op = BatchNormalization()(op)
op = Dense(128, activation="relu")(op)
op = Dropout(0.5)(op)
op = BatchNormalization()(op)
op = Dense(3, activation="softmax")(op)
model = Model(inputs=inp, outputs=op)
model.compile(loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=[
'sparse_categorical_accuracy',
km.sparse_categorical_f1_score()
])
return model, 'LSTMCNN_{}.hdf5'.format(embed_type)
def get_model(embed_type):
def count_parameters(model):
return sum(p.numel() for p in model.parameters() if p.requires_grad)
embedding_dim = utils.get_embedding_dim(embed_type)
hidden_dim = 512
output_dim = 3
n_layer = 2
bidirectional = True
dropout = 0.5
model = RNN(embedding_dim,
hidden_dim,
output_dim,
n_layer,
bidirectional,
dropout)
print(model)
print('The model has {count_parameters(model):,} trainable parameters')
return model, 'rnn_{}.pt'.format(embed_type)
def __init__(self,
max_seq_len=100,
embedding_dim=20,
embeddings_path=None,
optimizer='adam',
batch_size=32,
epochs=10,
vocab=None,
vocab_size=None,
class_count=None,
**kwargs):
self.vocab_size = vocab_size
self.vocab = vocab
self.class_count = class_count
self.embedding_dim = embedding_dim
self.max_seq_len = max_seq_len
self.embeddings_path = embeddings_path
if embeddings_path is not None:
self.embedding_dim = get_embedding_dim(embeddings_path)
self.optimizer = optimizer
self.epochs = epochs
self.batch_size = batch_size
self.patience = 3
self.history = None
self.model = None
self.pos_tag = None
self.params = {
'epochs': self.epochs,
'max_seq_len': self.max_seq_len,
'embedding_dim': self.embedding_dim,
'embeddings_path': self.embeddings_path,
'optimizer': self.optimizer,
'batch_size': self.batch_size,
'vocab': self.vocab,
'vocab_size': self.vocab_size,
'class_count': self.class_count
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--model',
type=str,
default='rnn',
help=
"Available models are: 'rnn', 'cnn', 'bilstm', 'fasttext', and 'distilbert'\nDefault is 'rnn'"
)
parser.add_argument('--train_data_path',
type=str,
default="./data/train_clean.csv",
help="Path to the training data")
parser.add_argument('--test_data_path',
type=str,
default="./data/dev_clean.csv",
help="Path to the test data")
parser.add_argument('--seed', type=int, default=1234)
parser.add_argument('--vectors',
type=str,
default='fasttext.simple.300d',
help="""
Pretrained vectors:
Visit
https://github.com/pytorch/text/blob/9ce7986ddeb5b47d9767a5299954195a1a5f9043/torchtext/vocab.py#L146
for more
""")
parser.add_argument('--max_vocab_size', type=int, default=750)
parser.add_argument('--batch_size', type=int, default=32)
parser.add_argument('--bidirectional', type=bool, default=True)
parser.add_argument('--dropout', type=float, default=0.5)
parser.add_argument('--hidden_dim', type=int, default=64)
parser.add_argument('--output_dim', type=int, default=1)
parser.add_argument('--n_layers', type=int, default=2)
parser.add_argument('--lr', type=float, default=1e-3)
parser.add_argument('--n_epochs', type=int, default=5)
parser.add_argument('--n_filters', type=int, default=100)
parser.add_argument('--filter_sizes', type=list, default=[3, 4, 5])
args = parser.parse_args()
torch.manual_seed(args.seed)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
########## BILSTM ##########
if args.model == "bilstm":
print('\nBiLSTM')
TEXT = Field(tokenize='spacy')
LABEL = LabelField(dtype=torch.float)
data_fields = [("text", TEXT), ("label", LABEL)]
train_data = TabularDataset(args.train_data_path,
format='csv',
fields=data_fields,
skip_header=True,
csv_reader_params={'delimiter': ","})
test_data = TabularDataset(args.test_data_path,
format='csv',
fields=data_fields,
skip_header=True,
csv_reader_params={'delimiter': ","})
train_data, val_data = train_data.split(split_ratio=0.8,
random_state=random.seed(
args.seed))
TEXT.build_vocab(train_data,
max_size=args.max_vocab_size,
vectors=args.vectors,
unk_init=torch.Tensor.normal_)
LABEL.build_vocab(train_data)
train_iterator, valid_iterator, test_iterator = BucketIterator.splits(
(train_data, val_data, test_data),
batch_size=args.batch_size,
sort_key=lambda x: len(x.text),
device=device)
input_dim = len(TEXT.vocab)
embedding_dim = get_embedding_dim(args.vectors)
pad_idx = TEXT.vocab.stoi[TEXT.pad_token]
unk_idx = TEXT.vocab.stoi[TEXT.unk_token]
model = BiLSTM(input_dim, embedding_dim, args.hidden_dim,
args.output_dim, args.n_layers, args.bidirectional,
args.dropout, pad_idx)
pretrained_embeddings = TEXT.vocab.vectors
model.embedding.weight.data.copy_(pretrained_embeddings)
model.embedding.weight.data[unk_idx] = torch.zeros(embedding_dim)
model.embedding.weight.data[pad_idx] = torch.zeros(embedding_dim)
optimizer = optim.Adam(model.parameters(), lr=args.lr)
criterion = nn.BCEWithLogitsLoss()
model.to(device)
criterion.to(device)
best_valid_loss = float('inf')
print("\nTraining...")
print("===========")
for epoch in range(1, args.n_epochs + 1):
start_time = time.time()
train_loss, train_acc = train(model, train_iterator, optimizer,
criterion)
valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)
end_time = time.time()
epoch_mins, epoch_secs = epoch_time(start_time, end_time)
if valid_loss < best_valid_loss:
best_valid_loss = valid_loss
torch.save(model.state_dict(),
'./checkpoints/{}-model.pt'.format(args.model))
print(
f'[Epoch: {epoch:02}] | Epoch Time: {epoch_mins}m {epoch_secs}s'
)
print(
f'\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%'
)
print(
f'\t Val. Loss: {valid_loss:.3f} | Val. Acc: {valid_acc*100:.2f}%'
)
model.load_state_dict(
torch.load('./checkpoints/{}-model.pt'.format(args.model)))
test_loss, test_acc = evaluate(model, test_iterator, criterion)
print('\nEvaluating...')
print("=============")
print(f'Test Loss: {test_loss:.3f} | Test Acc: {test_acc*100:.2f}%'
) # Test Loss: 0.139, Test Acc: 95.27%
########## VANILLA RNN ##########
else:
print('\nVanilla RNN')
TEXT = Field(tokenize='spacy')
LABEL = LabelField(dtype=torch.float)
data_fields = [("text", TEXT), ("label", LABEL)]
train_data = TabularDataset(args.train_data_path,
format='csv',
fields=data_fields,
skip_header=True,
csv_reader_params={'delimiter': ","})
test_data = TabularDataset(args.test_data_path,
format='csv',
fields=data_fields,
skip_header=True,
csv_reader_params={'delimiter': ","})
train_data, val_data = train_data.split(split_ratio=0.8,
random_state=random.seed(
args.seed))
TEXT.build_vocab(train_data,
max_size=args.max_vocab_size,
vectors=args.vectors)
LABEL.build_vocab(train_data)
train_iterator, valid_iterator, test_iterator = BucketIterator.splits(
(train_data, val_data, test_data),
batch_size=args.batch_size,
sort_key=lambda x: len(x.text),
device=device)
input_dim = len(TEXT.vocab)
embedding_dim = get_embedding_dim(args.vectors)
model = RNN(input_dim, embedding_dim, args.hidden_dim, args.output_dim)
pretrained_embeddings = TEXT.vocab.vectors
model.embedding.weight.data.copy_(pretrained_embeddings)
optimizer = optim.Adam(model.parameters(), lr=args.lr)
criterion = nn.BCEWithLogitsLoss()
model.to(device)
criterion.to(device)
best_valid_loss = float('inf')
print("\nTraining...")
print("===========")
for epoch in range(1, args.n_epochs + 1):
start_time = time.time()
train_loss, train_acc = train(model, train_iterator, optimizer,
criterion)
valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)
end_time = time.time()
epoch_mins, epoch_secs = epoch_time(start_time, end_time)
if valid_loss < best_valid_loss:
best_valid_loss = valid_loss
torch.save(model.state_dict(),
'./checkpoints/{}-model.pt'.format(args.model))
print(
f'[Epoch: {epoch:02}] | Epoch Time: {epoch_mins}m {epoch_secs}s'
)
print(
f'\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%'
)
print(
f'\t Val. Loss: {valid_loss:.3f} | Val. Acc: {valid_acc*100:.2f}%'
)
model.load_state_dict(
torch.load('./checkpoints/{}-model.pt'.format(args.model)))
test_loss, test_acc = evaluate(model, test_iterator, criterion)
print('\nEvaluating...')
print("=============")
print(f'Test Loss: {test_loss:.3f} | Test Acc: {test_acc*100:.2f}%'
) # Test Loss: 0.138, Test Acc: 95.05%
max_idf = max(tfidf.idf_)
self.word2weight = defaultdict(lambda: max_idf)
for w, i in tfidf.vocabulary_.items():
self.word2weight[w] = tfidf.idf_[i]
def transform(self, X):
return np.array([
np.mean([
self.fasttext_model.get_word_vector(w) * self.word2weight[w]
for w in words
] or [np.zeros(self.dim)],
axis=0) for words in X
])
if __name__ == '__main__':
descriptions_df = pnd.read_csv(utils.get_drugs_indication_path(),
encoding=params.UTF_8)
fasttext_model = FastText.load_model(MODEL_PATH)
tfidf_emb = TfidfEmbeddingVectorizer(fasttext_model,
utils.get_embedding_dim())
tfidf_emb.fit(descriptions_df.descriptions)
embeddings = tfidf_emb.transform(descriptions_df.descriptions)
embeddings_dict = {}
for i in range(descriptions_df.shape[0]):
embeddings_dict[descriptions_df.loc[i, 'drug_names']] = embeddings[i]
with open(utils.get_drug_embedding_path(), 'wb') as f:
pickle.dump(embeddings_dict, f)