def sentence_prediction(sentence):
sentence = preprocess(sentence)
model_path = config.MODEL_PATH
test_dataset = dataset.BERTDataset(
review=[sentence],
target=[0]
)
test_data_loader = torch.utils.data.DataLoader(
test_dataset,
batch_size=config.VALID_BATCH_SIZE,
num_workers=3
)
device = config.device
model = BERTBaseUncased()
model.load_state_dict(torch.load(
model_path, map_location=torch.device(device)))
model.to(device)
outputs, [] = engine.predict_fn(test_data_loader, model, device)
print(outputs)
return outputs[0]
def main(_):
input = config.EVAL_PROC
output = 'predictions.csv'
model_path = config.MODEL_PATH
if FLAGS.input:
input = FLAGS.input
if FLAGS.output:
output = FLAGS.input
if FLAGS.model_path:
model_path = FLAGS.model_path
df_test = pd.read_fwf(input)
logger.info(f"Bert Model: {config.BERT_PATH}")
logger.info(
f"Current date and time :{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')} "
)
logger.info(f"Test file: {input}")
logger.info(f"Test size : {len(df_test):.4f}")
trg = []
for i in range(len(df_test.values)):
trg.append(0)
test_dataset = dataset.BERTDataset(text=df_test.values, target=trg)
test_data_loader = torch.utils.data.DataLoader(
test_dataset, batch_size=config.VALID_BATCH_SIZE, num_workers=3)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = BERTBaseUncased(config.DROPOUT)
model.load_state_dict(
torch.load(model_path, map_location=torch.device(device)))
model.to(device)
outputs, extracted_features = engine.predict_fn(
test_data_loader, model, device, extract_features=FLAGS.features)
df_test["predicted"] = outputs
# save file
df_test.to_csv(output, header=None, index=False)
@app.route("/")
def home():
return render_template("index.html")
@app.route("/predict")
def predict():
sentence = request.args.get("sentence")
start_time = time.time()
positive_prediction = sentence_prediction(sentence)
negative_prediction = 1 - positive_prediction
response = {}
response["response"] = {
"positive": str(positive_prediction),
"negative": str(negative_prediction),
"sentence": str(sentence),
"time_taken": str(time.time() - start_time),
}
return flask.jsonify(response)
if __name__ == "__main__":
MODEL = BERTBaseUncased()
# MODEL = nn.DataParallel(MODEL)
MODEL.load_state_dict(torch.load(config.MODEL_PATH))
MODEL.to(DEVICE)
MODEL.eval()
app.run()
from model import BERTBaseUncased
if __name__ == '__main__':
device='cuda'
review = ['this is an amzaing place']
dataset = BERTDataset(
review = review,target=[0]
)
model = BERTBaseUncased()
#model = nn.DataParallel(model) #if you have used model as DataParallel-model then inside torch.onnx.export use 'model.module' instead of model
#====>>> question is from where 'module' arises ??=>>>> print model(uncomment line 25) here .You will output as Dataparallel({module:BERTBaseUncased())
# ====> that means 'module' is key of 'BERTBaseUncased' model's value
model.load_state_dict(torch.load(config.MODEL_PATH))
model.eval()
#print(model)
ids = dataset[0]['ids'].unsqueeze(0)
mask = dataset[0]['mask'].unsqueeze(0)
token_type_ids = dataset[0][''token_type_ids].unsqueeze(0)
torch.onnx.export(
model,
#model.module [===>> if dataparallel-model ==>> see above commented line 20]
#we have 3 inputs so name them here -- ordering is important
(ids,mask,token_type_ids),
#export it to model.onnx
)
outputs = torch.sigmoid(outputs).cpu().detach().numpy()
return outputs[0][0]
@app.route("/predict")
def predict():
sentence = request.args.get("sentence")
start_time = time.time()
positive_prediction = sentence_prediction(sentence, model=MODEL)
negative_prediction = 1 - positive_prediction
response = {}
response["response"] = {
'positive': str(positive_prediction),
'negative': str(negative_prediction),
'sentence': str(sentence),
'time_taken': str(time.time() - start_time)
}
return flask.jsonify(response)
if __name__ == "__main__":
MODEL = BERTBaseUncased()
MODEL = nn.DataParallel(MODEL)
MODEL.load_state_dict(torch.load(config.MODEL_PATH, map_location=torch.device(config.DEVICE)))
MODEL.to(config.DEVICE)
MODEL.eval()
app.run()
def main(_):
test_file = config.EVAL_PROC
model_path = config.MODEL_PATH
if FLAGS.test_file:
test_file = FLAGS.test_file
if FLAGS.model_path:
model_path = FLAGS.model_path
df_test = pd.read_csv(test_file).fillna("none")
logger.info(f"Bert Model: {config.BERT_PATH}")
logger.info(
f"Current date and time :{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')} "
)
logger.info(f"Test file: {test_file}")
logger.info(f"Test size : {len(df_test):.4f}")
test_dataset = dataset.BERTDataset(review=df_test.text.values,
target=df_test.label.values)
test_data_loader = torch.utils.data.DataLoader(
test_dataset, batch_size=config.VALID_BATCH_SIZE, num_workers=3)
device = config.device
model = BERTBaseUncased()
model.load_state_dict(
torch.load(model_path, map_location=torch.device(device)))
model.to(device)
outputs, extracted_features = engine.predict_fn(
test_data_loader, model, device, extract_features=FLAGS.features)
df_test["predicted"] = outputs
# save file
df_test.to_csv(model_path.split("/")[-2] + '.csv',
header=None,
index=False)
if FLAGS.features:
pca = PCA(n_components=50, random_state=7)
X1 = pca.fit_transform(extracted_features)
tsne = TSNE(n_components=2,
perplexity=10,
random_state=6,
learning_rate=1000,
n_iter=1500)
X1 = tsne.fit_transform(X1)
# if row == 0: print("Shape after t-SNE: ", X1.shape)
X = pd.DataFrame(np.concatenate([X1], axis=1), columns=["x1", "y1"])
X = X.astype({"x1": float, "y1": float})
# Plot for layer -1
plt.figure(figsize=(20, 15))
p1 = sns.scatterplot(x=X["x1"], y=X["y1"], palette="coolwarm")
# p1.set_title("development-"+str(row+1)+", layer -1")
x_texts = []
for output, value in zip(outputs, df_test.label.values):
if output == value:
x_texts.append("@" + label_decoder(output)[0] +
label_decoder(output))
else:
x_texts.append(
label_decoder(value) + "-" + label_decoder(output))
X["texts"] = x_texts
# X["texts"] = ["@G" + label_decoder(output) if output == value else "@R-" + label_decoder(value) + "-" + label_decoder(output)
# for output, value in zip(outputs, df_test.label.values)]
# df_test.label.astype(str)
#([str(output)+"-" + str(value)] for output, value in zip(outputs, df_test.label.values))
# Label each datapoint with the word it corresponds to
for line in X.index:
text = X.loc[line, "texts"] + "-" + str(line)
if "@U" in text:
p1.text(X.loc[line, "x1"] + 0.2,
X.loc[line, "y1"],
text[2:],
horizontalalignment='left',
size='medium',
color='blue',
weight='semibold')
elif "@P" in text:
p1.text(X.loc[line, "x1"] + 0.2,
X.loc[line, "y1"],
text[2:],
horizontalalignment='left',
size='medium',
color='green',
weight='semibold')
elif "@N" in text:
p1.text(X.loc[line, "x1"] + 0.2,
X.loc[line, "y1"],
text[2:],
horizontalalignment='left',
size='medium',
color='red',
weight='semibold')
else:
p1.text(X.loc[line, "x1"] + 0.2,
X.loc[line, "y1"],
text,
horizontalalignment='left',
size='medium',
color='black',
weight='semibold')
plt.show()
plt.savefig(model_path.split("/")[-2] + '-figure.svg', format="svg")
def run():
train_filename, label = sys.argv[1:3]
model_path = "models2/" + label + "_best.pt"
assert 'train' in train_filename
filenames = {'train': train_filename,
'dev': train_filename.replace('train', 'dev'),
'test':train_filename.replace('train', 'test')}
dataframes = {}
num_classes = 0
for subset, filename in filenames.items():
dataframes[subset] = preprocess(filename, label)
num_classes = max(num_classes, max(dataframes[subset].ENCODE_CAT) + 1)
dataloaders = {}
for subset, filename in filenames.items():
if subset == 'train':
batch_size = config.TRAIN_BATCH_SIZE
num_workers = 4
else:
batch_size = config.VALID_BATCH_SIZE
num_worker = 1
dataloaders[subset] = process_dataset(
dataframes[subset], batch_size, num_workers)
device = torch.device(config.DEVICE)
model = BERTBaseUncased(num_classes)
model.to(device)
param_optimizer = list(model.named_parameters())
no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
optimizer_parameters = [
{
"params": [
p for n, p in param_optimizer if not any(nd in n for nd in no_decay)
],
"weight_decay": 0.001,
},
{
"params": [
p for n, p in param_optimizer if any(nd in n for nd in no_decay)
],
"weight_decay": 0.0,
},
]
optimizer = AdamW(optimizer_parameters, lr=3e-5)
scheduler = get_linear_schedule_with_warmup(
optimizer, num_warmup_steps=0,
num_training_steps=get_num_train_steps(filenames["train"], label)
)
best_val_accuracy = float('-inf')
best_val_epoch = None
best_accuracy = 0
for epoch in range(config.EPOCHS):
engine.train_fn(
dataloaders["train"], model, optimizer, device, scheduler, epoch)
outputs, targets = engine.eval_fn(
dataloaders['dev'], model, device, epoch)
accuracy = metrics.accuracy_score(outputs, targets)
print(f"Validation Accuracy = {accuracy}")
if accuracy > best_val_accuracy:
torch.save(model.state_dict(), model_path)
best_val_accuracy = accuracy
best_val_epoch = epoch
print("Best val accuracy till now {}".format(best_val_accuracy))
if best_val_epoch < (epoch - config.PATIENCE):
break
model.load_state_dict(torch.load(model_path))
for subset in ['train', 'dev', 'test']:
outputs, targets = engine.eval_fn(
dataloaders[subset], model, device, epoch)
result_df_dicts = []
for o, t in zip(outputs, targets):
result_df_dicts.append({"output":o, "target":t})
result_df = pd.DataFrame.from_dict(result_df_dicts)
final_df = pd.concat([dataframes[subset], result_df], axis=1)
for i in final_df.itertuples():
assert i.ENCODE_CAT == i.target
result_file = "results2/" + subset + "_" + label + ".csv"
final_df.to_csv(result_file)
def main(_):
LEARNING_RATE = config.LEARNING_RATE
DROPOUT = config.DROPOUT
SAVE = config.SAVE
TUNE = False
ESTOP = 5
if FLAGS.lr:
LEARNING_RATE = FLAGS.lr
if FLAGS.dropout:
DROPOUT = FLAGS.dropout
if FLAGS.save:
SAVE = FLAGS.save
if FLAGS.tune:
TUNE = FLAGS.tune
if FLAGS.estop:
ESTOP = FLAGS.estop
train_file = config.TRAIN_PROC
df_train = pd.read_csv(train_file).fillna("none")
valid_file = config.DEVEL_PROC
df_valid = pd.read_csv(valid_file).fillna("none")
test_file = config.EVAL_PROC
df_test = pd.read_csv(test_file).fillna("none")
logger.info(f"Bert Model: {config.BERT_PATH}")
logger.info(
f"Current date and time :{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')} "
)
logger.info(f"Train file: {train_file}")
logger.info(f"Valid file: {valid_file}")
logger.info(f"Test file: {test_file}")
logger.info(f"Train size : {len(df_train):.4f}")
logger.info(f"Valid size : {len(df_valid):.4f}")
logger.info(f"Test size : {len(df_test):.4f}")
valid_dataset = dataset.BERTDataset(text=df_valid.text.values,
target=df_valid.label.values)
valid_data_loader = torch.utils.data.DataLoader(
valid_dataset, batch_size=config.VALID_BATCH_SIZE, num_workers=1)
test_dataset = dataset.BERTDataset(text=df_test.text.values,
target=df_test.label.values)
test_data_loader = torch.utils.data.DataLoader(
test_dataset, batch_size=config.VALID_BATCH_SIZE, num_workers=1)
device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu') #torch.device("cuda")
model = BERTBaseUncased(DROPOUT)
if TUNE:
model.load_state_dict(
torch.load(configtune.MODEL_PATH,
map_location=torch.device(device)))
model.to(device)
param_optimizer = list(model.named_parameters())
no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
optimizer_parameters = [
{
'params': [
p for n, p in param_optimizer
if not any(nd in n for nd in no_decay)
],
'weight_decay':
0.001
},
{
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
},
]
num_train_steps = int(
len(df_train) / config.TRAIN_BATCH_SIZE * config.EPOCHS)
optimizer = AdamW(optimizer_parameters, lr=LEARNING_RATE)
scheduler = get_linear_schedule_with_warmup(
optimizer, num_warmup_steps=0, num_training_steps=num_train_steps)
# model = nn.DataParallel(model)
best_accuracy = 0
best_path = ""
es = 1
for epoch in range(config.EPOCHS):
if es > ESTOP:
break
df_train = shuffle(df_train)
chunks = np.array_split(df_train, round(len(df_train) / SAVE))
for chunk in chunks:
train_dataset = dataset.BERTDataset(text=chunk.text.values,
target=chunk.label.values)
train_data_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=config.TRAIN_BATCH_SIZE,
num_workers=4,
shuffle=True)
logger.info(f"Epoch = {epoch}")
train_loss, train_acc = engine.train_fn(train_data_loader, model,
optimizer, device,
scheduler)
for tag, parm in model.named_parameters():
if parm.grad is not None:
writer.add_histogram(tag,
parm.grad.data.cpu().numpy(), epoch)
outputs, targets, val_loss, val_acc = engine.eval_fn(
valid_data_loader, model, device)
val_mcc = metrics.matthews_corrcoef(outputs, targets)
logger.info(f"val_MCC_Score = {val_mcc:.4f}")
outputs, targets, test_loss, test_acc = engine.eval_fn(
test_data_loader, model, device)
test_mcc = metrics.matthews_corrcoef(outputs, targets)
logger.info(f"test_MCC_Score = {test_mcc:.4f}")
logger.info(
f"train_loss={train_loss:.4f}, val_loss={val_loss:.4f}, test_loss={test_loss:.4f}"
)
writer.add_scalar('loss/train', train_loss, epoch)
writer.add_scalar('loss/val', val_loss, epoch)
writer.add_scalar('loss/test', test_loss, epoch)
logger.info(
f"train_acc={train_acc:.4f}, val_acc={val_acc:.4f}, test_acc={test_acc:.4f}"
)
writer.add_scalar('acc/train', train_acc, epoch)
writer.add_scalar('acc/val', val_acc, epoch)
writer.add_scalar('acc/test', test_acc, epoch)
logger.info(f"val_mcc={val_acc:.4f}, test_mcc={test_acc:.4f}")
writer.add_scalar('mcc/val', val_mcc, epoch)
writer.add_scalar('mcc/test', test_mcc, epoch)
accuracy = metrics.accuracy_score(targets, outputs)
logger.info(f"Accuracy Score = {accuracy:.4f}")
if accuracy < 0.4:
logger.info(
f"Something is very wrong! Accuracy is only {accuracy:.4f} Stopping..."
)
break
if accuracy > best_accuracy:
logger.info(
f"Saving model with Accuracy Score = {accuracy:.4f}")
if len(best_path) > 0 and os.path.exists(best_path):
#Delete previous best
os.remove(best_path)
best_path = config.MODEL_PATH[:-4] + "." + str(
round(accuracy * 100, 2)) + ".bin"
torch.save(model.state_dict(), best_path)
best_accuracy = accuracy
es = 0
else:
es += 1
logger.info(
f"Not improved for {es} times of {ESTOP}. Best so far - {best_accuracy:.4f}"
)
if es > ESTOP:
logger.info(
f"Early stopping with best accuracy: {best_accuracy:.4f} and accuracy for this epoch: {accuracy:.4f} ..."
)
break
outputs = MODEL(ids=ids, mask=mask, token_type_ids=token_type_ids)
outputs = torch.sigmoid(outputs).cpu().detach().numpy()
return outputs[0][0]
@app.route("/predict")
def predict():
sentence = request.args.get("sentence")
pos_prediction = sentence_predict(sentence)
neg_prediction = 1 - pos_prediction
response = {}
response['response'] = {
'positive': str(pos_prediction),
'negative': str(neg_prediction),
'sentence': str(sentence)
}
return flask.jsonify(response)
if __name__ == "__main__":
MODEL = BERTBaseUncased()
# MODEL.load_state_dict(torch.load(config.MODEL_PATH))
my_model = torch.load(config.MODEL_PATH, map_location=torch.device(DEVICE))
MODEL.load_state_dict(my_model)
torch.cuda.empty_cache()
MODEL.to(DEVICE)
MODEL.eval()
app.run(debug=True)
return outputs[0][0]
@app.route('/')
def home():
return render_template('home.html')
@app.route("/predict")
def predict():
sentence = request.args.get("sentence")
start_time = time.time()
positive_prediction = sentence_prediction(sentence)
negative_prediction = 1 - positive_prediction
response = {}
##response["response"] = {
rlist=[{
"positive": str(positive_prediction),
"negative": str(negative_prediction),
"sentence": str(sentence),
"time_taken": str(time.time() - start_time),
}]
##return flask.jsonify(response)
return render_template('home.html',rlist=rlist)
if __name__ == "__main__":
MODEL = BERTBaseUncased()
MODEL.load_state_dict(torch.load('model.bin'))
MODEL.to(DEVICE)
MODEL.eval()
app.run(host="0.0.0.0", port="9999")
@app.route("/predict")
def predict():
sentence = request.args.get("sentence")
start_time = time.time()
positive_prediction = sentence_prediction(sentence)
negative_prediction = 1 - positive_prediction
response = {}
response["response"] = {
"positive": str(positive_prediction),
"negative": str(negative_prediction),
"sentence": str(sentence),
"time_taken": str(time.time() - start_time),
}
return flask.jsonify(response)
if __name__ == "__main__":
MODEL = BERTBaseUncased()
state_dict = torch.load(config.MODEL_PATH)
#unexpected key module.* due to model saved as DataParallel model, need to remove module. from each state_dict
new_state_dict = OrderedDict()
for k, v in state_dict.items():
module_name = k.replace('module.', '') #remove 'module.'
new_state_dict[module_name] = v
MODEL.load_state_dict(new_state_dict)
MODEL.to(DEVICE)
MODEL.eval()
app.run(host="127.0.0.1", port="9999")
outputs = MODEL(ids=ids, mask=mask, token_type_ids=token_type_ids)
outputs = torch.sigmoid(outputs).cpu().detach().numpy()
return outputs[0][0]
@app.route("/predict")
def predict():
sentence = request.args.get("sentence")
start_time = time.time()
positive_prediction = sentence_prediction(sentence)
negative_prediction = 1 - positive_prediction
response = {}
response["response"] = {
'positive': str(positive_prediction),
'negative': str(negative_prediction),
'sentence': str(sentence),
'time_taken': str(time.time() - start_time)
}
return flask.jsonify(response)
if __name__ == "__main__":
MODEL = BERTBaseUncased()
MODEL = nn.DataParallel(MODEL)
MODEL.load_state_dict(torch.load(config.MODEL_PATH, map_location='cpu'))
MODEL.to(DEVICE)
MODEL.eval()
app.run()
