def runTest(file1, version, model, mode='words'):
"""Gets training or test file for stance detection SemiVal 2016 competition and prints prediction results.
Parameters
----------
file1 : list
a list with text tokens on index (0) and hashtags list on index (1)
istest : Boolean
specifies if the dataset is for test or training
version : int
0: Training dataset, 1: Test dataset, 2:Other domain dataset
mode : str
choose either (words) or (hashtags)
"""
indata = readfile(file1, version)
data = preprocesstweets(indata,
ignoreNONE=False,
version=version,
lowerCase=True)
tfidfAdded = getTfidfRepresentation(data, version, mode)
labels = [d[7] for d in data]
encoder = LabelEncoder()
y = encoder.fit_transform(labels)
print(encoder.classes_)
if version == 0:
x_train, x_test, y_train, y_test = train_test_split(tfidfAdded,
y,
test_size=0.2)
y_test = np_utils.to_categorical(y_test, num_classes=3)
y_train = np_utils.to_categorical(y_train, num_classes=3)
print(x_train.shape[1])
print(model.summary())
model.fit(x_train,
y_train,
epochs=10,
verbose=2,
validation_data=(x_test, y_test))
loss, acc = model.evaluate(x_test, y_test, verbose=0)
ypred = model.predict(x_test)
print('Training Accuracy: %f' % (acc * 100))
print('Training F-Score: ', f1(y_test, ypred) * 100)
if version == 1 or version == 2:
y = np_utils.to_categorical(y, num_classes=3)
loss, acc = model.evaluate(tfidfAdded, y)
ypred = model.predict(tfidfAdded)
otherdomain = ''
if version == 2:
otherdomain = '(other domain)'
print('TEST Accuracy ' + otherdomain + ': %f' % ((acc * 100)))
print('TEST F-Score ' + otherdomain + ': ', (f1(y, ypred) * 100))
def link(self, m1, m2, hypothetical=False, beta=1):
if m1 == -1:
return self.get_f1(beta=beta) if hypothetical else None
c1, c2 = self.mention_to_cluster[m1], self.mention_to_cluster[m2]
assert c1 != c2
new_c = c1 + c2
p_num, r_num, p_den, r_den = self.p_num, self.r_num, self.p_den, self.r_den
if len(c1) == 1:
self.p_den += 1
if len(c2) == 1:
self.p_den += 1
self.update_b3(new_c, hypothetical=hypothetical)
if hypothetical:
f1 = evaluation.f1(self.p_num, self.p_den, self.r_num, self.r_den, beta=beta)
self.p_num, self.r_num, self.p_den, self.r_den = p_num, r_num, p_den, r_den
return f1
else:
self.ana_to_ant[m2] = m1
self.ant_to_anas[m1].append(m2)
self.clusters.remove(c1)
self.clusters.remove(c2)
self.clusters.append(new_c)
for m in new_c:
self.mention_to_cluster[m] = new_c
def link(self, m1, m2, hypothetical=False, beta=1):
timer.start("link")
if m1 == -1:
return self.get_f1(beta=beta) if hypothetical else None
c1, c2 = self.mention_to_cluster[m1], self.mention_to_cluster[m2]
assert c1 != c2
new_c = c1 + c2
p_num, r_num, p_den, r_den = self.p_num, self.r_num, self.p_den, self.r_den
if len(c1) == 1:
self.p_den += 1
if len(c2) == 1:
self.p_den += 1
self.update_b3(new_c, hypothetical=hypothetical)
if hypothetical:
f1 = evaluation.f1(self.p_num, self.p_den, self.r_num, self.r_den, beta=beta)
self.p_num, self.r_num, self.p_den, self.r_den = p_num, r_num, p_den, r_den
timer.stop("link")
return f1
else:
self.ana_to_ant[m2] = m1
self.ant_to_anas[m1].append(m2)
self.clusters.remove(c1)
self.clusters.remove(c2)
self.clusters.append(new_c)
for m in new_c:
self.mention_to_cluster[m] = new_c
timer.stop("link")
def main():
# reading in
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--data_dir",
default='data/sampling',
help='determine the base dir of the dataset document')
parser.add_argument("--sample_n",
default=1000,
type=int,
help='starting image index of preprocessing')
parser.add_argument("--evidence_n",
default=20,
type=int,
help='how many top/bottom tiles to pick from')
parser.add_argument("--repl_n",
default=3,
type=int,
help='how many resampled replications')
parser.add_argument("--image_split",
action='store_true',
help='if use image_split')
parser.add_argument("--batch_size",
default=50,
type=int,
help="batch size")
parser.add_argument("--stage_two",
action='store_true',
help='if only use stage two patients')
parser.add_argument("--changhai",
action='store_true',
help='if use additional data')
args = parser.parse_args()
feature_size = 32
#gpu = "cuda:0"
gpu = None
# 5-folds cross validation
dataloader = CVDataLoader(args, gpu, feature_size)
n_epoch = 800
lr = 0.0005
if args.stage_two:
weight_decay = 0.008
else:
weight_decay = 0.005
manytimes_n = 8
if not os.path.isdir('figure'):
os.mkdir('figure')
if not os.path.isdir(os.path.join(args.data_dir, 'model')):
os.mkdir(os.path.join(args.data_dir, 'model'))
acc_folds = []
auc_folds = []
c_index_folds = []
f1_folds = []
f1_folds_pos = []
total_round = 0
model_count = 0
loss_function = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(0.8))
for _ in range(manytimes_n): # averaging
for i in range(5):
train_history = []
test_history = []
minimum_loss = None
auc_fold = None
acc_fold = None
early_stop_count = 0
model = Predictor(evidence_size=args.evidence_n,
layers=(100, 50, 1),
feature_size=feature_size)
# model.apply(weight_init)
if gpu:
model = model.to(gpu)
optimizer = torch.optim.RMSprop(model.parameters(),
lr=lr,
weight_decay=weight_decay)
# optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)
dataloader.set_fold(i)
X_test, Y_test, df_test = dataloader.get_test()
# X_train, Y_train, df_train = dataloader.get_train()
print('starting fold %d' % i)
for epoch in range(n_epoch):
#result = model(X_train)
#loss = nn.functional.binary_cross_entropy(result, Y_train) + nn.functional.mse_loss(result, Y_train)
# loss = nn.functional.mse_loss(result, Y_train)
#loss.backward()
#optimizer.step()
#optimizer.zero_grad()
# batch input
for X_train_batch, Y_train_batch, df_train_batch in dataloader:
# print(X_train_batch.shape)
result = model(X_train_batch)
loss = loss_function(result, Y_train_batch)
loss.backward()
optimizer.step()
optimizer.zero_grad()
X_train, Y_train, df_train = X_train_batch, Y_train_batch, df_train_batch
if epoch % 20 == 0:
result_test = model(X_test)
loss_test = loss_function(result_test, Y_test)
#loss_test = nn.functional.mse_loss(result_test, Y_test)
acc_train, acc_test = accuracy(result, Y_train), accuracy(
result_test, Y_test)
auc_train, auc_test = auc(result, Y_train), auc(
result_test, Y_test)
if args.changhai:
c_index_train, c_index_test = 0, 0
else:
c_index_train, c_index_test = c_index(
result, df_train), c_index(result_test, df_test)
recall_train, recall_test = recall(result,
Y_train), recall(
result_test, Y_test)
precision_train, precision_test = precision(
result, Y_train), precision(result_test, Y_test)
f1_train_pos, f1_test_pos = f1(result, Y_train), f1(
result_test, Y_test)
f1_train, f1_test = f1(result, Y_train,
negative=True), f1(result_test,
Y_test,
negative=True)
train_history.append(
(epoch, loss, acc_train, auc_train, c_index_train))
test_history.append(
(epoch, loss_test, acc_test, auc_test, c_index_test))
if epoch % 40 == 0:
print(
"%s epoch:%d loss:%.3f/%.3f acc:%.3f/%.3f auc:%.3f/%.3f c_index:%.3f/%.3f recall:%.3f/%.3f prec:%.3f/%.3f f1:%.3f/%.3f f1(neg):%.3f/%.3f"
% (time.strftime(
'%m.%d %H:%M:%S', time.localtime(
time.time())), epoch, loss, loss_test,
acc_train, acc_test, auc_train, auc_test,
c_index_train, c_index_test, recall_train,
recall_test, precision_train, precision_test,
f1_train_pos, f1_test_pos, f1_train, f1_test))
# early stop
if minimum_loss is None or minimum_loss * 0.995 > loss_test:
# if minimum_loss is None or minimum_loss > loss_test:
if f1_train == 0:
continue
minimum_loss = loss_test
auc_fold = auc_test
acc_fold = acc_test
c_index_fold = c_index_test
f1_fold_pos = f1_test_pos
f1_fold = f1_test
early_stop_count = 0
elif auc_test > auc_fold and auc_test > 0.5 and acc_test >= acc_fold:
minimum_loss = loss_test
auc_fold = auc_test
acc_fold = acc_test
c_index_fold = c_index_test
f1_fold_pos = f1_test_pos
f1_fold = f1_test
early_stop_count = 0
else:
early_stop_count += 1
if early_stop_count > 2 and epoch > 100:
if args.stage_two:
if auc_fold > 0.55:
print('early stop at epoch %d' % epoch)
break
elif early_stop_count > 3:
print('early stop at epoch %d' % epoch)
break
if epoch > 500:
optimizer = torch.optim.RMSprop(
model.parameters(),
lr * 0.6,
weight_decay=weight_decay * 1.2)
train_history = np.array(train_history)
test_history = np.array(test_history)
acc_folds.append(acc_fold)
auc_folds.append(auc_fold)
f1_folds.append(f1_fold)
f1_folds_pos.append(f1_fold_pos)
c_index_folds.append(c_index_fold)
plt.plot(train_history[:, 0], train_history[:, 1], label='train')
plt.plot(test_history[:, 0], test_history[:, 1], label='test')
plt.legend()
plt.savefig('figure/sample_%d_fold%d.png' % (args.sample_n, i))
plt.cla()
if acc_fold > 0.7 and auc_fold > 0.6 and model_count < 10:
model.save(args.data_dir + "/model/model_%d" % model_count)
model_count += 1
print("acc:%.3f\tauc:%.3f\tc_index:%.3f\tf1:%.3f" %
(acc_fold, auc_fold, c_index_fold, f1_fold))
total_round += 1
if gpu:
del dataloader.X_train, dataloader.Y_train, dataloader.X_test, dataloader.Y_test
del X_test, Y_test, X_train, Y_train, model, optimizer
torch.cuda.empty_cache()
print('CV-acc:%.3f CV-auc:%.3f CV-c-index:%.3f f1:%.3f f1(neg):%.3f' %
(sum(acc_folds) / 5 / manytimes_n, sum(auc_folds) / 5 / manytimes_n,
sum(c_index_folds) / 5 / manytimes_n, sum(f1_folds_pos) / 5 /
manytimes_n, sum(f1_folds) / 5 / manytimes_n))
def get_f1(self, beta=1):
return evaluation.f1(self.p_num, self.p_den, self.r_num, self.r_den, beta=beta)
def main():
"""Main function for training and testing."""
# Parse command line arguments and cache
opt = opts.Opts().args
utils.savecmd(opt.resume, sys.argv)
utils.print_color_msg("==> Setting up data loader")
train_loader, val_loader, test_loader = dataloader.create(opt)
# Load checkpoint if specified, None otherwise
utils.print_color_msg("==> Checking checkpoints")
checkpoint = checkpoints.load(opt)
utils.print_color_msg("==> Setting up model and criterion")
model, optim_state = init.setup(opt, checkpoint)
loss_fn = criterion.setup(opt, checkpoint)
utils.print_color_msg("==> Loading trainer")
trainer = train.create_trainer(model, loss_fn, opt, optim_state)
best_loss = float('Inf')
val_loss = float('Inf')
start_epoch = max([1, opt.epochNum])
if checkpoint is not None:
start_epoch = checkpoint['epoch'] + 1
best_loss = checkpoint['loss']
print("".ljust(4) + "Previous best loss: " +
utils.color_msg('%.5f' % best_loss))
if opt.valOnly:
assert start_epoch > 1, "There must be at least one epoch"
utils.print_color_msg("==> Validation:")
print("".ljust(4) + "=> Epoch %i" % (start_epoch - 1))
trainer.val(val_loader, start_epoch - 1)
sys.exit()
if opt.testOnly:
assert start_epoch > 1, "There must be at least one epoch"
utils.print_color_msg("==> Testing:")
print("".ljust(4) + "=> Epoch %i" % (start_epoch - 1))
_, prediction, reference, post, seq_length = trainer.test(
test_loader, start_epoch - 1)
prediction = F.sigmoid(torch.Tensor(prediction)).numpy()
nce = evaluation.nce(reference, prediction)
precision, recall, area, threshold = evaluation.pr(
reference, prediction)
precision_bl, recall_bl, area_bl, _ = evaluation.pr(reference, post)
f1, f1_precision, f1_recall, f1_threshold = evaluation.f1(
precision, recall, threshold)
tpr, fpr, roc_area = evaluation.roc(reference, prediction)
# Calculate stats for sequences binned by the posterior
limits = np.linspace(0, 1, 11).tolist()
utils.print_color_msg('\n\nEffect of Input Posterior on Performance')
for i in range(len(limits) - 1):
ref, pred, p = evaluation.bin_results(reference, prediction, post, measure=post, \
lower_limit=limits[i], upper_limit=limits[i+1])
if ref.size:
nce_post = evaluation.nce(ref, pred)
nce_post_bl = evaluation.nce(ref, p)
precision_post, recall_post, area_post, threshold_post = evaluation.pr(
ref, pred)
precision_post_bl, recall_post_bl, area_post_bl, threshold_post_bl = evaluation.pr(
ref, p)
f1_post, _, _, _ = evaluation.f1(precision_post, recall_post,
threshold_post)
f1_post_bl, _, _, _ = evaluation.f1(precision_post_bl,
recall_post_bl,
threshold_post_bl)
_, _, roc_area_post = evaluation.roc(ref, pred)
print('%.1f. - %.1f. %d Results (model/bl) NCE: %.4f. , %.4f. AUC(PR): %.4f. , %.4f. F-1: %.4f. , %.4f. AUC(ROC): %.4f.'\
%(limits[i], limits[i+1], int(ref.size), nce_post, nce_post_bl, area_post, area_post_bl, f1_post, f1_post_bl, roc_area_post))
else:
print('%.1f. - %.1f. Empty' % (limits[i], limits[i + 1]))
# Caluclate stats for sequences binned by sequence length
limits = [0, 2, 3, 6, 10, 20, 40]
utils.print_color_msg('\n\nEffect of Sequence Length on Performance')
for i in range(len(limits) - 1):
ref, pred, p = evaluation.bin_results(reference, prediction, post, measure=seq_length, \
lower_limit=limits[i], upper_limit=limits[i+1])
if ref.size:
nce_len = evaluation.nce(ref, pred)
nce_len_bl = evaluation.nce(ref, p)
precision_len, recall_len, area_len, threshold_len = evaluation.pr(
ref, pred)
precision_len_bl, recall_len_bl, area_len_bl, threshold_len_bl = evaluation.pr(
ref, p)
f1_len, _, _, _ = evaluation.f1(precision_len, recall_len,
threshold_len)
f1_len_bl, _, _, _ = evaluation.f1(precision_len_bl,
recall_len_bl,
threshold_len_bl)
_, _, roc_area_len = evaluation.roc(ref, pred)
print(f'%d - %d %d Results (model/bl) NCE: %.4f. , %.4f. AUC: %.4f. , %.4f. F-1: %.4f. , %.4f. AUC(ROC): %.4f.'\
%(limits[i], limits[i+1], int(ref.size), nce_len, nce_len_bl, area_len, area_len_bl, f1_len, f1_len_bl, roc_area_len))
else:
print('%d - %d Empty' % (limits[i], limits[i + 1]))
# Calulate calibration stats
limits = np.linspace(0, 1, 11).tolist()
print('\n\nCalibration Stats')
ece = 0
for i in range(len(limits) - 1):
ref, pred, p = evaluation.bin_results(reference, prediction, post, measure=prediction, \
lower_limit=limits[i], upper_limit=limits[i+1])
if ref.size:
accuracy_bin = np.mean(ref)
confidence_bin = np.mean(pred)
posterior_bin = np.mean(p)
ece += abs(accuracy_bin -
confidence_bin) * len(ref) / len(reference)
print(
f'%.1f. - %.1f. %d Reference: %.4f. , Prediction: %.4f. , Posterior: %.4f.'
% (limits[i], limits[i + 1], int(ref.size), accuracy_bin,
confidence_bin, posterior_bin))
else:
print('%.1f. - %.1f. Empty' % (limits[i], limits[i + 1]))
# Print Test Stats
print('\n\nTest Stats')
print(
"".ljust(7) + "\nNCE: %.4f. \nAUC(PR): %.4f. \nF-1: %.4f. p: %.4f. r: %.4f. t: %.4f. \nAUC(ROC): %.4f. \nECE: %.4f. " \
%(nce, area, f1, f1_precision, f1_recall, f1_threshold, roc_area, nce))
trainer.logger['test'].write('NCE: %f\nAUC(PR): %f\n' % (nce, area))
evaluation.plot_pr([precision, precision_bl], [recall, recall_bl],
[area, area_bl], ['BiLatticeRNN', 'posterior'],
opt.resume)
np.savez(os.path.join(opt.resume, 'result.npz'),
prediction=prediction,
reference=reference,
posteriors=post)
sys.exit()
utils.print_color_msg("==> Training:")
for epoch in range(start_epoch, opt.nEpochs + 1):
print("".ljust(4) + "=> Epoch %i" % epoch)
best_model = False
_ = trainer.train(train_loader, epoch, val_loss)
if not opt.debug:
val_loss = trainer.val(val_loader, epoch)
if val_loss < best_loss:
best_model = True
print("".ljust(4) + "** Best model: " +
utils.color_msg('%.4f' % val_loss))
best_loss = val_loss
checkpoints.save(epoch, trainer.model, loss_fn,
trainer.optim_state, best_model, val_loss, opt)
if not opt.debug:
utils.print_color_msg("==> Testing:")
_, prediction, reference, _, _ = trainer.test(test_loader, opt.nEpochs)
prediction = F.sigmoid(torch.Tensor(prediction)).numpy()
nce = evaluation.nce(reference, prediction)
precision, recall, area, _ = evaluation.pr(reference, prediction)
utils.print_color_msg("".ljust(7) + "NCE: %.4f. AUC(PR): %.4f" %
(nce, area))
trainer.logger['test'].write('NCE: %f\nAUC(PR): %f\n' % (nce, area))
evaluation.plot_pr([precision], [recall], [area], ['BiLatticeRNN'],
opt.resume)
# Flush write out and reset pointer
for open_file in trainer.logger.values():
open_file.flush()
open_file.seek(0)
plot.plot(opt.resume, opt.onebest)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--data_dir",
default='data/sampling',
help='determine the base dir of the dataset document')
parser.add_argument("--sample_n",
default=2000,
type=int,
help='starting image index of preprocessing')
parser.add_argument("--evidence_n",
default=500,
type=int,
help='how many top/bottom tiles to pick from')
parser.add_argument("--repl_n",
default=3,
type=int,
help='how many resampled replications')
parser.add_argument("--image_split",
action='store_true',
help='if use image_split')
parser.add_argument("--batch_size",
default=200,
type=int,
help="batch size")
parser.add_argument("--stage_two",
action='store_true',
help='if only use stage two patients')
parser.add_argument("--threshold",
default=25,
type=float,
help='threshold')
parser.add_argument("--changhai",
action='store_true',
help='if use additional data')
parser.add_argument("--TH", action='store_true')
args = parser.parse_args()
gpu = "cuda:0"
n_epoch = 80
acc_folds = []
auc_folds = []
c_index_folds = []
f1_folds = []
f1_folds_pos = []
unsuccessful_count = 0
model_count = 0
n_manytimes = 8
# caching
if False:
# if os.path.exists(os.path.join(args.data_dir, 'graph', 'graph_dataset.pkl')) and os.path.exists(os.path.join(args.data_dir, 'graph', 'graph_df.pkl')):
print("loading cached graph data")
with open(os.path.join(args.data_dir, 'graph', 'graph_dataset.pkl'),
'rb') as file:
dataset = pickle.load(file)
with open(os.path.join(args.data_dir, 'graph', 'graph_df.pkl'),
'rb') as file:
df = pickle.load(file)
else:
if not os.path.exists(os.path.join(args.data_dir, 'graph')):
os.mkdir(os.path.join(args.data_dir, 'graph'))
dataset, df = construct_graph_dataset(args, gpu)
with open(os.path.join(args.data_dir, 'graph', 'graph_dataset.pkl'),
'wb') as file:
pickle.dump(dataset, file)
with open(os.path.join(args.data_dir, 'graph', 'graph_df.pkl'),
'wb') as file:
pickle.dump(df, file)
splitter = CrossValidationSplitter(dataset,
df,
n=5,
n_manytimes=n_manytimes)
# criterion = torch.nn.CrossEntropyLoss()
criterion = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(0.4))
fold_num = 0
if not os.path.isdir(os.path.join(args.data_dir, 'model')):
os.mkdir(os.path.join(args.data_dir, 'model'))
for train_dataset, test_dataset, train_df, test_df in splitter:
print("starting fold %d-%d" % (fold_num // 5, fold_num % 5))
train_loader = DataLoader(train_dataset, batch_size=args.batch_size)
test_loader = DataLoader(test_dataset, batch_size=args.batch_size)
train_history = []
test_history = []
minimum_loss = None
auc_fold = None
acc_fold = None
early_stop_count = 0
model = GNN(32).to(gpu)
optimizer = torch.optim.RMSprop(model.parameters(),
lr=0.0004,
weight_decay=0.001)
for epoch in range(n_epoch):
model.train()
for data in train_loader: # Iterate in batches over the training dataset.
y_pred = model(data.x, data.edge_index,
data.batch.to(gpu)).view(
-1) # Perform a single forward pass.
loss = criterion(y_pred, data.y) # Compute the loss.
loss.backward() # Derive gradients.
optimizer.step() # Update parameters based on gradients.
optimizer.zero_grad() # Clear gradients.
if epoch % 1 == 0:
model.eval()
y_pred_train, y_train = concat_result(train_loader, model, gpu)
y_pred_test, y_test = concat_result(test_loader, model, gpu)
loss_train, loss_test = criterion(y_pred_train,
y_train), criterion(
y_pred_test, y_test)
#loss_test = nn.functional.mse_loss(result_test, Y_test)
acc_train, acc_test = accuracy(y_pred_train,
y_train), accuracy(
y_pred_test, y_test)
auc_train, auc_test = auc(y_pred_train,
y_train), auc(y_pred_test, y_test)
if False:
c_index_train, c_index_test = 0, 0
else:
c_index_train, c_index_test = c_index(
y_pred_train, train_df), c_index(y_pred_test, test_df)
f1_train, f1_test = f1(y_pred_train, y_train,
negative=True), f1(y_pred_test,
y_test,
negative=True)
if epoch % 5 == 0:
print(
f'Epoch:{epoch:03d} Loss:{loss_train:.3f}/{loss_test:.3f} ACC:{acc_train:.3f}/{acc_test:.3f} AUC:{auc_train:.3f}/{auc_test:.3f} CI:{c_index_train:.3f}/{c_index_test:.3f} f1(neg):{f1_train:.3f}/{f1_test:.3f}'
)
# early stop
if minimum_loss is None or minimum_loss * 0.997 > loss_test:
# if minimum_loss is None or minimum_loss > loss_test:
if f1_train == 0:
continue
minimum_loss = loss_test
auc_fold = auc_test
acc_fold = acc_test
c_index_fold = c_index_test
f1_fold = f1_test
early_stop_count = 0
if acc_fold > 0.75 and auc_fold > 0.75:
model.save(args.data_dir +
"/model/graph_%d" % model_count)
#elif auc_test > auc_fold and auc_test>0.5 and acc_test >= acc_fold:
# minimum_loss = loss_test
# auc_fold = auc_test
# acc_fold = acc_test
# c_index_fold = c_index_test
# f1_fold = f1_test
# early_stop_count = 0\
elif auc_fold + acc_fold + c_index_fold < auc_test + acc_test + c_index_fold:
minimum_loss = loss_test
auc_fold = auc_test
acc_fold = acc_test
c_index_fold = c_index_test
f1_fold = f1_test
early_stop_count = 0
if acc_fold > 0.75 and auc_fold > 0.75:
model.save(args.data_dir +
"/model/graph_%d" % model_count)
else:
early_stop_count += 1
if abs(auc_fold - 1) < 0.0001:
pass
#print('wtf')
if early_stop_count > 3 and epoch > 25:
if args.stage_two:
if auc_fold > 0.55 and acc_fold > 0.55:
print('early stop at epoch %d' % epoch)
if acc_fold > 0.75 and auc_fold > 0.75:
model.load(args.data_dir +
"/model/graph_%d" % model_count)
model_count += 1
break
elif early_stop_count > 3:
print('early stop at epoch %d' % epoch)
break
acc_folds.append(acc_fold)
auc_folds.append(auc_fold)
f1_folds.append(f1_fold)
c_index_folds.append(c_index_fold)
fold_num += 1
print("acc:%.3f\tauc:%.3f\tc_index:%.3f\tf1:%.3f" %
(acc_fold, auc_fold, c_index_fold, f1_fold))
total_count = 5 * n_manytimes
print('CV-acc:%.3f CV-auc:%.3f CV-c-index:%.3f f1(neg):%.3f' %
(sum(acc_folds) / total_count, sum(auc_folds) / total_count,
sum(c_index_folds) / total_count, sum(f1_folds) / total_count))
def convModel(tweets, stances, tweets_test, stances_test):
#General Parameters
global max
embeding_dim = 200
dropout_prob = (0.0, 0.5)
batch_size = 64
num_epochs = 20
print('Fitting tokenizer')
tokenizer = Tokenizer()
tokenizer.fit_on_sequences(tweets + tweets2)
max_length = max([len(s.split()) for s in tweets + tweets2])
print('max_length', max_length)
vocab_size = len(tokenizer.word_index) + 1
#Train and test split
print('Train and test split')
x_train, x_test, y_train, y_test = train_test_split(tweets, stances, test_size=0.2)
print('x_train: ', len(x_train), 'x_test', len(x_test))
#Training data
#traindata = np.array(x_train)
#testdata = np.array(x_test)
trainTokens = tokenizer.texts_to_sequences(x_train)
Xtrain = pad_sequences(trainTokens, maxlen=max_length, padding='post')
XtestTokens = tokenizer.texts_to_sequences(x_test)
Xtest = pad_sequences(XtestTokens, maxlen=max_length, padding='post')
#============ TEST DATA =============================================
#testgroup = np.array(tweets_test)
#testGroupTokens = tokenizer.texts_to_sequences(tweets_test)
#XtestGroup = pad_sequences(testGroupTokens, maxlen=max_length, padding='post')
#print('Xtrain padding: ', len(Xtrain), 'Xtest padding: ', len(Xtest), 'XtestGroup padding: ', len(XtestGroup))
#Convert stances to categorical output
y_test = np_utils.to_categorical(y_test, num_classes=3)
y_train = np_utils.to_categorical(y_train, num_classes=3)
y_testGroup = np_utils.to_categorical(stances_test, num_classes=3)
print('y_test: ', len(y_test), 'y_train: ', len(y_train), 'y_testGroup: ', len(stances_test))
print('Loading embeddings..')
#load word2vec and create embedding layer
wv_from_bin = KeyedVectors.load_word2vec_format(datapath('E:/glove/glove.twitter.27B.200dGINSIM.txt'),binary=False)
embedding_vectors = get_weight_matrix2(wv_from_bin, tokenizer.word_index.items())
embedding_layer = Embedding(vocab_size, embeding_dim, weights=[embedding_vectors], input_length=max_length, trainable=False)
#Create the model
print('Create and compile the model..')
model = createModelC(max_length, embedding_layer)
model.compile(loss="categorical_hinge", optimizer="adam", metrics=[f1])
model.summary(85)
print('Fitting the model..')
history = model.fit(Xtrain, y_train, batch_size=batch_size, epochs=num_epochs,
validation_data=(Xtest, y_test), verbose=2)
print('History', history.history)
# evaluate
print('Predicting (training)..')
ypred = model.predict(Xtest)
print('Accuracy (TRAIN): %f' % (model.evaluate(Xtest,y_test)[0]*100))
print('FScore (TRAIN): %f' % (f1(y_test, ypred)*100))
print('Predicting (testing)..')
def get_f1(self, beta=1):
return evaluation.f1(self.p_num, self.p_den, self.r_num, self.r_den, beta=beta)