def test(data_matrix, saved_model_path, title):
tf.reset_default_graph()
pred, input_placeholder, labels_placeholder, _, loss_op = build_model(data_matrix, train=False)
saver = tf.train.Saver()
loss_list = []
label_list= []
pred_list = []
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver.restore(sess, saved_model_path)
print("Model restored.")
minibatches = util.get_minibatches_lm(data_matrix, batch_size)
for tup in minibatches:
#label_data = np.zeros((batch_size, util.vocab_size))
#label_data[np.arange(batch_size), tup[1]] = 1
pred_temp, loss, labels_temp = sess.run([pred, loss_op, labels_placeholder], feed_dict={input_placeholder: tup[0], labels_placeholder: tup[1]})
for i, row in enumerate(pred_temp):
pred_list.append(np.where(row == max(row))[0][0])
for i, row in enumerate(labels_temp):
label_list.append(np.where(row == max(row))[0][0])
loss_list.append(loss)
print "Loss: " + str(np.mean(loss_list)) + "\n"
#util.outputConfusionMatrix(pred_list, label_list, "confusion_matrix " + title + " " + today)
util.get_accuracy(pred_list, label_list)
def test(data_matrix, data_labels, saved_model_path, batch_size=1000):
tf.reset_default_graph()
pred, input_placeholder, labels_placeholder, _, loss_op = build_model(
data_matrix, data_labels)
saver = tf.train.Saver()
loss_list = []
label_list = []
pred_list = []
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver.restore(sess, saved_model_path)
print("Model restored.")
minibatches = util.get_minibatches(data_matrix, data_labels,
batch_size)
for tup in minibatches:
pred_temp, loss, labels_temp = sess.run(
[pred, loss_op, labels_placeholder],
feed_dict={
input_placeholder: tup[0],
labels_placeholder: tup[1]
})
for i, row in enumerate(pred_temp):
pred_list.append(np.where(row == max(row))[0][0])
for i, row in enumerate(labels_temp):
label_list.append(np.where(row == max(row))[0][0])
loss_list.append(loss)
print "Loss: " + str(np.mean(loss_list)) + "\n"
util.outputConfusionMatrix(pred_list, label_list,
"confusion_matrix_baseline_test")
util.get_accuracy(pred_list, label_list)
def validate(validation_loader, model, criterion_is_noise, criterion_is_next):
model.eval()
losses = []
accs_is_noise = []
accs_is_next = []
start = time.time()
for i, (noise_type, continuity_type, origin_sentence, enc_sentence, mask) in enumerate(validation_loader):
# enc_sentence: (batch_size, len_sentence, len_morpheme, len_phoneme)
# print(num_morpheme, origin_sentence, enc_sentence.size())
enc_sentence = enc_sentence.to(device)
mask = mask.to(device)
outputs_is_noise, outputs_is_next = model(enc_sentence, mask)
if noise:
noise_type = np.array(noise_type)
noise_type[noise_type == 'no'] = 0
noise_type[noise_type != '0'] = 1
target_is_noise = torch.FloatTensor(noise_type.astype(float)).to(device)
ce_is_noise = criterion_is_noise(outputs_is_noise, target_is_noise)
acc_is_noise = get_accuracy(outputs_is_noise, target_is_noise)
accs_is_noise.append(acc_is_noise)
if continuous:
continuity_type = np.array(continuity_type)
continuity_type[continuity_type == 'no'] = 0
continuity_type[continuity_type != '0'] = 1
target_is_next = torch.FloatTensor(continuity_type.astype(float)).to(device)
ce_is_next = criterion_is_next(outputs_is_next, target_is_next)
acc_is_next = get_accuracy(outputs_is_next, target_is_next)
accs_is_next.append(acc_is_next)
if 'ce_is_noise' in locals() and 'ce_is_next' in locals():
loss = ce_is_noise + ce_is_next
elif 'ce_is_noise' in locals():
loss = ce_is_noise
elif 'ce_is_next' in locals():
loss = ce_is_next
else:
raise ValueError('There is no loss')
losses.append(loss)
trace_validation = '\nValidation Loss {loss_avg:.4f}\t'.format(loss_avg=sum(losses)/len(losses))
if 'acc_is_noise' in locals():
trace_validation += 'Noise Accuracy {acc_is_noise_avg:.4f}\t'.format(
acc_is_noise_avg=sum(accs_is_noise)/len(accs_is_noise))
if 'acc_is_next' in locals():
trace_validation += 'Continuity Accuracy {acc_is_next_avg:.4f}\t'.format(
acc_is_next_avg=sum(accs_is_next)/len(accs_is_next))
trace_validation += '\n'
print(trace_validation)
return sum(losses)/len(losses)
def test_classifier(data_matrix,
data_labels,
saved_model_path,
title,
max_sequence_length=70,
batch_size=60):
tf.reset_default_graph()
pred, input_placeholder, labels_placeholder, train_op, loss_op = build_classifier(
data_matrix, data_labels, max_sequence_length)
saver = tf.train.Saver()
loss_list = []
label_list = []
pred_list = []
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver.restore(sess, saved_model_path)
print("Model restored.")
minibatches = util.get_minibatches_seq_test(data_matrix, data_labels,
batch_size,
max_sequence_length)
for tup in minibatches:
pred_temp, loss, labels_temp = sess.run(
[pred, loss_op, labels_placeholder],
feed_dict={
input_placeholder: np.transpose(tup[0]),
labels_placeholder: tup[1]
})
for i, row in enumerate(pred_temp):
pred_list.append(np.where(row == max(row))[0][0])
for i, row in enumerate(labels_temp):
label_list.append(np.where(row == max(row))[0][0])
loss_list.append(loss)
count = 0
for i in range(len(pred_list)):
if pred_list[i] != label_list[i]:
count += 1
print "sentence: ", reconstruct_sentence(
(tup[0][i:i + 1, :max_sequence_length]).tolist())
print "predicted label: ", pred_list[i]
print "correct label: ", label_list[i]
if count > 4:
break
break
print "Loss: " + str(np.mean(loss_list)) + "\n"
util.outputConfusionMatrix(pred_list, label_list,
"confusion_matrix " + title + " " + today)
util.get_accuracy(pred_list, label_list)
def do_ref(num_features, X, y, X_test, y_test):
s = [i for i in range(num_features)]
r = []
while len(s) is not 0:
X_new = np.c_[X[:, s], X[:, -1]]
w = solve_svm(X_new, y)
w = np.asarray(w).reshape(-1)
print(len(s))
accuracy = get_accuracy(w, np.c_[X_test[:, s], X_test[:, -1]], y_test)
print("Node Id: " + str(node_id) + ", Accuracy: " + str(accuracy))
w = w[:-1]
c = np.array([w_i**2 for w_i in w])
c_list = list(c)
idx = c.argmin()
# Remove similar values across other places
min_c = c[idx]
min_real_indices = []
min_indices = np.array(np.where(c == min_c)).reshape(-1)
for ind in sorted(min_indices, reverse=True):
s_removed = s.pop(ind)
min_real_indices.append(s_removed)
c_list.pop(ind)
# s_f = s.pop(idx)
# c_list.pop(idx)
r = np.append(min_real_indices, r)
return np.array(r, dtype=int)
def test_dummy_baseline(alg):
n = 3
m = 2
true_labels = [0, 30, 57]
data = np.zeros((n, m))
alg.train(data, true_labels)
labels = alg.evaluate(data)
acc = util.get_accuracy(labels, true_labels)
print("accuracy on dummy data with " + str(alg) + ": " + str(acc))
def run_train():
input_features, labels, optional = define_placeholder(use_img_feat=True) # 입력값 정의
# model = baseline.Baseline(input_features, labels, optional) # 모델 구축
model = model_mgr.get_model('umsoimg_v2')(input_features, labels, optional)
# 최적화 알고리즘 정의
optimizer = tf.train.AdamOptimizer(config.learning_rate)
grads_and_vars = optimizer.compute_gradients(model.loss)
train_op = optimizer.apply_gradients(grads_and_vars, global_step=model.global_step, name="train_op")
_dataset = DatasetAll(target=None)
sess = tf.InteractiveSession()
init_op = tf.global_variables_initializer()
saver = tf.train.Saver(max_to_keep=20)
sess.run(init_op)
for i in range(config.epochs):
for j, _dict in enumerate(_dataset.batch_loader(config.batch_size)):
_label_weight = util.make_label_weights(_dict, b_weight=1., m_weight=1.2, s_weight=1.3,
d_weight=1.4) # label 별 weight 생성
feed_dict = {
input_features['product']: _dict['product'],
input_features['img_feat']: _dict['img_feat'],
labels['b']: _dict['bcateid'],
labels['m']: _dict['mcateid'],
labels['s']: _dict['scateid'],
labels['d']: _dict['dcateid'],
optional['label_weight']: _label_weight,
optional['keep_prob']: 0.7
}
_, step, _loss, _prediction = sess.run([train_op, model.global_step, model.loss, model.prediction], feed_dict)
_b, _m, _s, _d = _prediction
if j % 100 == 0:
result = util.get_accuracy(_dict, _b, _m, _s, _d)
print("epoch : {} [t:{}], global_step: {}, Acc: {}, loss: {}".format(
i + 1, _dataset.now_dataset, step,
[str(result[0])[:6], str(result[1])[:6], str(result[2])[:6], str(result[3])[:6]],
str(_loss)[:6]))
model_name = '{}_e{}.ckpt'.format(model.__class__.__name__, str(i) if i >= 10 else ('0' + str(i)))
print("save model... {}".format(model_name))
# save_path = saver.save(sess, "../tmp/{}".format(model_name))
save_path = saver.save(sess, os.path.join(config.result_model_dir, model_name))
def test_model(keys=1):
model = tf.keras.models.load_model('model.h5')
scaler = joblib.load('scaler.pkl')
trainer = get_pyspectre(keys).getTrainerStr()
secret = get_pyspectre(keys).getSecretStr()
repetitions = 5
samples = 3
print("Testing model with keysize " + str(keys) + ". " + str(repetitions) + " repetitions of " + str(samples) + " samples each.")
total_acc = 0
total_runtime = 0
for s in range(repetitions):
start_time = time.time()
guessed_secrets = []
weights = np.zeros((len(secret), len(trainer)))
for _ in range(samples):
# print("Sampling...")
X = np.zeros((len(secret), len(trainer)))
for i in range(len(secret)):
X[i] = np.take(np.array(get_pyspectre(keys).readMemoryByte(i, False)), [ord(x) for x in trainer])
X = scaler.transform(X)
prediction = model.predict(X)
guessed_chars = np.argmax(prediction, axis=1)
guessed_secret = "".join([trainer[x] for x in guessed_chars])
weights += prediction
guessed_secrets.append(guessed_secret)
weight_to_char_map = {char: index for index, char in enumerate(trainer)}
majority_guess = util.majority_vote(guessed_secrets, weights, weight_to_char_map)
acc = util.get_accuracy(majority_guess, secret)
total_acc += acc
elapsed = time.time()-start_time
total_runtime += elapsed
print(str(total_acc / repetitions * 100) + ",\t" + str(total_runtime / repetitions))
inference_logits, {
input_data: source_batch,
source_sequence_length: sources_lengths,
target_sequence_length: targets_lengths,
keep_prob: 1.0
})
batch_valid_logits = sess.run(
inference_logits, {
input_data: valid_sources_batch,
source_sequence_length: valid_sources_lengths,
target_sequence_length: valid_targets_lengths,
keep_prob: 1.0
})
train_acc = get_accuracy(target_batch, batch_train_logits)
valid_acc = get_accuracy(valid_targets_batch,
batch_valid_logits)
print(
'Epoch {:>3} Batch {:>4}/{} - Train Accuracy: {:>6.4f}, Validation Accuracy: {:>6.4f}, Loss: {:>6.4f}'
.format(epoch_i, batch_i,
len(en.text_as_ids) // batch_size, train_acc,
valid_acc, loss))
saver = tf.train.Saver()
saver.save(sess, 'dev')
print('Model Trained and Saved')
with open('preprocess.p', mode='rb') as in_file:
def get_baseline_predictions(train, test, x_cols, y_col):
avg = train[y_col].sum() / len(train)
prediction = [avg] * len(test)
return prediction
with open("train_data.csv", 'r') as csvfile:
train = pandas.read_csv(csvfile)
csvfile.close()
with open("validation_data.csv", 'r') as csvfile:
validate = pandas.read_csv(csvfile)
csvfile.close()
print('Training set size = ' + str(len(train)))
print('Validation set size = ' + str(len(validate)))
# print(data.columns.values) #list of all column names
x_cols = train.columns.values[3:]
y_col = train.columns.values[0]
actual = validate[y_col]
prediction = get_baseline_predictions(train, validate, x_cols, y_col)
acc = util.get_accuracy(prediction, actual)
util.make_accuracy_plot(prediction, actual)
util.write_accuracy_data(prediction, actual, 'baseline_results.csv')
print(acc)
y_train = np.array(df_training[str(num_features)])
X_train = np.array(X_train_int)
X_train = np.c_[X_train, np.ones(len(y_train))]
X_test_int = df_test.drop(columns=[str(num_features)])
y_test = np.array(df_test[str(num_features)])
X_test = np.array(X_test_int)
X_test = np.c_[X_test, np.ones(len(y_test))]
rankings = do_ref(num_features, X_train, y_train, X_test, y_test)
feature_counter = -1
accuracies = []
while feature_counter > (-1) * (len(rankings) + 1):
sub_rankings = rankings[feature_counter:]
w = solve_svm(np.c_[X_train[:, sub_rankings], X_train[:, -1]], y_train)
acc = get_accuracy(w, np.c_[X_test[:, sub_rankings], X_test[:, -1]],
y_test)
accuracies.append(acc)
feature_counter -= 1
print(-1 * feature_counter)
# w = solve_svm(X_train, y_train)
x = [i + 1 for i in range(len(rankings))]
y = accuracies
print(len(x), len(y))
plt.plot(x, y, c=np.random.rand(3, ))
plt.xlabel("Num of features")
plt.ylabel("Accuracy - %")
plt.show()
sigma,
K,
W,
bound_,
SLK_option,
C_init,
bound_lambda=lmbda,
bound_iterations=200)
if ts:
trivial[count] = 1
continue
# Evaluate the performance on validation set
current_nmi = nmi(gnd_val, l[val_ind])
acc, _ = get_accuracy(gnd_val, l[val_ind])
print('lambda = ', lmbda, ' : NMI= %0.4f' % current_nmi)
print('accuracy %0.4f' % acc)
if current_nmi > bestnmi:
bestnmi = current_nmi
best_lambda_nmi = lmbda
if acc > bestacc:
bestacc = acc
best_lambda_acc = lmbda
best_C_init = C_init.copy()
print('Best result: NMI= %0.4f' % bestnmi, '|NMI lambda = ',
best_lambda_nmi)
def test(test_loader, model, criterion_is_noise, criterion_is_next):
model.eval()
# if not self.noise:
# noise_threshold = 1
# if noise_threshold < 0.05:
# noise_type = 'removing_phoneme'
# elif noise_threshold < 0.1:
# noise_type = 'replacing_phoneme'
# elif noise_threshold < 0.15:
# noise_type = 'removing_syllable'
# elif noise_threshold < 0.2:
# noise_type = 'replacing_syllable'
# elif noise_threshold < 0.25:
# noise_type = 'removing_morpheme'
# elif noise_threshold < 0.3:
# noise_type = 'replacing_morpheme'
# elif noise_threshold < 0.35:
# noise_type = 'removing_word_phrase'
# elif noise_threshold < 0.4:
# noise_type = 'replacing_word_phrase'
# else:
# noise_type = 'no'
#
# continuity_threshold = np.random.uniform(0, 1, 1)
# if continuity_threshold < 0.5:
# continuity_type = 'no'
# else:
# continuity_type = 'yes'
losses = []
accs_is_noise = []
accs_is_next = []
confusions_is_noise = {'True_no': 0,
'False_no': 0,
'True_removing_phoneme': 0,
'False_removing_phoneme': 0,
'True_replacing_phoneme': 0,
'False_replacing_phoneme': 0,
'True_removing_syllable': 0,
'False_removing_syllable': 0,
'True_replacing_syllable': 0,
'False_replacing_syllable': 0,
'True_removing_morpheme': 0,
'False_removing_morpheme': 0,
'True_replacing_morpheme': 0,
'False_replacing_morpheme': 0,
'True_removing_word_phrase': 0,
'False_removing_word_phrase': 0,
'True_replacing_word_phrase': 0,
'False_replacing_word_phrase': 0,
}
confusions_is_next = [[0, 0], [0, 0]]
data_size = len(test_loader.dataset)
start = time.time()
for i, (noise_type, continuity_type, origin_sentence, enc_sentence, mask) in enumerate(test_loader):
if i % print_freq == 0:
print('PROGESS: %.2f' % (i*batch_size/data_size))
# enc_sentence: (batch_size, len_sentence, len_morpheme, len_phoneme)
# print(num_morpheme, origin_sentence, enc_sentence.size())
enc_sentence = enc_sentence.to(device)
mask = mask.to(device)
outputs_is_noise, outputs_is_next = model(enc_sentence, mask)
if noise:
noise_type = np.array(noise_type)
noise_type_copy = noise_type.copy()
noise_type[noise_type == 'no'] = 0
noise_type[noise_type != '0'] = 1
target_is_noise = torch.FloatTensor(noise_type.astype(float)).to(device)
ce_is_noise = criterion_is_noise(outputs_is_noise, target_is_noise)
acc_is_noise = get_accuracy(outputs_is_noise, target_is_noise)
accs_is_noise.append(acc_is_noise)
t = Variable(torch.Tensor([0.5])).to(device)
binary_outputs = (outputs_is_noise > t).float() * 1
for j, (y_actual, y_predict) in enumerate(zip(noise_type_copy, binary_outputs)):
if 'no' in y_actual:
if y_predict:
confusions_is_noise['False_no'] += 1
else:
confusions_is_noise['True_no'] += 1
else:
if 'phoneme' in y_actual:
if 'removing' in y_actual:
if y_predict:
confusions_is_noise['True_removing_phoneme'] += 1
else:
confusions_is_noise['False_removing_phoneme'] += 1
else:
if y_predict:
confusions_is_noise['True_replacing_phoneme'] += 1
else:
confusions_is_noise['False_replacing_phoneme'] += 1
elif 'syllable' in y_actual:
if 'removing' in y_actual:
if y_predict:
confusions_is_noise['True_removing_syllable'] += 1
else:
confusions_is_noise['False_removing_syllable'] += 1
else:
if y_predict:
confusions_is_noise['True_replacing_syllable'] += 1
else:
confusions_is_noise['False_replacing_syllable'] += 1
elif 'morpheme' in y_actual:
if 'removing' in y_actual:
if y_predict:
confusions_is_noise['True_removing_morpheme'] += 1
else:
confusions_is_noise['False_removing_morpheme'] += 1
else:
if y_predict:
confusions_is_noise['True_replacing_morpheme'] += 1
else:
confusions_is_noise['False_replacing_morpheme'] += 1
else:
if 'removing' in y_actual:
if y_predict:
confusions_is_noise['True_removing_word_phrase'] += 1
else:
confusions_is_noise['False_removing_word_phrase'] += 1
else:
if y_predict:
confusions_is_noise['True_replacing_word_phrase'] += 1
else:
confusions_is_noise['False_replacing_word_phrase'] += 1
if continuous:
continuity_type = np.array(continuity_type)
continuity_type[continuity_type == 'no'] = 0
continuity_type[continuity_type != '0'] = 1
target_is_next = torch.FloatTensor(continuity_type.astype(float)).to(device)
ce_is_next = criterion_is_next(outputs_is_next, target_is_next)
acc_is_next = get_accuracy(outputs_is_next, target_is_next)
accs_is_next.append(acc_is_next)
t = Variable(torch.Tensor([0.5])).to(device)
binary_outputs = (outputs_is_next > t).float() * 1
confusions_is_next += confusion_matrix(continuity_type.astype(int), np.array(binary_outputs.to("cpu")).astype(int))
if 'ce_is_noise' in locals() and 'ce_is_next' in locals():
loss = ce_is_noise + ce_is_next
elif 'ce_is_noise' in locals():
loss = ce_is_noise
elif 'ce_is_next' in locals():
loss = ce_is_next
else:
raise ValueError('There is no loss')
losses.append(loss)
# TODO: 각각 최종 accuracy, confusion matrix,
trace_test = '\nTest Loss {loss_avg:.4f}\t'.format(loss_avg=sum(losses)/len(losses))
if 'acc_is_noise' in locals():
trace_test += 'Noise Accuracy {acc_is_noise_avg:.4f}\t'.format(
acc_is_noise_avg=sum(accs_is_noise)/len(accs_is_noise))
print(confusions_is_noise)
if 'acc_is_next' in locals():
trace_test += 'Continuity Accuracy {acc_is_next_avg:.4f}\t'.format(
acc_is_next_avg=sum(accs_is_next)/len(accs_is_next))
print(confusions_is_next)
trace_test += '\n'
print(trace_test)
return sum(losses)/len(losses)
def train_val(config):
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
train_loader = get_dataloader(img_dir=config.train_img_dir,
mask_dir=config.train_mask_dir,
mode="train",
batch_size=config.batch_size,
num_workers=config.num_workers)
val_loader = get_dataloader(img_dir=config.val_img_dir,
mask_dir=config.val_mask_dir,
mode="val",
batch_size=config.batch_size,
num_workers=config.num_workers)
writer = SummaryWriter(
comment="LR_%f_BS_%d_MODEL_%s_DATA_%s" %
(config.lr, config.batch_size, config.model_type, config.data_type))
if config.model_type not in [
'UNet', 'R2UNet', 'AUNet', 'R2AUNet', 'SEUNet', 'SEUNet++',
'UNet++', 'DAUNet', 'DANet', 'AUNetR', 'RendDANet', "BASNet"
]:
print('ERROR!! model_type should be selected in supported models')
print('Choose model %s' % config.model_type)
return
if config.model_type == "UNet":
model = UNet()
elif config.model_type == "AUNet":
model = AUNet()
elif config.model_type == "R2UNet":
model = R2UNet()
elif config.model_type == "SEUNet":
model = SEUNet(useCSE=False, useSSE=False, useCSSE=True)
elif config.model_type == "UNet++":
model = UNetPP()
elif config.model_type == "DANet":
model = DANet(backbone='resnet101', nclass=1)
elif config.model_type == "AUNetR":
model = AUNet_R16(n_classes=1, learned_bilinear=True)
elif config.model_type == "RendDANet":
model = RendDANet(backbone='resnet101', nclass=1)
elif config.model_type == "BASNet":
model = BASNet(n_channels=3, n_classes=1)
else:
model = UNet()
if torch.cuda.device_count() > 1:
print("Let's use", torch.cuda.device_count(), "GPUs!")
model = nn.DataParallel(model)
model = model.to(device, dtype=torch.float)
if config.optimizer == "sgd":
optimizer = SGD(model.parameters(),
lr=config.lr,
weight_decay=1e-6,
momentum=0.9)
else:
optimizer = torch.optim.Adam(model.parameters(), lr=config.lr)
if config.loss == "dice":
criterion = DiceLoss()
elif config.loss == "bce":
criterion = nn.BCELoss()
elif config.loss == "bas":
criterion = BasLoss()
else:
criterion = MixLoss()
scheduler = lr_scheduler.StepLR(optimizer, step_size=40, gamma=0.1)
global_step = 0
best_dice = 0.0
for epoch in range(config.num_epochs):
epoch_loss = 0.0
with tqdm(total=config.num_train,
desc="Epoch %d / %d" % (epoch + 1, config.num_epochs),
unit='img') as train_pbar:
model.train()
for image, mask in train_loader:
image = image.to(device, dtype=torch.float)
mask = mask.to(device, dtype=torch.float)
d0, d1, d2, d3, d4, d5, d6, d7 = model(image)
loss = criterion(d0, d1, d2, d3, d4, d5, d6, d7, mask)
epoch_loss += loss.item()
writer.add_scalar('Loss/train', loss.item(), global_step)
train_pbar.set_postfix(**{'loss (batch)': loss.item()})
optimizer.zero_grad()
loss.backward()
optimizer.step()
train_pbar.update(image.shape[0])
global_step += 1
# if global_step % 100 == 0:
# writer.add_images('masks/true', mask, global_step)
# writer.add_images('masks/pred', d0 > 0.5, global_step)
scheduler.step()
epoch_dice = 0.0
epoch_acc = 0.0
epoch_sen = 0.0
epoch_spe = 0.0
epoch_pre = 0.0
current_num = 0
with tqdm(total=config.num_val,
desc="Epoch %d / %d validation round" %
(epoch + 1, config.num_epochs),
unit='img') as val_pbar:
model.eval()
locker = 0
for image, mask in val_loader:
current_num += image.shape[0]
image = image.to(device, dtype=torch.float)
mask = mask.to(device, dtype=torch.float)
d0, d1, d2, d3, d4, d5, d6, d7 = model(image)
batch_dice = dice_coeff(mask, d0).item()
epoch_dice += batch_dice * image.shape[0]
epoch_acc += get_accuracy(pred=d0, true=mask) * image.shape[0]
epoch_sen += get_sensitivity(pred=d0,
true=mask) * image.shape[0]
epoch_spe += get_specificity(pred=d0,
true=mask) * image.shape[0]
epoch_pre += get_precision(pred=d0, true=mask) * image.shape[0]
if locker == 200:
writer.add_images('masks/true', mask, epoch + 1)
writer.add_images('masks/pred', d0 > 0.5, epoch + 1)
val_pbar.set_postfix(**{'dice (batch)': batch_dice})
val_pbar.update(image.shape[0])
locker += 1
epoch_dice /= float(current_num)
epoch_acc /= float(current_num)
epoch_sen /= float(current_num)
epoch_spe /= float(current_num)
epoch_pre /= float(current_num)
epoch_f1 = get_F1(SE=epoch_sen, PR=epoch_pre)
if epoch_dice > best_dice:
best_dice = epoch_dice
writer.add_scalar('Best Dice/test', best_dice, epoch + 1)
torch.save(
model, config.result_path + "/%s_%s_%d.pth" %
(config.model_type, str(epoch_dice), epoch + 1))
logging.info('Validation Dice Coeff: {}'.format(epoch_dice))
print("epoch dice: " + str(epoch_dice))
writer.add_scalar('Dice/test', epoch_dice, epoch + 1)
writer.add_scalar('Acc/test', epoch_acc, epoch + 1)
writer.add_scalar('Sen/test', epoch_sen, epoch + 1)
writer.add_scalar('Spe/test', epoch_spe, epoch + 1)
writer.add_scalar('Pre/test', epoch_pre, epoch + 1)
writer.add_scalar('F1/test', epoch_f1, epoch + 1)
writer.close()
print("Training finished")
# Q4.4.2 Majority Baseline
################################################################################
print(
"\n************************************************************************"
)
print(
"*************************** Majority Label *****************************")
print(
"************************************************************************")
(values, counts) = np.unique(label_tr, return_counts=True)
majority_label = values[np.argmax(counts)]
prediction = np.ones(label_dv.shape) * majority_label
print("Accuracy for majority label on dev set = %.2f" %
(util.get_accuracy(label_dv, prediction)))
prediction = np.ones(label_te.shape) * majority_label
print("Accuracy for majority label on test set = %.2f" %
(util.get_accuracy(label_te, prediction)))
################################################################################
# Simple Perceptron with fixed learning rate
################################################################################
print(
"\n************************************************************************"
)
print(
"************* Simple Perceptron (Fixed learning rate) ******************")
print(
"************************************************************************")
##############
# ONE VS ONE #
##############
print "[One VS One]"
t0 = time.time()
clf_1v1 = OneVsOneClassifier(
svm.LinearSVC(random_state=42, class_weight='balanced'))
clf_1v1.fit(X_train_svd, newsgroup_train.target)
y_pred = clf_1v1.predict(X_test_svd)
util.print_time_in_miliseconds(t0, time.time())
y_true = newsgroup_test.target
print "Accuracy (overall): %.6f" % util.get_accuracy(y_pred, y_true)
print metrics.classification_report(y_true, y_pred, target_names=categories)
print "Confusion Matrix:"
print metrics.confusion_matrix(newsgroup_test.target, y_pred)
print ""
#--------------------------------
###################
# ONE VS THE REST #
###################
print "[One Vs Rest]"
clf_1vr = OneVsRestClassifier(svm.LinearSVC(random_state=42))
def train(train_loader, model, optimizer, criterion_is_noise, criterion_is_next, epoch):
model.train()
losses = []
accs_is_noise = []
accs_is_next = []
start = time.time()
for i, (noise_type, continuity_type, origin_sentence, enc_sentence, mask) in enumerate(train_loader):
# enc_sentence: (batch_size, len_sentence, len_morpheme, len_phoneme)
# print(num_morpheme, origin_sentence, enc_sentence.size())
enc_sentence = enc_sentence.to(device)
mask = mask.to(device)
outputs_is_noise, outputs_is_next = model(enc_sentence, mask)
if noise:
noise_type = np.array(noise_type)
noise_type[noise_type == 'no'] = 0
noise_type[noise_type != '0'] = 1
target_is_noise = torch.FloatTensor(noise_type.astype(float)).to(device)
ce_is_noise = criterion_is_noise(outputs_is_noise, target_is_noise)
acc_is_noise = get_accuracy(outputs_is_noise, target_is_noise)
accs_is_noise.append(acc_is_noise)
if continuous:
continuity_type = np.array(continuity_type)
continuity_type[continuity_type == 'no'] = 1
continuity_type[continuity_type != '1'] = 0
target_is_next = torch.FloatTensor(continuity_type.astype(float)).to(device)
ce_is_next = criterion_is_next(outputs_is_next, target_is_next)
acc_is_next = get_accuracy(outputs_is_next, target_is_next)
accs_is_next.append(acc_is_next)
if 'ce_is_noise' in locals() and 'ce_is_next' in locals():
loss = ce_is_noise + ce_is_next
elif 'ce_is_noise' in locals():
loss = ce_is_noise
elif 'ce_is_next' in locals():
loss = ce_is_next
else:
raise ValueError('There is no loss')
a = list(model.parameters())[0].clone()
optimizer.zero_grad()
loss.backward()
clip_gradient(optimizer, grad_clip)
optimizer.step()
b = list(model.parameters())[0].clone()
losses.append(loss)
if i % print_freq == 0:
trace_training = 'Epoch: [{0}][{1}/{2}]\t' \
'Loss {loss:.4f} ({loss_avg:.4f})\t'.format(epoch, i, len(train_loader),
loss=loss, loss_avg=sum(losses)/len(losses))
if 'acc_is_noise' in locals():
trace_training += 'Noise Accuracy {acc_is_noise:.4f} ({acc_is_noise_avg:.4f})\t'.format(
acc_is_noise=acc_is_noise, acc_is_noise_avg=sum(accs_is_noise)/len(accs_is_noise))
if 'acc_is_next' in locals():
trace_training += 'Continuity Accuracy {acc_is_next:.4f} ({acc_is_next_avg:.4f})\t'.format(
acc_is_next=acc_is_next, acc_is_next_avg=sum(accs_is_next)/len(accs_is_next))
# TODO: 값 추정하기
print(trace_training, torch.equal(a, b))