def batch():
print("Tesing the accuracy of LogisticRegression(batch)...")
# Train model
clf = LogisticRegression()
clf.fit(X=X_train, y=y_train, lr=0.008, epochs=5000)
# Model accuracy
get_acc(clf, X_test, y_test)
def stochastic():
print("Tesing the accuracy of LogisticRegression(stochastic)...")
# Train model
clf = LogisticRegression()
clf.fit(X=X_train, y=y_train, lr=0.01, epochs=200,
method="stochastic", sample_rate=0.5)
# Model accuracy
get_acc(clf, X_test, y_test)
def main():
print("Tesing the accuracy of NaiveBayes...")
# Load data
X, y = load_breast_cancer()
# Split data randomly, train set rate 70%
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=10)
# Train model
clf = GaussianNB()
clf.fit(X_train, y_train)
# Model accuracy
get_acc(clf, X_test, y_test)
def main():
print("Tesing the accuracy of DecisionTree...")
# Load data
X, y = load_breast_cancer()
# Split data randomly, train set rate 70%
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=10)
# Train model
clf = DecisionTree()
clf.fit(X_train, y_train, max_depth=4)
# Show rules
clf.print_rules()
# Model accuracy
get_acc(clf, X_test, y_test)
def main():
print("Tesing the accuracy of GBDT Classifier...")
# Load data
X, y = load_breast_cancer()
# Split data randomly, train set rate 70%
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=20)
# Train model
clf = GradientBoostingClassifier()
clf.fit(X_train,
y_train,
n_estimators=2,
lr=0.8,
max_depth=3,
min_samples_split=2)
# Model accuracy
get_acc(clf, X_test, y_test)
def train(epoch):
with torch.autograd.set_detect_anomaly(True):
t = time.time()
model.train()
optimizer.zero_grad()
output = model(features, adj_norm)
loss = loss_function_ae(preds=output,
labels=adj_label,
norm=norm,
pos_weight=pos_weight)
losses.append(loss)
indices.append(epoch)
loss.backward()
curr_loss = loss.item()
optimizer.step()
acc_train = get_acc(output, adj_label)
hidden_emb = model.mu.data.numpy()
roc_curr, ap_curr = get_roc_score(hidden_emb, adj_orig, val_edges,
val_edges_false)
print('Epoch: {:04d}'.format(epoch + 1),
'loss_train: {:.4f}'.format(curr_loss),
'acc_train: {:.4f}'.format(acc_train), "val_ap=",
"{:.5f}".format(ap_curr), "val_roc=", "{:.5f}".format(roc_curr),
'time: {:.4f}s'.format(time.time() - t))
def eval_model(model: nn.Module, task_classifier: nn.Module, dataset: ParentDataset, loss: nn.Module, binary: bool,
disp_tqdm: bool = True, special_binary: bool = False) -> Tuple[float, float, Mapping]:
# Set all models to evaluation mode
model.eval()
task_classifier.eval()
results = 0
avg_loss = 0
f1_acc = AccumulatorF1() if binary or special_binary else None
# Prevents the gradients from being computed
with torch.no_grad():
for i, (x, label) in tqdm(enumerate(dataset), total=len(dataset), position=0, disable=not disp_tqdm):
# For each document compute the output
out = task_classifier(model(x))
grad = loss(out, label)
results += get_acc(out, label, binary)
if binary or special_binary:
f1_acc.add(out, label)
avg_loss = (avg_loss * i + grad.item()) / (i + 1)
f1_stats = f1_acc.reduce() if binary or special_binary else None
return results / len(dataset), avg_loss, f1_stats
def main():
print("Comparing RandomForest with DecisionTree...")
# Load data
X, y = load_breast_cancer()
# Split data randomly, train set rate 70%
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=40)
# Train RandomForest model
rf = RandomForest()
rf.fit(X_train, y_train, n_samples=300, max_depth=3, n_estimators=20)
# RandomForest Model accuracy
print("RandomForest:", end=' ')
get_acc(rf, X_test, y_test)
# Train DecisionTree model
dt = DecisionTree()
dt.fit(X_train, y_train, max_depth=4)
# DecisionTree Model accuracy
print("DecisionTree:", end=' ')
get_acc(dt, X_test, y_test)
def get_results_nb():
"""
Run the NB algorithm
:return: acc,tags
"""
tags = []
nb = NBModel()
for example in utils.examples_test:
tags.append(nb.predict(example))
nb_acc = utils.get_acc(utils.tags_test, tags)
return nb_acc, tags
def get_results_knn():
"""
Run the KNN algorithm
:return: acc, tags
"""
tags = []
knn = KnnModel()
for example in utils.examples_test:
tags.append(knn.predict(example))
knn_acc = utils.get_acc(utils.tags_test, tags)
return knn_acc, tags
def get_results_dt():
"""
Run the Decision Tree algorithm
:return: acc, tags, tree
"""
tags_dtl = []
dt = DTModel()
for example in utils.examples_test:
tags_dtl.append(dt.predict(example))
dt_acc = utils.get_acc(utils.tags_test, tags_dtl)
tree = dt.get_constructed_dt(dt.root)
tree = tree[:len(tree) - 1]
return dt_acc, tags_dtl, tree
def train_model(model: nn.Module, task_classifier: nn.Module, dataset: ParentDataset,
loss: nn.Module, optim: torch.optim.Optimizer, binary: bool, disp_tqdm: bool = True) -> Tuple[
float, float]:
"""Performs an epoch of training on the provided model
:param conv_model: the ConvNet model. Takes care of transforming the sentence embeddings into a document embedding
:param sent_embedder: produces sentence embedding
:param task_classifier: the task-specific classifier. The output must be consistent with the task loss.
:param dataset: the dataset the models are trained on
:param loss: the loss function
:param optim: the optimizer the method should call after each batch
:param binary: if the task is binary or not
:param disp_tqdm: whether to display the progress bar or not (default: True).
:return: tuple containing average accuracy and average loss
"""
# important for BatchNorm layer
model.train()
task_classifier.train()
avg_acc = 0
avg_loss = 0
# display line
display_log = tqdm(dataset, total=0, position=1, bar_format='{desc}', disable=not disp_tqdm)
for i, (x, label) in tqdm(enumerate(dataset), total=len(dataset), position=0, disable=not disp_tqdm):
# Reset gradients
optim.zero_grad()
# Compute output
# x will be unpacked for compatibility with the finetuning mode
out = task_classifier(model(x))
# Compute loss
grad = loss(out, label)
# Backpropagate and update weights
grad.backward()
optim.step()
# Display results
acc = get_acc(out, label, binary)
avg_acc = (avg_acc * i + acc) / (i + 1)
avg_loss = (avg_loss * i + grad.item()) / (i + 1)
display_log.set_description_str(f"Batch {i:02d}:0 acc: {acc:.4f} loss: {grad.item():.4f}")
display_log.close()
return avg_acc, avg_loss
def test_acc(self, inp, resuse=True):
with tf.name_scope("test_acc"):
weights = self.weights
support_x, support_y, query_x, query_y = inp
# output_s = self.forward(self.support_x, weights, reuse=resuse)
# output_q = self.forward(self.query_x, weights, reuse=resuse)
output_s = vectorlize(self.forward(support_x, weights, reuse=resuse))
output_q = vectorlize(self.forward(query_x, weights, reuse=resuse))
if FLAGS.support_weight:
output_s = output_s + support_weight(output_s, support_y)
predict = category_choose(output_q, output_s, support_y)
accurcy = get_acc(predict, query_y)
return accurcy
def eval(epoch):
net.eval()
test_loss = 0.0
test_acc = 0.0
for batch_index, (imgs, labels) in enumerate(cifar100_test_loader):
imgs, labels = imgs.cuda(), labels.cuda()
outputs = net(imgs)
loss = criterion(outputs, labels)
test_loss += loss.item()
test_acc += get_acc(outputs, labels)
print("Test set: Loss: %.4f, Accuracy: %.4f" %
(test_loss / len(cifar100_test_loader),
test_acc / len(cifar100_test_loader)))
print("===============================================================")
def validate(xvalid, yvalid, model, loss, optimizer, device):
'''
Function for the validation step of the training loop
'''
model.eval()
xvalid = xvalid.to(device)
yvalid = yvalid.to(device)
# forward pass
yhat = model(xvalid)
# cost
total_loss = loss(yhat, yvalid)
# valid accuracy
accuracy = get_acc(yvalid, yhat)
return model, total_loss.item(), accuracy
def __call__(self, query_set, support_set, train='train'):
with tf.name_scope('loss_function'):
qx, qy, qm = query_set
sx, sy, sm = support_set
output_s = vectorlize(self.backbone(sx))
output_q = vectorlize(self.backbone(qx))
if FLAGS.prototype:
prototype_s, prototype_sy = get_prototype(output_s, sy)
predict = category_choose(output_q, prototype_s, prototype_sy)
accs = get_acc(predict, qy)
losses = classify_loss = self.loss_function(predict, qy)
if FLAGS.eps_loss and train == 'train':
epsloss = self.eps_loss(output_s, output_q, sm, qm, sy, qy,
FLAGS.margin)
losses = classify_loss * (
1.0 - FLAGS.weight) + FLAGS.weight * epsloss
loss = tf.reduce_mean(losses)
if train == 'train':
self.trainop(loss)
return [loss, accs]
def get_loss(self, inp, resuse=True, dist_weight=0.2):
with tf.name_scope("compute_loss"):
weights = self.weights
support_x, support_y, query_x, query_y, support_m, query_m = inp
output_s = vectorlize(self.forward(support_x, weights, reuse=resuse))
output_q = vectorlize(self.forward(query_x, weights, reuse=resuse))
if FLAGS.support_weight:
output_s = output_s + support_weight(output_s, support_y)
self.predict = predict = category_choose(output_q, output_s, support_y)
accurcy = get_acc(predict, query_y)
# task_losses = tf.map_fn(fn=lambda qxy: self.loss_function(qxy, output_s, support_y, 0.4),
# elems=(output_q, query_y), dtype=tf.float32,
# parallel_iterations=FLAGS.model * FLAGS.way_num * FLAGS.query_num)
# task_losses = self.loss_function((output_q, query_y), output_s, support_y)
task_losses = self.loss_function(predict, query_y)
# print("task_losses shape is:", task_losses.shape)
if FLAGS.intra_var:
task_losses = task_losses + intra_var(output_s, support_y) * 0.1 * tf.ones(shape=(FLAGS.model * FLAGS.way_num * FLAGS.query_num))
if FLAGS.inter_var:
task_losses = task_losses + (intra_var(output_s, support_y)/inter_var(output_s, support_y)) * 0.1 * tf.ones(shape=(FLAGS.model * FLAGS.way_num * FLAGS.query_num))
if FLAGS.eps_loss and FLAGS.category_loss:
self.losses_eps = losses_eps = self.eps(output_s, output_q, support_m, query_m, support_y,
query_y, FLAGS.margin)
losses = (1 - self.w) * task_losses + self.w * losses_eps
if not FLAGS.same_class_dist:
return losses, accurcy
else:
same_class_dist = scd(output_s, output_q, support_y, query_y)
# losses = (1 - self.w) * task_losses + (self.w - dist_weight) * losses_eps + dist_weight * same_class_dist
losses = (1 - self.w) * task_losses + (self.w) * losses_eps + dist_weight * same_class_dist
return losses, accurcy
elif FLAGS.category_loss:
return task_losses, accurcy
elif FLAGS.eps_loss:
self.losses_eps = losses_eps = loss_eps(output_s, output_q, support_m, query_m, support_y,
query_y, FLAGS.margin)
return losses_eps, accurcy
def train(xtrain, ytrain, model, loss, optimizer, device):
'''
Function for the training step of the training loop
'''
model.train()
xtrain = xtrain.to(device)
# forward pass
optimizer.zero_grad()
yhat = model(xtrain)
# cost
total_loss = loss(yhat, ytrain)
# train accuracy
accuracy = get_acc(ytrain, yhat)
# backward pass
total_loss.backward()
optimizer.step()
return model, optimizer, total_loss.item(), accuracy
def train(epoch):
net.train()
train_loss = 0.0
train_acc = 0.0
for batch_index, (imgs, labels) in enumerate(cifar100_train_loader):
if epoch <= args.warm:
warmup_scheduler.step()
#imgs, labels = imgs.to(device), labels.to(device)
imgs, labels = imgs.cuda(), labels.cuda()
outputs = net(imgs)
loss = criterion(outputs, labels)
optimizer.zero_grad()
loss.backward()
optimizer.step()
train_loss += loss.item()
train_acc += get_acc(outputs, labels)
print("Epoch %d/%d, train loss: %.4f, train acc: %.4f, LR: %.6f" %
(epoch, settings.EPOCH, train_loss / len(cifar100_train_loader),
train_acc / len(cifar100_train_loader),
optimizer.param_groups[0]['lr']))
# check summary shape , and value
val_acc, val_loss, preds = sess.run([accuracy, cost, pred_op],
feed_dict=test_feedDict)
val_loss_mean.append(val_loss)
pred_all.extend(preds)
test_feedDict = {
x_: test_imgs[(i + 1) * batch_size:],
y_: test_labs[(i + 1) * batch_size:],
phase_train: False
}
val_acc, val_loss, preds = sess.run([accuracy, cost, pred_op],
feed_dict=test_feedDict)
val_loss_mean.append(val_loss)
pred_all.extend(preds)
assert len(test_labs) == len(pred_all)
val_acc_mean = utils.get_acc(test_labs, pred_all)
val_loss_mean = np.mean(np.asarray(val_loss_mean))
summary = sess.run(merged, feed_dict=test_feedDict)
writer.add_summary(summary, step)
conv1_summary, topconv_summary, fc_summary = sess.run(
[conv1_summary_tensor, topconv_summary_tensor, fc_summary_tensor],
feed_dict=test_feedDict)
#print 'conv1 summary : ', conv1_summary
#print 'topconv summary : ', topconv_summary
#print 'FC summary : ', fc_summary
summary = tf.Summary(value=[
tf.Summary.Value(tag='Test batch : {} loss'.format(batch_size),
simple_value=float(val_loss_mean)),
tf.Summary.Value(tag='Test batch : {} acc'.format(batch_size),
simple_value=float(val_acc_mean)),
for epoch in range(epoch_count):
for step, (batch_x, batch_y) in enumerate(train_data_loader):
if if_cuda:
batch_x = batch_x.to(device)
batch_y = batch_y.to(device)
optimizer.zero_grad()
pred = net(batch_x)
loss = loss_function(pred, batch_y)
loss.backward()
optimizer.step()
# 计算准确率
this_test_acc[i] = get_acc(net.cpu(), test_x.cpu(), test_y.cpu())
this_train_acc[i] = get_acc(net.cpu(), train_x.cpu(),
train_y.cpu())
# 计算f1
this_test_f1[i] = get_f1_score(net.cpu(), test_x.cpu(),
test_y.cpu())
this_train_f1[i] = get_f1_score(net.cpu(), train_x.cpu(),
train_y.cpu())
mean_test_acc_list.append(this_test_acc.mean())
mean_train_acc_list.append(this_train_acc.mean())
np.save(training_path + 'test_acc_for_epoch_{}'.format(k_fold_time),
np.array(mean_test_acc_list))
np.save(training_path + 'train_acc_for_epoch_{}'.format(k_fold_time),
np.array(mean_train_acc_list))
# ben_david = E_s+MMD_loss
# E_ps = np.mean(w_source_pq * source_y_loss)
# iw = np.mean(src_qp * source_y_loss)
# d_supp = np.mean(np.logical_and((tgt_qp>=1),(target_pq[:,0]<epsilon)))-np.mean(np.logical_and((src_qp>=1),(source_pq[:,0]<epsilon)))
# bound = E_ps + M * d_supp# + beta * np.mean(w_source_pq) - beta
# print 'bound: %.4f E src: %.4f M: %.4f d supp: %.4f iw: %.4f E_s: %.4f MMD: %.4f' %(bound, E_ps, M, d_supp, iw, E_s, MMD_loss)
# source_train_pred = utils.get_data_pred(sess, model, 'y', source_train['images'], source_train['labels'])
# source_train_acc = utils.get_acc(source_train_pred, source_train['labels'])
source_valid_pred = utils.get_data_pred(sess, model, 'y',
source_valid['images'],
source_valid['labels'])
source_valid_acc = utils.get_acc(source_valid_pred,
source_valid['labels'])
target_valid_pred = utils.get_data_pred(sess, model, 'y',
target_valid['images'],
target_valid['labels'])
target_valid_acc = utils.get_acc(target_valid_pred,
target_valid['labels'])
# source_test_pred = utils.get_data_pred(sess, model, 'y', source_test['images'], source_test['labels'])
# source_test_acc = utils.get_acc(source_test_pred, source_test['labels'])
target_test_pred = utils.get_data_pred(sess, model, 'y',
target_test['images'],
target_test['labels'])
target_test_acc = utils.get_acc(target_test_pred,
target_test['labels'])
if i % 200 == 0:
print '%s iter %d loss: %f d_loss: %f p_acc: %f p: %f l: %f lr: %f' % \
(description, i, batch_loss, dloss, tp_acc, p, l, lr)
if i % valid_steps == 0:
train_pred = []
labtrain = []
for key in train.keys():
train_pred.append(
utils.get_data_pred(sess, model, 'y', train[key]['images'],
train[key]['labels']))
labtrain.append(train[key]['labels'])
train_pred = np.concatenate(train_pred, axis=0)
labtrain = np.concatenate(labtrain, axis=0)
train_acc = utils.get_acc(train_pred, labtrain)
valid_pred = []
labvalid = []
for key in valid.keys():
valid_pred.append(
utils.get_data_pred(sess, model, 'y', valid[key]['images'],
valid[key]['labels']))
labvalid.append(valid[key]['labels'])
valid_pred = np.concatenate(valid_pred, axis=0)
labvalid = np.concatenate(labvalid, axis=0)
valid_acc = utils.get_acc(valid_pred, labvalid)
test_pred = utils.get_data_pred(sess, model, 'y', test['images'],
test['labels'])
test_acc = utils.get_acc(test_pred, test['labels'])
seq8 = onehot_to_seq(pred, q8_list)
seq_true_3 = onehot_to_seq2(true, q3_list)
seq_true_8 = onehot_to_seq2(true, q8_list)
if i:
print('Q3 prediction, first pred then true: ')
print(seq3[:60])
print(seq_true_3[:60])
print('Q8 prediction, first pred then true: ')
print(seq8[:60])
print(seq_true_8[:60])
i = False
corr3, len3 = get_acc(seq_true_3, seq3)
corr8, len8 = get_acc(seq_true_8, seq8)
q8_accs.append(get_acc2(seq_true_8, seq8))
q3_accs.append(get_acc2(seq_true_3, seq3))
q3_pred += corr3
q8_pred += corr8
q3_len += len3
q8_len += len8
print("Accuracy #sum(correct per proteins)/#sum(len_proteins):")
print("Q3 " + test + " test accuracy: " + str(q3_pred / q3_len))
print("Q8 " + test + " test accuracy: " + str(q8_pred / q8_len))
print("\nAccuracy mean(#correct per protein/#len_protein):")
print("Q3 " + test + " test accuracy: " + str(np.mean(q3_accs)))
print("Q8 " + test + " test accuracy: " + str(np.mean(q8_accs)))
def main():
# read the train file from first arugment
train_file = sys.argv[1]
#train_file='../data/covtype.scale.trn.libsvm'
# read the test file from second argument
test_file = sys.argv[2]
#test_file = '../data/covtype.scale.tst.libsvm'
# You can use load_svmlight_file to load data from train_file and test_file
X_train, y_train = load_svmlight_file(train_file)
X_test, y_test = load_svmlight_file(test_file)
# You can use cg.ConjugateGradient(X, I, grad, lambda_)
# Main entry point to the program
X_train = sparse.hstack([X_train, np.ones((X_train.shape[0], 1))])
X_test = sparse.hstack([X_test, np.ones((X_test.shape[0], 1))])
X = sparse.csr_matrix(X_train)
X_test = sparse.csr_matrix(X_test)
y = sparse.csr_matrix(y_train).transpose()
y_test = sparse.csr_matrix(y_test).transpose()
#set global hyper parameter
if sys.argv[1] == "covtype.scale.trn.libsvm":
lambda_ = 3631.3203125
optimal_loss = 2541.664519
five_fold_CV = 75.6661
optimal_function_value = 2541.664519
else:
lambda_ = 7230.875
optimal_loss = 669.664812
five_fold_CV = 97.3655
optimal_function_value = 669.664812
#SGD
#set local sgd hyper parameter
print('starting SGD...')
n_batch = 1000
beta = 0
lr = 0.001
w = np.zeros((X_train.shape[1]))
n = X_train.shape[0]
sgd_grad = []
sgd_time = []
sgd_rel = []
sgd_test_acc = []
epoch = 180
start = time.time()
#redefine learaning rate
for i in range(epoch):
gamma_t = lr / (1 + beta * i)
batch_ = np.random.permutation(n) #shuffle
for j in range(n // n_batch):
#make batch
idx = batch_[j * n_batch:(j + 1) * n_batch]
X_bc = X[idx]
y_bc = y[idx]
grad = get_grad(w, lambda_, n, X_bc, y_bc,
n_batch) #comput gradient
w = w - gamma_t * grad #update gradient
t = time.time() - start
sgd_time.append(t) # append to time list
grad_ = np.linalg.norm(grad) # get gradient value
sgd_grad.append(grad_)
rel = (get_loss(w, lambda_, X_test, y_test, n_batch) -
optimal_loss) / optimal_loss # get relative func value
sgd_rel.append(rel)
test_acc = get_acc(w, lambda_, X_test, y_test,
n_batch) # get test accuracy
sgd_test_acc.append(test_acc)
print("SGD : final_time: {}, fina_test_acc: {}".format(
time.time() - start, sgd_test_acc[-1]))
#plot SGD
'''
plt.plot(sgd_time, sgd_grad)
plt.xlabel("time")
plt.ylabel("grad")
plt.title("SGD")
plt.show()
plt.plot(sgd_time, sgd_rel)
plt.xlabel("time")
plt.ylabel("relative function")
plt.title("SGD")
plt.show()
plt.plot(sgd_time, sgd_test_acc)
plt.xlabel("time")
plt.ylabel("test_acc")
plt.title("SGD")
plt.show()
'''
print('starting Newton...')
#Newton
#set local newton hyper parameter
epoch = 50
n_batch = 1000
beta = 0.0001
lr = 0.001
w = np.zeros((X_train.shape[1]))
n = X_train.shape[0]
nt_grad = []
nt_time = []
nt_rel = []
newton_time = time.time()
nt_test_acc = []
w = np.zeros((X_train.shape[1]))
n = X_train.shape[0]
for i in range(epoch):
gamma_t = lr / (1 + beta * i)
hessian_total = np.zeros(w.shape)
I_ = [] #init I list to compute conjgate gradient
for j in range(n // n_batch):
X_bc = X[j * n_batch:(j + 1) * n_batch] #make X_batch
y_bc = y[j * n_batch:(j + 1) * n_batch] #make y_batch
hessian, I = get_hessian(w, lambda_, n, X_bc, y_bc) # get hessian
hessian_total += hessian
I_.append(I)
I_ = np.concatenate(I_)
hessian_total += w
delta, _ = cg.conjugateGradient(
X, I_, hessian_total,
lambda_) #get update value from conjugateGradient
w = w + delta #update w
t = time.time() - newton_time
nt_time.append(t) # append to time list
grad_ = np.linalg.norm(hessian_total) # get gradient value
nt_grad.append(grad_)
rel = (get_loss(w, lambda_, X_test, y_test, n_batch) -
optimal_loss) / optimal_loss # get relative func value
nt_rel.append(rel)
test_acc = get_acc(w, lambda_, X_test, y_test,
n_batch) # get test accuracy
nt_test_acc.append(test_acc)
final_time = time.time() - newton_time
print("final_time: {}, fina_test_acc: {}".format(final_time,
nt_test_acc[-1]))
#plot
'''
for epoch in range(epoch_count):
for step, (batch_x, batch_y) in enumerate(train_data_loader):
if if_cuda:
batch_x = batch_x.to(device)
batch_y = batch_y.to(device)
optimizer.zero_grad()
pred = net(batch_x)
loss = loss_function(pred, batch_y)
loss.backward()
optimizer.step()
# 计算准确率
this_test_acc[i] = get_acc(net, test_x, test_y)
this_train_acc[i] = get_acc(net, train_x, train_y)
# 计算f1
this_test_f1[i] = get_f1_score(net, test_x, test_y)
this_train_f1[i] = get_f1_score(net, train_x, train_y)
mean_test_acc_list.append(this_test_acc.mean())
mean_train_acc_list.append(this_train_acc.mean())
np.save('test_acc_for_epoch_{}'.format(k_fold_time),
np.array(mean_test_acc_list))
np.save('train_acc_for_epoch_{}'.format(k_fold_time),
np.array(mean_train_acc_list))
mean_test_f1_list.append(this_test_f1.mean())
mean_train_f1_list.append(this_train_f1.mean())
def train_one_epoch(epoch, net):
print('\n Training for Epoch: %d' % epoch)
net.train()
# learning rate schedule
if epoch < args.decay_epoch1:
lr = args.lr
elif epoch < args.decay_epoch2:
lr = args.lr * args.decay_rate
else:
lr = args.lr * args.decay_rate * args.decay_rate
for param_group in optimizer.param_groups:
param_group['lr'] = lr
iterator = tqdm(trainloader, ncols=0, leave=False)
for batch_idx, (inputs, targets) in enumerate(iterator):
start_time = time.time()
inputs, targets = inputs.to(device), targets.to(device)
targets_onehot = one_hot_tensor(targets, args.num_classes, device)
x_tilde, y_tilde = adv_interp(inputs, targets_onehot, net,
args.num_classes,
config_adv_interp['epsilon'],
config_adv_interp['label_adv_delta'],
config_adv_interp['v_min'],
config_adv_interp['v_max'])
outputs = net(x_tilde, mode='logits')
loss = soft_xent_loss(outputs, y_tilde)
optimizer.zero_grad()
loss.backward()
optimizer.step()
train_loss = loss.detach().item()
duration = time.time() - start_time
if batch_idx % args.log_step == 0:
adv_acc = utils.get_acc(outputs, targets)
# natural
net_cp = copy.deepcopy(net)
nat_outputs = net_cp(inputs, mode='logits')
nat_acc = utils.get_acc(nat_outputs, targets)
print(
"Epoch %d, Step %d, lr %.4f, Duration %.2f, Training nat acc %.2f, Training adv acc %.2f, Training adv loss %.4f"
% (epoch, batch_idx, lr, duration, 100 * nat_acc,
100 * adv_acc, train_loss))
if epoch % args.save_epochs == 0 or epoch >= args.max_epoch - 2:
print('Saving..')
f_path = os.path.join(args.model_dir, ('checkpoint-%s' % epoch))
state = {
'net': net.state_dict(),
'epoch': epoch,
#'optimizer': optimizer.state_dict()
}
if not os.path.isdir(args.model_dir):
os.makedirs(args.model_dir)
torch.save(state, f_path)
if epoch >= 1:
print('Saving latest model for epoch %s..' % (epoch))
f_path = os.path.join(args.model_dir, 'latest')
state = {
'net': net.state_dict(),
'epoch': epoch,
#'optimizer': optimizer.state_dict()
}
if not os.path.isdir(args.model_dir):
os.mkdir(args.model_dir)
torch.save(state, f_path)
