def eval_search_engine(res_fname, format, th=50): ir = read_res_file(res_fname, format) # evaluate IR rec = metrics.recall_of_1(ir, th) acc = metrics.accuracy(ir, th) acc1 = metrics.accuracy1(ir, th) acc2 = metrics.accuracy2(ir, th) mrr = metrics.mrr(ir, th) # MAP map_ir = metrics.map(ir) print "%10s" %"IR" print "MRR: %5.2f" % mrr print "MAP: %5.2f" % map_ir for i, (r, a, a1, a2) in enumerate(zip(rec, acc, acc1, acc2), 1): print "REC-1@%02d: %6.2f ACC@%02d: %6.2f AC1@%02d: %6.2f AC2@%02d: %4.0f" %(i, r, i, a, i, a1, i, a2) print print "REC-1 - percentage of questions with at least 1 correct answer in the top @X positions (useful for tasks were questions have at most one correct answer)" print "ACC - accuracy, i.e. number of correct answers retrieved at rank @X normalized by the rank and the total number of questions" print "AC1 - the number of correct answers at @X normalized by the number of maximum possible answers (perfect re-ranker)" print "AC2 - the absolute number of correct answers at @X"
def eval_reranker(res_fname="svm.test.res", pred_fname="svm.train.pred",
format="trec",
th=50,
verbose=False,
reranking_th=0.0,
ignore_noanswer=False):
ir, svm = read_res_pred_files(res_fname, pred_fname, format, verbose,
reranking_th=reranking_th,
ignore_noanswer=ignore_noanswer)
# evaluate IR
prec_se = metrics.recall_of_1(ir, th)
acc_se = metrics.accuracy(ir, th)
acc_se1 = metrics.accuracy1(ir, th)
acc_se2 = metrics.accuracy2(ir, th)
# evaluate SVM
prec_svm = metrics.recall_of_1(svm, th)
acc_svm = metrics.accuracy(svm, th)
acc_svm1 = metrics.accuracy1(svm, th)
acc_svm2 = metrics.accuracy2(svm, th)
mrr_se = metrics.mrr(ir, th)
mrr_svm = metrics.mrr(svm, th)
map_se = metrics.map(ir)
map_svm = metrics.map(svm)
avg_acc1_svm = metrics.avg_acc1(svm, th)
avg_acc1_ir = metrics.avg_acc1(ir, th)
print "%13s %5s" %("IR", "SVM")
print "MRR: %5.2f %5.2f" %(mrr_se, mrr_svm)
print "MAP: %5.4f %5.4f" %(map_se, map_svm)
print "AvgRec: %5.2f %5.2f" %(avg_acc1_ir, avg_acc1_svm)
print "%16s %6s %14s %6s %14s %6s %12s %4s" % ("IR", "SVM", "IR", "SVM", "IR", "SVM", "IR", "SVM")
for i, (p_se, p_svm, a_se, a_svm, a_se1, a_svm1, a_se2, a_svm2) in enumerate(zip(prec_se, prec_svm, acc_se, acc_svm, acc_se1, acc_svm1, acc_se2, acc_svm2), 1):
print "REC-1@%02d: %6.2f %6.2f ACC@%02d: %6.2f %6.2f AC1@%02d: %6.2f %6.2f AC2@%02d: %4.0f %4.0f" %(i, p_se, p_svm, i, a_se, a_svm, i, a_se1, a_svm1, i, a_se2, a_svm2)
print
print "REC-1 - percentage of questions with at least 1 correct answer in the top @X positions (useful for tasks were questions have at most one correct answer)"
print "ACC - accuracy, i.e. number of correct answers retrieved at rank @X normalized by the rank and the total number of questions"
print "AC1 - the number of correct answers at @X normalized by the number of maximum possible answers (perfect re-ranker)"
print "AC2 - the absolute number of correct answers at @X"
def train_teacher(dataset, nb_teachers, teacher_id):
"""
This function trains a teacher (teacher id) among an ensemble of nb_teachers
models for the dataset specified.
:param dataset: string corresponding to dataset (svhn, cifar10)
:param nb_teachers: total number of teachers in the ensemble
:param teacher_id: id of the teacher being trained
:return: True if everything went well
"""
# If working directories do not exist, create them
assert input.create_dir_if_needed(FLAGS.data_dir)
assert input.create_dir_if_needed(FLAGS.train_dir)
# Load the dataset
if dataset == 'svhn':
train_data,train_labels,test_data,test_labels = input.ld_svhn(extended=True)
elif dataset == 'cifar10':
train_data, train_labels, test_data, test_labels = input.ld_cifar10()
elif dataset == 'mnist':
train_data, train_labels, test_data, test_labels = input.ld_mnist()
else:
print("Check value of dataset flag")
return False
# Retrieve subset of data for this teacher
data, labels = input.partition_dataset(train_data,
train_labels,
nb_teachers,
teacher_id)
print("Length of training data: " + str(len(labels)))
# Define teacher checkpoint filename and full path
if FLAGS.deeper:
filename = str(nb_teachers) + '_teachers_' + str(teacher_id) + '_deep.ckpt'
else:
filename = str(nb_teachers) + '_teachers_' + str(teacher_id) + '.ckpt'
ckpt_path = FLAGS.train_dir + '/' + str(dataset) + '_' + filename
# Perform teacher training
assert deep_cnn.train(data, labels, ckpt_path)
# Append final step value to checkpoint for evaluation
ckpt_path_final = ckpt_path + '-' + str(FLAGS.max_steps - 1)
# Retrieve teacher probability estimates on the test data
teacher_preds = deep_cnn.softmax_preds(test_data, ckpt_path_final)
# Compute teacher accuracy
precision = metrics.accuracy(teacher_preds, test_labels)
print('Precision of teacher after training: ' + str(precision))
return True
def compute_all_metrics(execution_id, path_input, path_output, formula, append):
from metrics import accuracy, precision, recall, f1, specificity
"""
Computes all metrics and persistes in a csv
Args:
execution_id (int): identifier of the execution
path_input (string): path of the file that contains the classifications
path_out (string): path of the file that will persist the metrics
formula (string): mean_max | mean_mean
append (boolean): true | false
"""
# loading results
with open(path_input) as data_file:
data = json.load(data_file)
# computing metrics
tp = tn = fp = fn = 0
for i in range(0, len(data)):
if (data[i]['values'][formula]['positive'] >= data[i]['values'][formula]['negative']):
if data[i]['values']['label'] == 'positive':
tp += 1
else:
fp += 1
elif (data[i]['values'][formula]['positive'] < data[i]['values'][formula]['negative']):
if (data[i]['values']['label'] == 'negative'):
tn += 1
else:
fn += 1
else:
raise Exception("Positive similarity equals to negative similarity to news " + data[i]['id'])
accuracy = accuracy(tp, tn, fp, fn)
recall = recall(tp, fn)
precision = precision(tp, fp)
f1 = f1(precision, recall);
specificity = specificity(tn, fp);
# persiting the results
with open(path_output, 'a' if append else 'w') as csvfile:
spamwriter = csv.writer(csvfile, delimiter=',')
if (not append):
spamwriter.writerow(
['execution_id', 'tp', 'tn', 'fp', 'fn', 'accuracy', 'precision', 'recall', 'f1', 'specificity'])
spamwriter.writerow([execution_id, tp, tn, fp, fn, accuracy, precision, recall, f1, specificity])
def train_student(dataset, nb_teachers):
"""
This function trains a student using predictions made by an ensemble of
teachers. The student and teacher models are trained using the same
neural network architecture.
:param dataset: string corresponding to mnist, cifar10, or svhn
:param nb_teachers: number of teachers (in the ensemble) to learn from
:return: True if student training went well
"""
assert input.create_dir_if_needed(FLAGS.train_dir)
# Call helper function to prepare student data using teacher predictions
stdnt_dataset = prepare_student_data(dataset, nb_teachers, save=True)
# Unpack the student dataset
stdnt_data, stdnt_labels, stdnt_test_data, stdnt_test_labels = stdnt_dataset
# Prepare checkpoint filename and path
if FLAGS.deeper:
ckpt_path = FLAGS.train_dir + '/' + str(dataset) + '_' + str(nb_teachers) + '_student_deeper.ckpt' #NOLINT(long-line)
else:
ckpt_path = FLAGS.train_dir + '/' + str(dataset) + '_' + str(nb_teachers) + '_student.ckpt' # NOLINT(long-line)
# Start student training
assert deep_cnn.train(stdnt_data, stdnt_labels, ckpt_path)
# Compute final checkpoint name for student (with max number of steps)
ckpt_path_final = ckpt_path + '-' + str(FLAGS.max_steps - 1)
# Compute student label predictions on remaining chunk of test set
student_preds = deep_cnn.softmax_preds(stdnt_test_data, ckpt_path_final)
# Compute teacher accuracy
precision = metrics.accuracy(student_preds, stdnt_test_labels)
print('Precision of student after training: ' + str(precision))
return True
def prepare_student_data(dataset, nb_teachers, save=False):
"""
Takes a dataset name and the size of the teacher ensemble and prepares
training data for the student model, according to parameters indicated
in flags above.
:param dataset: string corresponding to mnist, cifar10, or svhn
:param nb_teachers: number of teachers (in the ensemble) to learn from
:param save: if set to True, will dump student training labels predicted by
the ensemble of teachers (with Laplacian noise) as npy files.
It also dumps the clean votes for each class (without noise) and
the labels assigned by teachers
:return: pairs of (data, labels) to be used for student training and testing
"""
assert input.create_dir_if_needed(FLAGS.train_dir)
# Load the dataset
if dataset == 'svhn':
test_data, test_labels = input.ld_svhn(test_only=True)
elif dataset == 'cifar10':
test_data, test_labels = input.ld_cifar10(test_only=True)
elif dataset == 'mnist':
test_data, test_labels = input.ld_mnist(test_only=True)
else:
print("Check value of dataset flag")
return False
# Make sure there is data leftover to be used as a test set
assert FLAGS.stdnt_share < len(test_data)
# Prepare [unlabeled] student training data (subset of test set)
stdnt_data = test_data[:FLAGS.stdnt_share]
# Compute teacher predictions for student training data
teachers_preds = ensemble_preds(dataset, nb_teachers, stdnt_data)
# Aggregate teacher predictions to get student training labels
if not save:
stdnt_labels = aggregation.noisy_max(teachers_preds, FLAGS.lap_scale)
else:
# Request clean votes and clean labels as well
stdnt_labels, clean_votes, labels_for_dump = aggregation.noisy_max(teachers_preds, FLAGS.lap_scale, return_clean_votes=True) #NOLINT(long-line)
# Prepare filepath for numpy dump of clean votes
filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(nb_teachers) + '_student_clean_votes_lap_' + str(FLAGS.lap_scale) + '.npy' # NOLINT(long-line)
# Prepare filepath for numpy dump of clean labels
filepath_labels = FLAGS.data_dir + "/" + str(dataset) + '_' + str(nb_teachers) + '_teachers_labels_lap_' + str(FLAGS.lap_scale) + '.npy' # NOLINT(long-line)
# Dump clean_votes array
with gfile.Open(filepath, mode='w') as file_obj:
np.save(file_obj, clean_votes)
# Dump labels_for_dump array
with gfile.Open(filepath_labels, mode='w') as file_obj:
np.save(file_obj, labels_for_dump)
# Print accuracy of aggregated labels
ac_ag_labels = metrics.accuracy(stdnt_labels, test_labels[:FLAGS.stdnt_share])
print("Accuracy of the aggregated labels: " + str(ac_ag_labels))
# Store unused part of test set for use as a test set after student training
stdnt_test_data = test_data[FLAGS.stdnt_share:]
stdnt_test_labels = test_labels[FLAGS.stdnt_share:]
if save:
# Prepare filepath for numpy dump of labels produced by noisy aggregation
filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(nb_teachers) + '_student_labels_lap_' + str(FLAGS.lap_scale) + '.npy' #NOLINT(long-line)
# Dump student noisy labels array
with gfile.Open(filepath, mode='w') as file_obj:
np.save(file_obj, stdnt_labels)
return stdnt_data, stdnt_labels, stdnt_test_data, stdnt_test_labels
x_values = [] x_values_fscore = [] # Para cada porcentaje de confianza for i in xrange(100): # Obtengo las predicciones con una confianza mayor a cierto umbral porcentaje = float(i)/100 aux = result[result['trust'] > porcentaje] # matrix = metrics.confusion_matrix(aux) matrix = metrics.hard_matrix(aux) # Si la precision es menor que cero, es porque no habian datos que superaran tal nivel de confianza precision = metrics.accuracy(matrix, clase) if precision >= 0: valores_accuracy.append(precision) valores_recall.append(metrics.recall(matrix, clase)) x_values.append(porcentaje) # Si el f_score es menor que cero, es porque no habian datos que superaran tal nivel de confianza f_score = metrics.f_score(matrix, clase) if f_score >= 0: valores_fscore.append(f_score) x_values_fscore.append(porcentaje) #graf(clase, x_values, valores_accuracy, 'Accuracy') graf(clase, x_values, valores_recall, 'Recall') #graf(clase, x_values_fscore, valores_fscore, 'F-Score') print 'a'
# длина периодов сезонных составляющих
p1 = 24
p2 = 168
# тренировочный датасет train_set, данные с 14.01.2019 по 25.11.2019
train_start_date = datetime.datetime(2019, 1, 14, 0, 0, 0)
train_end_date = datetime.datetime(2019, 11, 25, 23, 0, 0)
train_set = fact[train_start_date:train_end_date]
# тестовый датасет test_set, данные за прогнозный день 27.11.2019
pred_date = datetime.date(2019, 11, 27)
pred_end_date = train_end_date + datetime.timedelta(days=2)
pred_start_date = pred_end_date - datetime.timedelta(hours=23)
test_set = fact[pred_start_date:pred_end_date]
# план предприятия plan_set для сверки, данные за прогнозный день 27.11.2019
plan_set = plan[pred_start_date:pred_end_date]
# выбор метода
method = double_seasonal_multiplicativeHoltWinters
# найденные постоянные сглаживания
best_params = [[0.334097458, 0.000291639411, 0.278284232, 0.123083456]]
# прогнозирование
pred = method(best_params[0], train_set, p1, p2, 48)[24:]
# оценка прогноза по метрикам
nnf_val = nnfmetrics(pred, test_set, plan_set)
mse_val = mse(test_set, pred)
mape_val = mape(test_set, pred)
acc_val = accuracy(test_set, pred)
print('Оценка по NNFMETRICS = ', nnf_val)
print('Оценка по MSE = ', mse_val)
print('Оценка по MAPE = ', mape_val)
print('Точность прогноза = ', acc_val)
# отрисовка графика
plot_results(pred_date.strftime('%d-%m-%Y'), test_set, pred, plan_set)
def metrics_for_eval(model,
data_loader,
device,
loss_fn,
topk=2,
get_balanced_acc=False):
"""
This function returns accuracy, topk accuracy and loss for the evaluation partition
:param model (nn.Model): the model you'd like to evaluate
:param data_loader (DataLoader): the DataLoader containing the data partition
:param checkpoint_path(string, optional): string with a checkpoint to load the model. If None, none checkpoint is
loaded. Default is None.
:param loss_fn (nn.Loss): the loss function used in the training
:param get_balanced_acc (bool, optional); if you'd like to compute the balanced accuracy for the eval partition
set it as True. The defaulf is False because it may impact in the training phase.
:return: a instance of the classe metrics
"""
# setting the model to evaluation mode
model.eval()
print("\nEvaluating...")
# Setting tqdm to show some information on the screen
with tqdm(total=len(data_loader), ascii=True, ncols=100) as t:
# Setting require_grad=False in order to dimiss the gradient computation in the graph
with torch.no_grad():
loss_avg = AVGMetrics()
acc_avg = AVGMetrics()
topk_avg = AVGMetrics()
if get_balanced_acc:
# Setting the metrics object
metrics = Metrics(['balanced_accuracy'])
else:
metrics = None
for data in data_loader:
# In data we may have imgs, labels and extra info. If extra info is [], it means we don't have it
# for the this training case. Imgs came in data[0], labels in data[1] and extra info in data[2]
try:
images_batch, labels_batch, extra_info_batch, _ = data
except ValueError:
images_batch, labels_batch = data
extra_info_batch = []
if (len(extra_info_batch)):
# Moving the data to the deviced that we set above
images_batch, labels_batch = images_batch.to(
device), labels_batch.to(device)
extra_info_batch = extra_info_batch.to(device)
# Doing the forward pass using the extra info
pred_batch = model(images_batch, extra_info_batch)
else:
# Moving the data to the deviced that we set above
images_batch, labels_batch = images_batch.to(
device), labels_batch.to(device)
# Doing the forward pass without the extra info
pred_batch = model(images_batch)
# Computing the loss
L = loss_fn(pred_batch, labels_batch)
# Computing the accuracy
acc, topk_acc = accuracy(pred_batch,
labels_batch,
topk=(1, topk))
loss_avg.update(L.item())
acc_avg.update(acc.item())
topk_avg.update(topk_acc.item())
if metrics is not None:
# Moving the data to CPU and converting it to numpy in order to compute the metrics
pred_batch_np = nnF.softmax(pred_batch,
dim=1).cpu().numpy()
labels_batch_np = labels_batch.cpu().numpy()
# updating the scores
metrics.update_scores(labels_batch_np, pred_batch_np)
# Updating tqdm
t.set_postfix(loss='{:05.3f}'.format(loss_avg()))
t.update()
if metrics is not None:
# Getting the metrics
metrics.compute_metrics()
bal_acc = metrics.metrics_values['balanced_accuracy']
else:
bal_acc = None
return {
"loss": loss_avg(),
"accuracy": acc_avg(),
"topk_acc": topk_avg(),
"balanced_accuracy": bal_acc
}
net = net.to(device)
net_dict = net.state_dict()
pretrained_dict = {k: v for k, v in checkpoint['net'].items() if k in net_dict}
if 'anchors' not in pretrained_dict.keys():
pretrained_dict['anchors'] = checkpoint['net']['means']
net.load_state_dict(pretrained_dict)
net.eval()
#find mean anchors for each class
anchor_means = find_anchor_means(net, mapping, args.dataset, trial_num, cfg, only_correct = True)
net.set_anchors(torch.Tensor(anchor_means))
print('==> Evaluating open set network accuracy for trial {}..'.format(trial_num))
x, y = gather_outputs(net, mapping, knownloader, data_idx = 1, calculate_scores = True)
accuracy = metrics.accuracy(x, y)
all_accuracy += [accuracy]
print('==> Evaluating open set network AUROC for trial {}..'.format(trial_num))
xK, yK = gather_outputs(net, mapping, knownloader, data_idx = 1, calculate_scores = True)
xU, yU = gather_outputs(net, mapping, unknownloader, data_idx = 1, calculate_scores = True, unknown = True)
auroc = metrics.auroc(xK, xU)
all_auroc += [auroc]
mean_auroc = np.mean(all_auroc)
mean_acc = np.mean(all_accuracy)
print('Raw Top-1 Accuracy: {}'.format(all_accuracy))
print('Raw AUROC: {}'.format(all_auroc))
print('Average Top-1 Accuracy: {}'.format(mean_acc))
pcaModel.fit(normalisedTrainData)
pcaTrainData = pcaModel.transform(normalisedTrainData)
pcaTestData = pcaModel.transform(normalisedTestData)
trainLabels = pp.convertLabels(trainLabels)
testLabels = pp.convertLabels(testLabels)
if (useNaiveBayes):
NB = MultinomialNB()
nbModel = NB.fit(np.array(normalisedTrainData),
np.array(trainLabels))
predictionsMade = nbModel.predict(
np.array(normalisedTestData)).tolist()
tempAcc = met.accuracy(predictionsMade, testLabels)
#tempF1 = met.precision_score(predictionsMade, testLabels)
tempF1 = f1_score(testLabels, predictionsMade, average=None)
nbAccuracy += tempAcc
nbF1Score += tempF1
if (useSupportVectorMachine):
svmModel = svm.SVC(gamma="scale")
svmModel.fit(np.array(pcaTrainData), np.array(trainLabels))
predictionsMade = svmModel.predict(
np.array(pcaTestData)).tolist()
tempAcc = met.accuracy(predictionsMade, testLabels)
#tempF1 = met.precision_score(predictionsMade, testLabels)
tempF1 = f1_score(testLabels, predictionsMade, average=None)
svmAccuracy += tempAcc
def main():
visualiseTrainingExamples()
nn = NeuralNetwork(config.numLayers, config.numClasses,
config.weightInitialisation, config.activationFn,
config.weightDecay)
nn.initialiseParams(len(x_train[0]) * len(x_train[0]), config.numNeurons)
sample = np.random.randint(3 * len(x_train) / 4)
nn.forwardPropagate(x_train[sample])
if config.optimizer == "sgd":
nn.stochasticGradDesc(x_train, y_train, config.maxIterations,
config.learningRate, config.batchSize)
elif config.optimizer == "momentum":
nn.momentumGradDesc(x_train, y_train, config.maxIterations,
config.learningRate, config.batchSize,
config.gamma)
elif config.optimizer == "nesterov":
nn.nesterovAcceleratedGradDesc(x_train, y_train, config.maxIterations,
config.learningRate, config.batchSize,
config.gamma)
elif config.optimizer == "rmsprop":
nn.rmsprop(x_train, y_train, config.maxIterations, config.learningRate,
config.batchSize, config.eps, config.beta1)
elif config.optimizer == "adam":
nn.adam(x_train, y_train, config.maxIterations, config.learningRate,
config.batchSize, config.eps, config.beta1, config.beta2)
elif config.optimizer == "nadam":
nn.nadam(x_train, y_train, config.maxIterations, config.learningRate,
config.batchSize, config.eps, config.beta1, config.beta2)
else:
print("No such optimizer available.")
predictions = []
predProbs = []
test_acc = 0
test_entropy = 0
test_mse = 0
for i in range(len(x_test)):
nn.forwardPropagate(x_test[i])
predictions.append(nn.predictedClass)
predProbs.append(nn.output[nn.predictedClass])
test_acc = accuracy(y_test, predictions)
test_entropy = crossEntropyLoss(y_test, predProbs)
test_mse = MSEloss(y_test, predictions)
predictions = []
predProbs = []
valid_acc = 0
valid_entropy = 0
valid_mse = 0
for i in range(len(x_valid)):
nn.forwardPropagate(x_valid[i])
predictions.append(nn.predictedClass)
predProbs.append(nn.output[nn.predictedClass])
valid_acc = accuracy(y_valid, predictions)
valid_entropy = crossEntropyLoss(y_valid, predProbs)
valid_mse = MSEloss(y_valid, predictions)
print(
f"Test Set:\nAccuracy = {test_acc}\nLoss = {test_entropy}\nMSE = {test_mse}"
)
print(
f"Validation Set:\nAccuracy = {valid_acc}\nLoss = {valid_entropy}\nMSE = {valid_mse}"
)
# #Log in wandb
metrics = {
'test_acc': test_acc,
# 'test_entropy': test_entropy,
"test_mse": test_mse,
'valid_acc': valid_acc,
# 'valid_entropy': valid_entropy,
"valid_mse": valid_mse
}
wandb.log(metrics)
run.finish()
def eval_reranker(res_fname="svm.test.res",
pred_fname="svm.train.pred",
format="trec",
th=50,
verbose=False,
reranking_th=-100.0,
ignore_noanswer=False,
ignore_allanswer=False):
ir, svm = read_res_pred_files(res_fname,
pred_fname,
format,
verbose,
reranking_th=reranking_th,
ignore_noanswer=ignore_noanswer,
ignore_allanswer=ignore_allanswer)
# evaluate IR
prec_se = metrics.recall_of_1(ir, th)
acc_se = metrics.accuracy(ir, th)
acc_se1 = metrics.accuracy1(ir, th)
acc_se2 = metrics.accuracy2(ir, th)
# evaluate SVM
prec_svm = metrics.recall_of_1(svm, th)
acc_svm = metrics.accuracy(svm, th)
acc_svm1 = metrics.accuracy1(svm, th)
acc_svm2 = metrics.accuracy2(svm, th)
mrr_se = metrics.mrr(ir, th)
mrr_svm = metrics.mrr(svm, th)
map_se = metrics.map(ir)
map_svm = metrics.map(svm)
avg_acc1_svm = metrics.avg_acc1(svm, th)
avg_acc1_ir = metrics.avg_acc1(ir, th)
'''
print "%13s %5s" %("IR", "SVM")
print "MRR: %5.2f %5.2f" %(mrr_se, mrr_svm)
print "MAP: %5.4f %5.4f" %(map_se, map_svm)
print "AvgRec: %5.2f %5.2f" %(avg_acc1_ir, avg_acc1_svm)
print "%16s %6s %14s %6s %14s %6s %12s %4s" % ("IR", "SVM", "IR", "SVM", "IR", "SVM", "IR", "SVM")
'''
rec1_se = -10
rec1_svm = -10
for i, (p_se, p_svm, a_se, a_svm, a_se1, a_svm1, a_se2,
a_svm2) in enumerate(
zip(prec_se, prec_svm, acc_se, acc_svm, acc_se1, acc_svm1,
acc_se2, acc_svm2), 1):
#print "REC-1@%02d: %6.2f %6.2f ACC@%02d: %6.2f %6.2f AC1@%02d: %6.2f %6.2f AC2@%02d: %4.0f %4.0f" %(i, p_se, p_svm, i, a_se, a_svm, i, a_se1, a_svm1, i, a_se2, a_svm2)
if (rec1_se < -5):
rec1_se = p_se
rec1_svm = p_svm
'''
print "REC-1 - percentage of questions with at least 1 correct answer in the top @X positions (useful for tasks were questions have at most one correct answer)"
print "ACC - accuracy, i.e. number of correct answers retrieved at rank @X normalized by the rank and the total number of questions"
print "AC1 - the number of correct answers at @X normalized by the number of maximum possible answers (perfect re-ranker)"
print "AC2 - the absolute number of correct answers at @X"
'''
print "Table view"
print " MRR MAP P@1"
print "REF_FILE %5.2f %5.2f %5.2f" % (mrr_se, map_se * 100, rec1_se)
print "SVM %5.2f %5.2f %5.2f" % (mrr_svm, map_svm * 100, rec1_svm)
def accuracy_fn(self, X, y):
preds = self.predict(X)
return metrics.accuracy(y, preds)
def main():
train_path = sys.argv[1] + '\\train\\'
test_path = sys.argv[1] + '\\test\\'
# load training data
print(f'[INFO] - Loading training data from {train_path}')
res = read_data(train_path)
train_data = res[0]
train_target = res[1]
print(f'[INFO] - Total train data: {len(train_data)}')
print(f'[INFO] - Loading testing data from {test_path}')
res = read_data(test_path)
test_data = res[0]
test_target = res[1]
print(f'[INFO] - Total test data: {len(test_data)}')
# 10% of training data will go to developer data set
print(f'[INFO] - Splitting training data into training data and developer data (keeping 10% for training data)')
res = train_test_split(train_data, train_target, test_size=0.1)
train_data = res[0]
train_target = res[2]
print(f'[INFO] - Total training data after split {len(train_data)}')
dev_data = res[1]
dev_target = res[3]
print(f'[INFO] - Total developer data {len(dev_data)}')
nb = MultinomialNaiveBayes()
accuracy_train = []
accuracy_test = []
counter = 1
for train_size in [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]:
print(f'\n[INFO] - Iteration No.{counter} (using {int(train_size*100)}% of 90% of train data).')
if train_size != 1.0:
res = train_test_split(train_data, train_target, train_size=train_size, shuffle=False)
fold_data = res[0]
fold_target = res[2]
else:
fold_data = train_data
fold_target = train_target
feature_size = 0.007
print(f'[INFO] - Fitting Multinomial Naive Bayes classifier using {feature_size*100:.1f}% of features...')
nb.fit(fold_data, fold_target, feature_size)
print(f'[INFO] - Predicting with Multinomial Naive Bayes classifier using train data...')
nb_targets, _ = nb.predict(fold_data)
accuracy_score = accuracy(fold_target, nb_targets)
accuracy_train.append(accuracy_score)
print(f'[INFO] - Accuracy: {accuracy_score}')
print(f'[INFO] - Predicting with Multinomial Naive Bayes classifier using developer data...')
nb_targets, _ = nb.predict(dev_data)
accuracy_score = accuracy(dev_target, nb_targets)
print(f'[INFO] - Accuracy: {accuracy_score}')
print(f'[INFO] - Predicting with Multinomial Naive Bayes classifier using test data...')
nb_targets, probabilities = nb.predict(test_data)
accuracy_score = accuracy(test_target, nb_targets)
accuracy_test.append(accuracy_score)
print(f'[INFO] - Accuracy: {accuracy_score}')
counter += 1
learning_curves_plot = plt.figure(1)
plt.plot([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], accuracy_train, label='train')
plt.plot([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], accuracy_test, label='test')
plt.title('Learning Curves (Multinomial Naive Bayes)')
plt.legend(loc='lower right')
plt.xlabel('Number of Train Data')
plt.ylabel('Accuracy')
precision_recall_plot = plt.figure(2)
average_precision, average_recall, thresholds = precision_recall(probabilities, test_target, 10)
plt.step(average_recall, average_precision, where='post')
plt.title('Precision-Recall Curve (Multinomial Naive Bayes)')
plt.xlabel('Recall')
plt.ylabel('Precision')
f1_plot = plt.figure(3)
f1_score = f1(average_precision, average_recall)
plt.plot(thresholds, f1_score)
plt.title('F1 Curve (Multinomial Naive Bayes)')
plt.xlabel('Thresholds')
plt.ylabel('F1 Measure')
plt.show()
def train_stochastic(model, train_loader, test_loader, args, device='cpu'):
if args['var_type'] == 'isotropic':
trainable_noise_params = {
'params': model.base.sigma,
'lr': args['lr'],
'weight_decay': args['wd']
}
elif args['var_type'] == 'anisotropic':
trainable_noise_params = {
'params': model.base.L,
'lr': args['lr'],
'weight_decay': args['wd']
}
optimizer = Adam([{
'params': model.base.gen.parameters(),
'lr': args['lr']
}, {
'params': model.base.fc1.parameters(),
'lr': args['lr']
}, trainable_noise_params, {
'params': model.proto.parameters(),
'lr': args['lr'],
'weight_decay': args['wd']
}])
# Uncomment for the "train model and noise separately" ablation. But first train a model with disable_noise=True.
# model.freeze_model_params()
loss_func = nn.CrossEntropyLoss()
if args['dataset'] == 'cifar10':
norm_func = normalize_cifar10
elif args['dataset'] == 'cifar100':
norm_func = normalize_cifar100
elif args['dataset'] == 'svhn':
norm_func = normalize_generic
elif args['dataset'] in ('mnist', 'fmnist'):
norm_func = None
best_test_acc = -1.
for epoch in range(args['num_epochs']):
for data, target in train_loader:
data = data.to(device)
target = target.to(device)
model.train()
if norm_func is not None:
data = norm_func(data)
logits = model(data)
optimizer.zero_grad()
if args['reg_type'] == 'wca':
if args['var_type'] == 'isotropic':
wca = (model.proto.weight @ model.sigma.diag()
@ model.proto.weight.T).diagonal().sum()
elif args['var_type'] == 'anisotropic':
wca = (model.proto.weight @ model.sigma
@ model.proto.weight.T).diagonal().sum()
loss = loss_func(logits, target) - torch.log(wca)
elif args['reg_type'] == 'max_entropy':
me = model.base.dist.entropy().mean()
loss = loss_func(logits, target) - torch.log(me)
elif args['reg_type'] == 'wca+max_entropy':
if args['var_type'] == 'isotropic':
wca = (model.proto.weight @ model.sigma.diag()
@ model.proto.weight.T).diagonal().sum()
elif args['var_type'] == 'anisotropic':
wca = (model.proto.weight @ model.sigma
@ model.proto.weight.T).diagonal().sum()
me = model.base.dist.entropy().mean()
loss = loss_func(logits,
target) - torch.log(wca) - torch.log(me)
loss.backward()
optimizer.step()
if args['var_type'] == 'anisotropic':
with torch.no_grad():
model.base.L.data = model.base.L.data.tril()
train_acc = accuracy(model,
train_loader,
device=device,
norm=norm_func)
test_acc = accuracy(model, test_loader, device=device, norm=norm_func)
print('Epoch {:03}, Train acc: {:.3f}, Test acc: {:.3f}'.format(
epoch + 1, train_acc, test_acc))
if test_acc > best_test_acc:
best_test_acc = test_acc
model.save(os.path.join(args['output_path']['models'],
'ckpt_best'))
print('Best accuracy achieved on epoch {}.'.format(epoch + 1))
def train(net,
net_size,
input_size,
feature_dim,
train_dataset,
val_dataset,
epochs,
learning_rate,
batch_size,
save_path,
pretrained_model=None):
# create dataloader
train_targets = [sampler[1] for sampler in train_dataset.imgs]
weighted_sampler = ScheduledWeightedSampler(len(train_dataset),
train_targets, 0.975, True)
train_loader = DataLoader(train_dataset,
batch_size=batch_size,
sampler=weighted_sampler,
drop_last=True)
val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)
# define model
model = net(net_size, input_size, feature_dim).cuda()
print_msg('Trainable layers: ',
['{}\t{}'.format(k, v) for k, v in model.layer_configs()])
# load pretrained weights
if pretrained_model:
pretrained_dict = model.load_weights(pretrained_model, ['fc', 'dense'])
print_msg('Loaded weights from {}: '.format(pretrained_model),
sorted(pretrained_dict.keys()))
# define loss and optimizier
MSELoss = torch.nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(),
lr=learning_rate,
momentum=0.9,
nesterov=True,
weight_decay=0.0005)
# optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=0.0005)
# learning rate warmup and decay
milestones = [160, 230]
warmup_epoch = 10
warmup_batch = (len(train_loader) // batch_size) * warmup_epoch
lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
milestones=milestones,
gamma=0.1)
warmup_scheduler = WarmupLRScheduler(optimizer, warmup_batch,
learning_rate)
# train
max_kappa = 0
record_epochs, accs, losses = [], [], []
model.train()
for epoch in range(1, epochs + 1):
# resampling weight update
weighted_sampler.step()
# learning rate update
lr_scheduler.step()
if epoch in milestones:
curr_lr = optimizer.param_groups[0]['lr']
print_msg('Learning rate decayed to {}'.format(curr_lr))
if epoch > 1 and epoch <= warmup_epoch:
curr_lr = optimizer.param_groups[0]['lr']
print_msg('Learning rate warmup to {}'.format(curr_lr))
epoch_loss = 0
correct = 0
total = 0
progress = tqdm(enumerate(train_loader))
for step, train_data in progress:
if epoch <= warmup_epoch:
warmup_scheduler.step()
X, y = train_data
X, y = X.cuda(), y.float().cuda()
# forward
y_pred = model(X)
loss = MSELoss(y_pred, y)
# backward
optimizer.zero_grad()
loss.backward()
optimizer.step()
# metrics
epoch_loss += loss.item()
total += y.size(0)
correct += accuracy(y_pred, y) * y.size(0)
avg_loss = epoch_loss / (step + 1)
avg_acc = correct / total
progress.set_description(
'epoch: {}, loss: {:.6f}, acc: {:.4f}'.format(
epoch, avg_loss, avg_acc))
# save model
c_matrix = np.zeros((5, 5), dtype=int)
acc = _eval(model, val_loader, c_matrix)
kappa = quadratic_weighted_kappa(c_matrix)
print('validation accuracy: {}, kappa: {}'.format(acc, kappa))
if kappa > max_kappa:
torch.save(model, save_path)
max_kappa = kappa
print_msg('Model save at {}'.format(save_path))
# record
record_epochs.append(epoch)
accs.append(acc)
losses.append(avg_loss)
return record_epochs, accs, losses
def KLR(K, y, Kval, yval, lambd, maxIter = 100, tresh = 1e-8):
# initialize the values
assert K.shape[0] == y.shape[0]
n = K.shape[0]
n_val = Kval.shape[0]
y_ = np.ones(n)
yval_ = np.ones(n_val)
y_[y == 0] = -1
yval_[yval == 0] = -1
alphas = []
for l in lambd :
cnt = 0
P_t, W_t = np.eye(n), np.eye(n)
z_t = K@ np.ones(n) - y_
alpha_t = np.ones(n)
diff_alpha = np.inf
while (diff_alpha > tresh) and (cnt < maxIter):
old_alpha = alpha_t
## Solving dual using CVXOpt
#P = matrix(2*((K @ W_t @ K)/n + l*K))
#q = matrix((-2*K@W_t@y_)/n)
#solvers.options['show_progress'] = False
#sol=solvers.qp(P, q)
#alpha_t = sol['x']
#alpha_t = np.reshape(alpha_t,-1)
alpha_t = solveWKRR(K, W_t, z_t, y_, l)
m_t = K@alpha_t
sigma_m = sigmoid(m_t)
sigma_my = sigmoid(-y_*m_t)
P_t = - np.diag(sigma_my)
W_t = np.diag(sigma_m * (1-sigma_m))
z_t = m_t - (P_t@y_)/(sigma_m * (1-sigma_m))
diff_alpha = np.linalg.norm(alpha_t - old_alpha)
cnt+=1
if cnt % 10 == 0:
print(l, cnt)
loss_lambda = logistic_loss(y_, K@alpha_t)
acc_lambda = accuracy(y,K@alpha_t, mode="SVM")
loss_lambdaval = logistic_loss(yval_, Kval@alpha_t)
acc_lambdaval = accuracy(yval,Kval@alpha_t, mode="SVM")
print(f"***********lambda = {l}***********")
print(f"Training: loss = {loss_lambda:.4f}, accuracy = {acc_lambda:.6f}")
print(f"Validation: loss = {loss_lambdaval:.4f}, accuracy = {acc_lambdaval:.6f}")
alphas +=[alpha_t]
return(alphas)
adfuller(train_set_diff)
acf_pacf(train_set_diff)
# ряд стационарен, можно переходить к поиску оптимальных параметров
# best_params.append(sarima_best_params(train_set, 7, 1, 0, 7, 3))
# построение модели
model = sm.tsa.SARIMAX(train_set,
order=(best_params[j][0], 1, best_params[j][1]),
seasonal_order=(best_params[j][2], 0,
best_params[j][3],
7)).fit(disp=False)
# вывод информации о модели
print(model.summary())
# проверка остатков модели на случайность
ljungbox(model.resid)
acf_pacf(model.resid)
# SARIMA прогноз на конкретный час
hour_pred = model.forecast(2)[-1]
# добавление прогноза на час к итоговому прогнозу
total_pred.append(hour_pred)
# оценка прогноза по метрикам
nnf = nnfmetrics(total_pred, test_set, plan_set)
mse = mse(test_set, total_pred)
mape = mape(test_set, total_pred)
acc = accuracy(test_set, total_pred)
print('Оценка по NNFMETRICS = ', nnf)
print('Оценка по MSE = ', mse)
print('Оценка по MAPE = ', mape)
print('Точность прогноза = ', acc)
# отрисовка графика
plot_results(pred_date.strftime('%d-%m-%Y'), test_set, total_pred, plan_set)
def test(dataset):
# # load BERT and GAN
# load_gan_model(D, G, config['gan_save_path'])
# if args.fine_tune:
# load_model(E, path=config['bert_save_path'], model_name='bert')
#
test_dataloader = DataLoader(dataset,
batch_size=args.predict_batch_size,
shuffle=False,
num_workers=2)
n_sample = len(test_dataloader)
result = dict()
# Loss function
detection_loss = torch.nn.CrossEntropyLoss().to(device)
model.eval()
all_detection_preds = []
all_detection_logit = []
total_loss = 0
for sample in tqdm(test_dataloader):
sample = (i.to(device) for i in sample)
token, mask, type_ids, y = sample
batch = len(token)
# -------------------------evaluate D------------------------- #
# BERT encode sentence to feature vector
with torch.no_grad():
logit = model(token, mask, type_ids)
all_detection_logit.append(logit)
all_detection_preds.append(torch.argmax(logit, 1))
total_loss += detection_loss(logit, y.long())
all_y = LongTensor(
dataset.dataset[:, -1].astype(int)).cpu() # [length, n_class]
all_binary_y = (all_y != 0).long() # [length, 1] label 0 is oos
all_detection_preds = torch.cat(all_detection_preds,
0).cpu() # [length, 1]
# all_detection_binary_preds = convert_to_int_by_threshold(all_detection_preds.squeeze()) # [length, 1]
all_detection_logit = torch.cat(all_detection_logit, 0).cpu()
# 计算损失
result['detection_loss'] = total_loss
logger.info(
metrics.classification_report(all_binary_y,
all_detection_preds,
target_names=['oos', 'in']))
# report
oos_ind_precision, oos_ind_recall, oos_ind_fscore, _ = metrics.binary_recall_fscore(
all_detection_preds, all_binary_y)
detection_acc = metrics.accuracy(all_detection_preds, all_binary_y)
# y_score = all_detection_preds.squeeze().tolist()
y_score = all_detection_logit.softmax(1)[:, 1].tolist()
eer = metrics.cal_eer(all_binary_y, y_score)
test_logit = all_detection_logit.tolist()
result['test_logit'] = test_logit
result['eer'] = eer
result['all_detection_preds'] = all_detection_preds
result['detection_acc'] = detection_acc
result['all_binary_y'] = all_binary_y
result['oos_ind_precision'] = oos_ind_precision
result['oos_ind_recall'] = oos_ind_recall
result['oos_ind_f_score'] = oos_ind_fscore
result['y_score'] = y_score
result['auc'] = roc_auc_score(all_binary_y, y_score)
return result
print("Fold number " + str(test_fl))
training, training_class, testing, testing_class = importfile.get_datasets(dir_name[dataset],
class_name[dataset],
str(test_fl),
test_str, train_str)
training = np.transpose(utils.normalize_columns(np.transpose(training)))
testing = np.transpose(utils.normalize_columns(np.transpose(testing)))
# Basic KNN (baseline)
start = time.time()
predicted = kNNAlgorithm.nearest_neighbor((training, training_class), testing, k, dist, policy)
efficiencies[0] = time.time() - start
accuracies[0] = metrics.accuracy(testing_class, predicted)
recalls[0] = metrics.recall(testing_class, predicted)
# Weighted KNN
start = time.time()
weights_tree = kNNWeightedAlgorithm.calculate_weights(utils.encode_data(training), training_class,
'TreeClassifier')
predicted = kNNAlgorithm.nearest_neighbor((training, training_class), testing, k, dist, policy,
weights_tree)
efficiencies[1] = time.time() - start
accuracies[1] = metrics.accuracy(testing_class, predicted)
recalls[1] = metrics.recall(testing_class, predicted)
start = time.time()
weights_relieff = kNNWeightedAlgorithm.calculate_weights(utils.encode_data(training),
training_class, 'Relieff')
def run(*options, cfg=None):
update_config(config, options=options, config_file=cfg)
torch.backends.cudnn.benchmark = config.CUDNN.BENCHMARK
if torch.cuda.is_available():
torch.cuda.manual_seed_all(config.SEED)
np.random.seed(seed=config.SEED)
# Setup Augmentation/Transformation pipeline
input_size = config.TRAIN.INPUT_SIZE
resize_range_min = config.TRAIN.RESIZE_MIN
# Data-parallel
devices_lst = list(range(torch.cuda.device_count()))
print("Devices {}".format(devices_lst))
if (config.TEST.MODALITY == "RGB") or (config.TEST.MODALITY == "combined"):
rgb_loader = torch.utils.data.DataLoader(
I3DDataSet(data_root=config.DATASET.DIR,
split=config.DATASET.SPLIT,
modality="RGB",
train_mode=False,
sample_frames_at_test=False,
transform=torchvision.transforms.Compose([
GroupScale(config.TRAIN.RESIZE_MIN),
GroupCenterCrop(config.TRAIN.INPUT_SIZE),
GroupNormalize(modality="RGB"),
Stack(),
])),
batch_size=config.TEST.BATCH_SIZE,
shuffle=False,
num_workers=config.WORKERS,
pin_memory=config.PIN_MEMORY)
rgb_model_file = config.TEST.MODEL_RGB
if not os.path.exists(rgb_model_file):
raise FileNotFoundError(rgb_model_file, " does not exist")
rgb_model = load_model(modality="RGB", state_dict_file=rgb_model_file)
print("scoring with rgb model")
targets, rgb_predictions = test(rgb_model, rgb_loader, "RGB")
del rgb_model
targets = targets.cuda(non_blocking=True)
rgb_top1_accuracy = accuracy(rgb_predictions, targets, topk=(1, ))
print("rgb top1 accuracy: ",
rgb_top1_accuracy[0].cpu().numpy().tolist())
if (config.TEST.MODALITY == "flow") or (config.TEST.MODALITY
== "combined"):
flow_loader = torch.utils.data.DataLoader(
I3DDataSet(data_root=config.DATASET.DIR,
split=config.DATASET.SPLIT,
modality="flow",
train_mode=False,
sample_frames_at_test=False,
transform=torchvision.transforms.Compose([
GroupScale(config.TRAIN.RESIZE_MIN),
GroupCenterCrop(config.TRAIN.INPUT_SIZE),
GroupNormalize(modality="flow"),
Stack(),
])),
batch_size=config.TEST.BATCH_SIZE,
shuffle=False,
num_workers=config.WORKERS,
pin_memory=config.PIN_MEMORY)
flow_model_file = config.TEST.MODEL_FLOW
if not os.path.exists(flow_model_file):
raise FileNotFoundError(flow_model_file, " does not exist")
flow_model = load_model(modality="flow",
state_dict_file=flow_model_file)
print("scoring with flow model")
targets, flow_predictions = test(flow_model, flow_loader, "flow")
del flow_model
targets = targets.cuda(non_blocking=True)
flow_top1_accuracy = accuracy(flow_predictions, targets, topk=(1, ))
print("flow top1 accuracy: ",
flow_top1_accuracy[0].cpu().numpy().tolist())
if config.TEST.MODALITY == "combined":
predictions = torch.stack([rgb_predictions, flow_predictions])
predictions_mean = torch.mean(predictions, dim=0)
top1accuracy = accuracy(predictions_mean, targets, topk=(1, ))
print("combined top1 accuracy: ",
top1accuracy[0].cpu().numpy().tolist())
plot_points(x_test, y_test, class_colors=class_colors, title='Test - correct labels')
# Train the SVM classifier on the training data.
svm = SVM(kernel_func=kernel_func, C=C)
print('Training...')
svm.train(x_train, y_train)
# Plot the decision boundary.
print('Plotting...')
plot_svm_decision_boundary(svm, x_train, y_train,
title='SVM decision boundary on training data', output_path=output_path,
file_name=str(dataset_name) + '_support_vectors_train.png',
class_colors=class_colors)
# Make predictions on train and test data.
y_train_pred = svm.predict(x_train)
y_test_pred = svm.predict(x_test)
plot_points(x_train, y_train_pred, class_colors=class_colors,
title='Your predictions for training data')
plot_points(x_test, y_test_pred, class_colors=class_colors,
title='Your predictions for test data')
# Compute the classification accuracy for train and test.
acc_train = accuracy(y_train_pred, y_train)
acc_test = accuracy(y_test_pred, y_test)
print('Train accuracy: %.2f.' % acc_train)
print('Test accuracy: %.2f.' % acc_test)
print('Done.')
def score(self, X_test, y_test):
y_predict = self.predict(X_test)
return accuracy(y_predict.reshape(-1, 1), y_test)
def train(self):
"""
训练模型
:return:
"""
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9,
allow_growth=True)
sess_config = tf.ConfigProto(log_device_placement=False,
allow_soft_placement=True,
gpu_options=gpu_options)
with tf.Session(config=sess_config) as sess:
# 初始化变量值
sess.run(tf.global_variables_initializer())
current_step = 0
for epoch in range(self.config["epochs"]):
print("----- Epoch {}/{} -----".format(epoch + 1,
self.config["epochs"]))
for batch in self.data_obj.next_batch(
self.train_queries, self.config["batch_size"]):
loss, predictions = self.model.train(
sess, batch, self.config["keep_prob"])
acc = accuracy(predictions)
print("train: step: {}, loss: {}, acc: {}".format(
current_step, loss, acc))
current_step += 1
if current_step % self.config["checkpoint_every"] == 0:
eval_losses = []
eval_acc = []
for eval_batch in self.data_obj.next_batch(
self.eval_queries, self.config["batch_size"]):
eval_loss, eval_predictions = self.model.eval(
sess, eval_batch)
eval_losses.append(eval_loss)
acc = accuracy(eval_predictions)
eval_acc.append(acc)
print("\n")
print("eval: , loss: {}, acc: {}".format(
mean(eval_losses), mean(eval_acc)))
print("\n")
if self.config["ckpt_model_path"]:
save_path = os.path.join(
os.path.abspath(os.getcwd()),
self.config["ckpt_model_path"])
if not os.path.exists(save_path):
os.makedirs(save_path)
model_save_path = os.path.join(
save_path, self.config["model_name"])
self.model.saver.save(sess,
model_save_path,
global_step=current_step)
def test(dataset):
# # load BERT and GAN
# load_gan_model(D, G, config['gan_save_path'])
# if args.fine_tune:
# load_model(E, path=config['bert_save_path'], model_name='bert')
#
test_dataloader = DataLoader(dataset,
batch_size=args.predict_batch_size,
shuffle=False,
num_workers=2)
n_sample = len(test_dataloader)
result = dict()
# Loss function
detection_loss = torch.nn.BCELoss().to(device)
classified_loss = torch.nn.CrossEntropyLoss(ignore_index=0).to(device)
D_detect.eval()
E.eval()
all_detection_preds = []
all_class_preds = []
all_features = []
for sample in tqdm(test_dataloader):
sample = (i.to(device) for i in sample)
token, mask, type_ids, y = sample
batch = len(token)
# -------------------------evaluate D------------------------- #
# BERT encode sentence to feature vector
with torch.no_grad():
sequence_output, pooled_output = E(token, mask, type_ids)
real_feature = pooled_output
discriminator_output, f_vector = D_detect(real_feature)
all_detection_preds.append(discriminator_output)
if args.do_vis:
all_features.append(f_vector)
all_y = LongTensor(
dataset.dataset[:, -1].astype(int)).cpu() # [length, n_class]
all_binary_y = (all_y != 0).long() # [length, 1] label 0 is oos
all_detection_preds = torch.cat(all_detection_preds,
0).cpu() # [length, 1]
all_detection_binary_preds = convert_to_int_by_threshold(
all_detection_preds.squeeze()) # [length, 1]
# 计算损失
detection_loss = detection_loss(all_detection_preds,
all_binary_y.float())
result['detection_loss'] = detection_loss
logger.info(
metrics.classification_report(all_binary_y,
all_detection_binary_preds,
target_names=['oos', 'in']))
# report
oos_ind_precision, oos_ind_recall, oos_ind_fscore, _ = metrics.binary_recall_fscore(
all_detection_binary_preds, all_binary_y)
detection_acc = metrics.accuracy(all_detection_binary_preds,
all_binary_y)
y_score = all_detection_preds.squeeze().tolist()
eer = metrics.cal_eer(all_binary_y, y_score)
result['eer'] = eer
result['all_detection_binary_preds'] = all_detection_binary_preds
result['detection_acc'] = detection_acc
result['all_binary_y'] = all_binary_y
result['oos_ind_precision'] = oos_ind_precision
result['oos_ind_recall'] = oos_ind_recall
result['oos_ind_f_score'] = oos_ind_fscore
result['y_score'] = y_score
result['auc'] = roc_auc_score(all_binary_y, y_score)
if args.do_vis:
all_features = torch.cat(all_features, 0).cpu().numpy()
result['all_features'] = all_features
return result
def prepare_student_data(dataset, nb_teachers, save=False):
"""
Takes a dataset name and the size of the teacher ensemble and prepares
training data for the student model, according to parameters indicated
in flags above.
:param dataset: string corresponding to mnist, cifar10, or svhn
:param nb_teachers: number of teachers (in the ensemble) to learn from
:param save: if set to True, will dump student training labels predicted by
the ensemble of teachers (with Laplacian noise) as npy files.
It also dumps the clean votes for each class (without noise) and
the labels assigned by teachers
:return: pairs of (data, labels) to be used for student training and testing
"""
assert input.create_dir_if_needed(FLAGS.train_dir)
# Load the dataset
if dataset == 'svhn':
test_data, test_labels = input.ld_svhn(test_only=True)
elif dataset == 'cifar10':
test_data, test_labels = input.ld_cifar10(test_only=True)
elif dataset == 'mnist':
test_data, test_labels = input.ld_mnist(test_only=True)
else:
print("Check value of dataset flag")
return False
# Make sure there is data leftover to be used as a test set
assert FLAGS.stdnt_share < len(test_data)
# Prepare [unlabeled] student training data (subset of test set)
stdnt_data = test_data[:FLAGS.stdnt_share]
# Compute teacher predictions for student training data
teachers_preds = ensemble_preds(dataset, nb_teachers, stdnt_data)
# Aggregate teacher predictions to get student training labels
if not save:
stdnt_labels = aggregation.noisy_max(teachers_preds, FLAGS.lap_scale)
else:
# Request clean votes and clean labels as well
stdnt_labels, clean_votes, labels_for_dump = aggregation.noisy_max(
teachers_preds, FLAGS.lap_scale,
return_clean_votes=True) #NOLINT(long-line)
# Prepare filepath for numpy dump of clean votes
filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(
nb_teachers) + '_student_clean_votes_lap_' + str(
FLAGS.lap_scale) + '.npy' # NOLINT(long-line)
# Prepare filepath for numpy dump of clean labels
filepath_labels = FLAGS.data_dir + "/" + str(dataset) + '_' + str(
nb_teachers) + '_teachers_labels_lap_' + str(
FLAGS.lap_scale) + '.npy' # NOLINT(long-line)
# Dump clean_votes array
with tf.gfile.Open(filepath, mode='w') as file_obj:
np.save(file_obj, clean_votes)
# Dump labels_for_dump array
with tf.gfile.Open(filepath_labels, mode='w') as file_obj:
np.save(file_obj, labels_for_dump)
# Print accuracy of aggregated labels
ac_ag_labels = metrics.accuracy(stdnt_labels,
test_labels[:FLAGS.stdnt_share])
print("Accuracy of the aggregated labels: " + str(ac_ag_labels))
# Store unused part of test set for use as a test set after student training
stdnt_test_data = test_data[FLAGS.stdnt_share:]
stdnt_test_labels = test_labels[FLAGS.stdnt_share:]
if save:
# Prepare filepath for numpy dump of labels produced by noisy aggregation
filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(
nb_teachers) + '_student_labels_lap_' + str(
FLAGS.lap_scale) + '.npy' #NOLINT(long-line)
# Dump student noisy labels array
with tf.gfile.Open(filepath, mode='w') as file_obj:
np.save(file_obj, stdnt_labels)
return stdnt_data, stdnt_labels, stdnt_test_data, stdnt_test_labels
def eval_reranker(res_fname="svm.test.res", pred_fname="svm.train.pred",
format="trec",
th=10,
verbose=False,
reranking_th=0.0,
ignore_noanswer=False):
ir, svm, conf_matrix = read_res_pred_files(res_fname, pred_fname, format, verbose,
reranking_th=reranking_th,
ignore_noanswer=ignore_noanswer)
# Calculate standard P, R, F1, Acc
acc = 1.0 * (conf_matrix['true']['true'] + conf_matrix['false']['false']) / (conf_matrix['true']['true'] + conf_matrix['false']['false'] + conf_matrix['true']['false'] + conf_matrix['false']['true'])
p = 0
if (conf_matrix['true']['true'] + conf_matrix['false']['true']) > 0:
p = 1.0 * (conf_matrix['true']['true']) / (conf_matrix['true']['true'] + conf_matrix['false']['true'])
r = 0
if (conf_matrix['true']['true'] + conf_matrix['true']['false']) > 0:
r = 1.0 * (conf_matrix['true']['true']) / (conf_matrix['true']['true'] + conf_matrix['true']['false'])
f1 = 0
if (p + r) > 0:
f1 = 2.0 * p * r / (p + r)
# evaluate IR
prec_se = metrics.recall_of_1(ir, th)
acc_se = metrics.accuracy(ir, th)
acc_se1 = metrics.accuracy1(ir, th)
acc_se2 = metrics.accuracy2(ir, th)
# evaluate SVM
prec_svm = metrics.recall_of_1(svm, th)
acc_svm = metrics.accuracy(svm, th)
acc_svm1 = metrics.accuracy1(svm, th)
acc_svm2 = metrics.accuracy2(svm, th)
mrr_se = metrics.mrr(ir, th)
mrr_svm = metrics.mrr(svm, th)
map_se = metrics.map(ir, th)
map_svm = metrics.map(svm, th)
avg_acc1_svm = metrics.avg_acc1(svm, th)
avg_acc1_ir = metrics.avg_acc1(ir, th)
print ("")
print ("*** Official score (MAP for SYS): %5.4f" %(map_svm))
print ("")
print ("")
print( "******************************")
print( "*** Classification results ***")
print( "******************************")
print( "")
print( "Acc = %5.4f" %(acc))
print( "P = %5.4f" %(p))
print( "R = %5.4f" %(r))
print( "F1 = %5.4f" %(f1))
print( "")
print( "")
print( "********************************")
print( "*** Detailed ranking results ***")
print( "********************************")
print( "")
print( "IR -- Score for the output of the IR system (baseline).")
print( "SYS -- Score for the output of the tested system.")
print( "")
print( "%13s %5s" %("IR", "SYS"))
print( "MAP : %5.4f %5.4f" %(map_se, map_svm))
print( "AvgRec: %5.4f %5.4f" %(avg_acc1_ir, avg_acc1_svm))
print( "MRR : %6.2f %6.2f" %(mrr_se, mrr_svm))
print( "%16s %6s %14s %6s %14s %6s %12s %4s" % ("IR", "SYS", "IR", "SYS", "IR", "SYS", "IR", "SYS"))
for i, (p_se, p_svm, a_se, a_svm, a_se1, a_svm1, a_se2, a_svm2) in enumerate(zip(prec_se, prec_svm, acc_se, acc_svm, acc_se1, acc_svm1, acc_se2, acc_svm2), 1):
print( "REC-1@%02d: %6.2f %6.2f ACC@%02d: %6.2f %6.2f AC1@%02d: %6.2f %6.2f AC2@%02d: %4.0f %4.0f" %(i, p_se, p_svm, i, a_se, a_svm, i, a_se1, a_svm1, i, a_se2, a_svm2))
print( "REC-1 - percentage of questions with at least 1 correct answer in the top @X positions (useful for tasks where questions have at most one correct answer)")
print( "ACC - accuracy, i.e., number of correct answers retrieved at rank @X normalized by the rank and the total number of questions")
print( "AC1 - the number of correct answers at @X normalized by the number of maximum possible answers (perfect re-ranker)")
print( "AC2 - the absolute number of correct answers at @X")
return map_svm
def train_teacher(dataset, nb_teachers, teacher_id):
"""
This function trains a teacher (teacher id) among an ensemble of nb_teachers
models for the dataset specified.
:param dataset: string corresponding to dataset (svhn, cifar10)
:param nb_teachers: total number of teachers in the ensemble
:param teacher_id: id of the teacher being trained
:return: True if everything went well
"""
# If working directories do not exist, create them
assert input.create_dir_if_needed(FLAGS.data_dir)
assert input.create_dir_if_needed(FLAGS.train_dir)
print("teacher {}:".format(teacher_id))
# Load the dataset
if dataset == 'svhn':
train_data,train_labels,test_data,test_labels = input.ld_svhn(extended=True)
elif dataset == 'cifar10':
train_data, train_labels, test_data, test_labels = input.ld_cifar10()
elif dataset == 'mnist':
train_data, train_labels, test_data, test_labels = input.ld_mnist()
else:
print("Check value of dataset flag")
return False
path = os.path.abspath('.')
path1 = path + '\\plts_nodisturb\\'
# 对标签进行干扰
import copy
train_labels1 = copy.copy(train_labels)
train_labels2 = disturb(train_labels, 0.3)
disturb(test_labels, 0.3)
#path1 = path + '\\plts_withdisturb\\'
# Retrieve subset of data for this teacher
#干扰前
data, labels = input.partition_dataset(train_data,
train_labels,
nb_teachers,
teacher_id)
from pca import K_S
import operator
print(operator.eq(train_labels1, train_labels2))
print("干扰前: ", K_S.tst_norm(train_labels1))
print("干扰后: ", K_S.tst_norm(train_labels2))
print(K_S.tst_samp(train_labels1, train_labels2))
print("Length of training data: " + str(len(labels)))
# Define teacher checkpoint filename and full path
if FLAGS.deeper:
filename = str(nb_teachers) + '_teachers_' + str(teacher_id) + '_deep.ckpt'
else:
filename = str(nb_teachers) + '_teachers_' + str(teacher_id) + '.ckpt'
ckpt_path = FLAGS.train_dir + '/' + str(dataset) + '_' + filename
# Perform teacher training
losses = deep_cnn.train(data, labels, ckpt_path)
# Append final step value to checkpoint for evaluation
ckpt_path_final = ckpt_path + '-' + str(FLAGS.max_steps - 1)
# Retrieve teacher probability estimates on the test data
teacher_preds = deep_cnn.softmax_preds(test_data, ckpt_path_final)
# Compute teacher accuracy
precision = metrics.accuracy(teacher_preds, test_labels)
print('Precision of teacher after training: ' + str(precision))
print("each n step loss: ", losses)
#x = list(range(1, len(losses)+1))
#plt.plot(x, losses, 'bo-', markersize=20)
#plt.savefig(path1 + 'loss' + str(teacher_id) + '.jpg')
#plt.show()
#print("x: ",x)
#print("loss: ", losses)
return True
def train(args, train_loader, all_models, optimizer, epoch):
"""
Trains for one epoch of the train_loader dataset.
"""
# switch all models to train mode
for m in all_models:
m.train()
(trunk_model, incident_layer, place_layer) = all_models
# holds some metrics
a_v_batch_time = AverageMeter()
a_v_data_time = AverageMeter()
a_v_losses = AverageMeter()
a_v_incident_top1 = AverageMeter()
a_v_place_top1 = AverageMeter()
a_v_incident_top5 = AverageMeter()
a_v_place_top5 = AverageMeter()
# set end time as current time before training on a batch
end_time = time.time()
for batch_iteration, (input_data, target_p_v, target_d_v, weight_p_v,
weight_d_v) in enumerate(train_loader):
# measure data loading time
a_v_data_time.update(time.time() - end_time)
image_v = input_data.cuda(non_blocking=True)
target_p_v = target_p_v.cuda(non_blocking=True)
target_d_v = target_d_v.cuda(non_blocking=True)
weight_p_v = weight_p_v.cuda(non_blocking=True)
weight_d_v = weight_d_v.cuda(non_blocking=True)
# input_v = torch.autograd.Variable(image_v)
# target_p_v = torch.autograd.Variable(target_p_v)
# target_d_v = torch.autograd.Variable(target_d_v)
# weight_p_v = torch.autograd.Variable(weight_p_v)
# weight_d_v = torch.autograd.Variable(weight_d_v)
# compute output
output = trunk_model(image_v)
place_output = place_layer(output)
incident_output = incident_layer(output)
# get the loss according to parameters
loss, incident_output, place_output = get_loss(args, incident_output,
target_d_v, weight_d_v,
place_output,
target_p_v, weight_p_v)
# measure accuracy and record loss
incident_prec1, incident_prec5 = accuracy(incident_output.data, target_d_v, topk=1), \
accuracy(incident_output.data, target_d_v, topk=5)
place_prec1, place_prec5 = accuracy(place_output.data, target_p_v, topk=1), \
accuracy(place_output.data, target_p_v, topk=5)
a_v_losses.update(loss.data, input_data.size(0))
a_v_place_top1.update(place_prec1, input_data.size(0))
a_v_incident_top1.update(incident_prec1, input_data.size(0))
a_v_place_top5.update(place_prec5, input_data.size(0))
a_v_incident_top5.update(incident_prec5, input_data.size(0))
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
a_v_batch_time.update(time.time() - end_time)
end_time = time.time()
if batch_iteration % args.print_freq == 0:
print(
'Epoch: [{0}][{1}/{2}]\t'
'Time {a_v_batch_time.val:.3f} ({a_v_batch_time.avg:.3f})\t'
'Data {a_v_data_time.val:.3f} ({a_v_data_time.avg:.3f})\t'
'Loss {a_v_losses.val:.4f} ({a_v_losses.avg:.4f})\t'
'Incident Prec@1 {a_v_incident_top1.val:.3f} ({a_v_incident_top1.avg:.3f})\t'
'Place Prec@1 {a_v_place_top1.val:.3f} ({a_v_place_top1.avg:.3f})\t'
'Place Prec@5 {a_v_place_top5.val:.3f} ({a_v_place_top5.avg:.3f})\t'
'Incident Prec@5 {a_v_incident_top5.val:.3f} ({a_v_incident_top5.avg:.3f})\t'
.format(epoch,
batch_iteration,
len(train_loader),
a_v_batch_time=a_v_batch_time,
a_v_data_time=a_v_data_time,
a_v_losses=a_v_losses,
a_v_incident_top1=a_v_incident_top1,
a_v_place_top1=a_v_place_top1,
a_v_incident_top5=a_v_incident_top5,
a_v_place_top5=a_v_place_top5))
# TODO: add more metrics here
writer.add_scalar('Loss/train', a_v_losses.avg,
batch_iteration + epoch * len(train_loader))
writer.add_scalar('Accuracy/train_place_1', a_v_place_top1.avg,
batch_iteration + epoch * len(train_loader))
writer.add_scalar('Accuracy/train_place_5', a_v_place_top5.avg,
batch_iteration + epoch * len(train_loader))
writer.add_scalar('Accuracy/train_incident_1', a_v_incident_top1.avg,
batch_iteration + epoch * len(train_loader))
writer.add_scalar('Accuracy/train_incident_5', a_v_incident_top5.avg,
batch_iteration + epoch * len(train_loader))
with np.load(weight_path) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
nlayers = len(lasagne.layers.get_all_params(simple_net_output))
lasagne.layers.set_all_param_values(simple_net_output, param_values[:nlayers])
print 'Done assigning weights'
# In[33]:
print "Defining and compiling test functions"
test_prediction = lasagne.layers.get_output(simple_net_output[0],
deterministic=True)
test_loss = categorical_crossentropy(test_prediction, target_var)
test_loss = test_loss.mean()
test_acc, test_acc_per_sample = accuracy(test_prediction, target_var,
void_labels)
test_jacc = jaccard(test_prediction, target_var, n_classes)
test_fn = theano.function(
[input_var, target_var],
[test_loss, test_acc, test_jacc, test_acc_per_sample])
print "Done"
# In[34]:
#Function computing the prediction with current parameters (for visualization)
pred = theano.function([input_var],
lasagne.layers.get_output(net['probs_reshape'],
deterministic=True))
# In[35]:
def train():
"""Training"""
opt = FLAGS
tf.logging.info("Build CleanNet...")
batch_size = opt.batch_size_sup + opt.batch_size_unsup
model = CleanNet(opt.num_ref, opt.img_dim, opt.embed_norm, opt.dropout_rate, opt.weight_decay)
# phi_s: class embedding (batch_size, embed_size)
# v_q: query image feature (batch_size, img_dim)
# phi_q: query embedding (batch_size, embed_size)
# v_qr: reconstructed query image feature (batch_size, img_dim)
phi_s, v_q, phi_q, v_qr = model.forward(is_training=True)
# verification labels
vlabel = tf.placeholder(tf.float32, shape=(None,), name="vlabel")
# verification flags indicating a sample is for supervised(1) or unsupervised(0) training
vflag = tf.placeholder(tf.float32, shape=(None,), name="vflag")
cos_sim = similarity(phi_s, phi_q)
acc = accuracy(vlabel[:opt.batch_size_sup], cos_sim[:opt.batch_size_sup], threshold=0.1, scope="train_acc")
val_acc = accuracy(vlabel, cos_sim, threshold=opt.val_sim_thres, scope="val_acc_at_{}".format(opt.val_sim_thres))
tf.summary.scalar('train/accuracy', acc)
objective_loss = tf.reduce_mean(total_loss(vlabel, cos_sim, phi_s, v_q, phi_q, v_qr, vflag, opt.neg_weight, beta=0.1, gamma=0.1))
tf.summary.scalar('train/objective_loss', objective_loss)
regularization_loss = tf.reduce_sum(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
tf.summary.scalar('train/regularization_loss', regularization_loss)
loss = objective_loss + regularization_loss
tf.summary.scalar('train/loss', loss)
lr = tf.train.exponential_decay(opt.learning_rate, model.global_step, opt.lr_update, opt.lr_decay, staircase=True)
tf.summary.scalar('train/lr', lr)
merged = tf.summary.merge_all()
optimizer = tf.train.MomentumOptimizer(lr, opt.momentum)
train_op = optimizer.minimize(loss, global_step=model.global_step)
tf.logging.info("Get data batcher...")
supervised_data = data_provider_factory.get_data_batcher('trainval', 'train', opt)
val_data = data_provider_factory.get_data_batcher('trainval', 'val', opt)
if opt.batch_size_unsup > 0:
unsupervised_data = data_provider_factory.get_data_batcher('trainval', 'unverified', opt)
saver = tf.train.Saver()
init_op = tf.global_variables_initializer()
with tf.Session() as sess:
train_summary_writer = tf.summary.FileWriter(opt.log_dir + '/train', sess.graph)
val_summary_writer = tf.summary.FileWriter(opt.log_dir + '/val')
cur_step = 0
best_avg_val_acc = 0.0
sess.run(init_op)
# recover from latest checkpoint and run validation if available
ckpt = tf.train.get_checkpoint_state(opt.checkpoint_dir)
if ckpt:
saver.restore(sess, ckpt.model_checkpoint_path)
saver.recover_last_checkpoints(ckpt.all_model_checkpoint_paths)
cur_step, avg_val_acc = validation(sess, model, loss, val_acc, vlabel, vflag, opt.val_batch_size, val_data, val_summary_writer)
best_avg_val_acc = avg_val_acc
tf.logging.info("Recover model at global step = %d.", cur_step)
else:
tf.logging.info("Training from scratch.")
while cur_step < opt.n_step:
# data for supervised training
_, batch_vlabel, batch_q, batch_vflag, batch_ref = supervised_data.get_batch(batch_size=opt.batch_size_sup)
# data for unsupervised training
if opt.batch_size_unsup > 0:
# ubatch_vlabel_u is a dummy zero tensor since unsupervised samples don't have verification labels
_, ubatch_vlabel_u, ubatch_q, ubatch_vflag, ubatch_ref = unsupervised_data.get_batch(batch_size=opt.batch_size_unsup)
# concate supervised and unsupervied training data
batch_vlabel = np.concatenate([batch_vlabel, ubatch_vlabel_u], axis=0)
batch_q = np.concatenate([batch_q, ubatch_q], axis=0)
batch_vflag = np.concatenate([batch_vflag, ubatch_vflag], axis=0)
batch_ref = np.concatenate([batch_ref, ubatch_ref], axis=0)
_, cur_step, cur_loss, cur_acc, summary = sess.run([train_op, model.global_step, loss, acc, merged],
feed_dict={model.reference: batch_ref,
model.query: batch_q,
vlabel: batch_vlabel,
vflag: batch_vflag})
train_summary_writer.add_summary(summary, cur_step)
if cur_step % opt.log_interval == 0:
tf.logging.info('step {}: train/loss = {}, train/acc = {}'.format(cur_step, cur_loss, cur_acc))
if cur_step % opt.val_interval == 0 and cur_step != 0:
_, avg_val_acc = validation(sess, model, loss, val_acc, vlabel, vflag, opt.val_batch_size, val_data, val_summary_writer)
if not os.path.exists(opt.checkpoint_dir):
os.mkdir(opt.checkpoint_dir)
save_path = saver.save(sess, opt.checkpoint_dir)
print("Model saved in path: %s" % save_path)
if avg_val_acc > best_avg_val_acc:
best_avg_val_acc = avg_val_acc
model_path = os.path.join(save_path, "checkpoint")
best_model_path = os.path.join(save_path, "best_model_{}".format(cur_step))
shutil.copy(model_path, best_model_path)
print("Best model saved in path: %s" % best_model_path)
def test_regression(model, test_features, test_labels):
model.eval()
return accuracy(model(test_features), test_labels)
""" Dataset Preprocessing """
from sklearn.datasets import load_breast_cancer
data = load_breast_cancer()
X = data.data
y = data.target
scalar = MinMaxScaler()
scalar.fit(X)
X = scalar.transform(X)
print("######### Q1 Part A")
LR = LogisticRegression(lr=0.1,epoch=1000)
LR.fit(X,y)
y_hat = LR.predict(X)
print("Accuracy: ",end = "")
print(accuracy(y_hat,y))
print("######### Q1 Part B")
LR = LogisticRegression(lr=0.1,epoch=1000)
LR.fit_autograd(X,y)
y_hat = LR.predict(X)
print("Accuracy: ",end = "")
print(accuracy(y_hat,y))
print("######### Q1 Part C")
acc=0
X = np.append(X, np.matrix(y).T, axis=1)
k_fold = KFold(3, shuffle=True, random_state=1)
for train, test in k_fold.split(X):
LR = LogisticRegression(lr=0.1,epoch=1000)
train_all = X[train]
def prepare_student_data(dataset, nb_teachers, save=False):
"""
Takes a dataset name and the size of the teacher ensemble and prepares
training data for the student model, according to parameters indicated
in flags above.
:param dataset: string corresponding to mnist, cifar10, or svhn
:param nb_teachers: number of teachers (in the ensemble) to learn from
:param save: if set to True, will dump student training labels predicted by
the ensemble of teachers (with Laplacian noise) as npy files.
It also dumps the clean votes for each class (without noise) and
the labels assigned by teachers
:return: pairs of (data, labels) to be used for student training and testing
"""
assert input.create_dir_if_needed(FLAGS.train_dir)
# Load the dataset
if dataset == 'svhn':
test_data, test_labels = input.ld_svhn(test_only=True)
elif dataset == 'cifar10':
test_data, test_labels = input.ld_cifar10(test_only=True)
elif dataset == 'mnist':
test_data, test_labels = input.ld_mnist(test_only=True)
elif dataset == 'digit':
test_data, test_labels = input.ld_digit_test(test_name=FLAGS.test_name,
num=2000)
else:
print("Check value of dataset flag")
return False
# Make sure there is data leftover to be used as a test set
assert FLAGS.stdnt_share < len(test_data)
# Prepare [unlabeled] student training data (subset of test set)
if (FLAGS.d_stu > -1):
# stdnt_data = []
# for i in range(FLAGS.stdnt_share):
# new_img = transform.resize(skimage.img_as_ubyte(test_data[i].astype(int)),(28,28))
# if FLAGS.d_stu == 3:
# new_img = color.rgb2gray(new_img)
# else:
# new_img = new_img[ :,:, FLAGS.d_stu]
# stdnt_data.append(new_img.reshape(28,28,1).astype(np.float32))
# stdnt_data = np.array(stdnt_data)
trimmed = test_data[:FLAGS.stdnt_share, 2:30, 2:30, :]
# grey scale
if (FLAGS.d_stu == 3):
stdnt_data = 0.2125 * trimmed[:, :, :,
0] + 0.7154 * trimmed[:, :, :,
1] + 0.0721 * trimmed[:, :, :,
2]
else:
stdnt_data = trimmed[:, :, :, FLAGS.d_stu]
stdnt_data = stdnt_data.reshape((-1, 28, 28, 1))
else:
stdnt_data = test_data[:FLAGS.stdnt_share]
# Compute teacher predictions for student training data
teachers_preds = ensemble_preds(dataset, nb_teachers, stdnt_data)
# Aggregate teacher predictions to get student training labels
if not save:
stdnt_labels = aggregation.noisy_max(teachers_preds, FLAGS.lap_scale)
else:
# Request clean votes and clean labels as well
stdnt_labels, clean_votes, labels_for_dump = aggregation.noisy_max(
teachers_preds, FLAGS.lap_scale,
return_clean_votes=True) #NOLINT(long-line)
# Prepare filepath for numpy dump of clean votes
filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(
nb_teachers) + '_student_clean_votes_lap_' + str(
FLAGS.lap_scale) + '.npy' # NOLINT(long-line)
# Prepare filepath for numpy dump of clean labels
filepath_labels = FLAGS.data_dir + "/" + str(dataset) + '_' + str(
nb_teachers) + '_teachers_labels_lap_' + str(
FLAGS.lap_scale) + '.npy' # NOLINT(long-line)
# Dump clean_votes array
with tf.gfile.Open(filepath, mode='w') as file_obj:
np.save(file_obj, clean_votes)
# Dump labels_for_dump array
with tf.gfile.Open(filepath_labels, mode='w') as file_obj:
np.save(file_obj, labels_for_dump)
# Print accuracy of aggregated labels
ac_ag_labels = metrics.accuracy(stdnt_labels,
test_labels[:FLAGS.stdnt_share])
print("Accuracy of the aggregated labels: " + str(ac_ag_labels))
# Store unused part of test set for use as a test set after student training
if FLAGS.dataset_teacher == 'mnist':
test_data, test_labels = input.ld_mnist(test_only=True)
else:
assert 0 == 1, "Non implemented error: dataset_teacher not equals to mnist"
# if FLAGS.d_stu > -1:
# stdnt_test_data = test_data[FLAGS.stdnt_share:, 2:30, 2:30, FLAGS.d_stu : FLAGS.d_stu+1]
# else:
stdnt_test_data = test_data[FLAGS.stdnt_share:]
stdnt_test_labels = test_labels[FLAGS.stdnt_share:]
if save:
# Prepare filepath for numpy dump of labels produced by noisy aggregation
filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(
nb_teachers) + '_student_labels_lap_' + str(
FLAGS.lap_scale) + '.npy' #NOLINT(long-line)
# Dump student noisy labels array
with tf.gfile.Open(filepath, mode='w') as file_obj:
np.save(file_obj, stdnt_labels)
return stdnt_data, stdnt_labels, stdnt_test_data, stdnt_test_labels
from sklearn.model_selection import train_test_split
from metrics import accuracy
import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve
digits = datasets.load_digits()
# print(digits.keys())
X = digits.data
y = digits.target.copy()
y[digits.target == 9] = 1
y[digits.target != 9] = 0
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666)
lr = LogisticRegression()
lr.fit(X_train, y_train)
y_predict = lr.predict(X_test)
print(accuracy(y_test, y_predict))
def TN(y_true, y_predict):
assert len(y_true) == len(
y_predict), "the size of y_true must be equal to y_predict"
return np.sum((y_true == 0) & (y_predict == 0))
# print(TN(y_test, y_predict))
def TP(y_true, y_predict):
assert len(y_true) == len(y_predict)
return np.sum((y_true == 1) & (y_predict == 1))
def FN(y_true, y_predict):
x_shared.set_value(x_chunk_eval)
outputs_chunk = []
for b in xrange(num_batches_chunk_eval):
out = compute_output(b)
outputs_chunk.append(out)
outputs_chunk = np.vstack(outputs_chunk)
outputs_chunk = outputs_chunk[:chunk_length_eval]
outputs.append(outputs_chunk)
outputs = np.vstack(outputs)
outputs_labels = np.argmax(outputs, axis=1)
loss = np.mean(log_losses(outputs, labels))
acc = accuracy(outputs_labels, labels)
kappa_eval = continuous_kappa(
outputs_labels,
labels,
)
metric, conf_mat, \
hist_rater_a, hist_rater_b, \
nom, denom = kappa_eval
try:
kappa_cont_eval = continuous_kappa(outputs, labels,
y_pow=model.y_pow)
except:
kappa_cont_eval = [1, 1, 1, 1, 1, 1]
def test(dataset):
# load BERT and GAN
load_gan_model(D, G, config['gan_save_path'])
if args.fine_tune:
load_model(E, path=config['bert_save_path'], model_name='bert')
test_dataloader = DataLoader(dataset,
batch_size=args.predict_batch_size,
shuffle=False,
num_workers=2)
n_sample = len(test_dataloader)
result = dict()
# Loss function
detection_loss = torch.nn.BCELoss().to(device)
classified_loss = torch.nn.CrossEntropyLoss(ignore_index=0).to(device)
G.eval()
D.eval()
E.eval()
all_detection_preds = []
all_class_preds = []
all_features = []
for sample in tqdm.tqdm(test_dataloader):
sample = (i.to(device) for i in sample)
token, mask, type_ids, y = sample
batch = len(token)
# -------------------------evaluate D------------------------- #
# BERT encode sentence to feature vector
with torch.no_grad():
sequence_output, pooled_output = E(token, mask, type_ids)
real_feature = pooled_output
# 大于2表示除了训练判别器还要训练分类器
if n_class > 2:
f_vector, discriminator_output, classification_output = D(
real_feature, return_feature=True)
all_detection_preds.append(discriminator_output)
all_class_preds.append(classification_output)
# 只预测判别器
else:
f_vector, discriminator_output = D.detect_only(
real_feature, return_feature=True)
all_detection_preds.append(discriminator_output)
if args.do_vis:
all_features.append(f_vector)
all_y = LongTensor(
dataset.dataset[:, -1].astype(int)).cpu() # [length, n_class]
all_binary_y = (all_y != 0).long() # [length, 1] label 0 is oos
all_detection_preds = torch.cat(all_detection_preds,
0).cpu() # [length, 1]
all_detection_binary_preds = convert_to_int_by_threshold(
all_detection_preds.squeeze()) # [length, 1]
# 计算损失
detection_loss = detection_loss(all_detection_preds,
all_binary_y.float())
result['detection_loss'] = detection_loss
if n_class > 2:
class_one_hot_preds = torch.cat(all_class_preds,
0).detach().cpu() # one hot label
class_loss = classified_loss(class_one_hot_preds,
all_y) # compute loss
all_class_preds = torch.argmax(class_one_hot_preds, 1) # label
class_acc = metrics.ind_class_accuracy(
all_class_preds, all_y, oos_index=0) # accuracy for ind class
logger.info(
metrics.classification_report(
all_y, all_class_preds,
target_names=processor.id_to_label))
logger.info(
metrics.classification_report(all_binary_y,
all_detection_binary_preds,
target_names=['oos', 'in']))
# report
oos_ind_precision, oos_ind_recall, oos_ind_fscore, _ = metrics.binary_recall_fscore(
all_detection_binary_preds, all_binary_y)
detection_acc = metrics.accuracy(all_detection_binary_preds,
all_binary_y)
y_score = all_detection_preds.squeeze().tolist()
eer = metrics.cal_eer(all_binary_y, y_score)
result['eer'] = eer
result['all_detection_binary_preds'] = all_detection_binary_preds
result['detection_acc'] = detection_acc
result['all_binary_y'] = all_binary_y
result['all_y'] = all_y
result['oos_ind_precision'] = oos_ind_precision
result['oos_ind_recall'] = oos_ind_recall
result['oos_ind_f_score'] = oos_ind_fscore
result['score'] = y_score
result['y_score'] = y_score
result['auc'] = roc_auc_score(all_binary_y, y_score)
if n_class > 2:
result['class_loss'] = class_loss
result['class_acc'] = class_acc
if args.do_vis:
all_features = torch.cat(all_features, 0).cpu().numpy()
result['all_features'] = all_features
freeze_data['test_all_y'] = all_y.tolist()
freeze_data['test_all_pred'] = all_detection_binary_preds.tolist()
freeze_data['test_score'] = y_score
return result
def main(args):
"""Run training and validation.
1. Build graphs
1.1 Training graph to run on multiple GPUs
1.2 Validation graph to run on multiple GPUs
2. Configure sessions
2.1 Train
2.2 Validate
3. Main loop
3.1 Train
3.2 Write summary
3.3 Save model
3.4 Validate model
Author:
Ashley Gritzman
"""
# Set reproduciable random seed
tf.set_random_seed(1234)
# Directories
train_dir, train_summary_dir = conf.setup_train_directories()
# Logger
conf.setup_logger(logger_dir=train_dir, name="logger_train.txt")
# Hyperparameters
conf.load_or_save_hyperparams(train_dir)
# Get dataset hyperparameters
logger.info('Using dataset: {}'.format(FLAGS.dataset))
dataset_size_train = conf.get_dataset_size_train(FLAGS.dataset)
dataset_size_val = conf.get_dataset_size_validate(FLAGS.dataset)
build_arch = conf.get_dataset_architecture(FLAGS.dataset)
num_classes = conf.get_num_classes(FLAGS.dataset)
create_inputs_train = conf.get_create_inputs(FLAGS.dataset, mode="train")
create_inputs_train_wholeset = conf.get_create_inputs(FLAGS.dataset,
mode="train_whole")
if dataset_size_val > 0:
create_inputs_val = conf.get_create_inputs(FLAGS.dataset,
mode="validate")
#*****************************************************************************
# 1. BUILD GRAPHS
#*****************************************************************************
#----------------------------------------------------------------------------
# GRAPH - TRAIN
#----------------------------------------------------------------------------
logger.info('BUILD TRAIN GRAPH')
g_train = tf.Graph()
with g_train.as_default(), tf.device('/cpu:0'):
# Get global_step
global_step = tf.train.get_or_create_global_step()
# Get batches per epoch
num_batches_per_epoch = int(dataset_size_train / FLAGS.batch_size)
# In response to a question on OpenReview, Hinton et al. wrote the
# following:
# "We use an exponential decay with learning rate: 3e-3, decay_steps: 20000, # decay rate: 0.96."
# https://openreview.net/forum?id=HJWLfGWRb¬eId=ryxTPFDe2X
lrn_rate = tf.train.exponential_decay(learning_rate=FLAGS.lrn_rate,
global_step=global_step,
decay_steps=20000,
decay_rate=0.96)
tf.summary.scalar('learning_rate', lrn_rate)
opt = tf.train.AdamOptimizer(learning_rate=lrn_rate)
# Get batch from data queue. Batch size is FLAGS.batch_size, which is then
# divided across multiple GPUs
input_dict = create_inputs_train()
batch_x = input_dict['image']
batch_labels = input_dict['label']
# AG 03/10/2018: Split batch for multi gpu implementation
# Each split is of size FLAGS.batch_size / FLAGS.num_gpus
# See: https://github.com/naturomics/CapsNet-Tensorflow/blob/master/
# dist_version/distributed_train.py
splits_x = tf.split(axis=0,
num_or_size_splits=FLAGS.num_gpus,
value=batch_x)
splits_labels = tf.split(axis=0,
num_or_size_splits=FLAGS.num_gpus,
value=batch_labels)
#--------------------------------------------------------------------------
# MULTI GPU - TRAIN
#--------------------------------------------------------------------------
# Calculate the gradients for each model tower
tower_grads = []
tower_losses = []
tower_logits = []
reuse_variables = None
for i in range(FLAGS.num_gpus):
with tf.device('/gpu:%d' % i):
with tf.name_scope('tower_%d' % i) as scope:
logger.info('TOWER %d' % i)
#with slim.arg_scope([slim.model_variable, slim.variable],
# device='/cpu:0'):
with slim.arg_scope([slim.variable], device='/cpu:0'):
loss, logits = tower_fn(
build_arch,
splits_x[i],
splits_labels[i],
scope,
num_classes,
reuse_variables=reuse_variables,
is_train=True)
# Don't reuse variable for first GPU, but do reuse for others
reuse_variables = True
# Compute gradients for one GPU
grads = opt.compute_gradients(loss)
# Keep track of the gradients across all towers.
tower_grads.append(grads)
# Keep track of losses and logits across for each tower
tower_logits.append(logits)
tower_losses.append(loss)
# Loss for each tower
tf.summary.scalar("loss", loss)
# We must calculate the mean of each gradient. Note that this is the
# synchronization point across all towers.
grad = average_gradients(tower_grads)
# See: https://stackoverflow.com/questions/40701712/how-to-check-nan-in-
# gradients-in-tensorflow-when-updating
grad_check = ([
tf.check_numerics(g, message='Gradient NaN Found!')
for g, _ in grad if g is not None
] + [tf.check_numerics(loss, message='Loss NaN Found')])
# Apply the gradients to adjust the shared variables
with tf.control_dependencies(grad_check):
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = opt.apply_gradients(grad, global_step=global_step)
# Calculate mean loss
loss = tf.reduce_mean(tower_losses)
# Calculate accuracy
logits = tf.concat(tower_logits, axis=0)
acc = met.accuracy(logits, batch_labels)
# Prepare predictions and one-hot labels
probs = tf.nn.softmax(logits=logits)
labels_oh = tf.one_hot(batch_labels, num_classes)
# Group metrics together
# See: https://cs230-stanford.github.io/tensorflow-model.html
trn_metrics = {
'loss': loss,
'labels': batch_labels,
'labels_oh': labels_oh,
'logits': logits,
'probs': probs,
'acc': acc,
}
# Reset and read operations for streaming metrics go here
trn_reset = {}
trn_read = {}
# Logging
tf.summary.scalar('batch_loss', loss)
tf.summary.scalar('batch_acc', acc)
# Set Saver
# AG 26/09/2018: Save all variables including Adam so that we can continue
# training from where we left off
# max_to_keep=None should keep all checkpoints
saver = tf.train.Saver(tf.global_variables(), max_to_keep=None)
# Display number of parameters
train_params = np.sum([
np.prod(v.get_shape().as_list()) for v in tf.trainable_variables()
]).astype(np.int32)
logger.info('Trainable Parameters: {}'.format(train_params))
# Set summary op
trn_summary = tf.summary.merge_all()
#----------------------------------------------------------------------------
# GRAPH - TRAINING SET ACCURACY
#----------------------------------------------------------------------------
logger.info('BUILD TRAINING SET ACCURACY GRAPH')
g_trn_acc = tf.Graph()
with g_trn_acc.as_default():
# Get global_step
global_step = tf.train.get_or_create_global_step()
# Get data
input_dict = create_inputs_train_wholeset()
batch_x = input_dict['image']
batch_labels = input_dict['label']
# AG 10/12/2018: Split batch for multi gpu implementation
# Each split is of size FLAGS.batch_size / FLAGS.num_gpus
# See: https://github.com/naturomics/CapsNet-
# Tensorflow/blob/master/dist_version/distributed_train.py
splits_x = tf.split(axis=0,
num_or_size_splits=FLAGS.num_gpus,
value=batch_x)
splits_labels = tf.split(axis=0,
num_or_size_splits=FLAGS.num_gpus,
value=batch_labels)
#--------------------------------------------------------------------------
# MULTI GPU - TRAINING SET ACCURACY
#--------------------------------------------------------------------------
# Calculate the logits for each model tower
tower_logits = []
reuse_variables = None
for i in range(FLAGS.num_gpus):
with tf.device('/gpu:%d' % i):
with tf.name_scope('tower_%d' % i) as scope:
with slim.arg_scope([slim.variable], device='/cpu:0'):
loss, logits = tower_fn(
build_arch,
splits_x[i],
splits_labels[i],
scope,
num_classes,
reuse_variables=reuse_variables,
is_train=False)
# Don't reuse variable for first GPU, but do reuse for others
reuse_variables = True
# Keep track of losses and logits across for each tower
tower_logits.append(logits)
# Loss for each tower
tf.summary.histogram("train_set_logits", logits)
# Combine logits from all towers
logits = tf.concat(tower_logits, axis=0)
# Calculate metrics
train_set_loss = mod.spread_loss(logits, batch_labels)
train_set_acc = met.accuracy(logits, batch_labels)
# Prepare predictions and one-hot labels
train_set_probs = tf.nn.softmax(logits=logits)
train_set_labels_oh = tf.one_hot(batch_labels, num_classes)
# Group metrics together
# See: https://cs230-stanford.github.io/tensorflow-model.html
train_set_metrics = {
'loss': train_set_loss,
'labels': batch_labels,
'labels_oh': train_set_labels_oh,
'logits': logits,
'probs': train_set_probs,
'acc': train_set_acc,
}
# Reset and read operations for streaming metrics go here
train_set_reset = {}
train_set_read = {}
saver = tf.train.Saver(max_to_keep=None)
tf.summary.scalar("train_set_loss", train_set_loss)
tf.summary.scalar("train_set_acc", train_set_acc)
trn_acc_summary = tf.summary.merge_all()
if dataset_size_val > 0:
#----------------------------------------------------------------------------
# GRAPH - VALIDATION
#----------------------------------------------------------------------------
logger.info('BUILD VALIDATION GRAPH')
g_val = tf.Graph()
with g_val.as_default():
# Get global_step
global_step = tf.train.get_or_create_global_step()
num_batches_val = int(dataset_size_val / FLAGS.batch_size)
# Get data
input_dict = create_inputs_val()
batch_x = input_dict['image']
batch_labels = input_dict['label']
# AG 10/12/2018: Split batch for multi gpu implementation
# Each split is of size FLAGS.batch_size / FLAGS.num_gpus
# See: https://github.com/naturomics/CapsNet-
# Tensorflow/blob/master/dist_version/distributed_train.py
splits_x = tf.split(axis=0,
num_or_size_splits=FLAGS.num_gpus,
value=batch_x)
splits_labels = tf.split(axis=0,
num_or_size_splits=FLAGS.num_gpus,
value=batch_labels)
#--------------------------------------------------------------------------
# MULTI GPU - VALIDATE
#--------------------------------------------------------------------------
# Calculate the logits for each model tower
tower_logits = []
reuse_variables = None
for i in range(FLAGS.num_gpus):
with tf.device('/gpu:%d' % i):
with tf.name_scope('tower_%d' % i) as scope:
with slim.arg_scope([slim.variable], device='/cpu:0'):
loss, logits = tower_fn(
build_arch,
splits_x[i],
splits_labels[i],
scope,
num_classes,
reuse_variables=reuse_variables,
is_train=False)
# Don't reuse variable for first GPU, but do reuse for others
reuse_variables = True
# Keep track of losses and logits across for each tower
tower_logits.append(logits)
# Loss for each tower
tf.summary.histogram("val_logits", logits)
# Combine logits from all towers
logits = tf.concat(tower_logits, axis=0)
# Calculate metrics
val_loss = mod.spread_loss(logits, batch_labels)
val_acc = met.accuracy(logits, batch_labels)
# Prepare predictions and one-hot labels
val_probs = tf.nn.softmax(logits=logits)
val_labels_oh = tf.one_hot(batch_labels, num_classes)
# Group metrics together
# See: https://cs230-stanford.github.io/tensorflow-model.html
val_metrics = {
'loss': val_loss,
'labels': batch_labels,
'labels_oh': val_labels_oh,
'logits': logits,
'probs': val_probs,
'acc': val_acc,
}
# Reset and read operations for streaming metrics go here
val_reset = {}
val_read = {}
tf.summary.scalar("val_loss", val_loss)
tf.summary.scalar("val_acc", val_acc)
# Saver
saver = tf.train.Saver(max_to_keep=None)
# Set summary op
val_summary = tf.summary.merge_all()
#****************************************************************************
# 2. SESSIONS
#****************************************************************************
#----- SESSION TRAIN -----#
# Session settings
#sess_train = tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
# log_device_placement=False),
# graph=g_train)
# Perry: added in for RTX 2070 incompatibility workaround
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
config.gpu_options.allow_growth = True
sess_train = tf.Session(config=config, graph=g_train)
# Debugger
# AG 05/06/2018: Debugging using either command line or TensorBoard
if FLAGS.debugger is not None:
# sess = tf_debug.LocalCLIDebugWrapperSession(sess)
sess_train = tf_debug.TensorBoardDebugWrapperSession(
sess_train, FLAGS.debugger)
with g_train.as_default():
sess_train.run([
tf.global_variables_initializer(),
tf.local_variables_initializer()
])
# Restore previous checkpoint
# AG 26/09/2018: where should this go???
if FLAGS.load_dir is not None:
prev_step = load_training(saver, sess_train, FLAGS.load_dir)
else:
prev_step = 0
# Create summary writer, and write the train graph
summary_writer = tf.summary.FileWriter(train_summary_dir,
graph=sess_train.graph)
#----- SESSION TRAIN SET ACCURACY -----#
#sess_val = tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
# log_device_placement=False),
# graph=g_val)
# Perry: added in for RTX 2070 incompatibility workaround
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
config.gpu_options.allow_growth = True
sess_train_acc = tf.Session(config=config, graph=g_trn_acc)
with g_trn_acc.as_default():
sess_train_acc.run([
tf.local_variables_initializer(),
tf.global_variables_initializer()
])
if dataset_size_val > 0:
#----- SESSION VALIDATION -----#
#sess_val = tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
# log_device_placement=False),
# graph=g_val)
# Perry: added in for RTX 2070 incompatibility workaround
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
config.gpu_options.allow_growth = True
sess_val = tf.Session(config=config, graph=g_val)
with g_val.as_default():
sess_val.run([
tf.local_variables_initializer(),
tf.global_variables_initializer()
])
#****************************************************************************
# 3. MAIN LOOP
#****************************************************************************
SUMMARY_FREQ = 100
SAVE_MODEL_FREQ = num_batches_per_epoch # 500
VAL_FREQ = num_batches_per_epoch # 500
PROFILE_FREQ = 5
for step in range(prev_step, FLAGS.epoch * num_batches_per_epoch + 1):
#for step in range(0,3):
# AG 23/05/2018: limit number of iterations for testing
# for step in range(100):
epoch_decimal = step / num_batches_per_epoch
epoch = int(np.floor(epoch_decimal))
# TF queue would pop batch until no file
try:
# TRAIN
with g_train.as_default():
# With profiling
if (FLAGS.profile is True) and ((step % PROFILE_FREQ) == 0):
logger.info("Train with Profiling")
run_options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
# Without profiling
else:
run_options = None
run_metadata = None
# Reset streaming metrics
if step % (num_batches_per_epoch / 4) == 1:
logger.info("Reset streaming metrics")
sess_train.run([trn_reset])
# MAIN RUN
tic = time.time()
train_op_v, trn_metrics_v, trn_summary_v = sess_train.run(
[train_op, trn_metrics, trn_summary],
options=run_options,
run_metadata=run_metadata)
toc = time.time()
# Read streaming metrics
trn_read_v = sess_train.run(trn_read)
# Write summary for profiling
if run_options is not None:
summary_writer.add_run_metadata(
run_metadata, 'epoch{:f}'.format(epoch_decimal))
# Logging
#logger.info('TRN'
# + ' e-{:d}'.format(epoch)
# + ' stp-{:d}'.format(step)
# + ' {:.2f}s'.format(toc - tic)
# + ' loss: {:.4f}'.format(trn_metrics_v['loss'])
# + ' acc: {:.2f}%'.format(trn_metrics_v['acc']*100)
# )
except KeyboardInterrupt:
sess_train.close()
sess_val.close()
sys.exit()
except tf.errors.InvalidArgumentError as e:
logger.warning('%d iteration contains NaN gradients. Discard.' %
step)
logger.error(str(e))
continue
else:
# WRITE SUMMARY
if (step % SUMMARY_FREQ) == 0:
logger.info("Write Train Summary")
with g_train.as_default():
# Summaries from graph
summary_writer.add_summary(trn_summary_v, step)
# SAVE MODEL
if (step % SAVE_MODEL_FREQ) == 0:
logger.info("Save Model")
with g_train.as_default():
train_checkpoint_dir = train_dir + '/checkpoint'
if not os.path.exists(train_checkpoint_dir):
os.makedirs(train_checkpoint_dir)
# Save ckpt from train session
ckpt_path = os.path.join(train_checkpoint_dir,
'model.ckpt' + str(epoch))
saver.save(sess_train, ckpt_path, global_step=step)
if (step % VAL_FREQ) == 0:
# calculate metrics every epoch
with g_trn_acc.as_default():
logger.info("Start Train Set Accuracy")
# Restore ckpt to val session
latest_ckpt = tf.train.latest_checkpoint(
train_checkpoint_dir)
saver.restore(sess_train_acc, latest_ckpt)
# Reset accumulators
accuracy_sum = 0
loss_sum = 0
sess_train_acc.run(train_set_reset)
for i in range(num_batches_per_epoch):
train_set_metrics_v, train_set_summary_str_v = sess_train_acc.run(
[train_set_metrics, trn_acc_summary])
# Update
accuracy_sum += train_set_metrics_v['acc']
loss_sum += train_set_metrics_v['loss']
# Read
trn_read_v = sess_train_acc.run(val_read)
# Get checkpoint number
ckpt_num = re.split('-', latest_ckpt)[-1]
# Average across batches
ave_acc = accuracy_sum / num_batches_per_epoch
ave_loss = loss_sum / num_batches_per_epoch
logger.info('TRN ckpt-{}'.format(ckpt_num) +
' avg_acc: {:.2f}%'.format(ave_acc * 100) +
' avg_loss: {:.4f}'.format(ave_loss))
logger.info("Write Train Summary")
summary_train = tf.Summary()
summary_train.value.add(tag="trn_acc",
simple_value=ave_acc)
summary_train.value.add(tag="trn_loss",
simple_value=ave_loss)
summary_writer.add_summary(summary_train, epoch)
if dataset_size_val > 0:
#----- Validation -----#
with g_val.as_default():
logger.info("Start Validation")
# Restore ckpt to val session
latest_ckpt = tf.train.latest_checkpoint(
train_checkpoint_dir)
saver.restore(sess_val, latest_ckpt)
# Reset accumulators
accuracy_sum = 0
loss_sum = 0
sess_val.run(val_reset)
for i in range(num_batches_val):
val_metrics_v, val_summary_str_v = sess_val.run(
[val_metrics, val_summary])
# Update
accuracy_sum += val_metrics_v['acc']
loss_sum += val_metrics_v['loss']
# Read
val_read_v = sess_val.run(val_read)
# Get checkpoint number
ckpt_num = re.split('-', latest_ckpt)[-1]
# Logging
#logger.info('VAL ckpt-{}'.format(ckpt_num)
# + ' bch-{:d}'.format(i)
# + ' cum_acc: {:.2f}%'.format(accuracy_sum/(i+1)*100)
# + ' cum_loss: {:.4f}'.format(loss_sum/(i+1))
# )
# Average across batches
ave_acc = accuracy_sum / num_batches_val
ave_loss = loss_sum / num_batches_val
logger.info('VAL ckpt-{}'.format(ckpt_num) +
' avg_acc: {:.2f}%'.format(ave_acc * 100) +
' avg_loss: {:.4f}'.format(ave_loss))
logger.info("Write Val Summary")
summary_val = tf.Summary()
summary_val.value.add(tag="val_acc",
simple_value=ave_acc)
summary_val.value.add(tag="val_loss",
simple_value=ave_loss)
summary_writer.add_summary(summary_val, epoch)
# Close (main loop)
sess_train.close()
sess_val.close()
sys.exit()
training_steps = config.training_steps
start_train = time.perf_counter()
start_step = time.perf_counter()
for epoch in range(epochs_finished, epochs):
for step, (input_data, labels) in enumerate(train_loader):
current_step = step + epoch * steps_per_epoch
start_step = time.perf_counter()
losses, logits = train_model(input_data, labels)
step_length = time.perf_counter() - start_step
scheduler.step()
train_model.setOptimizer(optimizer)
mean_loss = losses.mean().item()
preds = torch.argmax(logits, dim=-1)
acc = accuracy(preds, labels)
step_throughput = config.samples_per_step / step_length
msg = ("Epoch: {:.2f}/{} "
"Step: {}/{} "
"Lr: {:.6f} "
"Loss: {:.3f} "
"Acc: {:.3f} "
"Throughput: {:.2f} samples/sec").format(
epoch, epochs, current_step, training_steps,
scheduler.get_last_lr()[0], mean_loss, acc,
step_throughput)
logger.info(msg)
if config.wandb:
wandb.log({
"LR": scheduler.get_last_lr()[0],
