def evaluate(model, generator, data_type, devices, max_iteration, cuda): """Evaluate Args: model: object. generator: object. data_type: 'train' | 'validate'. devices: list of devices, e.g. ['a'] | ['a', 'b', 'c'] max_iteration: int, maximum iteration for validation cuda: bool. Returns: accuracy: float """ # Generate function generate_func = generator.generate_validate(data_type=data_type, devices=devices, shuffle=True, max_iteration=max_iteration) # Forward dict = forward(model=model, generate_func=generate_func, cuda=cuda, return_target=True) outputs_device = dict['output_device'] # (audios_num, classes_num) devices = dict['device'] # (audios_num, classes_num) outputs = dict['output'] # (audios_num, classes_num) targets = dict['target'] # (audios_num, classes_num) predictions = np.argmax(outputs, axis=-1) # (audios_num,) predictions_device = np.argmax(outputs_device, axis=-1) # (audios_num,) # Evaluate classes_num = outputs.shape[-1] devices_num = outputs_device.shape[-1] loss = F.nll_loss(Variable(torch.Tensor(outputs)), Variable(torch.LongTensor(targets))).data.numpy() loss = float(loss) loss_device = F.nll_loss(Variable(torch.Tensor(outputs_device)), Variable(torch.LongTensor(devices))).data.numpy() loss_device = float(loss_device) confusion_matrix = calculate_confusion_matrix(targets, predictions, classes_num) accuracy = calculate_accuracy(targets, predictions, classes_num, average='macro') accuracy_device = calculate_accuracy(devices, predictions_device, devices_num, average='macro') return accuracy, loss, accuracy_device, loss_device
def fit(self, train_data, batch_sz): dataset, capacity = make_dataset(train_data, batch_sz) dataset = dataset.shuffle(buffer_size=100) dataset = dataset.batch(batch_sz) dataset = dataset.repeat(5) iterator = dataset.make_one_shot_iterator() next_examples, next_labels = iterator.get_next() logits, predicted_classes = self.forward_classifier(next_examples) accuracy = calculate_accuracy(predicted_classes, next_labels) loss = tf.losses.softmax_cross_entropy(onehot_labels=next_labels, logits=logits) self.global_step = tf.train.get_or_create_global_step() train_op = tf.train.AdamOptimizer().minimize(loss, global_step=self.global_step) if not os.path.exists('tmp/'): os.makedirs('tmp/') with tf.train.MonitoredTrainingSession( hooks=[self.hook], config=self.config) as sess: while not sess.should_stop(): sess.run(train_op) acc = sess.run(accuracy) print("Train accuracy: " + str(acc))
def evaluate(model, generator, data_type, max_iteration, cuda): """Evaluate. Args: model: object generator: object data_type: string, 'train' | 'validate' max_iteration: int, maximum iteration for validation cuda: bool Returns: accuracy: float auc: float """ generate_func = generator.generate_validate(data_type=data_type, shuffle=True, max_iteration=max_iteration) # Inference dict = forward(model=model, generate_func=generate_func, cuda=cuda, return_target=True, return_bottleneck=False) outputs = dict['output'] # (audios_num, classes_num) targets = dict['target'] # (audios_num, classes_num) # Evaluate accuracy = calculate_accuracy(targets, outputs) auc = calculate_auc(targets, outputs) return accuracy, auc
def evaluate( model, generator, data_type, devices, max_iteration, ): """Evaluate Args: model: object. generator: object. data_type: 'train' | 'validate'. devices: list of devices, e.g. ['a'] | ['a', 'b', 'c'] max_iteration: int, maximum iteration for validation Returns: accuracy: float """ # Generate function # if data_type == 'train': generate_func = generator.generate_validate(data_type=data_type, devices=devices, shuffle=True, max_iteration=max_iteration) # Forward dict = forward(model=model, generate_func=generate_func, return_target=True) outputs = dict['output'] # (audios_num, classes_num) targets = dict['target'] # (audios_num, classes_num) predictions = np.argmax(outputs, axis=-1) # (audios_num,) # Evaluate classes_num = outputs.shape[-1] # categorical_crossentropy 必须配合softmax使用 binary_crossentropy 配合sigmoid 使用 loss = K.mean( keras.metrics.categorical_crossentropy(K.constant(targets), K.constant(outputs))) loss = K.eval(loss) # confusion_matrix = calculate_confusion_matrix( # targets, predictions, classes_num) targets = np.argmax(targets, axis=-1) accuracy = calculate_accuracy(targets, predictions, classes_num, average='macro') return accuracy, loss
def evaluate(data, model, device): with torch.no_grad(): data_loader = DataLoader(data, batch_size=512) model.to(device) model.eval() total_accuracy = 0 for index, batch in enumerate(data_loader): predicted, labels = perform_forward_loop(model, batch, device) total_accuracy += calculate_accuracy(predicted, labels) return total_accuracy / (index + 1)
def inference_validation(args): # Arugments & parameters workspace = args.workspace holdout_fold = args.holdout_fold iteration = args.iteration filename = args.filename cuda = args.cuda validate = True # Paths hdf5_path = os.path.join(workspace, 'features', 'logmel', 'development.h5') validation_csv = os.path.join(workspace, 'validation.csv') model_path = os.path.join(workspace, 'models', filename, 'holdout_fold{}'.format(holdout_fold), 'md_{}_iters.tar'.format(iteration)) # Load model model = Model() checkpoint = torch.load(model_path) model.load_state_dict(checkpoint['state_dict']) if cuda: model.cuda() # Data generator generator = DataGenerator(hdf5_path=hdf5_path, batch_size=batch_size, validation_csv=validation_csv, holdout_fold=holdout_fold) generate_func = generator.generate_validate(data_type='validate', shuffle=False, max_iteration=None) # Inference dict = forward(model=model, generate_func=generate_func, cuda=cuda, return_target=True, return_bottleneck=False) outputs = dict['output'] targets = dict['target'] itemids = dict['itemid'] # Evaluate va_acc = calculate_accuracy(targets, outputs) va_auc = calculate_auc(targets, outputs) logging.info('va_acc: {:.3f}, va_auc: {:.3f}'.format(va_acc, va_auc))
def evaluate(model, generator, data_type, cuda): """Evaluate Args: model: object. generator: object. data_type: 'train' | 'validate'. cuda: bool. Returns: accuracy: float mapk: float """ if data_type == 'train': max_audios_num = 1000 # A small portion of training data to evaluate elif data_type == 'validate': max_audios_num = None # All evaluation data to evaluate generate_func = generator.generate_validate_slices( data_type=data_type, manually_verified_only=True, shuffle=True, max_audios_num=max_audios_num) # Forward dict = forward(model=model, generate_func=generate_func, cuda=cuda, return_target=True) outputs = dict['output'] # (audios_num, patches_num, classes_num) targets = dict['target'] # (audios_num,) agg_outputs = aggregate_outputs(outputs) '''(audios_num, classes_num)''' predictions = np.argmax(agg_outputs, axis=-1) '''(audios_num,)''' sorted_indices = np.argsort(agg_outputs, axis=-1)[:, ::-1][:, :kmax] '''(audios_num, kmax)''' # Accuracy accuracy = calculate_accuracy(predictions, targets) # mAP mapk_value = mapk(actual=[[e] for e in targets], predicted=[e.tolist() for e in sorted_indices], k=kmax) return accuracy, mapk_value
def model_validate(classifier, class_wise_accuracy=False, plot_confusion_matrix=False): x_val, y_val = prepare_data(datatype='validate') if class_wise_accuracy: predict = classifier.predict(x_val) if plot_confusion_matrix: cm = calculate_confusion_matrix(y_val, predict, 10) plot_confusion_matrix2(cm, "svm", cfg.labels) class_wise_accuracy = calculate_accuracy(y_val, predict, 10) print_accuracy(class_wise_accuracy, cfg.labels) score = np.mean(class_wise_accuracy) else: score = classifier.score(x_val, y_val) print('The accuracy of validation: {:.4f}'.format(score)) return score
def evaluate(self, test_data, batch_sz): dataset, capacity = make_dataset(test_data, batch_sz) dataset = dataset.batch(batch_sz) self.hook.is_training = False iterator = dataset.make_one_shot_iterator() next_examples, next_labels = iterator.get_next() logits, predicted_classes = self.forward_classifier(next_examples) accuracy = calculate_accuracy(predicted_classes, next_labels) with tf.train.MonitoredTrainingSession(hooks=[self.hook]) as sess: while not sess.should_stop(): acc = sess.run(accuracy) print("Test accuracy: " + str(acc))
def stack_two_stage(): X, y = prepare_data('train') X_predict, y_predict = prepare_data('validate') dataset_blend_train = np.load('data.npz.npy') dataset_blend_test = np.load('label.npz.npy') clf = GradientBoostingClassifier(learning_rate=0.005, subsample=0.8, random_state=10, n_estimators=600) clf.fit(dataset_blend_train, y) y_submission = clf.predict(dataset_blend_test) # print("Linear stretch of predictions to [0,1]") # y_submission = (y_submission - y_submission.min()) / (y_submission.max() - y_submission.min()) print("blend result") print("auc Score: %f" % (calculate_accuracy( y_predict, y_submission, classes_num=10, average='macro')))
def evaluate(model, generator, data_type, max_iteration, cuda): """Evaluate. Args: model: object generator: object data_type: string, 'train' | 'validate' max_iteration: int, maximum iteration for validation cuda: bool Returns: accuracy: float f1_score: float loss: float """ generate_func = generator.generate_validate(data_type=data_type, shuffle=True, max_iteration=max_iteration) # Inference dict = forward(model=model, generate_func=generate_func, cuda=cuda, return_target=True) outputs = dict['output'] # (audios_num, classes_num) targets = dict['target'] # (audios_num, classes_num) # Metrics loss = F.nll_loss(torch.Tensor(outputs), torch.LongTensor(targets)).numpy() loss = float(loss) predictions = np.argmax(outputs, axis=-1) accuracy = calculate_accuracy(targets, predictions) f1_score = calculate_f1_score(targets, predictions, average='macro') return accuracy, f1_score, loss
def evaluate(model, generator, data_type, max_iteration, cuda): """Evaluate Args: model: object. generator: object. data_type: 'train' | 'validate'. max_iteration: int, maximum iteration for validation cuda: bool. Returns: accuracy: float """ # Generate function generate_func = generator.generate_validate(data_type=data_type, shuffle=True, max_iteration=max_iteration) # Forward dict = forward(model=model, generate_func=generate_func, cuda=cuda, return_target=True) outputs = dict['output'] # (audios_num, classes_num) targets = dict['target'] # (audios_num, classes_num) predictions = np.argmax(outputs, axis=-1) # (audios_num,) # Evaluate classes_num = outputs.shape[-1] loss = F.nll_loss(torch.Tensor(outputs), torch.LongTensor(targets)).numpy() loss = float(loss) accuracy = calculate_accuracy(targets, predictions) return accuracy, loss
def inference_validation_data(args): # Arugments & parameters dataset_dir = args.dataset_dir subdir = args.subdir workspace = args.workspace holdout_fold = args.holdout_fold iteration = args.iteration filename = args.filename cuda = args.cuda labels = config.labels if 'mobile' in subdir: devices = ['a', 'b', 'c'] else: devices = ['a'] validation = True classes_num = len(labels) # Paths hdf5_path = os.path.join(workspace, 'features', 'logmel', subdir, 'development.h5') dev_train_csv = os.path.join(dataset_dir, subdir, 'evaluation_setup', 'fold1_train.txt') dev_validate_csv = os.path.join(dataset_dir, subdir, 'evaluation_setup', 'fold{}_evaluate.txt'.format(holdout_fold)) model_path = os.path.join(workspace, 'models', subdir, filename, 'holdout_fold={}'.format(holdout_fold), 'md_{}_iters.tar'.format(iteration)) # Load model model = Model(classes_num) checkpoint = torch.load(model_path) model.load_state_dict(checkpoint['state_dict']) if cuda: model.cuda() # Predict & evaluate for device in devices: print('Device: {}'.format(device)) # Data generator generator = DataGenerator(hdf5_path=hdf5_path, batch_size=batch_size, dev_train_csv=dev_train_csv, dev_validate_csv=dev_validate_csv) generate_func = generator.generate_validate(data_type='validate', devices=device, shuffle=False) # Inference dict = forward(model=model, generate_func=generate_func, cuda=cuda, return_target=True) outputs = dict['output'] # (audios_num, classes_num) targets = dict['target'] # (audios_num, classes_num) predictions = np.argmax(outputs, axis=-1) classes_num = outputs.shape[-1] # Evaluate confusion_matrix = calculate_confusion_matrix(targets, predictions, classes_num) class_wise_accuracy = calculate_accuracy(targets, predictions, classes_num) # Print print_accuracy(class_wise_accuracy, labels) print('confusion_matrix: \n', confusion_matrix) logging.info('confusion_matrix: \n', confusion_matrix)
def inference_validation_data(args): # Arugments & parameters dataset_dir = args.dataset_dir subdir = args.subdir workspace = args.workspace holdout_fold = args.holdout_fold iteration = args.iteration filename = args.filename # data_type = args.data_type labels = config.labels if 'mobile' in subdir: devices = ['a', 'b', 'c'] else: devices = ['a'] # Paths hdf5_path = os.path.join(workspace, 'features', 'logmel', subdir, 'development_hpss_lrad.h5') dev_train_csv = os.path.join(dataset_dir, subdir, 'evaluation_setup', 'fold{}_train.txt'.format(holdout_fold)) dev_validate_csv = os.path.join(dataset_dir, subdir, 'evaluation_setup', 'fold{}_evaluate.txt'.format(holdout_fold)) # model_path = os.path.join(workspace, 'models', subdir, filename, # 'holdout_fold={}'.format(holdout_fold), # 'md_{}_iters_max_attention2_2019-05-31 10:33:46.h5'.format(iteration)) # model_path = os.path.join(workspace, 'appendixes', # 'md_{}_iters_max_attention2_2019-05-31 00:48:09.h5'.format(iteration)) model_path = '/home/r506/Downloads/dcase2018_task1-master/models/' \ 'TUT-urban-acoustic-scenes-2018-development/main_keras/' \ 'new/DAN-DFF/md_9700_iters_max_attention2_78.1.h5' model = keras.models.load_model(model_path) # Predict & evaluate for device in devices: print('Device: {}'.format(device)) # Data generator generator = DataGenerator(hdf5_path=hdf5_path, batch_size=batch_size, dev_train_csv=dev_train_csv, dev_validate_csv=dev_validate_csv) generate_func = generator.generate_validate(data_type='validate', devices=device, shuffle=False) # Inference dict = forward(model=model, generate_func=generate_func, return_target=True) outputs = dict['output'] # (audios_num, classes_num) targets = dict['target'] # (audios_num, classes_num) # 多分类交叉熵 predictions = np.argmax(outputs, axis=-1) classes_num = outputs.shape[-1] # Evaluate targets = np.argmax(targets, axis=-1) confusion_matrix = calculate_confusion_matrix(targets, predictions, classes_num) class_wise_accuracy = calculate_accuracy(targets, predictions, classes_num) # Print print_accuracy(class_wise_accuracy, labels) print('confusion_matrix: \n', confusion_matrix) # Plot confusion matrix # plot_confusion_matrix( # confusion_matrix, # title='Device {}'.format(device.upper()), # labels=labels, # values=class_wise_accuracy) np.save('data5', confusion_matrix) plot_confusion_matrix2( confusion_matrix, title='The best performance of the proposed DAN-DFF method', labels=labels, )
def inference_validation_data(args): # Arugments & parameters dataset_dir = args.dataset_dir workspace = args.workspace holdout_fold = args.holdout_fold iteration = args.iteration cuda = args.cuda filename = args.filename validation = True labels = config.labels classes_num = len(config.labels) # Paths hdf5_path = os.path.join(workspace, 'features', 'logmel', 'development.h5') train_txt = os.path.join(dataset_dir, 'evaluation_setup', 'fold{}_train.txt'.format(holdout_fold)) validate_txt = os.path.join(dataset_dir, 'evaluation_setup', 'fold{}_evaluate.txt'.format(holdout_fold)) model_path = os.path.join(workspace, 'models', filename, 'holdout_fold{}'.format(holdout_fold), 'md_{}_iters.tar'.format(iteration)) # Model model = Model(classes_num) checkpoint = torch.load(model_path) model.load_state_dict(checkpoint['state_dict']) if cuda: model.cuda() # Data generator generator = DataGenerator(hdf5_path=hdf5_path, batch_size=batch_size, train_txt=train_txt, validate_txt=validate_txt) generate_func = generator.generate_validate(data_type='validate', shuffle=False, max_iteration=None) # Inference inference_time = time.time() dict = forward(model=model, generate_func=generate_func, cuda=cuda, return_target=True) outputs = dict['output'] targets = dict['target'] audio_names = dict['audio_name'] logging.info('Inference time: {:.4f} s'.format(time.time() - inference_time)) predictions = np.argmax(outputs, axis=-1) # Calculate statistics accuracy = calculate_accuracy(targets, predictions) class_wise_f1_score = calculate_f1_score(targets, predictions, average=None) # Print statistics logging.info('Averaged accuracy: {:.4f}'.format(accuracy)) logging.info('{:<30}{}'.format('class_name', 'f1_score')) logging.info('---------------------------------------') for (n, lb) in enumerate(labels): logging.info('{:<30}{:.4f}'.format(lb, class_wise_f1_score[n])) logging.info('---------------------------------------') logging.info('{:<30}{:.4f}'.format('Average', np.mean(class_wise_f1_score)))
def inference_validation_data(args): # Arugments & parameters dataset_dir = args.dataset_dir subdir = args.subdir workspace = args.workspace holdout_fold = args.holdout_fold iteration = args.iteration filename = args.filename cuda = args.cuda labels = config.labels if 'mobile' in subdir: devices = ['a', 'b', 'c'] else: devices = ['a'] validation = True classes_num = len(labels) devices_num = len(devices) # Paths hdf5_path = os.path.join(workspace, 'features', 'logmel', subdir, 'development.h5') dev_train_csv = os.path.join(dataset_dir, subdir, 'evaluation_setup', 'fold1_train.csv') dev_validate_csv = os.path.join(dataset_dir, subdir, 'evaluation_setup', 'fold{}_evaluate.csv'.format(holdout_fold)) model_path = os.path.join(workspace, 'models', subdir, filename, 'holdout_fold={}'.format(holdout_fold), 'md_{}_iters.tar'.format(iteration)) # Load model model = Model(classes_num, devices_num, cond_layer) checkpoint = torch.load(model_path) model.load_state_dict(checkpoint['state_dict']) if cuda: model.cuda() # Predict & evaluate for device in devices: print('Device: {}'.format(device)) # Data generator generator = DataGenerator(hdf5_path=hdf5_path, batch_size=batch_size, dev_train_csv=dev_train_csv, dev_validate_csv=dev_validate_csv) generate_func = generator.generate_validate(data_type='validate', devices=device, shuffle=False) # Inference dict = forward(model=model, generate_func=generate_func, cuda=cuda, return_target=True) outputs = dict['output'] # (audios_num, classes_num) targets = dict['target'] # (audios_num, classes_num) # (outputs, targets, audio_names, outputs_heatmap) = forward_heatmap(model=model,################### # generate_func=generate_func, # cuda=cuda, # has_target=True) predictions = np.argmax(outputs, axis=-1) classes_num = outputs.shape[-1] ########################################################################## # heatmaps = [] # classes = [] # for i in range(0, len(predictions)): # pred_num = predictions[i] # if pred_num == targets[i]: # if not (pred_num in classes): # classes.append(pred_num) # print 'classes:' # print classes # logging.info('\n') # logging.info(outputs_heatmap[i][pred_num]) # logging.info('class num: ') # logging.info(pred_num) # heatmaps.append(outputs_heatmap[i][pred_num]) ############################################################################ # Evaluate confusion_matrix = calculate_confusion_matrix(targets, predictions, classes_num) class_wise_accuracy = calculate_accuracy(targets, predictions, classes_num) # save # np.save(os.path.join(workspace, 'logs', 'main_pytorch', str(device)+"heatmap.npy"),heatmaps)############## # np.save(os.path.join(workspace, 'logs', 'main_pytorch', str(device)+"confusionMat.npy"),confusion_matrix)######### # Print print_accuracy(class_wise_accuracy, labels) print('confusion_matrix: \n', confusion_matrix) logging.info('confusion_matrix: \n', confusion_matrix)
def inference_validation_data(args): # Arguments & parameters workspace = args.workspace holdout_fold = args.holdout_fold iteration = args.iteration filename = args.filename cuda = args.cuda classes_num = len(config.labels) # Paths model_path = os.path.join(workspace, 'models', filename, 'holdout_fold{}'.format(holdout_fold), 'md_{}_iters.tar'.format(iteration)) hdf5_path = os.path.join(workspace, 'features', 'logmel', 'development.h5') validation_csv = os.path.join(workspace, 'validate_meta.csv') stats_pickle_path = os.path.join(workspace, 'stats', filename, 'holdout_fold{}'.format(holdout_fold), '{}_iters.p'.format(iteration)) create_folder(os.path.dirname(stats_pickle_path)) # Model model = Model(classes_num) checkpoint = torch.load(model_path) model.load_state_dict(checkpoint['state_dict']) if cuda: model.cuda() # Data generator generator = DataGenerator(hdf5_path=hdf5_path, batch_size=batch_size, time_steps=time_steps, validation_csv=validation_csv, holdout_fold=holdout_fold) generate_func = generator.generate_validate_slices( data_type='validate', manually_verified_only=True, shuffle=False, max_audios_num=None) # Forward dict = forward(model=model, generate_func=generate_func, cuda=cuda, return_target=True) outputs = dict['output'] # (audios_num, patches_num, classes_num) targets = dict['target'] # (audios_num,) agg_outputs = aggregate_outputs(outputs) '''(audios_num, classes_num)''' predictions = np.argmax(agg_outputs, axis=-1) '''(audios_num,)''' sorted_indices = np.argsort(agg_outputs, axis=-1)[:, ::-1][:, :kmax] '''(audios_num, kmax)''' # Accuracy accuracy = calculate_accuracy(predictions, targets) # mAP mapk_value = mapk(actual=[[e] for e in targets], predicted=[e.tolist() for e in sorted_indices], k=kmax) # Print logging.info('') logging.info('iteration: {}'.format(iteration)) logging.info('accuracy: {:.3f}'.format(accuracy)) logging.info('mapk: {:.3f}'.format(mapk_value)) (class_wise_accuracy, correctness, total) = print_class_wise_accuracy(predictions, targets) # Save stats for current holdout training dict = { 'correctness': correctness, 'total': total, 'accuracy': accuracy, 'mapk': mapk_value } pickle.dump(dict, open(stats_pickle_path, 'wb')) logging.info('Write out stat to {}'.format(stats_pickle_path))
def main(): args = parse_args() ## Create an output dir output_dir_path = args.od + args.en if not os.path.exists(output_dir_path): os.makedirs(output_dir_path) dir_name=output_dir_path tb_dirname = output_dir_path + '/tb_logdir' else: counter=1 dir_name = output_dir_path new_dir_name = dir_name while os.path.exists(new_dir_name): new_dir_name = dir_name + "_" + str(counter) counter +=1 os.makedirs(new_dir_name) dir_name=new_dir_name tb_dirname = dir_name + "/tb_logdir" print("===>> Output folder = {}".format(dir_name)) args.cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) if args.cuda: torch.backends.cudnn.benchmark = True torch.cuda.manual_seed(args.seed) loaders = Cifar10Loaders() train_loader = loaders.train_loader() test_loader = loaders.test_loader() if args.netqat: base_model = cnn() model=cnn(qat_mode=True) ## Loading checkpoints here checkpoint = torch.load(args.load_ckpt) ##making sure that the checkpoint was loaded from disk correctly base_model.load_state_dict(checkpoint['model_state_dict'],strict=True) base_model_sd = base_model.state_dict() ##renaming the keys only to match the model with qat nodes. Only convs and BNs's will be changed base_model_sd_new = map_ckpt_names(base_model_sd) ##Updating the values of keys for qat model for k in base_model_sd_new.keys(): try: model.state_dict()[k].copy_(base_model_sd_new[k]) print("{} successfully loaded".format(k)) except: print("{} didnt load".format(k)) else: model=cnn() ## Instantiate tensorboard logs writer = SummaryWriter(tb_dirname) images, labels = iter(train_loader).next() img_grid = torchvision.utils.make_grid(images) writer.add_image('cifar-10', img_grid) #writer.add_graph(model,images) if args.cuda: model = model.cuda() if args.parallel: model = nn.DataParallel(model,device_ids=range(torch.cuda.device_count())) criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.wd) best_test_accuracy=0 print("===>> Training started") optimizer.load_state_dict(checkpoint['optimizer_state_dict']) epoch = checkpoint['epoch'] loss = checkpoint['loss'] print("===>>> Checkpoint loaded successfully from {} at epoch {} ".format(args.load_ckpt,epoch)) print(model) for epoch in range(args.start_epoch, args.start_epoch + args.num_epochs): running_loss=0.0 start=time.time() model.train() for i, data in enumerate(train_loader,0): inputs, labels = data if args.cuda: inputs = inputs.cuda() labels = labels.cuda() optimizer.zero_grad() outputs = model(inputs) loss = criterion(outputs,labels) loss.backward() optimizer.step() running_loss +=loss.item() if i %1000 == 999: writer.add_scalar('train_loss',running_loss/1000, epoch * len(train_loader) + i) writer.add_scalar('learning_rate',optimizer.param_groups[0]['lr'],epoch * len(train_loader) + i) if epoch > 0 and epoch % args.lrdt == 0: print("===>> decaying learning rate at epoch {}".format(epoch)) for param_group in optimizer.param_groups: param_group['lr'] = param_group['lr'] * 0.94 running_loss /= len(train_loader) end = time.time() test_accuracy = calculate_accuracy(model,test_loader) writer.add_scalar('test_accuracy',test_accuracy, epoch) ## Adding histograms after every epoch for tag, value in model.named_parameters(): tag = tag.replace('.','/') writer.add_histogram(tag,value.data.cpu().numpy(),epoch) print("Epoch: {0} | Loss: {1} | Test accuracy: {2}| Time Taken (sec): {3} ".format(epoch+1, np.around(running_loss,6), test_accuracy, np.around((end-start),4))) best_ckpt_filename = dir_name + "/ckpt_" + str(epoch) +'.pth' ##Save the best checkpoint if test_accuracy > best_test_accuracy: best_test_accuracy = test_accuracy torch.save({ 'epoch': epoch, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'loss': running_loss, }, best_ckpt_filename) writer.close() print("Training finished")