def validate(args, epoch, model, loader, criterion, logger): steps = len(loader) local_loss = AverageMeter() local_acc = AverageMeter() local_recall = AverageMeter() aver_loss = AverageMeter() aver_acc = AverageMeter() aver_recall = AverageMeter() model.eval() if args.verbose: logger.info("Validating") with torch.no_grad(): for i, (images, targets) in enumerate(loader, start=1): if args.gpu is not None: images = images.cuda(args.gpu, non_blocking=True) targets = targets.cuda(args.gpu, non_blocking=True) outputs = model(images) if args.multi: outputs = torch.sigmoid(outputs) loss = criterion(outputs, targets) if args.multi: precision, recall = calculate_metrics( outputs.detach().cpu().numpy(), targets.detach().cpu().numpy(), args.threshold) else: precision = accuracy(outputs, targets)[0].item() recall = precision local_loss.update(loss.item(), images.size(0)) local_acc.update(precision, images.size(0)) local_recall.update(recall, images.size(0)) if args.distributed: running_metrics = torch.FloatTensor( [loss.item(), precision, recall]).cuda(args.gpu) running_metrics /= args.world_size dist.all_reduce(running_metrics, op=dist.ReduceOp.SUM) aver_loss.update(running_metrics[0].item()) aver_acc.update(running_metrics[1].item()) aver_recall.update(running_metrics[2].item()) else: aver_loss.update(loss.item(), images.size(0)) aver_acc.update(precision, images.size(0)) aver_recall.update(recall, images.size(0)) if args.verbose and i % args.log_interval == 0: logger.info("Epoch: [{}] [{}]/[{}]({:.2%}) " "Loss: {:.4f} / {:.4f} / {:.4f} " "Acc: {:.2f} / {:.2f} / {:.2f} " "Recall: {:.2f} / {:.2f} / {:.2f}".format( epoch, i, steps, i / steps, loss, local_loss.avg, aver_loss.avg, precision, local_acc.avg, aver_acc.avg, recall, local_recall.avg, aver_recall.avg)) return aver_loss.avg, aver_acc.avg, aver_recall.avg
def test(args, model, device, test_loader): if test_loader.dataset.train: print("test on validation set\r\n") else: print("test on test set\r\n") # validate model.eval() # test_loss = 0 # correct = 0 # num_samples = 0 losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() with torch.no_grad(): for data in test_loader: inputs, labels = data[0].to(device), data[1].to(device) outputs = model(inputs) test_loss = criterion(outputs, labels).item() # pred = outputs.argmax(dim=1, keepdim=True) # correct += pred.eq(labels.view_as(pred)).sum().item() # # _, pred = torch.max(outputs, 1) # # correct += (pred == labels).sum().item() # num_samples += pred.size(0) prec1, prec5 = accuracy(outputs, labels, topk=(1, 5)) losses.update(test_loss, labels.size(0)) top1.update(prec1[0], labels.size(0)) top5.update(prec5[0], labels.size(0)) print( '\nTest set: Average loss: {:.4f}, Accuracy: Prec@1:{}/{} ({:.2f}%) Prec@5:{}/{} ({:.2f}%)\n' .format(losses.avg, top1.sum // 100, top1.count, top1.avg, top5.sum // 100, top1.count, top5.avg)) return top1.avg, top5.avg
def validate(val_loader, model, criterion): model.eval() all_acc = [] all_acc_top5 = [] all_loss = [] if args.model == "Elastic_InceptionV3": for ix in range((num_outputs - 1)): all_loss.append(AverageMeter()) all_acc.append(AverageMeter()) all_acc_top5.append(AverageMeter()) else: for ix in range(num_outputs): all_loss.append(AverageMeter()) all_acc.append(AverageMeter()) all_acc_top5.append(AverageMeter()) for i, (input, target) in enumerate(val_loader): target = target.cuda(async=True) input_var = torch.autograd.Variable(input) target_var = torch.autograd.Variable(target) losses = 0 outputs = model(input_var) with torch.no_grad(): for ix in range(len(outputs)): loss = criterion(outputs[ix], target_var) all_loss[ix].update(loss.item(), input.size(0)) losses += loss prec1 = accuracy(outputs[ix].data, target) all_acc[ix].update(prec1[0].data[0].item(), input.size(0)) # top 5 accuracy prec5 = accuracy(outputs[ix].data, target, topk=(5, )) all_acc_top5[ix].update(prec5[0].data[0].item(), input.size(0)) accs = [] ls = [] accs_top5 = [] for i, j, k in zip(all_acc, all_loss, all_acc_top5): accs.append(float(100 - i.avg)) ls.append(j.avg) accs_top5.append(float(100 - k.avg)) print("validation top 5 error: ", accs_top5) return accs, ls, accs_top5
def train(train_loader, model, criterion, optimizer, epoch, args, lth_pruner, cur_round, mask_applied): batch_time = helper.AverageMeter('Time', ':6.3f') data_time = helper.AverageMeter('Data', ':6.3f') losses = helper.AverageMeter('Loss', ':.4e') top1 = helper.AverageMeter('Acc@1', ':6.2f') top5 = helper.AverageMeter('Acc@5', ':6.2f') progress = helper.ProgressMeter( len(train_loader), [batch_time, data_time, losses, top1, top5], prefix="Epoch: [{}]".format(epoch)) # switch to train mode model.train() end = time.time() for i, (images, target) in enumerate(train_loader): data_time.update(time.time() - end) images, target = images.cuda(), target.cuda() output = model(images) # import pdb; # pdb.set_trace() loss = criterion(output, target) acc1, acc5 = helper.accuracy(output, target, topk=(1, 5)) losses.update(loss.item(), images[0].size(0)) top1.update(acc1[0], images[0].size(0)) top5.update(acc5[0], images[0].size(0)) # compute gradient and do SGD step optimizer.zero_grad() loss.backward() #aa = list(model.parameters()) #print(aa[0].mean()) #If in prune mode, block out gradients. #Basically after LTH pruning starts,maintain zero weights at zero. if cur_round > 0 or mask_applied: for k, (name, param) in enumerate(model.named_parameters()): #if 'weight' in name: if name in lth_pruner.mask: weight_copy = param.data.abs().clone() mask = weight_copy.gt(0).float().cuda() param.grad.data.mul_(mask) optimizer.step() batch_time.update(time.time() - end) end = time.time() #pdb.set_trace() if i % 10 == 0: progress.display(i) return top1.avg, top5.avg, losses.avg, model
def classifier_transfer(high_fid_data, batch_size, epoch, option=1): print("------------------in transfer----------------------") """ transfer learning: 1. reuse seq2seq and FNN weights and train both of them 2. reuse seq2seq and FNN weights and train FNN weights 3. reuse seq2seq weights, reinitialize FNN weights and train FNN only other: return ErrorMessage """ sess = tf.compat.v1.Session() saver = tf.compat.v1.train.import_meta_graph("pretrain.meta") saver.restore(sess, tf.train.latest_checkpoint("./")) graph = tf.compat.v1.get_default_graph() x = graph.get_tensor_by_name("x:0") y = graph.get_tensor_by_name("y:0") y_ = graph.get_tensor_by_name("y_:0") loss = graph.get_tensor_by_name("loss:0") train = tf.compat.v1.get_collection("optimizer")[0] new_train = tf.compat.v1.get_collection("optimizer")[1] if option == 1: optimizer = train elif option == 2: optimizer = new_train elif option == 3: optimizer = new_train var = tf.compat.v1.global_variables() var_to_init = [ val for val in var if ("fnn/W" in val.name or "fnn/b" in val.name) ] epoch *= 3 sess.run(tf.compat.v1.variables_initializer(var_to_init)) else: print("option not available, please assign 1 or 2 or 3 to option") return for i in range(epoch): for _ in range(int(high_fid_data.train_n / batch_size)): d = high_fid_data.train_next_batch(batch_size) sess.run(optimizer, feed_dict={ x: d[0], y_: threeclasses(d[1][:, None]) }) d = high_fid_data.test() err = sess.run(loss, feed_dict={ x: d[0], y_: threeclasses(d[1][:, None]) }) y_pred = sess.run(y, feed_dict={x: d[0]}) acc = accuracy(threeclasses(d[1][:, None]), y_pred) print("Epoch %d, %f, test acc: %f" % (i, err, acc)) return acc
def validate(val_loader, model, criterion, print_freq, colorization=False): batch_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() # switch to evaluate mode model.eval() end = time.time() for i, (input, target) in enumerate(val_loader): # target = target.cuda() # input = input.cuda() if args.mode=='pretrain': target = target.to(device, dtype=torch.float32) input = input.to(device, dtype=torch.float32) else: target = target.cuda() input = input.cuda() if colorization: input = transforms.Resize(500)(input) target = transforms.Resize(500)(target) input = input.repeat(1,3,1,1) if args.pretrain_task=='colorization': input = transforms.Resize(500)(input) with torch.no_grad(): # compute output output = model(input) loss = criterion(output, target) # measure accuracy and record loss losses.update(loss.item(), input.size(0)) if not colorization: prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) top1.update(prec1, input.size(0)) top5.update(prec5, input.size(0)) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if not colorization: print('Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( i, len(val_loader), batch_time=batch_time, loss=losses, top1=top1, top5=top5)) else: print('Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format( i, len(val_loader), batch_time=batch_time, loss=losses)) return losses.avg, top1.avg, top5.avg
def train(train_loader, model, criterion, optimizer, epoch, print_freq): device = xm.xla_device() batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() # switch to train mode model.train() end = time.time() for i, (input, target) in enumerate(train_loader): # measure data loading time data_time.update(time.time() - end) target = target.to(device) #.cuda(non_blocking=True) input = input.to(device) #.cuda(non_blocking=True) # compute output output = model(input) loss = criterion(output, target) # measure accuracy and record loss prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) losses.update(loss.item(), input.size(0)) top1.update(prec1[0], input.size(0)) top5.update(prec1[0], input.size(0)) # compute gradient and do SGD step optimizer.zero_grad() loss.backward() optimizer.step() # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % print_freq == 0: print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})\t'.format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses, top1=top1, top5=top5))
def validate(val_loader, model, criterion, print_freq, epoch): losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() # switch to evaluate mode model.eval() # model.apply(unfreeze_bn) for i, (input, target) in enumerate(val_loader): target = target.cuda(non_blocking=True) input = input.cuda(non_blocking=True) with torch.no_grad(): # compute output output = model(input) loss = criterion(output, target) # measure accuracy and record loss prec = accuracy(output.data, target, topk=(1, 5)) losses.update(loss.item(), input.size(0)) top1.update(prec[0].item(), input.size(0)) top5.update(prec[1].item(), input.size(0)) # measure elapsed time if i % print_freq == 0: print('Test: [{0}][{1}/{2}]\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( epoch, i, len(val_loader), loss=losses, top1=top1, top5=top5)) n_iter = (epoch) * len(train_loader) + i + 1 # if args.record: # writer.add_scalar('Test/Average loss', losses.val, n_iter) # writer.add_scalar('Test/Accuracy', top1.val, n_iter) model.train() f_loss.write('\n epoch {} test with loss {:.4f} \n'.format( epoch, losses.avg)) f_acc.write('\n epoch {} test with top1 {:.3f} and top5 {:.3f} \n'.format( epoch, top1.avg, top5.avg)) print(' * Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}'.format(top1=top1, top5=top5)) return top1.avg, top5.avg
def train(train_loader, model, criterion, optimizer, epoch, print_freq): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() # switch to train mode model.train() end = time.time() for i, (input, target) in enumerate(train_loader): # measure data loading time data_time.update(time.time() - end) target = target.cuda(async=True) input = input.cuda(async=True) # compute output output = model(input) loss = criterion(output, target) # measure accuracy and record loss prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) losses.update(loss.item(), input.size(0)) top1.update(prec1[0], input.size(0)) top5.update(prec5[0], input.size(0)) # compute gradient and do SGD step optimizer.zero_grad() loss.backward() optimizer.step() # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % print_freq == 0: printInfo = 'Epoch: [{0}][{1}/{2}],\t'.format(epoch,i,len(train_loader))+\ 'Time {batch_time.val:.3f}({batch_time.avg:.3f}),\t'.format(batch_time=batch_time) + \ 'Data {data_time.val:.3f} ({data_time.avg:.3f}),\t'.format(data_time=data_time) + \ 'Loss {loss.val:.4f} ({loss.avg:.4f}),\t'.format(loss=losses) + \ 'Prec@1 {top1.val:.3f} ({top1.avg:.3f}),\t'.format(top1=top1) + \ 'Prec@5 {top5.val:.3f} ({top5.avg:.3f}),\t'.format(top5=top5) logging.info(printInfo)
def validate(val_loader, model, criterion, print_freq): batch_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() # switch to evaluate mode model.eval() end = time.time() for i, (input, target) in enumerate(val_loader): target = target.cuda(async=True) input = input.cuda(async=True) with torch.no_grad(): # compute output output = model(input) loss = criterion(output, target) # measure accuracy and record loss prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) losses.update(loss.item(), input.size(0)) top1.update(prec1[0], input.size(0)) top5.update(prec5[0], input.size(0)) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % print_freq == 0: print('Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( i, len(val_loader), batch_time=batch_time, loss=losses, top1=top1, top5=top5)) print(' * Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}'.format(top1=top1, top5=top5)) return top1.avg, top5.avg
def validate(val_loader, model, criterion, args): batch_time = helper.AverageMeter('Time', ':6.3f') losses = helper.AverageMeter('Loss', ':.4e') top1 = helper.AverageMeter('Acc@1', ':6.2f') top5 = helper.AverageMeter('Acc@5', ':6.2f') progress = helper.ProgressMeter(len(val_loader), [batch_time, losses, top1, top5], prefix='Test: ') # switch to evaluate mode model.eval() with torch.no_grad(): end = time.time() #print(len(val_loader),args.batch_size) for i, (images, target) in enumerate(val_loader): images, target = images.cuda(), target.cuda() output = model(images) loss = criterion(output, target) # measure accuracy and record loss acc1, acc5 = helper.accuracy(output, target, topk=(1, 5)) losses.update(loss.item(), images.size(0)) top1.update(acc1[0], images.size(0)) top5.update(acc5[0], images.size(0)) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % 10 == 0: progress.display(i) # TODO: this should also be done with the helper.ProgressMeter print(' * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'.format(top1=top1, top5=top5)) return top1.avg, top5.avg, losses.avg
def validatetrain(val_loader, model, criterion): losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() model.eval() for i, (input, target) in enumerate(val_loader): target = target.cuda(non_blocking=True) input = input.cuda(non_blocking=True) with torch.no_grad(): # compute output output = model(input) loss = criterion(output, target) # measure accuracy and record loss prec = accuracy(output.data, target, topk=(1, 5)) losses.update(loss.item(), input.size(0)) top1.update(prec[0].item(), input.size(0)) top5.update(prec[1].item(), input.size(0)) model.train() print(' *TRAINMODE Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}'.format( top1=top1, top5=top5))
def batch_boost_train(train_loader, model,weak_model,samplewise_criterion,\ optimizer_1,optimizer_2,epoch, args): batch_time = helper.AverageMeter('Time', ':6.3f') data_time = helper.AverageMeter('Data', ':6.3f') losses_strong = helper.AverageMeter('Loss strong', ':.4e') top1_strong = helper.AverageMeter('Acc@1 strong', ':6.2f') losses_weak = helper.AverageMeter('Loss weak', ':.4e') top1_weak = helper.AverageMeter('Acc@1 weak', ':6.2f') progress = helper.ProgressMeter(len(train_loader), [ batch_time, data_time, losses_strong, top1_strong, losses_weak, top1_weak ], prefix="Epoch: [{}]".format(epoch)) samplewise_criterion_weak = nn.CrossEntropyLoss(reduce='none') # switch to train mode model.train() weak_model.train() end = time.time() for i, (images, target) in enumerate(train_loader): data_time.update(time.time() - end) images, target = images.cuda(), target.cuda() output = model(images) samplewise_loss_strong = samplewise_criterion(output, target) strong_loss = samplewise_loss_strong.mean() acc1, acc5 = helper.accuracy(output, target, topk=(1, 5)) losses_strong.update(strong_loss.item(), images[0].size(0)) top1_strong.update(acc1[0], images[0].size(0)) # compute gradient and do SGD step optimizer_1.zero_grad() strong_loss.backward() #Use the per sample loss from strong model as weights for the weak model. #Rescale sample wise loss to [0,1] weights = samplewise_loss_strong.clone().detach() weights -= weights.min() weights /= weights.max() output_2 = weak_model(images) samplewise_loss_weak = samplewise_criterion_weak(output_2, target) #Use it as weightsfor the weak model loss. samplewise_loss_weak = samplewise_loss_weak * weights weak_loss = samplewise_loss_weak.mean() acc1, acc5 = helper.accuracy(output_2, target, topk=(1, 5)) losses_weak.update(weak_loss.item(), images[0].size(0)) top1_weak.update(acc1[0], images[0].size(0)) optimizer_2.zero_grad() weak_loss.backward() #If in prune mode, block out gradients. #Basically after LTH pruning starts,maintain zero weights at zero. for k, (name, param) in enumerate(model.named_parameters()): if 'weight' in name and 'bn' not in name: weight_copy = param.data.abs().clone() mask = weight_copy.gt(0).float().cuda() param.grad.data.mul_(mask) for k, (name, param) in enumerate(weak_model.named_parameters()): if 'weight' in name and 'bn' not in name: weight_copy = param.data.abs().clone() mask = weight_copy.gt(0).float().cuda() param.grad.data.mul_(mask) optimizer_1.step() optimizer_2.step() #pdb.set_trace() batch_time.update(time.time() - end) end = time.time() if i % 10 == 0: progress.display(i) return top1_strong.avg, top1_weak.avg, losses_strong.avg, losses_weak.avg, model, weak_model
def meta_fit(self, meta_dataset_generator) -> Learner: """ Args: meta_dataset_generator : a DataGenerator object. We can access the meta-train and meta-validation episodes via its attributes. Refer to the metadl/data/dataset.py for more details. Returns: MyLearner object : a Learner that stores the meta-learner's learning object. (e.g. a neural network trained on meta-train episodes) """ meta_train_dataset = meta_dataset_generator.meta_train_pipeline meta_valid_dataset = meta_dataset_generator.meta_valid_pipeline meta_train_dataset = meta_train_dataset.batch(1) # meta_valid_dataset = meta_valid_dataset.batch(2) logging.info('Starting meta-fit for the proto-net ...') self.embedding_fn.eval() acc_list = [] for tasks_batch in meta_train_dataset: sup_set = tf.data.Dataset.from_tensor_slices(\ (tasks_batch[0][1], tasks_batch[0][0])) que_set = tf.data.Dataset.from_tensor_slices(\ (tasks_batch[0][4], tasks_batch[0][3])) new_ds = tf.data.Dataset.zip((sup_set, que_set)) self.count += 1 for ((spt_labs, spt_img), (qry_labs, qry_img)) in new_ds: spt_img = torch.tensor(np.array(spt_img)).permute( 0, 3, 1, 2).contiguous().to('cuda', non_blocking=True) spt_labs = torch.tensor(np.array(spt_labs)).to( 'cuda', non_blocking=True) with torch.no_grad(): spt_fea = self.embedding_fn(spt_img, True)[0] self.classifier.fit(spt_fea, spt_labs) qry_img = torch.tensor(np.array(qry_img)).permute( 0, 3, 1, 2).contiguous().to('cuda', non_blocking=True) qry_labs = torch.tensor(np.array(qry_labs)).to( 'cuda', non_blocking=True) torch.cuda.empty_cache() prob_list = [] for a_img in qry_img: a_img = a_img.unsqueeze(0) with torch.no_grad(): a_img = self.embedding_fn(a_img, True)[0] prob_list.append(self.classifier.predict(a_img)) probs = torch.cat(prob_list) # pdb.set_trace() acc_list.append(accuracy(probs.max(dim=1)[1], qry_labs)) # Backward Propagation self.optimizer.zero_grad() torch.cuda.empty_cache() qry_labs = torch.tensor(qry_labs, dtype=torch.long).cuda() ce_loss = F.cross_entropy(probs, qry_labs) torch.cuda.empty_cache() ce_loss.backward() torch.cuda.empty_cache() self.optimizer.step() torch.cuda.empty_cache() self.lr_schedule.step() if self.count % 50 == 0: acc, h_ = mean_confidence_interval(acc_list) acc = round(acc * 100, 2) h_ = round(h_ * 100, 2) print("meta_test aver_accuracy:", acc, "h:", h_) acc_list = [] if (self.count > self.meta_iterations): break return MyLearner(embedding_fn=self.embedding_fn, classifier=self.classifier)
def classifier( low_fid_data, high_fid_data, nblock, nn_size, nn_layer, learning_rate, batch_size, epoch, beta, ): batch_size = min(high_fid_data.train_n, batch_size) outdim = 2 tf.compat.v1.reset_default_graph() sampling_horizon = low_fid_data.sampling_horizon if low_fid_data.feature is not None: sampling_horizon += low_fid_data.feature[0].size x = tf.compat.v1.placeholder(tf.float32, [None, sampling_horizon], name="x") # weights = tf.compat.v1.placeholder(tf.float32, [sampling_horizon], name="weights") learn_rate = tf.constant(learning_rate, name="learn_rate") print(f"Learning rate: {learn_rate}") y = x for _ in range(nblock): y = attention_block(y, sampling_horizon) y = tf.squeeze(y) with tf.compat.v1.variable_scope("fnn"): W = tf.Variable(tf.random.normal([sampling_horizon, nn_size], stddev=0.1), name="W") b = tf.Variable(tf.random.normal([nn_size], stddev=0.1), name="b") y = tf.nn.relu(tf.tensordot(y, W, [[1], [0]]) + b) for _ in range(nn_layer - 1): W = tf.Variable(tf.random.normal([nn_size, nn_size], stddev=0.1), name="W") b = tf.Variable(tf.random.normal([nn_size], stddev=0.1), name="b") y = tf.nn.relu(tf.tensordot(y, W, [[1], [0]]) + b) W = tf.Variable(tf.random.normal([nn_size, outdim], stddev=0.1), name="W") b = tf.Variable(tf.random.normal([], stddev=0.1), name="b") y = tf.identity(tf.tensordot(y, W, [[1], [0]]) + b, name="y") y_ = tf.compat.v1.placeholder(tf.float32, [None, outdim], name="y_") loss = tf.compat.v1.losses.softmax_cross_entropy(y_, y) # add L2 regularization L2_var = [ var for var in tf.compat.v1.global_variables() if ("fnn/W" in var.name or "fnn/b" in var.name) and "Adam" not in var.name ] lossL2 = tf.math.add_n([tf.nn.l2_loss(v) for v in L2_var]) * beta loss = tf.identity(loss + lossL2, name="loss") train = tf.compat.v1.train.AdamOptimizer(learning_rate).minimize(loss) tf.compat.v1.add_to_collections("optimizer", train) saver = tf.compat.v1.train.Saver() sess = tf.compat.v1.Session() sess.run(tf.compat.v1.global_variables_initializer()) for i in range(epoch): low_fid_data.mixup("None") for _ in range(int(low_fid_data.train_n / batch_size)): d = low_fid_data.train_next_batch(batch_size) sess.run(train, feed_dict={ x: d[0], y_: hypoglycemia(d[1][:, None]) }) d = high_fid_data.test() err = sess.run(loss, feed_dict={ x: d[0], y_: hypoglycemia(d[1][:, None]) }) y_pred = sess.run(y, feed_dict={x: d[0]}) acc = accuracy(hypoglycemia(d[1][:, None]), y_pred) print("Epoch %d, %f, test acc: %f" % (i, err, acc)) saver.save(sess, "pretrain") return acc, np.hstack((np.argmax(hypoglycemia(d[1][:, None]), axis=1).reshape((-1, 1)), y_pred))
def classifier_transfer(high_fid_data, batch_size, epoch, option=1): print("------------------in transfer----------------------") """ transfer learning: 1. reuse attention and FNN weights and train both of them 2. reuse attention and FNN weights and train FNN weights 3. reuse attention weights, reinitialize FNN weights and train FNN only other: return ErrorMessage """ sess = tf.compat.v1.Session() saver = tf.compat.v1.train.import_meta_graph("pretrain.meta") saver.restore(sess, tf.train.latest_checkpoint("./")) graph = tf.compat.v1.get_default_graph() x = graph.get_tensor_by_name("x:0") y = graph.get_tensor_by_name("y:0") y_ = graph.get_tensor_by_name("y_:0") # weights = graph.get_tensor_by_name("weights:0") loss = graph.get_tensor_by_name("loss:0") learning_rate = graph.get_tensor_by_name("learn_rate:0") if option == 1: optimizer = tf.compat.v1.get_collection("optimizer")[0] elif option > 1: var_to_init = [ var for var in tf.compat.v1.global_variables() if ("fnn/W" in var.name or "fnn/b" in var.name) and "Adam" not in var.name ] optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate, name="newAdam").minimize( loss, var_list=var_to_init) uninitialized_vars = [] for var in tf.compat.v1.global_variables(): try: sess.run(var) except tf.errors.FailedPreconditionError: uninitialized_vars.append(var) sess.run(tf.compat.v1.variables_initializer(uninitialized_vars)) if option == 3: # reinitialize weights in FNN epoch *= 3 sess.run(tf.compat.v1.variables_initializer(var_to_init)) for i in range(epoch): for _ in range(int(high_fid_data.train_n / batch_size)): d = high_fid_data.train_next_batch(batch_size) sess.run(optimizer, feed_dict={ x: d[0], y_: hypoglycemia(d[1][:, None]) }) d = high_fid_data.test() err = sess.run(loss, feed_dict={ x: d[0], y_: hypoglycemia(d[1][:, None]) }) y_pred = sess.run(y, feed_dict={x: d[0]}) acc = accuracy(hypoglycemia(d[1][:, None]), y_pred) print("Epoch %d, %f, test acc: %f" % (i, err, acc)) return acc, y_pred
def classifier( low_fid_data, high_fid_data, k_size, nblock, nn_size, nn_layer, learning_rate, batch_size, epoch, beta, ): tf.compat.v1.reset_default_graph() batch_size = min(high_fid_data.train_n, batch_size) outdim = 3 learn_rate = tf.constant(learning_rate, name="learn_rate") print(f"Learning rate: {learn_rate}") sampling_horizon = low_fid_data.sampling_horizon if low_fid_data.feature is not None: sampling_horizon += low_fid_data.feature[0].size sess = tf.compat.v1.Session() x = tf.compat.v1.placeholder(tf.float32, [None, sampling_horizon], name="x") alpha = tf.Variable(tf.random.normal([], stddev=0.1)) p = tf.math.sin(tf.range(float(sampling_horizon))) y = x + alpha * p # weights = tf.compat.v1.placeholder(tf.float32, [sampling_horizon], name="weights") assert k_size < sampling_horizon for _ in range(nblock): x0 = tf.slice(y, [0, 0], [-1, 1]) x0s = tf.tile(x0, [1, k_size - 1]) xx = tf.concat([x0s, y], 1) data = tf.reshape(xx, [-1, sampling_horizon + k_size - 1, 1]) kernel1 = tf.Variable(tf.random.normal([k_size, 1, 1], stddev=0.1)) kernel2 = tf.Variable(tf.random.normal([k_size, 1, 1], stddev=0.1)) A = tf.squeeze( tf.nn.conv1d(input=data, filters=kernel1, stride=1, padding="VALID") ) B = tf.squeeze( tf.nn.conv1d(input=data, filters=kernel2, stride=1, padding="VALID") ) y = tf.math.multiply(A, tf.sigmoid(B)) + y # FNN with tf.compat.v1.variable_scope("fnn"): W = tf.Variable( tf.random.normal([sampling_horizon, nn_size], stddev=0.1), name="W" ) b = tf.Variable(tf.random.normal([nn_size], stddev=0.1), name="b") y = tf.nn.relu(tf.tensordot(y, W, [[1], [0]]) + b) for _ in range(nn_layer - 1): W = tf.Variable(tf.random.normal([nn_size, nn_size], stddev=0.1), name="W") b = tf.Variable(tf.random.normal([nn_size], stddev=0.1), name="b") y = tf.nn.relu(tf.tensordot(y, W, [[1], [0]]) + b) W = tf.Variable(tf.random.normal([nn_size, outdim], stddev=0.1), name="W") b = tf.Variable(tf.random.normal([], stddev=0.1), name="b") y = tf.tensordot(y, W, [[1], [0]]) + b y = tf.identity(y, name="y") y_ = tf.compat.v1.placeholder(tf.float32, [None, outdim], name="y_") loss = tf.compat.v1.losses.softmax_cross_entropy(y_, y) # add L2 regularization L2_var = [ var for var in tf.compat.v1.global_variables() if ("fnn/W" in var.name or "fnn/b" in var.name) and "Adam" not in var.name ] lossL2 = tf.math.add_n([tf.nn.l2_loss(v) for v in L2_var]) * beta loss = tf.identity(loss + lossL2, name="loss") train = tf.compat.v1.train.AdamOptimizer(learning_rate).minimize(loss) new_train = tf.compat.v1.train.AdamOptimizer(learning_rate).minimize( loss, var_list=[W, b] ) tf.compat.v1.add_to_collections("optimizer", train) tf.compat.v1.add_to_collections("optimizer", new_train) saver = tf.compat.v1.train.Saver() sess.run(tf.compat.v1.global_variables_initializer()) for i in range(epoch): for _ in range(int(low_fid_data.train_n / batch_size)): d = low_fid_data.train_next_batch(batch_size) sess.run(train, feed_dict={x: d[0], y_: threeclasses(d[1][:, None])}) d = high_fid_data.test() err = sess.run(loss, feed_dict={x: d[0], y_: threeclasses(d[1][:, None])}) y_pred = sess.run(y, feed_dict={x: d[0]}) acc = accuracy(threeclasses(d[1][:, None]), y_pred) print("Epoch %d, %f, test acc: %f" % (i, err, acc)) saver.save(sess, "pretrain") return acc
shuffle=True) testloader = torch.utils.data.DataLoader(test_data, batch_size=64) validloader = torch.utils.data.DataLoader(validate_data, batch_size=64) if args.arch == 'vgg': input_size = 25088 model = models.vgg16(pretrained=True) elif args.arch == 'resnet': input_size = 2048 model = models.alexnet(pretrained=True) for param in model.parameters(): param.requires_grad = False model.classifier = nn.Sequential(nn.Linear(input_size, args.hidden_layers), nn.ReLU(), nn.Dropout(p=0.5), nn.Linear(args.hidden_layers, 102), nn.LogSoftmax(dim=1)) print(model) criterion = nn.NLLLoss() device = args.gpu optimizer = optim.Adam(model.classifier.parameters(), args.lr) loss, accuracy = helper.validate(model, criterion, testloader, device) print(f"loss: {loss} \n Accuracy: {accuracy}") epochs = args.epochs model = helper.train(model, optimizer, criterion, epochs, trainloader, validloader, device) helper.accuracy(model, testloader, device) helper.save(model, train_data, args.arch, input_size, args.hidden_layers, epochs, args.lr)
def main(**kwargs): global args for arg, v in kwargs.items(): args.__setattr__(arg, v) print(args) program_start_time = time.time() instanceName = "classification_Accuracy" folder_path = os.path.dirname(os.path.abspath(__file__)) timestamp = datetime.datetime.now() ts_str = timestamp.strftime('%Y-%m-%d-%H-%M-%S') path = folder_path + os.sep + instanceName + os.sep + args.model + os.sep + ts_str + "_" + args.dataset + "_" + args.wordembedding if args.debug: print("[Debug mode]") path = folder_path + os.sep + instanceName + os.sep + "Debug-" + args.model + os.sep + ts_str + "_" + args.dataset + "_" + args.wordembedding else: path = folder_path + os.sep + instanceName + os.sep + args.model + os.sep + ts_str + "_" + args.dataset + "_" + args.wordembedding os.makedirs(path) args.savedir = path global logFile logFile = path + os.sep + "log.txt" if args.model == "BiLSTMConv": Model = models.BiLSTMConv # elif args.model == "BiGRU": # Model = models.BiGRU # elif args.model == "WordCNN": # Model = models.WordCNN # elif args.model == "BiGRUWithTimeDropout": # Model = models.BiGRUWithTimeDropout elif args.model == "CNN_Text_Model": Model = CNN_Text_Model.CNN_Text elif args.model == "VDCNN": Model = vcdnn.VDCNN else: NotImplementedError # process the input data. captionStrDict = {"fig_title": args.dataset, "x_label": "epoch"} train_iter, test_iter, net = data_preprocess.prepare_data_and_model( Model=Model, args=args, using_gpu=True) print("args: ", args) LOG(str(args), logFile) global device device = 'cuda' if torch.cuda.is_available() else 'cpu' device = 'cuda' net = net.to(device) if device == 'cuda': net = torch.nn.DataParallel(net).cuda() cudnn.benchmark = True optimizer = optim.Adam(params=net.parameters(), lr=1e-3, weight_decay=1e-4) lr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1000, gamma=.99) if device == "cuda": criterion = nn.CrossEntropyLoss().cuda() else: criterion = nn.CrossEntropyLoss() # criterion = nn.CrossEntropyLoss().cuda() best_test_acc = 0 best_test_results = [] ground_truth = [] epoch_train_accs = [] epoch_train_losses = [] epoch_test_accs = [] epoch_test_losses = [] epoch_lrs = [] for epoch in range(args.epochs): epoch_start_time = time.time() train_accs = [] train_losses = [] for batch in tqdm(train_iter): lr_scheduler.step() net.train() xs = batch.text ys = batch.label # # ys = ys.squeeze(1) # print("ys_train data type: ", type(ys)) # print("ys_train: ", ys) if device == 'cuda': ys = ys.cuda(async=True) # ys = torch.autograd.Variable(ys) xs = torch.autograd.Variable(xs) ys_var = torch.autograd.Variable(ys) # print(ys_var) logits = net(xs) loss = criterion(logits, ys_var) # print("loss: ", loss.item()) optimizer.zero_grad() loss.backward() optimizer.step() train_losses.append(loss.item() / int(args.batch_size)) train_accs.append(accuracy(logits.data, ys)) train_accs_normal = [i[0].item() for i in train_accs] # print("epoch ", epoch, " : training accumulated accuracy ", np.mean(train_accs_normal)) LOG("epoch: " + str(epoch), logFile) LOG("[TRAIN] accumulated accuracy: " + str(np.mean(train_accs_normal)), logFile) epoch_train_accs.append(np.mean(train_accs_normal)) epoch_train_losses.append(np.mean(train_losses)) test_accs = [] test_losses = [] test_predict_results = [] best_test_acc = 0 net.eval() pred_results = [] print("running testing.....") for batch in tqdm(test_iter): xs_test = batch.text ys_test = batch.label logits_test = net(xs_test) test_loss = criterion(logits_test, ys_test) test_losses.append(test_loss.item() / int(args.batch_size)) test_accs.append(accuracy(logits_test.data, ys_test)) pred_results = pred_results + logits_test.topk( 1, 1, True, True)[1].t().cpu().numpy().tolist()[0] if epoch == (args.epochs - 1): ground_truth = ground_truth + ys_test.cpu().numpy().tolist() test_accs_normal = [i[0].item() for i in test_accs] # print("epoch {} : testing accumulated accuracy {} %".format(epoch, np.mean(test_accs))) print("epoch ", epoch, " : testing accumulated accuracy ", np.mean(test_accs_normal)) # LOG("epoch: "+ str(epoch), logFile) LOG("[TEST] accumulated accuracy: " + str(np.mean(test_accs_normal)), logFile) if best_test_acc < np.mean(test_accs_normal): best_test_acc = np.mean(test_accs_normal) best_test_results = pred_results torch.save(net.state_dict(), path + os.sep + str(Model.name) + ".pkl") epoch_test_accs.append(np.mean(test_accs_normal)) epoch_test_losses.append(np.mean(test_losses)) # epoch_lrs.append(0.1) try: lr = float(str(optimizer[-1]).split("\n")[-5].split(" ")[-1]) except: lr = 100 epoch_lrs.append(lr) log_stats(path, [np.mean(train_accs_normal)], [np.mean(train_losses)], [np.mean(test_accs_normal)], [np.mean(test_losses)], lr) one_epoch_last_time = time.time() - epoch_start_time LOG("last time: " + str(one_epoch_last_time), logFile) df = pd.DataFrame(data={ "test_label": best_test_results, "ground truth": ground_truth }) df.to_csv(path + os.sep + "test_classification_result.csv", sep=',', index=True) # save the metrics report logFile = confusion_matrix(df["test_label"], df["ground truth"], logFile) # # # here plot figures # # algos\Classification_Accuracy\CNN_Text_Model\2019-01-23-14-58-01_tripadvisor\test_acc.txt # import pandas as pd # # algos\Classification_Accuracy\BiLSTMConv\\2019-01-22-10-29-54_tripadvisor\test_acc.txt # epoch_test_accs = list(pd.read_csv("algos\\Classification_Accuracy\\CNN_Text_Model\\2019-01-23-14-58-01_tripadvisor\\test_acc.txt", header=None).iloc[:,0]) # epoch_train_accs = list(pd.read_csv("algos\\Classification_Accuracy\\CNN_Text_Model\\2019-01-23-14-58-01_tripadvisor\\train_acc.txt", header=None).iloc[:,0]) # epoch_train_losses = list(pd.read_csv("algos\\Classification_Accuracy\\CNN_Text_Model\\2019-01-23-14-58-01_tripadvisor\\train_losses.txt", header=None).iloc[:,0]) # epoch_test_losses = list(pd.read_csv("algos\\Classification_Accuracy\\CNN_Text_Model\\2019-01-23-14-58-01_tripadvisor\\test_losses.txt", header=None).iloc[:,0]) plot_figs(epoch_train_accs, epoch_train_losses, epoch_test_accs, epoch_test_losses, args, captionStrDict) LOG("============Finish============", logFile)
def train(train_loader, model, criterion, optimizer, epoch, print_freq): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() # switch to train mode model.train() #model.apply(freeze_bn) dslen = len(train_loader) end = time.time() for k, (input, target) in enumerate(train_loader): if epoch < args.warm: warmup_scheduler.step() # measure data loading time if True: data_time.update(time.time() - end) target = target.cuda(non_blocking=True) input = input.cuda(non_blocking=True) # compute output output = model(input) loss = criterion(output, target) # measure accuracy and record loss prec = accuracy(output.data, target, topk=(1, 5)) losses.update(loss.item(), input.size(0)) top1.update(prec[0].item(), input.size(0)) top5.update(prec[1].item(), input.size(0)) # compute gradient and do SGD step optimizer.zero_grad() loss.backward() optimizer.step() # measure elapsed time batch_time.update(time.time() - end) end = time.time() if 'mean' in args.weight: model.apply(meanweigh) #(model) n_iter = (epoch) * len(train_loader) + k if n_iter == 1: os.system('nvidia-smi') if n_iter % print_freq == 1 and epoch < args.start_epoch + 5: print('Epoch: [{0}][{1}/{2}]\t' 'LR: {3:.5f}\t' 'Time {batch_time.avg:.3f}\t' 'Data {data_time.avg:.3f}\t' 'LossEpoch {loss.avg:.4f}\t' 'Prec@1 {top1.avg:.3f}\t' 'Prec@5 {top5.avg:.3f}\t'.format( epoch, n_iter, dslen, optimizer.param_groups[0]['lr'], batch_time=batch_time, data_time=data_time, loss=losses, top1=top1, top5=top5), flush=True) batch_time.reset() data_time.reset() losses.reset() top1.reset() top5.reset() #validatetrain(val_loader, model, criterion) elif k == dslen - 1: model.apply(inspect_bn) print('Epoch: [{0}][{1}/{2}]\t' 'LR: {3:.5f}\t' 'Time {batch_time.avg:.3f}\t' 'Data {data_time.avg:.3f}\t' 'LossEpoch {loss.avg:.4f}\t' 'Prec@1 {top1.avg:.3f}\t' 'Prec@5 {top5.avg:.3f}\t'.format( epoch, n_iter, dslen, optimizer.param_groups[0]['lr'], batch_time=batch_time, data_time=data_time, loss=losses, top1=top1, top5=top5), flush=True)
def train(train_loader, model, criterion, optimizers, epoch): model.train() lr = None all_acc = [] all_acc_top5 = [] all_loss = [] for ix in range(num_outputs): all_loss.append(AverageMeter()) all_acc.append(AverageMeter()) all_acc_top5.append(AverageMeter()) LOG("==> train ", logFile) # print("num_outputs: ", num_outputs) for i, (input, target) in enumerate(train_loader): # print("input: ", input, input.shape) # print("target: ", target, target.shape) # bp_1 if args.backpropagation == 1: # LOG("enter backpropagation method : " + str(args.backpropagation) +"\n", logFile) target = target.cuda(async=True) input_var = torch.autograd.Variable(input) target_var = torch.autograd.Variable(target) for ix in range(num_outputs): outputs = model(input_var) # optimizers[ix].zero_grad() loss = criterion(outputs[ix], target_var) loss.backward() optimizers[ix].step() # optimizer.zero_grad() # if ix == (num_outputs - 1): # loss.backward() # else: # loss.backward(retain_graph=True) # optimizer.step() all_loss[ix].update(loss.item(), input.size(0)) # top 1 accuracy prec1 = accuracy(outputs[ix].data, target) all_acc[ix].update(prec1[0].data[0].item(), input.size(0)) # # top 5 accuracy prec5 = accuracy(outputs[ix].data, target, topk=(5, )) # print("prec top 5-1: ", prec5) # print("prec top 5-2: ", prec5[0]) # print("prec top 5-3: ", prec5[0].data[0].item()) all_acc_top5[ix].update(prec5[0].data[0].item(), input.size(0)) # elif args.backpropagation == 2: # # LOG("enter backpropagation method : " + str(args.backpropagation) +"\n", logFile) # # bp_2 # for ix in range(num_outputs): # target = target.cuda(async=True) # input_var = torch.autograd.Variable(input) # target_var = torch.autograd.Variable(target) # optimizer.zero_grad() # outputs = model(input_var) # loss = criterion(outputs[ix], target_var) # loss.backward() # optimizer.step() # all_loss[ix].update(loss.item(), input.size(0)) # # top 1 accuracy # prec1 = accuracy(outputs[ix].data, target) # all_acc[ix].update(prec1[0].data[0].item(), input.size(0)) # # top 5 accuracy # prec5 = accuracy(outputs[ix].data, target, topk=(5,)) # all_acc_top5[ix].update(prec5[0].data[0].item(), input.size(0)) # elif args.backpropagation == 3: # # LOG("enter backpropagation method : " + str(args.backpropagation) +"\n", logFile) # # bp_3 # target = target.cuda(async=True) # input_var = torch.autograd.Variable(input) # target_var = torch.autograd.Variable(target) # optimizer.zero_grad() # outputs = model(input_var) # losses = 0 # for ix in range(len(outputs)): # # print("outputs[ix]: ", outputs[ix]) # loss = criterion(outputs[ix], target_var) # losses += loss # all_loss[ix].update(loss.item(), input.size(0)) # # top 1 accuracy # prec1 = accuracy(outputs[ix].data, target) # all_acc[ix].update(prec1[0].data[0].item(), input.size(0)) # # top 5 accuracy # prec5 = accuracy(outputs[ix].data, target, topk=(5,)) # all_acc_top5[ix].update(prec5[0].data[0].item(), input.size(0)) # # losses = losses/len(outputs) # losses.backward() # optimizer.step() else: NotImplementedError accs = [] accs_top5 = [] ls = [] for i, j, k in zip(all_acc, all_loss, all_acc_top5): accs.append(float(100 - i.avg)) ls.append(j.avg) accs_top5.append(float(100 - k.avg)) try: lr = float(str(optimizers[-1]).split("\n")[-5].split(" ")[-1]) except: lr = 100 print("train epoch top 5 error: ", accs_top5) return accs, ls, lr, accs_top5
def train(train_loader, model, criterion, optimizer, epoch, print_freq, colorization=False,scheduler=None): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() global cur_itrs # switch to train mode model.train() end = time.time() for i, (input, target) in enumerate(train_loader): cur_itrs+=1 # measure data loading time data_time.update(time.time() - end) # if torch.cuda.is_available(): # target = target.cuda() # input = input.cuda() target = target.to(device, dtype=torch.float32) input = input.to(device, dtype=torch.float32) if colorization: input = transforms.Resize(500)(input) target = transforms.Resize(500)(target) input = input.repeat(1,3,1,1) # compute output output = model(input) loss = criterion(output, target) # measure accuracy and record loss losses.update(loss.item(), input.size(0)) if not colorization: prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) top1.update(prec1, input.size(0)) top5.update(prec5, input.size(0)) # compute gradient and do SGD step optimizer.zero_grad() loss.backward() optimizer.step() if scheduler: scheduler.step() # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % print_freq == 0: if not colorization: print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})\t'.format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses, top1=top1, top5=top5)) else: print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses)) # return loss, top1, top5 corresponding to each epoch return losses.avg, top1.avg, top5.avg