def infer(test_queue, model, criterion): model.eval() objs = utils.AverageMeter() test_true = [] test_pred = [] with torch.no_grad(): for i, (data, label) in enumerate(test_queue): data, label = data.to(DEVICE), label.to(DEVICE).squeeze() data = data.permute(0, 2, 1).unsqueeze(3) out, out_aux = model(data) pred = out.max(dim=1)[1] test_true.append(label.cpu().numpy()) test_pred.append(pred.detach().cpu().numpy()) loss = criterion(out, label.squeeze()) n = label.size(0) objs.update(loss.item(), n) test_true = np.concatenate(test_true) test_pred = np.concatenate(test_pred) overall_acc = metrics.accuracy_score(test_true, test_pred) class_acc = metrics.balanced_accuracy_score(test_true, test_pred) return overall_acc, class_acc, objs.avg
def train_step(train_queue, model, criterion, optimizer, args): objs = utils.AverageMeter() train_true = [] train_pred = [] for step, (data, label) in enumerate(tqdm(train_queue)): model.train() data, label = data.to(DEVICE), label.to(DEVICE).squeeze() data = data.permute(0, 2, 1).unsqueeze(3) n = data.size(0) optimizer.zero_grad() out, out_aux = model(data) loss = criterion(out, label) if args.auxiliary: loss_aux = criterion(out_aux, label) loss += args.auxiliary_weight * loss_aux loss.backward() nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip) optimizer.step() pred = out.max(dim=1)[1] train_true.append(label.cpu().numpy()) train_pred.append(pred.detach().cpu().numpy()) objs.update(loss.item(), n) train_true = np.concatenate(train_true) train_pred = np.concatenate(train_pred) overall_acc = metrics.accuracy_score(train_true, train_pred) class_acc = metrics.balanced_accuracy_score(train_true, train_pred) return overall_acc, class_acc, objs.avg
def train_step(train_queue, model, criterion, optimizer): objs = utils.AverageMeter() micro_f1 = 0. count = 0. for step, input in enumerate(train_queue): model.train() input = input.to(DEVICE) target = input.y n = input.x.size(0) optimizer.zero_grad() logits, logits_aux = model(input) loss = criterion(logits, target) if args.auxiliary: loss_aux = criterion(logits_aux, target) loss += args.auxiliary_weight * loss_aux loss.backward() nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip) optimizer.step() micro_f1 += utils.mF1(logits, target) * n count += n objs.update(loss.item(), n) micro_f1 = float(micro_f1) / count return micro_f1, objs.avg
def infer(valid_queue, model, criterion): objs = utils.AverageMeter() model.eval() count = 0. micro_f1 = 0. with torch.no_grad(): for step, input in enumerate(valid_queue): input = input.to(DEVICE) target = input.y logits, _ = model(input) loss = criterion(logits, target) n = target.size(0) micro_f1 += utils.mF1(logits, target) * n count += n objs.update(loss.item(), n) micro_f1 = float(micro_f1) / count return micro_f1, objs.avg
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) if args.random_seed: args.seed = np.random.randint(0, 1000, 1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) # dataset modelnet pre_transform, transform = T.NormalizeScale(), T.SamplePoints( args.num_points) train_dataset = GeoData.ModelNet(os.path.join(args.data, 'modelnet10'), '10', True, transform, pre_transform) train_queue = DenseDataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.batch_size // 2) test_dataset = GeoData.ModelNet(os.path.join(args.data, 'modelnet10'), '10', False, transform, pre_transform) valid_queue = DenseDataLoader(test_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.batch_size // 2) n_classes = train_queue.dataset.num_classes criterion = torch.nn.CrossEntropyLoss().cuda() model = Network(args.init_channels, n_classes, args.num_cells, criterion, args.n_steps, in_channels=args.in_channels, emb_dims=args.emb_dims, dropout=args.dropout, k=args.k).cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) num_edges = model._steps * 2 post_train = 5 # import pdb;pdb.set_trace() args.epochs = args.warmup_dec_epoch + args.decision_freq * ( num_edges - 1) + post_train + 1 logging.info("total epochs: %d", args.epochs) optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs), eta_min=args.learning_rate_min) architect = Architect(model, args) normal_selected_idxs = torch.tensor(len(model.alphas_normal) * [-1], requires_grad=False, dtype=torch.int).cuda() normal_candidate_flags = torch.tensor(len(model.alphas_normal) * [True], requires_grad=False, dtype=torch.bool).cuda() logging.info('normal_selected_idxs: {}'.format(normal_selected_idxs)) logging.info('normal_candidate_flags: {}'.format(normal_candidate_flags)) model.normal_selected_idxs = normal_selected_idxs model.normal_candidate_flags = normal_candidate_flags print(F.softmax(torch.stack(model.alphas_normal, dim=0), dim=-1).detach()) count = 0 normal_probs_history = [] train_losses, valid_losses = utils.AverageMeter(), utils.AverageMeter() for epoch in range(args.epochs): lr = scheduler.get_lr()[0] logging.info('epoch %d lr %e', epoch, lr) # training # import pdb;pdb.set_trace() att = model.show_att() beta = model.show_beta() train_acc, train_losses = train(train_queue, valid_queue, model, architect, criterion, optimizer, lr, train_losses) valid_overall_acc, valid_class_acc, valid_losses = infer( valid_queue, model, criterion, valid_losses) logging.info( 'train_acc %f\tvalid_overall_acc %f \t valid_class_acc %f', train_acc, valid_overall_acc, valid_class_acc) logging.info('beta %s', beta.cpu().detach().numpy()) logging.info('att %s', att.cpu().detach().numpy()) # make edge decisions saved_memory_normal, model.normal_selected_idxs, \ model.normal_candidate_flags = edge_decision('normal', model.alphas_normal, model.normal_selected_idxs, model.normal_candidate_flags, normal_probs_history, epoch, model, args) if saved_memory_normal: del train_queue, valid_queue torch.cuda.empty_cache() count += 1 new_batch_size = args.batch_size + args.batch_increase * count logging.info("new_batch_size = {}".format(new_batch_size)) train_queue = DenseDataLoader(train_dataset, batch_size=new_batch_size, shuffle=True, num_workers=args.batch_size // 2) valid_queue = DenseDataLoader(test_dataset, batch_size=new_batch_size, shuffle=False, num_workers=args.batch_size // 2) # post validation if args.post_val: post_valid_overall_acc, post_valid_class_acc, valid_losses = infer( valid_queue, model, criterion, valid_losses) logging.info('post_valid_overall_acc %f', post_valid_overall_acc) writer.add_scalar('stats/train_acc', train_acc, epoch) writer.add_scalar('stats/valid_overall_acc', valid_overall_acc, epoch) writer.add_scalar('stats/valid_class_acc', valid_class_acc, epoch) utils.save(model, os.path.join(args.save, 'weights.pt')) scheduler.step() logging.info("#" * 30 + " Done " + "#" * 30) logging.info('genotype = %s', model.get_genotype())
def main(): if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) if args.random_seed: args.seed = np.random.randint(0, 1000, 1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) # dataset ppi train_dataset = GeoData.PPI(os.path.join(args.data, 'ppi'), split='train') train_queue = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True) valid_dataset = GeoData.PPI(os.path.join(args.data, 'ppi'), split='val') valid_queue = DataLoader(valid_dataset, batch_size=args.batch_size, shuffle=False) n_classes = train_queue.dataset.num_classes criterion = torch.nn.BCEWithLogitsLoss().cuda() model = Network(args.init_channels, n_classes, args.num_cells, criterion, args.n_steps, in_channels=args.in_channels).cuda() logging.info("param size = %fMB", utils.count_parameters_in_MB(model)) num_edges = model._steps * 2 post_train = 5 args.epochs = args.warmup_dec_epoch + args.decision_freq * (num_edges - 1) + post_train + 1 logging.info("total epochs: %d", args.epochs) optimizer = torch.optim.SGD( model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( optimizer, float(args.epochs), eta_min=args.learning_rate_min) architect = Architect(model, args) normal_selected_idxs = torch.tensor(len(model.alphas_normal) * [-1], requires_grad=False, dtype=torch.int).cuda() normal_candidate_flags = torch.tensor(len(model.alphas_normal) * [True], requires_grad=False, dtype=torch.bool).cuda() logging.info('normal_selected_idxs: {}'.format(normal_selected_idxs)) logging.info('normal_candidate_flags: {}'.format(normal_candidate_flags)) model.normal_selected_idxs = normal_selected_idxs model.normal_candidate_flags = normal_candidate_flags print(F.softmax(torch.stack(model.alphas_normal, dim=0), dim=-1).detach()) count = 0 normal_probs_history = [] train_losses, valid_losses = utils.AverageMeter(), utils.AverageMeter() for epoch in range(args.epochs): lr = scheduler.get_lr()[0] logging.info('epoch %d lr %e', epoch, lr) # training train_acc, train_losses = train(train_queue, valid_queue, model, architect, criterion, optimizer, lr, train_losses) valid_acc, valid_losses = infer(valid_queue, model, criterion, valid_losses) logging.info('train_acc %f\tvalid_acc %f', train_acc, valid_acc) # make edge decisions saved_memory_normal, model.normal_selected_idxs, \ model.normal_candidate_flags = edge_decision('normal', model.alphas_normal, model.normal_selected_idxs, model.normal_candidate_flags, normal_probs_history, epoch, model, args) if saved_memory_normal: del train_queue, valid_queue torch.cuda.empty_cache() count += 1 new_batch_size = args.batch_size + args.batch_increase * count logging.info("new_batch_size = {}".format(new_batch_size)) train_queue = DataLoader(train_dataset, batch_size=new_batch_size, shuffle=True) valid_queue = DataLoader(valid_dataset, batch_size=new_batch_size, shuffle=False) if args.post_val: valid_acc, valid_obj = infer(valid_queue, model, criterion) logging.info('post valid_acc %f', valid_acc) writer.add_scalar('stats/train_acc', train_acc, epoch) writer.add_scalar('stats/valid_acc', valid_acc, epoch) utils.save(model, os.path.join(args.save, 'weights.pt')) scheduler.step() logging.info("#" * 30 + " Done " + "#" * 30) logging.info('genotype = %s', model.get_genotype())