# layer.weight.grad.data.copy_(layer.calibration * meta_grad_dict[layer_name][1].data) # Get refine gradients for next computation optimizee.get_refine_gradient() # These gradient should be saved in next iteration's inference if len(meta_grad_dict) != 0: update_parameters(net, lr=optimizee.param_groups[0]['lr']) recorder.update(loss=losses.data.item(), acc=accuracy(outputs.data, targets.data, (1, 5)), batch_size=outputs.shape[0], cur_lr=optimizee.param_groups[0]['lr'], end=end) recorder.print_training_result(batch_idx, len(train_loader)) end = time.time() test_acc = test(net, quantized_type=quantized_type, test_loader=test_loader, dataset_name=dataset_name, n_batches_used=None) recorder.update(loss=None, acc=test_acc, batch_size=0, end=None, is_train=False) # Adjust learning rate recorder.adjust_lr(optimizer=optimizee, adjust_type=lr_adjust, epoch=epoch)
model.zero_grad() # global_step += 1 # ------ # Record # ------ preds = logits.data.cpu().numpy() preds = np.argmax(preds, axis=1) out_label_ids = inputs["labels"].data.cpu().numpy() result = glue_compute_metrics( task_name, preds, out_label_ids) # ['acc', 'f1', 'acc_and_f1'] if recorder is not None: recorder.update(losses.item(), acc=[result['acc_and_f1']], batch_size=args.train_batch_size, is_train=True) recorder.print_training_result(batch_idx=step, n_batch=len(train_dataloader)) else: train_loss += losses.item() progress_bar(step, len(train_dataloader), "Loss: %.3f" % (train_loss / (step + 1))) result = evaluate(task_name, model, eval_dataloader, model_type) print(result) if recorder is not None: recorder.update(acc=result['acc_and_f1'], is_train=False) if recorder is not None: recorder.close()
'%d, %.3f\n' % (batch_idx + epoch * len(train_loader), conv1_prune_rate)) conv1_pos_file.flush() conv1_neg_file.flush() conv1_prune_file.flush() recorder.update(loss=losses.item(), acc=accuracy(outputs.data, targets.data, (1, 5)), batch_size=outputs.shape[0], cur_lr=optimizer.param_groups[0]['lr'], end=end) recorder.print_training_result( batch_idx, len(train_loader), append= 'pos: %.3f, neg: %.3f, pos rate: %.3f, neg rate: %.3f, prune rate: %.3f' % (conv1_pos, conv1_neg, conv1_pos_rate, conv1_neg_rate, conv1_prune_rate)) end = time.time() test_acc = test(net, test_loader=test_loader, dataset_name=dataset_name) recorder.update(loss=None, acc=test_acc, batch_size=0, end=None, is_train=False) # Adjust lr recorder.adjust_lr(optimizer=optimizer)
outputs = net(inputs) losses = nn.CrossEntropyLoss()(outputs, targets) losses.backward() optimizer.step() optimizer_QIL.step() recorder.update(loss=losses.item(), acc=accuracy(outputs.data, targets.data, (1, 5)), batch_size=outputs.shape[0], cur_lr=optimizer.param_groups[0]['lr'], end=end) selected_p = net.layer1[0].conv1.pruning_point.data recorder.print_training_result(batch_idx, len(train_loader), append='%e' % (selected_p)) end = time.time() if batch_idx == 1: ds test_acc = test(net, test_loader=test_loader, dataset_name=dataset_name) recorder.update(loss=None, acc=test_acc, batch_size=0, end=None, is_train=False) # Adjust lr if dataset_name == 'ImageNet':