# layer.weight.grad.data.copy_(layer.calibration * meta_grad_dict[layer_name][1].data)
# Get refine gradients for next computation
optimizee.get_refine_gradient()
# These gradient should be saved in next iteration's inference
if len(meta_grad_dict) != 0:
update_parameters(net, lr=optimizee.param_groups[0]['lr'])
recorder.update(loss=losses.data.item(),
acc=accuracy(outputs.data, targets.data, (1, 5)),
batch_size=outputs.shape[0],
cur_lr=optimizee.param_groups[0]['lr'],
end=end)
recorder.print_training_result(batch_idx, len(train_loader))
end = time.time()
test_acc = test(net,
quantized_type=quantized_type,
test_loader=test_loader,
dataset_name=dataset_name,
n_batches_used=None)
recorder.update(loss=None,
acc=test_acc,
batch_size=0,
end=None,
is_train=False)
# Adjust learning rate
recorder.adjust_lr(optimizer=optimizee, adjust_type=lr_adjust, epoch=epoch)
model.zero_grad()
# global_step += 1
# ------
# Record
# ------
preds = logits.data.cpu().numpy()
preds = np.argmax(preds, axis=1)
out_label_ids = inputs["labels"].data.cpu().numpy()
result = glue_compute_metrics(
task_name, preds, out_label_ids) # ['acc', 'f1', 'acc_and_f1']
if recorder is not None:
recorder.update(losses.item(),
acc=[result['acc_and_f1']],
batch_size=args.train_batch_size,
is_train=True)
recorder.print_training_result(batch_idx=step,
n_batch=len(train_dataloader))
else:
train_loss += losses.item()
progress_bar(step, len(train_dataloader),
"Loss: %.3f" % (train_loss / (step + 1)))
result = evaluate(task_name, model, eval_dataloader, model_type)
print(result)
if recorder is not None:
recorder.update(acc=result['acc_and_f1'], is_train=False)
if recorder is not None:
recorder.close()
'%d, %.3f\n' %
(batch_idx + epoch * len(train_loader), conv1_prune_rate))
conv1_pos_file.flush()
conv1_neg_file.flush()
conv1_prune_file.flush()
recorder.update(loss=losses.item(),
acc=accuracy(outputs.data, targets.data, (1, 5)),
batch_size=outputs.shape[0],
cur_lr=optimizer.param_groups[0]['lr'],
end=end)
recorder.print_training_result(
batch_idx,
len(train_loader),
append=
'pos: %.3f, neg: %.3f, pos rate: %.3f, neg rate: %.3f, prune rate: %.3f'
% (conv1_pos, conv1_neg, conv1_pos_rate, conv1_neg_rate,
conv1_prune_rate))
end = time.time()
test_acc = test(net, test_loader=test_loader, dataset_name=dataset_name)
recorder.update(loss=None,
acc=test_acc,
batch_size=0,
end=None,
is_train=False)
# Adjust lr
recorder.adjust_lr(optimizer=optimizer)
outputs = net(inputs)
losses = nn.CrossEntropyLoss()(outputs, targets)
losses.backward()
optimizer.step()
optimizer_QIL.step()
recorder.update(loss=losses.item(),
acc=accuracy(outputs.data, targets.data, (1, 5)),
batch_size=outputs.shape[0],
cur_lr=optimizer.param_groups[0]['lr'],
end=end)
selected_p = net.layer1[0].conv1.pruning_point.data
recorder.print_training_result(batch_idx,
len(train_loader),
append='%e' % (selected_p))
end = time.time()
if batch_idx == 1:
ds
test_acc = test(net, test_loader=test_loader, dataset_name=dataset_name)
recorder.update(loss=None,
acc=test_acc,
batch_size=0,
end=None,
is_train=False)
# Adjust lr
if dataset_name == 'ImageNet':
