def testcase13(self):
'''传入格式错误的start_time添加发布会失败'''
self.data['start_time'] = get_time()
self.r = requests.post(Add_EventURL,self.data)
self.res = self.r.json()
sql = 'select * from sign_event where id="%d" and name="%s";'
self.result = self.em.execute_sql(sql %(self.data['eid'],self.data['name']))
self.assertEqual(self.res['status'],int(response_data[5][0]))
self.assertIn(response_data[5][1],self.res['message'])
self.assertEqual(self.result,0)
def OneEpoch(epoch, train_loader, OPTIMIZER, DISP_FREQ, NUM_EPOCH_WARM_UP, NUM_BATCH_WARM_UP):
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
batch = 0
#iterator = iter(train_loader)
start = time.time()
for inputs, labels in train_loader:
if (epoch + 1 <= NUM_EPOCH_WARM_UP) and (batch + 1 <= NUM_BATCH_WARM_UP): # adjust LR for each training batch during warm up
warm_up_lr(batch + 1, NUM_BATCH_WARM_UP, LR, OPTIMIZER)
# compute output
inputs = inputs.to(DEVICE, non_blocking=True)
labels = labels.to(DEVICE, non_blocking=True).long()
features = BACKBONE(inputs)
outputs = HEAD(features, labels)
loss = LOSS(outputs, labels)
# measure accuracy and record loss
prec1, prec5 = accuracy(outputs.data, labels, topk = (1, 5))
losses.update(loss.data.item(), inputs.size(0))
top1.update(prec1.data.item(), inputs.size(0))
top5.update(prec5.data.item(), inputs.size(0))
# compute gradient and do SGD step
OPTIMIZER.zero_grad()
loss.backward()
OPTIMIZER.step()
# dispaly training loss & acc every DISP_FREQ
if ((batch + 1) % DISP_FREQ == 0) and batch != 0:
print("=" * 60)
print('Epoch {}/{} Batch {}/{}\t'
'Training Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Training Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Training Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
epoch + 1, NUM_EPOCH, batch + 1, len(train_loader) * NUM_EPOCH, loss = losses, top1 = top1, top5 = top5))
print("Running speed in the last 100 batches: {:.3f} iter/s.".format(DISP_FREQ / (time.time() - start)))
start = time.time()
print("=" * 60)
batch += 1
epoch_loss = losses.avg
epoch_acc = top1.avg
writer.add_scalar("Training_Loss", epoch_loss, epoch + 1)
writer.add_scalar("Training_Accuracy", epoch_acc, epoch + 1)
print("=" * 60)
print('Epoch: {}/{}\t'
'Training Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Training Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'
'Training Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
epoch + 1, NUM_EPOCH, loss = losses, top1 = top1, top5 = top5))
print("=" * 60)
# perform validation & save checkpoints per epoch
# validation statistics per epoch (buffer for visualization)
print("=" * 60)
print("Perform Evaluation on LFW, CFP_FF, CFP_FP, AgeDB, CALFW, CPLFW and VGG2_FP, and Save Checkpoints...")
accuracy_lfw, best_threshold_lfw, roc_curve_lfw = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, lfw, lfw_issame)
buffer_val(writer, "LFW", accuracy_lfw, best_threshold_lfw, roc_curve_lfw, epoch + 1)
# accuracy_cfp_ff, best_threshold_cfp_ff, roc_curve_cfp_ff = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, cfp_ff, cfp_ff_issame)
# buffer_val(writer, "CFP_FF", accuracy_cfp_ff, best_threshold_cfp_ff, roc_curve_cfp_ff, epoch + 1)
# accuracy_cfp_fp, best_threshold_cfp_fp, roc_curve_cfp_fp = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, cfp_fp, cfp_fp_issame)
# buffer_val(writer, "CFP_FP", accuracy_cfp_fp, best_threshold_cfp_fp, roc_curve_cfp_fp, epoch + 1)
# accuracy_agedb, best_threshold_agedb, roc_curve_agedb = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, agedb, agedb_issame)
# buffer_val(writer, "AgeDB", accuracy_agedb, best_threshold_agedb, roc_curve_agedb, epoch + 1)
# accuracy_calfw, best_threshold_calfw, roc_curve_calfw = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, calfw, calfw_issame)
# buffer_val(writer, "CALFW", accuracy_calfw, best_threshold_calfw, roc_curve_calfw, epoch + 1)
# accuracy_cplfw, best_threshold_cplfw, roc_curve_cplfw = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, cplfw, cplfw_issame)
# buffer_val(writer, "CPLFW", accuracy_cplfw, best_threshold_cplfw, roc_curve_cplfw, epoch + 1)
accuracy_vgg2_fp, best_threshold_vgg2_fp, roc_curve_vgg2_fp = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, vgg2_fp, vgg2_fp_issame)
buffer_val(writer, "VGGFace2_FP", accuracy_vgg2_fp, best_threshold_vgg2_fp, roc_curve_vgg2_fp, epoch + 1)
print("=" * 60)
# save checkpoints per epoch
if MULTI_GPU:
torch.save(BACKBONE.module.state_dict(), os.path.join(MODEL_ROOT, "Backbone_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".format(BACKBONE_NAME, epoch + 1, batch, get_time())))
torch.save(HEAD.state_dict(), os.path.join(MODEL_ROOT, "Head_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".format(HEAD_NAME, epoch + 1, batch, get_time())))
else:
torch.save(BACKBONE.state_dict(), os.path.join(MODEL_ROOT, "Backbone_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".format(BACKBONE_NAME, epoch + 1, batch, get_time())))
torch.save(HEAD.state_dict(), os.path.join(MODEL_ROOT, "Head_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".format(HEAD_NAME, epoch + 1, batch, get_time())))
print(
"Batch {}/{}, Evaluation: LFW Acc: {}, CFP_FF Acc: {}, CFP_FP Acc: {}, AgeDB Acc: {}, CALFW Acc: {}, CPLFW Acc: {}, VGG2_FP Acc: {}"
.format(batch + 1,
len(train_loader) * NUM_EPOCH, accuracy_lfw,
accuracy_cfp_ff, accuracy_cfp_fp, accuracy_agedb,
accuracy_calfw, accuracy_cplfw, accuracy_vgg2_fp))
print("=" * 60)
if MULTI_GPU:
torch.save(
BACKBONE.module.state_dict(),
os.path.join(
MODEL_ROOT,
"Backbone_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth"
.format(BACKBONE_NAME, epoch + 1, batch,
get_time())))
torch.save(
HEAD.module.state_dict(),
os.path.join(
MODEL_ROOT,
"Head_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".
format(HEAD_NAME, epoch + 1, batch, get_time())))
else:
torch.save(
BACKBONE.state_dict(),
os.path.join(
MODEL_ROOT,
"Backbone_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth"
.format(BACKBONE_NAME, epoch + 1, batch,
get_time())))
torch.save(
def main_worker(gpu, ngpus_per_node, cfg):
cfg['GPU'] = gpu
if gpu != 0:
def print_pass(*args):
pass
builtins.print = print_pass
cfg['RANK'] = cfg['RANK'] * ngpus_per_node + gpu
dist.init_process_group(backend=cfg['DIST_BACKEND'],
init_method=cfg["DIST_URL"],
world_size=cfg['WORLD_SIZE'],
rank=cfg['RANK'])
# Data loading code
batch_size = int(cfg['BATCH_SIZE'] / ngpus_per_node)
workers = int((cfg['NUM_WORKERS'] + ngpus_per_node - 1) / ngpus_per_node)
DATA_ROOT = cfg[
'DATA_ROOT'] # the parent root where your train/val/test data are stored
RECORD_DIR = cfg['RECORD_DIR']
RGB_MEAN = cfg['RGB_MEAN'] # for normalize inputs
RGB_STD = cfg['RGB_STD']
train_transform = transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean=RGB_MEAN, std=RGB_STD),
])
dataset_train = FaceDataset(DATA_ROOT, RECORD_DIR, train_transform)
train_sampler = torch.utils.data.distributed.DistributedSampler(
dataset_train)
train_loader = torch.utils.data.DataLoader(dataset_train,
batch_size=batch_size,
shuffle=(train_sampler is None),
num_workers=workers,
pin_memory=True,
sampler=train_sampler,
drop_last=True)
SAMPLE_NUMS = dataset_train.get_sample_num_of_each_class()
NUM_CLASS = len(train_loader.dataset.classes)
print("Number of Training Classes: {}".format(NUM_CLASS))
#======= model & loss & optimizer =======#
BACKBONE_DICT = {
'ResNet_50': ResNet_50,
'ResNet_101': ResNet_101,
'ResNet_152': ResNet_152,
'IR_50': IR_50,
'IR_101': IR_101,
'IR_152': IR_152,
'IR_SE_50': IR_SE_50,
'IR_SE_101': IR_SE_101,
'IR_SE_152': IR_SE_152
}
BACKBONE_NAME = cfg['BACKBONE_NAME']
INPUT_SIZE = cfg['INPUT_SIZE']
assert INPUT_SIZE == [112, 112]
backbone = BACKBONE_DICT[BACKBONE_NAME](INPUT_SIZE)
print("=" * 60)
print(backbone)
print("{} Backbone Generated".format(BACKBONE_NAME))
print("=" * 60)
HEAD_DICT = {'ArcFace': ArcFace, 'CurricularFace': CurricularFace}
HEAD_NAME = cfg['HEAD_NAME']
EMBEDDING_SIZE = cfg['EMBEDDING_SIZE'] # feature dimension
head = HEAD_DICT[HEAD_NAME](in_features=EMBEDDING_SIZE,
out_features=NUM_CLASS)
print("=" * 60)
print(head)
print("{} Head Generated".format(HEAD_NAME))
print("=" * 60)
#--------------------optimizer-----------------------------
if BACKBONE_NAME.find("IR") >= 0:
backbone_paras_only_bn, backbone_paras_wo_bn = separate_irse_bn_paras(
backbone
) # separate batch_norm parameters from others; do not do weight decay for batch_norm parameters to improve the generalizability
else:
backbone_paras_only_bn, backbone_paras_wo_bn = separate_resnet_bn_paras(
backbone
) # separate batch_norm parameters from others; do not do weight decay for batch_norm parameters to improve the generalizability
LR = cfg['LR'] # initial LR
WEIGHT_DECAY = cfg['WEIGHT_DECAY']
MOMENTUM = cfg['MOMENTUM']
optimizer = optim.SGD(
[{
'params': backbone_paras_wo_bn + list(head.parameters()),
'weight_decay': WEIGHT_DECAY
}, {
'params': backbone_paras_only_bn
}],
lr=LR,
momentum=MOMENTUM)
print("=" * 60)
print(optimizer)
print("Optimizer Generated")
print("=" * 60)
# loss
LOSS_NAME = cfg['LOSS_NAME']
LOSS_DICT = {'Softmax': nn.CrossEntropyLoss()}
loss = LOSS_DICT[LOSS_NAME].cuda(gpu)
print("=" * 60)
print(loss)
print("{} Loss Generated".format(loss))
print("=" * 60)
torch.cuda.set_device(cfg['GPU'])
backbone.cuda(cfg['GPU'])
head.cuda(cfg['GPU'])
#optionally resume from a checkpoint
BACKBONE_RESUME_ROOT = cfg[
'BACKBONE_RESUME_ROOT'] # the root to resume training from a saved checkpoint
HEAD_RESUME_ROOT = cfg[
'HEAD_RESUME_ROOT'] # the root to resume training from a saved checkpoint
if BACKBONE_RESUME_ROOT:
print("=" * 60)
if os.path.isfile(BACKBONE_RESUME_ROOT):
print("Loading Backbone Checkpoint '{}'".format(
BACKBONE_RESUME_ROOT))
loc = 'cuda:{}'.format(cfg['GPU'])
backbone.load_state_dict(
torch.load(BACKBONE_RESUME_ROOT, map_location=loc))
if os.path.isfile(HEAD_RESUME_ROOT):
print("Loading Head Checkpoint '{}'".format(HEAD_RESUME_ROOT))
checkpoint = torch.load(HEAD_RESUME_ROOT, map_location=loc)
cfg['START_EPOCH'] = checkpoint['EPOCH']
head.load_state_dict(checkpoint['HEAD'])
optimizer.load_state_dict(checkpoint['OPTIMIZER'])
else:
print(
"No Checkpoint Found at '{}' and '{}'. Please Have a Check or Continue to Train from Scratch"
.format(BACKBONE_RESUME_ROOT, HEAD_RESUME_ROOT))
print("=" * 60)
backbone = torch.nn.parallel.DistributedDataParallel(
backbone, device_ids=[cfg['GPU']])
head = torch.nn.parallel.DistributedDataParallel(head,
device_ids=[cfg['GPU']])
# checkpoint and tensorboard dir
MODEL_ROOT = cfg['MODEL_ROOT'] # the root to buffer your checkpoints
LOG_ROOT = cfg['LOG_ROOT'] # the root to log your train/val status
STAGES = cfg['STAGES'] # epoch stages to decay learning rate
if not os.path.exists(MODEL_ROOT):
os.makedirs(MODEL_ROOT)
if not os.path.exists(LOG_ROOT):
os.makedirs(LOG_ROOT)
writer = SummaryWriter(
LOG_ROOT) # writer for buffering intermedium results
# train
for epoch in range(cfg['START_EPOCH'], cfg['NUM_EPOCH']):
train_sampler.set_epoch(epoch)
adjust_learning_rate(optimizer, epoch, cfg)
#train for one epoch
train(train_loader, backbone, head, loss, optimizer, epoch, cfg,
writer)
print("=" * 60)
print("Save Checkpoint...")
if cfg['RANK'] % ngpus_per_node == 0:
torch.save(
backbone.module.state_dict(),
os.path.join(
MODEL_ROOT,
"Backbone_{}_Epoch_{}_Time_{}_checkpoint.pth".format(
BACKBONE_NAME, epoch + 1, get_time())))
save_dict = {
'EPOCH': epoch + 1,
'HEAD': head.module.state_dict(),
'OPTIMIZER': optimizer.state_dict()
}
torch.save(
save_dict,
os.path.join(
MODEL_ROOT,
"Head_{}_Epoch_{}_Time_{}_checkpoint.pth".format(
HEAD_NAME, epoch + 1, get_time())))
import unittest
from util.utils import CASEPATH, REPORTPATH, get_time
from HTMLTestReportCN import HTMLTestRunner
tests = unittest.defaultTestLoader.discover(CASEPATH)
with open(REPORTPATH + '/' + get_time() + '.html', 'wb') as f:
runner = HTMLTestRunner(stream=f, title='guest-master接口测试报告', tester='杨劲松')
runner.run(tests)
buffer_val(writer, name, accuracy, std, xnorm,
best_threshold, roc_curve, batch + 1)
print('[%s][%d]XNorm: %1.5f' % (name, batch + 1, xnorm))
print('[%s][%d]Accuracy-Flip: %1.5f+-%1.5f' %
(name, batch + 1, accuracy, std))
print('[%s][%d]Best-Threshold: %1.5f' %
(name, batch + 1, best_threshold))
acc.append(accuracy)
# save checkpoints per epoch
if need_save(acc, highest_acc):
if MULTI_GPU:
torch.save(
BACKBONE.module.state_dict(),
os.path.join(
WORK_PATH,
"Backbone_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth"
.format(BACKBONE_NAME, epoch + 1, batch + 1,
get_time())))
else:
torch.save(
BACKBONE.state_dict(),
os.path.join(
WORK_PATH,
"Backbone_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth"
.format(BACKBONE_NAME, epoch + 1, batch + 1,
get_time())))
BACKBONE.train() # set to training mode
batch += 1 # batch index
# accuracy_vgg2_fp, best_threshold_vgg2_fp, roc_curve_vgg2_fp = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, vgg2_fp, vgg2_fp_issame)
# buffer_val(writer, "VGGFace2_FP", accuracy_vgg2_fp, best_threshold_vgg2_fp, roc_curve_vgg2_fp, epoch + 1)
# print("Epoch {}/{}, Evaluation: LFW Acc: {}, CFP_FF Acc: {}, CFP_FP Acc: {}, AgeDB Acc: {}, CALFW Acc: {}, CPLFW Acc: {}, VGG2_FP Acc: {}".format(epoch + 1, NUM_EPOCH, accuracy_lfw, accuracy_cfp_ff, accuracy_cfp_fp, accuracy_agedb, accuracy_calfw, accuracy_cplfw, accuracy_vgg2_fp))
# print("=" * 60)
print("Epoch {}/{}, Evaluation: CALFW Acc: {}, CPLFW Acc: {}".format(
epoch + 1, NUM_EPOCH, accuracy_calfw, accuracy_cplfw))
print("=" * 60)
# save checkpoints per epoch
if MULTI_GPU:
torch.save(
BACKBONE.module.state_dict(),
os.path.join(
MODEL_ROOT,
"Backbone_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".
format(BACKBONE_NAME, epoch + 1, batch, get_time())))
torch.save(
HEAD.state_dict(),
os.path.join(
MODEL_ROOT,
"Head_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".format(
HEAD_NAME, epoch + 1, batch, get_time())))
else:
torch.save(
BACKBONE.state_dict(),
os.path.join(
MODEL_ROOT,
"Backbone_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".
format(BACKBONE_NAME, epoch + 1, batch, get_time())))
torch.save(
HEAD.state_dict(),
print("=" * 60)
# perform validation & save checkpoints per epoch
# validation statistics per epoch (buffer for visualization)
print("=" * 60)
print("Perform Evaluation on LFW, CFP_FF, CFP_FP, AgeDB, CALFW, CPLFW and VGG2_FP, and Save Checkpoints...")
accuracy_lfw, best_threshold_lfw, roc_curve_lfw = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, lfw, lfw_issame)
buffer_val(writer, "LFW", accuracy_lfw, best_threshold_lfw, roc_curve_lfw, epoch + 1)
accuracy_cfp_ff, best_threshold_cfp_ff, roc_curve_cfp_ff = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, cfp_ff, cfp_ff_issame)
buffer_val(writer, "CFP_FF", accuracy_cfp_ff, best_threshold_cfp_ff, roc_curve_cfp_ff, epoch + 1)
accuracy_cfp_fp, best_threshold_cfp_fp, roc_curve_cfp_fp = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, cfp_fp, cfp_fp_issame)
buffer_val(writer, "CFP_FP", accuracy_cfp_fp, best_threshold_cfp_fp, roc_curve_cfp_fp, epoch + 1)
accuracy_agedb, best_threshold_agedb, roc_curve_agedb = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, agedb, agedb_issame)
buffer_val(writer, "AgeDB", accuracy_agedb, best_threshold_agedb, roc_curve_agedb, epoch + 1)
accuracy_calfw, best_threshold_calfw, roc_curve_calfw = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, calfw, calfw_issame)
buffer_val(writer, "CALFW", accuracy_calfw, best_threshold_calfw, roc_curve_calfw, epoch + 1)
accuracy_cplfw, best_threshold_cplfw, roc_curve_cplfw = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, cplfw, cplfw_issame)
buffer_val(writer, "CPLFW", accuracy_cplfw, best_threshold_cplfw, roc_curve_cplfw, epoch + 1)
accuracy_vgg2_fp, best_threshold_vgg2_fp, roc_curve_vgg2_fp = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, vgg2_fp, vgg2_fp_issame)
buffer_val(writer, "VGGFace2_FP", accuracy_vgg2_fp, best_threshold_vgg2_fp, roc_curve_vgg2_fp, epoch + 1)
print("Epoch {}/{}, Evaluation: LFW Acc: {}, CFP_FF Acc: {}, CFP_FP Acc: {}, AgeDB Acc: {}, CALFW Acc: {}, CPLFW Acc: {}, VGG2_FP Acc: {}".format(epoch + 1, NUM_EPOCH, accuracy_lfw, accuracy_cfp_ff, accuracy_cfp_fp, accuracy_agedb, accuracy_calfw, accuracy_cplfw, accuracy_vgg2_fp))
print("=" * 60)
# save checkpoints per epoch
if MULTI_GPU:
torch.save(BACKBONE.module.state_dict(), os.path.join(MODEL_ROOT, "Backbone_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".format(BACKBONE_NAME, epoch + 1, batch, get_time())))
torch.save(HEAD.state_dict(), os.path.join(MODEL_ROOT, "Head_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".format(HEAD_NAME, epoch + 1, batch, get_time())))
else:
torch.save(BACKBONE.state_dict(), os.path.join(MODEL_ROOT, "Backbone_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".format(BACKBONE_NAME, epoch + 1, batch, get_time())))
torch.save(HEAD.state_dict(), os.path.join(MODEL_ROOT, "Head_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".format(HEAD_NAME, epoch + 1, batch, get_time())))
.format(epoch, NUM_EPOCH - 1, batch,
len(train_loader) * NUM_EPOCH - 1,
accuracy_agedb_30, accuracy_lfw, accuracy_cfp_fp))
print("=" * 60)
if (
batch % SAVE_FREQ
) == 0 and batch != 0: # save checkpoints (only save BACKBONE) every SAVE_FREQ
torch.save(
BACKBONE.state_dict(),
os.path.join(
MODEL_ROOT,
"Backbone_{}_Head_{}_Loss_{}_agedb_30_acc_{}_lfw_acc_{}_cfp_fp_acc_{}_epoch_{}_batch_{}_time_{}"
.format(BACKBONE_NAME, HEAD_NAME, LOSS_NAME,
accuracy_agedb_30, accuracy_lfw,
accuracy_cfp_fp, epoch, batch, get_time())))
running_loss += loss.data.item() * inputs.data.size(
0) # compute training loss & acc every epoch
running_corrects += torch.sum(preds == labels.data)
batch += 1 # batch index
# training statistics per epoch (buffer for visualization)
epoch_loss = running_loss / len(train_loader.dataset)
epoch_acc = running_corrects.double() / len(train_loader.dataset)
writer.add_scalar("Training_Loss", epoch_loss, epoch)
writer.add_scalar("Training_Accuracy", epoch_acc, epoch)
print("=" * 60)
if epoch <= NUM_EPOCH_WARM_UP - 1:
print("During Warm Up Process, Epoch {}/{}".format(
accuracy_agedb, best_threshold_agedb, roc_curve_agedb = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, agedb, agedb_issame)
buffer_val(writer, "AgeDB", accuracy_agedb, best_threshold_agedb, roc_curve_agedb, batch + 1)
accuracy_calfw, best_threshold_calfw, roc_curve_calfw = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, calfw, calfw_issame)
buffer_val(writer, "CALFW", accuracy_calfw, best_threshold_calfw, roc_curve_calfw, batch + 1)
accuracy_cplfw, best_threshold_cplfw, roc_curve_cplfw = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, cplfw, cplfw_issame)
buffer_val(writer, "CPLFW", accuracy_cplfw, best_threshold_cplfw, roc_curve_cplfw, batch + 1)
accuracy_vgg2_fp, best_threshold_vgg2_fp, roc_curve_vgg2_fp = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, vgg2_fp, vgg2_fp_issame)
buffer_val(writer, "VGGFace2_FP", accuracy_vgg2_fp, best_threshold_vgg2_fp, roc_curve_vgg2_fp, batch + 1)
print("Batch {}/{}, Evaluation: LFW Acc: {}, CFP_FF Acc: {}, CFP_FP Acc: {}, AgeDB Acc: {}, CALFW Acc: {}, CPLFW Acc: {}, VGG2_FP Acc: {}".format(batch + 1, len(train_loader) * NUM_EPOCH, accuracy_lfw, accuracy_cfp_ff, accuracy_cfp_fp, accuracy_agedb, accuracy_calfw, accuracy_cplfw, accuracy_vgg2_fp))
print("=" * 60)
if MULTI_GPU:
torch.save(BACKBONE.module.state_dict(), os.path.join(MODEL_ROOT,
"Backbone_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".format(
BACKBONE_NAME, epoch + 1, batch,
get_time())))
torch.save(HEAD.module.state_dict(), os.path.join(MODEL_ROOT,
"Head_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".format(
HEAD_NAME, epoch + 1, batch, get_time())))
else:
torch.save(BACKBONE.state_dict(), os.path.join(MODEL_ROOT,
"Backbone_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".format(
BACKBONE_NAME, epoch + 1, batch, get_time())))
torch.save(HEAD.state_dict(), os.path.join(MODEL_ROOT,
"Head_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".format(
HEAD_NAME, epoch + 1, batch, get_time())))
batch += 1 # batch index
# training statistics per epoch (buffer for visualization)
# epoch_loss = losses.avg
print("=" * 60)
# perform validation & save checkpoints per epoch
# validation statistics per epoch (buffer for visualization)
print("=" * 60)
print("Perform Evaluation on LFW, CFP_FF, CFP_FP, AgeDB, CALFW, CPLFW and VGG2_FP, and Save Checkpoints...")
accuracy_lfw, best_threshold_lfw, roc_curve_lfw = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, lfw, lfw_issame)
buffer_val(writer, "LFW", accuracy_lfw, best_threshold_lfw, roc_curve_lfw, epoch + 1)
accuracy_cfp_ff, best_threshold_cfp_ff, roc_curve_cfp_ff = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, cfp_ff, cfp_ff_issame)
buffer_val(writer, "CFP_FF", accuracy_cfp_ff, best_threshold_cfp_ff, roc_curve_cfp_ff, epoch + 1)
accuracy_cfp_fp, best_threshold_cfp_fp, roc_curve_cfp_fp = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, cfp_fp, cfp_fp_issame)
buffer_val(writer, "CFP_FP", accuracy_cfp_fp, best_threshold_cfp_fp, roc_curve_cfp_fp, epoch + 1)
accuracy_agedb, best_threshold_agedb, roc_curve_agedb = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, agedb, agedb_issame)
buffer_val(writer, "AgeDB", accuracy_agedb, best_threshold_agedb, roc_curve_agedb, epoch + 1)
accuracy_calfw, best_threshold_calfw, roc_curve_calfw = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, calfw, calfw_issame)
buffer_val(writer, "CALFW", accuracy_calfw, best_threshold_calfw, roc_curve_calfw, epoch + 1)
accuracy_cplfw, best_threshold_cplfw, roc_curve_cplfw = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, cplfw, cplfw_issame)
buffer_val(writer, "CPLFW", accuracy_cplfw, best_threshold_cplfw, roc_curve_cplfw, epoch + 1)
accuracy_vgg2_fp, best_threshold_vgg2_fp, roc_curve_vgg2_fp = perform_val(MULTI_GPU, DEVICE, EMBEDDING_SIZE, BATCH_SIZE, BACKBONE, vgg2_fp, vgg2_fp_issame)
buffer_val(writer, "VGGFace2_FP", accuracy_vgg2_fp, best_threshold_vgg2_fp, roc_curve_vgg2_fp, epoch + 1)
print("Epoch {}/{}, Evaluation: LFW Acc: {}, CFP_FF Acc: {}, CFP_FP Acc: {}, AgeDB Acc: {}, CALFW Acc: {}, CPLFW Acc: {}, VGG2_FP Acc: {}".format(epoch + 1, NUM_EPOCH, accuracy_lfw, accuracy_cfp_ff, accuracy_cfp_fp, accuracy_agedb, accuracy_calfw, accuracy_cplfw, accuracy_vgg2_fp))
print("=" * 60)
# save checkpoints per epoch
if MULTI_GPU:
torch.save(BACKBONE.module.state_dict(), os.path.join(MODEL_ROOT, "Backbone_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".format(BACKBONE_NAME, epoch + 1, batch, get_time())))
torch.save(HEAD.state_dict(), os.path.join(MODEL_ROOT, "Head_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".format(HEAD_NAME, epoch + 1, batch, get_time())))
else:
torch.save(BACKBONE.state_dict(), os.path.join(MODEL_ROOT, "Backbone_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".format(BACKBONE_NAME, epoch + 1, batch, get_time())))
torch.save(HEAD.state_dict(), os.path.join(MODEL_ROOT, "Head_{}_Epoch_{}_Batch_{}_Time_{}_checkpoint.pth".format(HEAD_NAME, epoch + 1, batch, get_time())))