def validate_detection(self, epoch, iteration):
print('Start val')
self.setModelState('test')
val_loader = self.val_loader
val_iter = iter(val_loader)
input_json_path = self.res2json()
gt_json_path = self.opt.ADDRESS.GT_JSON_DIR
# loss
losses = AverageMeter()
for i in range(len(val_loader)):
data = val_iter.next()
pretreatmentData = self.pretreatment(data)
img = self.get_img_data(pretreatmentData)
modelResult = self.model(img)
loss = self.posttreatment(modelResult, pretreatmentData, data, True)
print("No.%d, loss:%f" % (i, loss))
file_summary(self.opt.ADDRESS.LOGGER_DIR, self.opt.BASE.MODEL + "_result.txt",
"No.%d, loss:%f \n" % (i, loss))
losses.update(loss.item(), img.size(0))
tqdm.write('Validate Loss - Avg Loss {0}'.format(losses.avg))
# Precision / Recall / F_score
precision, recall, f_score = \
eval_func(input_json_path, gt_json_path, self.opt)
# Generate log
total_index = epoch * (
len(self.train_loader) // self.opt.FREQ.VAL_FREQ + 1) + iteration // self.opt.FREQ.VAL_FREQ
self.Logger.scalar_summary('Precision', precision, total_index)
self.Logger.scalar_summary('Recall', recall, total_index)
self.Logger.scalar_summary('F_score', f_score, total_index)
self.Logger.scalar_summary('Avg Loss', losses.avg, total_index)
self.setModelState('train')
return precision
def valid_fn(model, criterion, valid_dataloader, config, device):
assert hasattr(
config, "gradient_accumulation_steps"
), "Please create gradient_accumulation_steps(int default=1) attribute"
model.eval()
losses = AverageMeter()
preds = []
all_targets = []
for step, (images, targets) in enumerate(valid_dataloader):
images = images.to(device)
targets = targets.to(device)
batch_size = targets.size(0)
with torch.no_grad():
y_preds = model(images)
loss = criterion(y_preds.view(-1), targets)
losses.update(loss.item(), batch_size)
preds.append(y_preds.to('cpu').numpy())
all_targets.append(targets.detach().cpu().numpy())
if config.gradient_accumulation_steps > 1:
loss = loss / config.gradient_accumulation_steps
del loss
gc.collect()
predictions = np.concatenate(preds)
all_labels = np.concatenate(all_targets)
return losses.avg, predictions, all_labels
def run():
siamese_wrapper = SiameseNNWrapper()
end = time.time()
if loc.params["mode"] == "train":
train_ds = SiameseDataset(loc.train_dir, number_of_class=80, number_per_class=600)
train_dataloader = torch.utils.data.DataLoader(
train_ds, batch_size=loc.params["batch_size"], shuffle=True, num_workers=4, pin_memory=True)
eval_ds = SiameseEvalDataset(loc.train_dir, number_of_class=20, number_per_class=200)
eval_dataloader = torch.utils.data.DataLoader(
eval_ds, batch_size=1, shuffle=True, num_workers=4, pin_memory=True)
print("Size of Batch: {}, Spent Time: {}".format(len(train_dataloader), time.time()-end))
siamese_wrapper.train(train_dataloader=train_dataloader, eval_dataloader = eval_dataloader, fine_tune=False)
else:
num_of_class = 20
shot = 5
print("way = {}, shot={}".format(num_of_class, shot))
result = AverageMeter()
for i in range(10):
ds = SiameseEvalDataset(loc.val_dir, number_of_class=num_of_class, number_per_class=50, shot=shot)
dataloader = torch.utils.data.DataLoader(
ds, batch_size=48, shuffle=True, num_workers=4, pin_memory=True)
# print("Size of Eval Batch: {}, Spent Time: {}".format(len(dataloader), time.time() - end))
result.update(siamese_wrapper.evaluate(dataloader))
return result.avg
def gen_benchmark(pose_estimator, video_folder):
logging.basicConfig(filename='pose_estimation_benchmarks.log', filemode='w', format=LOG_FORMAT, level=logging.INFO)
logging.info(f'Evaluating {pose_estimator} method...')
tic = time.time()
net = model_zoo.get_model(pose_estimator)
fps_meter = AverageMeter()
videos_list = os.listdir(video_folder)
for video in videos_list:
logging.info(f'Running video {video}...')
video_filepath = os.path.join(video_folder, video)
video_reader = cv2.VideoCapture(video_filepath)
while(video_reader.isOpened()):
ret, frame = video_reader.read()
if ret:
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
model_in = time.time()
net.get_poses(frame)
fps_meter.update(1.0 / (time.time() - model_in))
else:
break
logging.info(f'Average FPS: {fps_meter.avg}')
tac = time.time()
logging.info(f'Done in {(tac - tic):.3f} seconds.')
def validate_function(data_loader):
validation_loss = AverageMeter()
validation_acc = AverageMeter()
net.eval()
# TODO
with torch.no_grad():
for idx, (sketch, label) in enumerate(tqdm(data_loader, ascii=True)):
sketch = sketch.cuda()
label = label.cuda()
#output, _ = net(sketch)
output = net(sketch)
batch_loss = loss_function(output, label)
validation_loss.update(batch_loss.item(), sketch.size(0))
validation_acc.update(
accuracy(output, label, topk=(1, ))[0].item(), sketch.size(0))
logger.info("==> Evaluation Result: ")
logger.info("loss: {} acc:{}".format(validation_loss.avg,
validation_acc.avg))
return validation_loss, validation_acc
def test(args):
runtime = AverageMeter()
ckpt_path = args.checkpoint_path
try:
names = os.listdir(ckpt_path)
for name in names:
out = re.findall("ResNetNMS_.*", name)
if out != []:
ckpt_path = out[0]
break
ckpt_path = os.path.join(args.checkpoint_path, ckpt_path)
except Exception:
print("There is no checkpoint in ", args.checkpoint)
exit
model = RC3D_resnet_learn_nms.RC3D(num_classes, cfg.Test.Image_shape,
args.feature_path)
model = model.cuda()
model.zero_grad()
model.load(ckpt_path)
#test_batch = utils.Batch_Generator(name_to_id, num_classes, args.image_path, args.annotation_path, mode = 'test')
test_batch = utils.new_Batch_Generator(name_to_id, num_classes,
args.image_path,
args.annotation_path)
tic = time.time()
data, gt = next(test_batch)
with torch.no_grad():
#pdb.set_trace()
print(gt)
_, _, _, _, nms_score = model.forward(data)
#bbox = utils.nms(model.proposal_bbox, object_cls_score, object_offset, model.num_classes, model.im_info)
pdb.set_trace()
num_bbox = nms_score.shape[0]
label = torch.arange(1, num_classes).cuda()
label = label.repeat(num_bbox, 1)
idx = torch.nonzero(nms_score > cfg.Network.nms_threshold[0])
if idx.shape[0] == 0:
exit
bbox = utils.subscript_index(model.sorted_bbox, idx)
cls_score = utils.subscript_index(nms_score, idx)
cls_label = utils.subscript_index(label, idx)
toc = time.time()
torch.cuda.empty_cache()
runtime.update(toc - tic)
print('Time {runtime.val:.3f} ({runtime.avg:.3f})\t'.format(
runtime=runtime))
for _cls, score, proposal in zip(cls_label, cls_score, bbox):
print(
"class:{:}({:})\t score:{:.6f}\t start:{:.2f}\t end:{:.2f}\t"
.format(id_to_name[int(_cls)], _cls, score, proposal[0],
proposal[1]))
def validate_function():
validation_loss = AverageMeter()
validation_acc = AverageMeter()
net.eval()
with torch.no_grad():
for idx, (coordinate, label, flag_bits, _, _, _, position_encoding) in enumerate(tqdm(val_loader, ascii=True)):
coordinate = coordinate.cuda()
label = label.cuda()
flag_bits = flag_bits.cuda()
position_encoding = position_encoding.cuda()
flag_bits.squeeze_(2)
position_encoding.squeeze_(2)
output, _ = net(coordinate, flag_bits, position_encoding)
batch_loss = loss_function(output, label)
validation_loss.update(batch_loss.item(), coordinate.size(0))
validation_acc.update(accuracy(output, label, topk = (1,))[0].item(), coordinate.size(0))
logger.info("==> Evaluation Result: ")
logger.info("loss: {} acc:{}".format(validation_loss.avg, validation_acc.avg))
return validation_loss, validation_acc
def train_function(epoch):
training_loss = AverageMeter()
training_acc = AverageMeter()
net.train()
lr = next((lr for (max_epoch, lr) in lr_protocol if max_epoch > epoch), lr_protocol[-1][1])
for param_group in optimizer.param_groups:
param_group['lr'] = lr
logger.info("set learning rate to: {}".format(lr))
for idx, (coordinate, label, flag_bits, _, _, _, position_encoding) in enumerate(tqdm(train_loader, ascii=True)):
coordinate = coordinate.cuda()
label = label.cuda()
flag_bits = flag_bits.cuda()
position_encoding = position_encoding.cuda()
flag_bits.squeeze_(2)
position_encoding.squeeze_(2)
optimizer.zero_grad()
output, _ = net(coordinate, flag_bits, position_encoding)
batch_loss = loss_function(output, label)
batch_loss.backward()
optimizer.step()
training_loss.update(batch_loss.item(), coordinate.size(0))
training_acc.update(accuracy(output, label, topk = (1,))[0].item(), coordinate.size(0))
if (idx + 1) % basic_configs["display_step"] == 0:
logger.info(
"==> Iteration [{}][{}/{}]:".format(epoch + 1, idx + 1, len(train_loader)))
logger.info("current batch loss: {}".format(
batch_loss.item()))
logger.info("average loss: {}".format(
training_loss.avg))
logger.info("average acc: {}".format(training_acc.avg))
logger.info("Begin Evaluating")
validation_loss, validation_acc = validate_function()
writer.add_scalars("loss", {
"training_loss":training_loss.avg,
"validation_loss":validation_loss.avg
}, epoch+1)
writer.add_scalars("acc", {
"training_acc":training_acc.avg,
"validation_acc":validation_acc.avg
}, epoch+1)
def train_function(epoch):
training_loss = AverageMeter()
training_acc = AverageMeter()
net.train()
lr = next((lr for (max_epoch, lr) in lr_protocol if max_epoch > epoch),
lr_protocol[-1][1])
for param_group in optimizer.param_groups:
param_group['lr'] = lr
logger.info("set learning rate to: {}".format(lr))
# TODO
for idx, (sketch, label) in enumerate(tqdm(train_loader, ascii=True)):
sketch = sketch.cuda()
label = label.cuda()
optimizer.zero_grad()
output = net(sketch)
output = output[0]
# ipdb.set_trace()
batch_loss = loss_function(output, label)
batch_loss.backward()
optimizer.step()
training_loss.update(batch_loss.item(), sketch.size(0))
training_acc.update(
accuracy(output, label, topk=(1, ))[0].item(), sketch.size(0))
if (idx + 1) % basic_configs["display_step"] == 0:
logger.info("==> Iteration [{}][{}/{}]:".format(
epoch + 1, idx + 1, len(train_loader)))
logger.info("current batch loss: {}".format(batch_loss.item()))
logger.info("average loss: {}".format(training_loss.avg))
logger.info("average acc: {}".format(training_acc.avg))
# TODO
logger.info("Begin Evaluating")
validation_loss, validation_acc = validate_function(val_loader)
test_loss, test_acc = validate_function(test_loader)
writer.add_scalars(
"loss", {
"training_loss": training_loss.avg,
"validation_loss": validation_loss.avg,
"test_loss": test_loss.avg
}, epoch + 1)
writer.add_scalars(
"acc", {
"training_acc": training_acc.avg,
"validation_acc": validation_acc.avg,
"test_acc": test_acc.avg
}, epoch + 1)
def train(conf):
"""Total training procedure.
"""
conf.device = torch.device('cuda:0')
criterion = torch.nn.CrossEntropyLoss().cuda(conf.device)
backbone_factory = BackboneFactory(conf.backbone_type, conf.backbone_conf_file)
probe_net = backbone_factory.get_backbone()
gallery_net = backbone_factory.get_backbone()
head_factory = HeadFactory(conf.head_type, conf.head_conf_file)
prototype = head_factory.get_head().cuda(conf.device)
probe_net = torch.nn.DataParallel(probe_net).cuda()
gallery_net = torch.nn.DataParallel(gallery_net).cuda()
optimizer = optim.SGD(probe_net.parameters(), lr=conf.lr, momentum=conf.momentum, weight_decay=5e-4)
lr_schedule = optim.lr_scheduler.MultiStepLR(optimizer, milestones=conf.milestones, gamma=0.1)
if conf.resume:
probe_net.load_state_dict(torch.load(args.pretrain_model))
moving_average(probe_net, gallery_net, 0)
probe_net.train()
gallery_net.eval().apply(train_BN)
exclude_id_set = set()
loss_meter = AverageMeter()
for epoch in range(conf.epoches):
data_loader = DataLoader(
ImageDataset_SST(conf.data_root, conf.train_file, exclude_id_set),
conf.batch_size, True, num_workers = 4, drop_last = True)
exclude_id_set = train_one_epoch(data_loader, probe_net, gallery_net,
prototype, optimizer, criterion, epoch, conf, loss_meter)
lr_schedule.step()
def train(conf):
"""Total training procedure.
"""
data_loader = DataLoader(ImageDataset(conf.data_root, conf.train_file),
conf.batch_size,
True,
num_workers=4)
conf.device = torch.device('cuda:0')
criterion = torch.nn.CrossEntropyLoss().cuda(conf.device)
backbone_factory = BackboneFactory(conf.backbone_type,
conf.backbone_conf_file)
head_factory = HeadFactory(conf.head_type, conf.head_conf_file)
model = FaceModel(backbone_factory, head_factory)
ori_epoch = 0
if conf.resume:
ori_epoch = torch.load(args.pretrain_model)['epoch'] + 1
state_dict = torch.load(args.pretrain_model)['state_dict']
model.load_state_dict(state_dict)
model = model.cuda()
parameters = [p for p in model.parameters() if p.requires_grad]
optimizer = optim.SGD(parameters,
lr=conf.lr,
momentum=conf.momentum,
weight_decay=1e-4)
model, optimizer = amp.initialize(model, optimizer, opt_level="O1")
model = torch.nn.DataParallel(model).cuda()
lr_schedule = optim.lr_scheduler.MultiStepLR(optimizer,
milestones=conf.milestones,
gamma=0.1)
loss_meter = AverageMeter()
model.train()
for epoch in range(ori_epoch, conf.epoches):
train_one_epoch(data_loader, model, optimizer, criterion, epoch,
loss_meter, conf)
lr_schedule.step()
def evaluate(self, dataloader):
self.nn.eval()
eval_acc = AverageMeter()
output2 = None
for i, (x1, x2, label) in enumerate(dataloader):
# x1 = torch.squeeze(x1, 0)
x2 = x2[0]
if loc.params["cuda"]:
x1, x2 = x1.cuda(), x2.cuda()
x1 = Variable(x1)
if output2 is None:
x2 = Variable(x2)
output2 = self.nn.forward_once(x2)
output1 = self.nn.forward_once(x1)
acc = self.get_acc(output1, output2, label)
eval_acc.update(acc, x1.shape[0])
# print("Siamese Test, acc[{}]({})".format(eval_acc.val, eval_acc.avg))
return eval_acc.avg
def train_fn(train_loader, model, criterion, optimizer, scheduler, config,
device):
assert hasattr(
config, "gradient_accumulation_steps"
), "Please create gradient_accumulation_steps(int default=1) attribute"
assert hasattr(config,
"apex"), "Please create apex(bool default=False) attribute"
assert hasattr(
config, "batch_scheduler"
), "Please create batch_scheduler(bool default=False) attribute"
model.train()
scaler = GradScaler(enabled=config.apex)
losses = AverageMeter()
tk0 = tqdm(train_loader, total=len(train_loader))
for step, (inputs, mask, labels) in enumerate(tk0):
inputs = inputs.to(device)
mask = mask.to(device)
labels = labels.to(device)
batch_size = labels.size(0)
with torch.cuda.amp.autocast(enabled=config.apex):
y_preds = model(inputs, mask)
loss = criterion(y_preds.view(-1, 1), labels.view(-1, 1))
if config.gradient_accumulation_steps > 1:
loss = loss / config.gradient_accumulation_steps
losses.update(loss.item(), batch_size)
if config.apex:
scaler.scale(loss).backward()
else:
loss.backward()
grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(),
config.max_grad_norm)
if (step + 1) % config.gradient_accumulation_steps == 0:
if config.apex:
scaler.step(optimizer)
scaler.update()
else:
optimizer.step()
optimizer.zero_grad()
if config.batch_scheduler:
scheduler.step()
del loss
gc.collect()
return losses.avg
def eval_model(args, model, device, test_loader):
"""
Evaluation function for supervised learning. Metrics are balanced (macro) and unbalanced (micro) top-1 accuracies.
:param args:
:param model:
:param device:
:param test_loader:
:return:
"""
model.eval()
test_loss = AverageMeter()
top1 = AverageMeter()
# init vars for per class metrics
correct = np.zeros((args['learn']['num_classes'],))
total = np.zeros((args['learn']['num_classes'],))
criterion = torch.nn.CrossEntropyLoss()
with torch.no_grad():
for batch_idx, (examples, labels, index) in enumerate(test_loader):
# index contains the clip ID, i.e. all TF patches loaded from the same clip share this ID (the label too)
examples, labels, index = examples.to(device), labels.to(device), index.to(device)
idx_batch = torch.unique(index)
counter = 0
for i in range(len(idx_batch)): # Each iteration averages softmax predictions to get each clip prediction
# process one evaluation clip
num_examples = len(torch.nonzero(index==idx_batch[i]))
output = model(examples[counter:counter+num_examples].unsqueeze(1)) # logits per patch
loss = criterion(output, labels[counter:counter+num_examples])
output = F.softmax(output, dim=1) # class probabilities per patch
output2 = torch.mean(output, dim=0) # class probabilities per clip
pred = output2.argmax() # get the index of the max log-probability
gt_label = labels[counter:counter + num_examples][0]
# update count for correct (0 or 1), and total (1)
correct[gt_label] += pred.eq(gt_label.view_as(pred)).sum().item()
total[gt_label] += 1
# compute unbalanced (micro) accuracy
prec1, prec5 = accuracy_v2(output2.unsqueeze(0), labels[counter:counter+num_examples][0], top=[1, 5])
test_loss.update(loss.item(), num_examples)
top1.update(prec1.item(), num_examples)
counter += num_examples
# after all evaluation set, finalize per class accuracy, and compute balanced (macro) accuracy
acc_per_class = 100*(correct/total)
balanced_accuracy = acc_per_class.mean()
return test_loss.avg, top1.avg, balanced_accuracy, acc_per_class
def train_epoch(self, logger):
loss_meter = {}
for s in self.sets:
loss_meter[s] = AverageMeter()
for s in self.sets:
# Iterate over data.
for data in tqdm(self.dataloaders[s]):
inputs_d, C, item_idxs, inputs_rgb, gt_depth = data
inputs_d = inputs_d.cuda()
gt_depth = gt_depth.cuda()
inputs_rgb = inputs_rgb.cuda()
outputs = self.net(inputs_d, inputs_rgb)
# Calculate loss for valid pixel in the ground truth
MSELoss11 = self.objective1(outputs[0], gt_depth)
MSELoss12 = self.objective1(outputs[1], gt_depth)
MSELoss14 = self.objective1(outputs[2], gt_depth)
MAELoss11 = self.objective2(outputs[0], gt_depth)
MAELoss12 = self.objective2(outputs[1], gt_depth)
MAELoss14 = self.objective2(outputs[2], gt_depth)
if self.epoch < 41:
gamma = 1
else:
gamma = 0
if self.epoch < 6:
delta = 1
elif self.epoch < 11:
delta = 0.1
else:
delta = 0
MSELoss = delta * MSELoss14 + delta * MSELoss12 + MSELoss11
MAELoss = delta * MAELoss14 + delta * MAELoss12 + MAELoss11
loss = MSELoss + gamma * MAELoss
# backward + optimize only if in training phase
loss.backward()
self.optimizer.step()
self.optimizer.zero_grad()
# statistics
loss_meter[s].update(MSELoss11.item(), inputs_d.size(0))
logger.write(f'{loss_meter[s].avg:^15.8f}')
torch.cuda.empty_cache()
return loss_meter
def test(args):
runtime = AverageMeter()
ckpt_path = args.checkpoint_path
try:
names = os.listdir(ckpt_path)
for name in names:
out = re.findall("ResNet_.*", name)
if out != []:
ckpt_path = out[0]
break
ckpt_path = os.path.join(args.checkpoint_path, ckpt_path)
except Exception:
print("There is no checkpoint in ", args.checkpoint)
exit
model = RC3D_resnet.RC3D(num_classes, cfg.Test.Image_shape,
args.feature_path)
model = model.cuda()
model.zero_grad()
model.load(ckpt_path)
#test_batch = utils.Batch_Generator(name_to_id, num_classes, args.image_path, args.annotation_path, mode = 'test')
test_batch = utils.new_Batch_Generator(name_to_id, num_classes,
args.image_path,
args.annotation_path)
tic = time.time()
data, gt = next(test_batch)
with torch.no_grad():
pdb.set_trace()
print(gt)
_, _, object_cls_score, object_offset = model.forward(data)
bbox = utils.nms(model.proposal_bbox, object_cls_score, object_offset,
model.num_classes, model.im_info)
toc = time.time()
torch.cuda.empty_cache()
runtime.update(toc - tic)
print('Time {runtime.val:.3f} ({runtime.avg:.3f})\t'.format(
runtime=runtime))
for _cls, score, proposal in zip(bbox['cls'], bbox['score'],
bbox['bbox']):
print(
"class:{:}({:})\t score:{:.6f}\t start:{:.2f}\t end:{:.2f}\t"
.format(id_to_name[int(_cls[0])], _cls[0], score[0],
proposal[0, 0], proposal[0, 1]))
def train(args):
"""Total training procedure.
"""
print("Use GPU: {} for training".format(args.local_rank))
if args.local_rank == 0:
writer = SummaryWriter(log_dir=args.tensorboardx_logdir)
args.writer = writer
if not os.path.exists(args.out_dir):
os.makedirs(args.out_dir)
dist.init_process_group(backend='nccl', init_method='env://')
torch.cuda.set_device(args.local_rank)
args.rank = dist.get_rank()
#print('args.rank: ', dist.get_rank())
#print('args.get_world_size: ', dist.get_world_size())
#print('is_nccl_available: ', dist.is_nccl_available())
args.world_size = dist.get_world_size()
trainset = ImageDataset(args.data_root, args.train_file)
train_sampler = torch.utils.data.distributed.DistributedSampler(
trainset, shuffle=True)
train_loader = DataLoader(dataset=trainset,
batch_size=args.batch_size,
sampler=train_sampler,
num_workers=0,
pin_memory=True,
drop_last=False)
backbone_factory = BackboneFactory(args.backbone_type,
args.backbone_conf_file)
head_factory = HeadFactory(args.head_type, args.head_conf_file)
model = FaceModel(backbone_factory, head_factory)
model = model.to(args.local_rank)
model.train()
for ps in model.parameters():
dist.broadcast(ps, 0)
# DDP
model = torch.nn.parallel.DistributedDataParallel(
module=model, broadcast_buffers=False, device_ids=[args.local_rank])
criterion = torch.nn.CrossEntropyLoss().to(args.local_rank)
ori_epoch = 0
parameters = [p for p in model.parameters() if p.requires_grad]
optimizer = optim.SGD(parameters,
lr=args.lr,
momentum=args.momentum,
weight_decay=1e-4)
lr_schedule = optim.lr_scheduler.MultiStepLR(optimizer,
milestones=args.milestones,
gamma=0.1)
loss_meter = AverageMeter()
model.train()
for epoch in range(ori_epoch, args.epoches):
train_one_epoch(train_loader, model, optimizer, criterion, epoch,
loss_meter, args)
lr_schedule.step()
dist.destroy_process_group()
def train(model: torch.nn.Module, train_loader: Iterable,
optimizer: torch.optim.Optimizer, epoch: int, summary: SummaryWriter):
model.train()
loss = nn.CrossEntropyLoss()
train_loss = AverageMeter()
for step, data in enumerate(train_loader):
img, label = data
label = label.cuda()
img = img.float().cuda()
pred = model(img)
losses = loss(pred, label)
train_loss.update(losses.item(), img.size()[0])
optimizer.zero_grad()
losses.backward()
optimizer.step()
#print("losses : {} , epoch : {}".format(losses, epoch))
summary.add_scalar('train/loss', train_loss.avg, epoch)
def train_fn(train_loader, model, criterion, optimizer, config, device):
assert hasattr(config, "apex"), "Please create apex(bool) attribute"
assert hasattr(
config, "gradient_accumulation_steps"
), "Please create gradient_accumulation_steps(int default=1) attribute"
model.train()
if config.apex:
scaler = GradScaler()
losses = AverageMeter()
for step, (images, labels) in enumerate(train_loader):
images = images.to(device)
labels = labels.to(device)
batch_size = labels.size(0)
if config.apex:
with autocast():
y_preds = model(images)
loss = criterion(y_preds.view(-1), labels)
else:
y_preds = model(images)
loss = criterion(y_preds.view(-1), labels)
# record loss
losses.update(loss.item(), batch_size)
if config.gradient_accumulation_steps > 1:
loss = loss / config.gradient_accumulation_steps
if config.apex:
scaler.scale(loss).backward()
else:
loss.backward()
if (step + 1) % config.gradient_accumulation_steps == 0:
if config.apex:
scaler.step(optimizer)
scaler.update()
else:
optimizer.step()
optimizer.zero_grad()
del loss
gc.collect()
return losses.avg
def train_epoch(self):
device = torch.device(
"cuda:" +
str(self.params['gpu_id']) if torch.cuda.is_available() else "cpu")
loss_meter = {}
for s in self.sets:
loss_meter[s] = AverageMeter()
for s in self.sets:
# Iterate over data.
for data in self.dataloaders[s]:
start_iter_time = time.time()
inputs_d, C, labels, item_idxs, inputs_rgb = data
inputs_d = inputs_d.to(device)
C = C.to(device)
labels = labels.to(device)
inputs_rgb = inputs_rgb.to(device)
outputs = self.net(inputs_d, inputs_rgb)
# Calculate loss for valid pixel in the ground truth
loss11 = self.objective(outputs[0], labels)
loss12 = self.objective(outputs[1], labels)
loss14 = self.objective(outputs[2], labels)
if self.epoch < 6:
loss = loss14 + loss12 + loss11
elif self.epoch < 11:
loss = 0.1 * loss14 + 0.1 * loss12 + loss11
else:
loss = loss11
# backward + optimize only if in training phase
loss.backward()
self.optimizer.step()
self.optimizer.zero_grad()
# statistics
loss_meter[s].update(loss11.item(), inputs_d.size(0))
end_iter_time = time.time()
iter_duration = end_iter_time - start_iter_time
if self.params['print_time_each_iter']:
print('finish the iteration in %.2f s.\n' %
(iter_duration))
print('Loss within the curt iter: {:.8f}\n'.format(
loss_meter[s].avg))
print('[{}] Loss: {:.8f}'.format(s, loss_meter[s].avg))
torch.cuda.empty_cache()
return loss_meter
def valid_fn(model, criterion, valid_loader, config, device):
assert hasattr(
config, "gradient_accumulation_steps"
), "Please create gradient_accumulation_steps(int default=1) attribute"
losses = AverageMeter()
model.eval()
preds = []
tk0 = tqdm(valid_loader, total=len(valid_loader))
for step, (inputs, attention_mask, labels) in enumerate(tk0):
labels = labels.to(device)
inputs = inputs.to(device)
attention_mask = attention_mask.to(device)
batch_size = labels.size(0)
with torch.no_grad():
y_preds = model(inputs, attention_mask)
loss = criterion(y_preds.view(-1, 1), labels.view(-1, 1))
if config.gradient_accumulation_steps > 1:
loss = loss / config.gradient_accumulation_steps
losses.update(loss.item(), batch_size)
preds.append(y_preds.sigmoid().to('cpu').numpy())
predictions = np.concatenate(preds)
predictions = np.concatenate(predictions)
return losses.avg, predictions
def test(model: torch.nn.Module, val_loader: Iterable, epoch: int,
summary: SummaryWriter):
model.eval()
val_acc = AverageMeter()
val_losses = AverageMeter()
temp = []
pred_list = []
with torch.no_grad():
for step, data in enumerate(val_loader):
image, label = data
image = image.float().cuda()
label = label.cuda()
pred = model(image)
prec1, preds = accuracy(pred.data, label)
temp.append(label.tolist())
pred_list.append(preds.tolist())
val_acc.update(prec1.item(), image.size()[0])
# confusion_matrix
y_true = reduce(lambda x, y: x + y, temp)
y_pred = []
for i in pred_list:
temp = i[0]
for k in temp:
y_pred.append(k)
confusion_matrixs = confusion_matrix(y_true, y_pred)
print(confusion_matrixs)
print(classification_report(y_true, y_pred))
def get_group_euclidean(group1, group2 = None):
eu = AverageMeter()
if group2 is None:
for i, image1 in enumerate(group1):
for j, image2 in enumerate(group1[i+1:]):
eu.update(loss.euclidean_loss(image1, image2)[0])
else:
for i, image1 in enumerate(group1):
for j, image2 in enumerate(group2):
eu.update(loss.euclidean_loss(image1, image2)[0])
return eu
def val(model: torch.nn.Module, val_loader: Iterable, epoch: int, summary: SummaryWriter):
model.eval()
val_acc = AverageMeter()
val_losses = AverageMeter()
with torch.no_grad():
loss = nn.CrossEntropyLoss()
for step, data in enumerate(val_loader):
img, label = data
label = label.cuda()
img = img.float()
pred = model(img)
losses = loss(pred, label)
prec1 = accuracy(pred.data, label)[0]
val_losses.update(losses.item(), img.size()[0])
val_acc.update(prec1.item(), img.size()[0])
print("val acc : {} epoch : {}".format(val_acc.avg, epoch))
summary.add_scalar('val/loss', val_losses.avg, epoch)
summary.add_scalar('val/acc', val_acc.avg, epoch)
def test_resnet():
model = Resnet50_new()
top1 = AverageMeter()
top5 = AverageMeter()
accu_20 = []
for i in range(80):
accu_20.append(AverageMeter())
for step, sample in enumerate(gd.val_loader):
weight = sample[0].shape[0]
s = Variable(sample[0].cuda())
pre = model(s)
prec1, prec5 = accuracy(pre.data, sample[1].cuda(), topk=(1, 5))
top1.update(prec1[0], n=weight)
top5.update(prec5[0], n=weight)
update_class_acc(accu_20, pre.data, sample[1].cuda())
print("Step: {step}, top1: {top1.avg:.3f}({top1.val:.3f}), "
"top5: {top5.avg:.3f}({top5.val:.3f})".format(step=step, top1=top1, top5=top5))
for k, j in enumerate(accu_20):
print("{k}: {top1.avg:.3f}({top1.val:.3f}), ".format(k=k, top1=j))
def train_epoch(self):
device = torch.device(
"cuda:" +
str(self.params['gpu_id']) if torch.cuda.is_available() else "cpu")
loss_meter = {}
for s in self.sets:
loss_meter[s] = AverageMeter()
for s in self.sets:
# Iterate over data.
for data in self.dataloaders[s]:
if self.load_rgb:
inputs_d, C, labels, item_idxs, inputs_rgb = data
inputs_d = inputs_d.to(device)
C = C.to(device)
labels = labels.to(device)
inputs_rgb = inputs_rgb.to(device)
outputs, cout = self.net(inputs_d, C, inputs_rgb)
else:
inputs_d, C, labels, item_idxs = data
inputs_d = inputs_d.to(device)
C = C.to(device)
labels = labels.to(device)
outputs, cout = self.net(inputs_d, C)
# Calculate loss for valid pixel in the ground truth
loss = self.objective(outputs, labels, cout, self.epoch)
# backward + optimize only if in training phase
if s == 'train':
loss.backward()
self.optimizer.step()
self.optimizer.zero_grad()
# statistics
loss_meter[s].update(loss.item(), inputs_d.size(0))
print('[{}] Loss: {:.8f}'.format(s, loss_meter[s].avg), end=' ')
return loss_meter
def train(conf):
"""Total training procedure.
"""
if conf.virtual_batch:
assert conf.batch_size % 64 == 0
update_interval = conf.batch_size // 64
batch_per_epoch = 64
else:
update_interval = 1
batch_per_epoch = conf.batch_size
data_loader = DataLoader(ImageDataset(conf.data_root, conf.train_file),
batch_per_epoch, True, num_workers = 6)
conf.device = torch.device('cuda:0')
criterion = torch.nn.CrossEntropyLoss().cuda(conf.device)
backbone_factory = BackboneFactory(conf.backbone_type, conf.backbone_conf_file)
head_factory = HeadFactory(conf.head_type, conf.head_conf_file)
model = FaceModel(backbone_factory, head_factory)
ori_epoch = 0
if conf.pretrain_model != '':
state_dict = torch.load(args.pretrain_model)['state_dict']
model.load_state_dict(state_dict)
if conf.resume:
ori_epoch = torch.load(args.pretrain_model)['epoch'] + 1
del state_dict
model = model.cuda()
# parameters = [p for p in model.parameters() if p.requires_grad]
backbone_parameters = [p for n, p in model.named_parameters() if ("backbone" in n) and (p.requires_grad)]
head_parameters = [p for n, p in model.named_parameters() if ("head" in n) and (p.requires_grad)]
optimizer = optim.AdamW(backbone_parameters + head_parameters, lr = conf.lr, weight_decay = 3e-5)
if conf.resume:
for param_group in optimizer.param_groups:
param_group['initial_lr'] = args.lr
scaler = torch.cuda.amp.GradScaler()
model = torch.nn.DataParallel(model).cuda()
lr_schedule = optim.lr_scheduler.MultiStepLR(
optimizer, milestones = conf.milestones, gamma = 0.1, last_epoch=ori_epoch-1)
loss_meter = AverageMeter()
model.train()
for epoch in range(ori_epoch, conf.epoches):
train_one_epoch(data_loader, model, optimizer,
criterion, epoch, loss_meter, backbone_parameters, conf, scaler, update_interval)
lr_schedule.step()
def train(self, train_dataloader, eval_dataloader, fine_tune=False):
self.nn.train()
optimizer = optim.Adam(self.nn.parameters())
batch_time = AverageMeter()
losses = AverageMeter()
end = time.time()
if fine_tune:
status = "fine_tune"
else:
status = "train"
for j in range(loc.params["epoch"]):
for i, (x1, x2, label) in enumerate(train_dataloader):
label = label.float()
if loc.params["cuda"]:
x1, x2, label = x1.cuda(), x2.cuda(), label.cuda()
x1 = Variable(x1)
x2 = Variable(x2)
label = Variable(label)
output1, output2 = self.nn.forward(x1, x2)
loss = contrastive_loss(output1, output2, label, margin=loc.params["margin"])
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
losses.update(loss.item(), x1.shape[0])
self.save_model()
print("Simese {}, Epoch[{}], Loss[{}]({})".format(status, j, losses.val, losses.avg))
self.evaluate(eval_dataloader)
def train(conf):
"""Total training procedure.
"""
data_loader = DataLoader(ImageDataset_SST(conf.data_root, conf.train_file),
conf.batch_size,
True,
num_workers=4)
conf.device = torch.device('cuda:0')
#criterion = OnlineContrastiveLoss(margin=2.5, pair_selector=HardNegativePairSelector(cpu=False)).cuda(torch.device('cuda:0'))
triplet_selector = FunctionNegativeTripletSelector(
margin=2.5, negative_selection_fn=random_hard_negative, cpu=False)
criterion = OnlineTripletLoss(margin=2.5,
triplet_selector=triplet_selector).cuda(
conf.device)
backbone_factory = BackboneFactory(conf.backbone_type,
conf.backbone_conf_file)
model = backbone_factory.get_backbone()
if conf.resume:
model.load_state_dict(torch.load(args.pretrain_model))
model = torch.nn.DataParallel(model).cuda()
parameters = [p for p in model.parameters() if p.requires_grad]
optimizer = optim.SGD(parameters,
lr=conf.lr,
momentum=conf.momentum,
weight_decay=1e-4)
lr_schedule = optim.lr_scheduler.MultiStepLR(optimizer,
milestones=conf.milestones,
gamma=0.1)
loss_meter = AverageMeter()
model.train()
for epoch in range(conf.epoches):
train_one_epoch(data_loader, model, optimizer, criterion, epoch,
loss_meter, conf)
lr_schedule.step()
if conf.evaluate:
conf.evaluator.evaluate(model)
def train(args):
is_am = True if args.is_am == 1 else False
model = MVFace(args.backbone, args.feat_dim, args.num_class, is_am)
ori_epoch = 0
if args.resume:
ori_epoch = torch.load(args.pretrain_model)['epoch'] + 1
state_dict = torch.load(args.pretrain_model)['state_dict']
model.load_state_dict(state_dict)
model = torch.nn.DataParallel(model).cuda()
criteria = torch.nn.CrossEntropyLoss().cuda()
parameters = [p for p in model.parameters() if p.requires_grad]
optimizer = optim.SGD(parameters, lr=args.lr, momentum=args.momentum, weight_decay=1e-4)
lr_schedule = optim.lr_scheduler.MultiStepLR(optimizer, milestones=args.milestones, gamma=0.1)
train_transform, _ = make_transform()
data_loader = DataLoader(ImageDataset(args.data_root, args.train_file, train_transform),
args.batch_size, shuffle=True, num_workers=4)
args.db_size = len(data_loader)
args.check_point_size = (args.db_size // 3, args.db_size // 2, args.db_size // 3 * 2)
loss_meter = AverageMeter()
model.train()
for epoch in range(ori_epoch, args.epochs):
train_one_epoch(data_loader, model, optimizer, criteria, epoch, loss_meter, args)
lr_schedule.step()