def singleTagLoss(pred_tag, keypoints):
"""
associative embedding loss for one image
"""
eps = 1e-6
tags = []
pull = 0
for i in keypoints:
tmp = []
for j in i:
if j[1] > 0:
tmp.append(pred_tag[j[0]])
if len(tmp) == 0:
continue
tmp = torch.stack(tmp)
tags.append(torch.mean(tmp, dim=0))
pull = pull + torch.mean((tmp - tags[-1].expand_as(tmp))**2)
if len(tags) == 0:
return make_input(torch.zeros([1]).float()), make_input(
torch.zeros([1]).float())
tags = torch.stack(tags)[:, 0]
num = tags.size()[0]
size = (num, num, tags.size()[1])
A = tags.unsqueeze(dim=1).expand(*size)
B = A.permute(1, 0, 2)
diff = A - B
diff = torch.pow(diff, 2).sum(dim=2)[:, :, 0]
push = torch.exp(-diff)
push = (torch.sum(push) - num)
return push / ((num - 1) * num + eps) * 0.5, pull / (num + eps)
def singleTagLoss(pred_tag, keypoints):
"""
associative embedding loss for one image
"""
eps = 1e-6
tags = []
pull = 0
for i in keypoints:
tmp = []
for j in i:
if j[1]>0:
tmp.append(pred_tag[j[0]])
if len(tmp) == 0:
continue
tmp = torch.stack(tmp)
tags.append(torch.mean(tmp, dim=0))
pull = pull + torch.mean((tmp - tags[-1].expand_as(tmp))**2)
if len(tags) == 0:
return make_input(torch.zeros([1]).float()), make_input(torch.zeros([1]).float())
tags = torch.stack(tags)[:,0]
num = tags.size()[0]
size = (num, num, tags.size()[1])
A = tags.unsqueeze(dim=1).expand(*size)
B = A.permute(1, 0, 2)
diff = A - B
diff = torch.pow(diff, 2).sum(dim=2)[:,:,0]
push = torch.exp(-diff)
push = (torch.sum(push) - num)
return push/((num - 1) * num + eps) * 0.5, pull/(num + eps)
def make_train(batch_id, config, phase, **inputs):
for i in inputs:
inputs[i] = make_input(inputs[i])
net = config['inference']['net']
config['batch_id'] = batch_id
if phase != 'inference':
result = net(inputs['imgs'],
**{i: inputs[i]
for i in inputs if i != 'imgs'})
num_loss = len(config['train']['loss'])
## I use the last outputs as the loss
## the weights of the loss are controlled by config['train']['loss']
losses = {
i[0]: result[-num_loss + idx] * i[1]
for idx, i in enumerate(config['train']['loss'])
}
loss = 0
toprint = '\n{}: '.format(batch_id)
for i in losses:
loss = loss + torch.mean(losses[i])
my_loss = make_output(losses[i])
my_loss = my_loss.mean(axis=0)
if my_loss.size == 1:
toprint += ' {}: {}'.format(i,
format(my_loss.mean(), '.8f'))
else:
toprint += '\n{}'.format(i)
for j in my_loss:
toprint += ' {}'.format(format(j.mean(), '.8f'))
logger.write(toprint)
logger.flush()
if batch_id == 200000:
## decrease the learning rate after 200000 iterations
for param_group in optimizer.param_groups:
param_group['lr'] = 1e-5
if phase == 'train':
optimizer = train_cfg['optimizer']
optimizer.zero_grad()
loss.backward()
optimizer.step()
return None
else:
out = {}
net = net.eval()
result = net(**inputs)
if type(result) != list and type(result) != tuple:
result = [result]
out['preds'] = [make_output(i) for i in result]
return out
def test_tag_loss():
t = make_input( torch.Tensor((1, 2)), requires_grad=True )
# register_hook is to take the backward gradient as input, when variable's gradient is updated in backward
# and this method will return a new tensor
t.register_hook(lambda x: print('t', x))
loss = singleTagLoss((t, [[[0,1]], [[1,1]]]))[0]
loss.backward()
print(loss)
def test_tag_loss():
def myhook(x):
print('t', x)
t = make_input( torch.Tensor((1, 2)), requires_grad=True )
t.register_hook(myhook)
loss = singleTagLoss((t, [[[0,1]], [[1,1]]]))[0]
loss.backward()
print(loss)
def do_train(epoch, config, loader):
logger = config['inference']['logger']
net = config['inference']['net']
batch_idx = 0
for inputs in tqdm(loader):
for i, input in enumerate(inputs):
try:
if type(inputs[i]) is list:
for ind, inp in enumerate(inputs[i]):
inputs[i][ind] = make_input(inp)
else:
inputs[i] = make_input(inputs[i])
except:
pass # for last input, which is a string (id_)
combined_preds = net(inputs[0])
combined_loss, labels_loss = config['inference']["lossLayers"](
combined_preds, **{
'heatmaps': inputs[1],
'labels': inputs[2]
})
num_loss = len(config['train']['loss'])
# losses = [all_loss[idx].cpu()*i[1] for idx, i in enumerate(config['train']['loss'])]
heatmap_loss = torch.sum(combined_loss.cpu().mul(
torch.Tensor(config['train']['stack_loss'])))
label_loss = torch.sum(labels_loss.cpu().mul(
torch.Tensor(config['train']['stack_loss'])))
toprint = '\n{} {}: '.format(epoch, batch_idx)
if batch_idx % 100 == 0:
toprint += ' \n{}'.format(str(combined_loss.cpu()))
toprint += ' \n{}'.format(str(labels_loss.cpu()))
else:
toprint += ' {},{}'.format(heatmap_loss, label_loss)
logger.write(toprint)
logger.flush()
optimizer = config['train']['optimizer']
optimizer.zero_grad()
loss = heatmap_loss + label_loss
loss.backward()
optimizer.step()
batch_idx += 1
def train_func(opts, model, optimizer, phase, **inputs):
for i in inputs:
inputs[i] = make_input(inputs[i])
if phase == 'train':
net = model.train()
else:
net = model.eval()
forward_net = DataParallel(net.cuda())
if phase != 'inference':
if phase == 'valid':
with torch.no_grad():
output = forward_net(inputs['imgs'])
losses = model.calc_loss(
output, **{i: inputs[i]
for i in inputs if i != 'imgs'})
losses = {
'push_loss': losses[0] * opts.push_loss,
'pull_loss': losses[1] * opts.pull_loss,
'detection_loss': losses[2] * opts.detection_loss
}
loss = 0
else:
output = forward_net(inputs['imgs'])
losses = model.calc_loss(
output, **{i: inputs[i]
for i in inputs if i != 'imgs'})
losses = {
'push_loss': losses[0] * opts.push_loss,
'pull_loss': losses[1] * opts.pull_loss,
'detection_loss': losses[2] * opts.detection_loss
}
loss = 0
for i in losses:
loss = loss + torch.mean(losses[i])
my_loss = make_output(losses[i])
my_loss = my_loss.mean(axis=0)
#if my_loss.size == 1:
# toprint += ' {}: {}'.format(i, format(my_loss.mean(), '.8f'))
#else:
# toprint += '\n{}'.format(i)
# for j in my_loss:
# toprint += ' {}'.format(format(j.mean(), '.8f'))
if phase == 'train':
optimizer.zero_grad()
loss.backward()
optimizer.step()
return loss
def make_train(batch_id, config, phase, **inputs):
for i in inputs:
inputs[i] = make_input(inputs[i])
net = config['inference']['net']
config['batch_id'] = batch_id
if phase != 'inference':
result = net(inputs['imgs'], **{i:inputs[i] for i in inputs if i!='imgs'})
num_loss = len(config['train']['loss'])
## I use the last outputs as the loss
## the weights of the loss are controlled by config['train']['loss']
losses = {i[0]: result[-num_loss + idx]*i[1] for idx, i in enumerate(config['train']['loss'])}
loss = 0
toprint = '\n{}: '.format(batch_id)
for i in losses:
loss = loss + torch.mean(losses[i])
my_loss = make_output( losses[i] )
my_loss = my_loss.mean(axis = 0)
if my_loss.size == 1:
toprint += ' {}: {}'.format(i, format(my_loss.mean(), '.8f'))
else:
toprint += '\n{}'.format(i)
for j in my_loss:
toprint += ' {}'.format(format(j.mean(), '.8f'))
logger.write(toprint)
logger.flush()
if batch_id == 200000:
## decrease the learning rate after 200000 iterations
for param_group in optimizer.param_groups:
param_group['lr'] = 1e-5
optimizer = train_cfg['optimizer']
optimizer.zero_grad()
loss.backward()
optimizer.step()
return None
else:
out = {}
net = net.eval()
result = net(**inputs)
if type(result)!=list and type(result)!=tuple:
result = [result]
out['preds'] = [make_output(i) for i in result]
return out
def test_func(model, **inputs):
for i in inputs:
inputs[i] = make_input(inputs[i])
net = model.eval()
forward_net = DataParallel(net.cuda())
out = {}
net = forward_net.eval()
result = net(**inputs)
if type(result) != list and type(result) != tuple:
result = [result]
out['preds'] = [make_output(i) for i in result]
return out
def singleTagLoss(pred_tag, keypoints):
"""
associative embedding loss for one image
"""
eps = 1e-6
tags = []
pull = 0
for i in keypoints:
tmp = []
for j in i:
if j[1]>0:
tmp.append(pred_tag[j[0]])
if len(tmp) == 0:
continue
# torch.stack is to concatenate sequence of tensors along a new dimension
tmp = torch.stack(tmp)
tags.append(torch.mean(tmp, dim=0))
# expand_as() is to expand this tensor to the size of tensor in parameter, same as expand()
pull = pull + torch.mean((tmp - tags[-1].expand_as(tmp))**2)
# if no tags then input zero value
if len(tags) == 0:
return make_input(torch.zeros([1]).float()), make_input(torch.zeros([1]).float())
tags = torch.stack(tags)[:,0]
num = tags.size()[0]
size = (num, num, tags.size()[1])
# unsqueeze() is to insert one dimension tensor into the specific position
A = tags.unsqueeze(dim=1).expand(*size)
# this to change the tensor's arrangement only
B = A.permute(1, 0, 2)
diff = A - B
diff = torch.pow(diff, 2).sum(dim=2)[:,:,0]
push = torch.exp(-diff)
push = (torch.sum(push) - num)
return push/((num - 1) * num + eps) * 0.5, pull/(num + eps)
def do_valid(epoch, config, loader):
logger = config['inference']['logger']
net = config['inference']['net']
run_loss = 0
with torch.no_grad():
batch_idx = 0
for inputs in tqdm(loader):
for i, input in enumerate(inputs):
if type(inputs[i]) is list:
for ind, inp in enumerate(inputs[i]):
inputs[i][ind] = make_input(inp)
else:
inputs[i] = make_input(inputs[i])
combined_preds = net(inputs[0])
combined_loss, labels_loss = config['inference']["lossLayers"](
combined_preds, **{
'heatmaps': inputs[1],
'labels': inputs[2]
})
loss = labels_loss[:, -1].sum() + combined_loss[:, -1].sum()
toprint = '\n{} {}: '.format(epoch, batch_idx)
heatmap_loss = torch.sum(combined_loss.cpu().mul(
torch.Tensor(config['train']['stack_loss'])))
label_loss = torch.sum(labels_loss.cpu().mul(
torch.Tensor(config['train']['stack_loss'])))
if batch_idx % 100 == 0:
toprint += ' \n{}'.format(str(combined_loss.cpu()))
toprint += ' \n{}'.format(str(labels_loss.cpu()))
else:
toprint += ' {},{}'.format(heatmap_loss, label_loss)
logger.write(toprint)
logger.flush()
batch_idx += 1
run_loss += loss.float().item()
return run_loss
def make_train(batch_id, config, phase, **inputs):
# get the gradient of the input and make it cuda data type
for i in inputs:
inputs[i] = make_input(inputs[i])
net = config['inference']['net']
config['batch_id'] = batch_id
# check current phase, set it train or evaluation
if phase == 'train':
net = net.train()
else:
net = net.eval()
# check if current stage is inference or not, it relate to 'train' and 'evaluation'
if phase != 'inference':
# if it is 'train'.
# {i: inputs[i] for i in inputs if i!='imgs'} is to separate inputs from images
result = net(inputs['imgs'], **{i: inputs[i] for i in inputs if i!='imgs'})
num_loss = len(config['train']['loss'])
"I use the last outputs as the loss"
"the weights of the loss are controlled by config['train']['loss'] "
losses = {i[0]: result[-num_loss + idx]*i[1] for idx, i in enumerate(config['train']['loss'])}
loss = 0
# this is to write the log of training process
toprint = '\n{}: '.format(batch_id)
for i in losses:
loss = loss + torch.mean(losses[i])
my_loss = make_output( losses[i] )
my_loss = my_loss.mean(axis = 0)
if my_loss.size == 1:
toprint += ' {}: {}'.format(i, format(my_loss.mean(), '.8f'))
else:
toprint += '\n{}'.format(i)
for j in my_loss:
toprint += ' {}'.format(format(j.mean(), '.8f'))
logger.write(toprint)
logger.flush()
if batch_id == 200000:
## decrease the learning rate after 200000 iterations
for param_group in optimizer.param_groups:
param_group['lr'] = 1e-5
if phase == 'train':
optimizer = train_cfg['optimizer']
# set the gradient to zero, before backpropragation. The PyTorch accumulates the gradients
# on subsequent backward pass, it suitable for RNN rather than CNN
optimizer.zero_grad()
loss.backward()
optimizer.step()
return None
else:
# if it is not 'train'. used when it is test.py. need it to return the predictions
# return matrix cpu data type
out = {}
net = net.eval()
result = net(**inputs)
if type(result)!=list and type(result)!=tuple:
result = [result]
out['preds'] = [make_output(i) for i in result]
return out
def test_tag_loss():
t = make_input( torch.Tensor((1, 2)), requires_grad=True )
t.register_hook(lambda x: print('t', x))
loss = singleTagLoss((t, [[[0,1]], [[1,1]]]))[0]
loss.backward()
print(loss)
