def singleTagLoss(pred_tag, keypoints): """ associative embedding loss for one image """ eps = 1e-6 tags = [] pull = 0 for i in keypoints: tmp = [] for j in i: if j[1] > 0: tmp.append(pred_tag[j[0]]) if len(tmp) == 0: continue tmp = torch.stack(tmp) tags.append(torch.mean(tmp, dim=0)) pull = pull + torch.mean((tmp - tags[-1].expand_as(tmp))**2) if len(tags) == 0: return make_input(torch.zeros([1]).float()), make_input( torch.zeros([1]).float()) tags = torch.stack(tags)[:, 0] num = tags.size()[0] size = (num, num, tags.size()[1]) A = tags.unsqueeze(dim=1).expand(*size) B = A.permute(1, 0, 2) diff = A - B diff = torch.pow(diff, 2).sum(dim=2)[:, :, 0] push = torch.exp(-diff) push = (torch.sum(push) - num) return push / ((num - 1) * num + eps) * 0.5, pull / (num + eps)
def singleTagLoss(pred_tag, keypoints): """ associative embedding loss for one image """ eps = 1e-6 tags = [] pull = 0 for i in keypoints: tmp = [] for j in i: if j[1]>0: tmp.append(pred_tag[j[0]]) if len(tmp) == 0: continue tmp = torch.stack(tmp) tags.append(torch.mean(tmp, dim=0)) pull = pull + torch.mean((tmp - tags[-1].expand_as(tmp))**2) if len(tags) == 0: return make_input(torch.zeros([1]).float()), make_input(torch.zeros([1]).float()) tags = torch.stack(tags)[:,0] num = tags.size()[0] size = (num, num, tags.size()[1]) A = tags.unsqueeze(dim=1).expand(*size) B = A.permute(1, 0, 2) diff = A - B diff = torch.pow(diff, 2).sum(dim=2)[:,:,0] push = torch.exp(-diff) push = (torch.sum(push) - num) return push/((num - 1) * num + eps) * 0.5, pull/(num + eps)
def make_train(batch_id, config, phase, **inputs): for i in inputs: inputs[i] = make_input(inputs[i]) net = config['inference']['net'] config['batch_id'] = batch_id if phase != 'inference': result = net(inputs['imgs'], **{i: inputs[i] for i in inputs if i != 'imgs'}) num_loss = len(config['train']['loss']) ## I use the last outputs as the loss ## the weights of the loss are controlled by config['train']['loss'] losses = { i[0]: result[-num_loss + idx] * i[1] for idx, i in enumerate(config['train']['loss']) } loss = 0 toprint = '\n{}: '.format(batch_id) for i in losses: loss = loss + torch.mean(losses[i]) my_loss = make_output(losses[i]) my_loss = my_loss.mean(axis=0) if my_loss.size == 1: toprint += ' {}: {}'.format(i, format(my_loss.mean(), '.8f')) else: toprint += '\n{}'.format(i) for j in my_loss: toprint += ' {}'.format(format(j.mean(), '.8f')) logger.write(toprint) logger.flush() if batch_id == 200000: ## decrease the learning rate after 200000 iterations for param_group in optimizer.param_groups: param_group['lr'] = 1e-5 if phase == 'train': optimizer = train_cfg['optimizer'] optimizer.zero_grad() loss.backward() optimizer.step() return None else: out = {} net = net.eval() result = net(**inputs) if type(result) != list and type(result) != tuple: result = [result] out['preds'] = [make_output(i) for i in result] return out
def test_tag_loss(): t = make_input( torch.Tensor((1, 2)), requires_grad=True ) # register_hook is to take the backward gradient as input, when variable's gradient is updated in backward # and this method will return a new tensor t.register_hook(lambda x: print('t', x)) loss = singleTagLoss((t, [[[0,1]], [[1,1]]]))[0] loss.backward() print(loss)
def test_tag_loss(): def myhook(x): print('t', x) t = make_input( torch.Tensor((1, 2)), requires_grad=True ) t.register_hook(myhook) loss = singleTagLoss((t, [[[0,1]], [[1,1]]]))[0] loss.backward() print(loss)
def do_train(epoch, config, loader): logger = config['inference']['logger'] net = config['inference']['net'] batch_idx = 0 for inputs in tqdm(loader): for i, input in enumerate(inputs): try: if type(inputs[i]) is list: for ind, inp in enumerate(inputs[i]): inputs[i][ind] = make_input(inp) else: inputs[i] = make_input(inputs[i]) except: pass # for last input, which is a string (id_) combined_preds = net(inputs[0]) combined_loss, labels_loss = config['inference']["lossLayers"]( combined_preds, **{ 'heatmaps': inputs[1], 'labels': inputs[2] }) num_loss = len(config['train']['loss']) # losses = [all_loss[idx].cpu()*i[1] for idx, i in enumerate(config['train']['loss'])] heatmap_loss = torch.sum(combined_loss.cpu().mul( torch.Tensor(config['train']['stack_loss']))) label_loss = torch.sum(labels_loss.cpu().mul( torch.Tensor(config['train']['stack_loss']))) toprint = '\n{} {}: '.format(epoch, batch_idx) if batch_idx % 100 == 0: toprint += ' \n{}'.format(str(combined_loss.cpu())) toprint += ' \n{}'.format(str(labels_loss.cpu())) else: toprint += ' {},{}'.format(heatmap_loss, label_loss) logger.write(toprint) logger.flush() optimizer = config['train']['optimizer'] optimizer.zero_grad() loss = heatmap_loss + label_loss loss.backward() optimizer.step() batch_idx += 1
def train_func(opts, model, optimizer, phase, **inputs): for i in inputs: inputs[i] = make_input(inputs[i]) if phase == 'train': net = model.train() else: net = model.eval() forward_net = DataParallel(net.cuda()) if phase != 'inference': if phase == 'valid': with torch.no_grad(): output = forward_net(inputs['imgs']) losses = model.calc_loss( output, **{i: inputs[i] for i in inputs if i != 'imgs'}) losses = { 'push_loss': losses[0] * opts.push_loss, 'pull_loss': losses[1] * opts.pull_loss, 'detection_loss': losses[2] * opts.detection_loss } loss = 0 else: output = forward_net(inputs['imgs']) losses = model.calc_loss( output, **{i: inputs[i] for i in inputs if i != 'imgs'}) losses = { 'push_loss': losses[0] * opts.push_loss, 'pull_loss': losses[1] * opts.pull_loss, 'detection_loss': losses[2] * opts.detection_loss } loss = 0 for i in losses: loss = loss + torch.mean(losses[i]) my_loss = make_output(losses[i]) my_loss = my_loss.mean(axis=0) #if my_loss.size == 1: # toprint += ' {}: {}'.format(i, format(my_loss.mean(), '.8f')) #else: # toprint += '\n{}'.format(i) # for j in my_loss: # toprint += ' {}'.format(format(j.mean(), '.8f')) if phase == 'train': optimizer.zero_grad() loss.backward() optimizer.step() return loss
def make_train(batch_id, config, phase, **inputs): for i in inputs: inputs[i] = make_input(inputs[i]) net = config['inference']['net'] config['batch_id'] = batch_id if phase != 'inference': result = net(inputs['imgs'], **{i:inputs[i] for i in inputs if i!='imgs'}) num_loss = len(config['train']['loss']) ## I use the last outputs as the loss ## the weights of the loss are controlled by config['train']['loss'] losses = {i[0]: result[-num_loss + idx]*i[1] for idx, i in enumerate(config['train']['loss'])} loss = 0 toprint = '\n{}: '.format(batch_id) for i in losses: loss = loss + torch.mean(losses[i]) my_loss = make_output( losses[i] ) my_loss = my_loss.mean(axis = 0) if my_loss.size == 1: toprint += ' {}: {}'.format(i, format(my_loss.mean(), '.8f')) else: toprint += '\n{}'.format(i) for j in my_loss: toprint += ' {}'.format(format(j.mean(), '.8f')) logger.write(toprint) logger.flush() if batch_id == 200000: ## decrease the learning rate after 200000 iterations for param_group in optimizer.param_groups: param_group['lr'] = 1e-5 optimizer = train_cfg['optimizer'] optimizer.zero_grad() loss.backward() optimizer.step() return None else: out = {} net = net.eval() result = net(**inputs) if type(result)!=list and type(result)!=tuple: result = [result] out['preds'] = [make_output(i) for i in result] return out
def test_func(model, **inputs): for i in inputs: inputs[i] = make_input(inputs[i]) net = model.eval() forward_net = DataParallel(net.cuda()) out = {} net = forward_net.eval() result = net(**inputs) if type(result) != list and type(result) != tuple: result = [result] out['preds'] = [make_output(i) for i in result] return out
def singleTagLoss(pred_tag, keypoints): """ associative embedding loss for one image """ eps = 1e-6 tags = [] pull = 0 for i in keypoints: tmp = [] for j in i: if j[1]>0: tmp.append(pred_tag[j[0]]) if len(tmp) == 0: continue # torch.stack is to concatenate sequence of tensors along a new dimension tmp = torch.stack(tmp) tags.append(torch.mean(tmp, dim=0)) # expand_as() is to expand this tensor to the size of tensor in parameter, same as expand() pull = pull + torch.mean((tmp - tags[-1].expand_as(tmp))**2) # if no tags then input zero value if len(tags) == 0: return make_input(torch.zeros([1]).float()), make_input(torch.zeros([1]).float()) tags = torch.stack(tags)[:,0] num = tags.size()[0] size = (num, num, tags.size()[1]) # unsqueeze() is to insert one dimension tensor into the specific position A = tags.unsqueeze(dim=1).expand(*size) # this to change the tensor's arrangement only B = A.permute(1, 0, 2) diff = A - B diff = torch.pow(diff, 2).sum(dim=2)[:,:,0] push = torch.exp(-diff) push = (torch.sum(push) - num) return push/((num - 1) * num + eps) * 0.5, pull/(num + eps)
def do_valid(epoch, config, loader): logger = config['inference']['logger'] net = config['inference']['net'] run_loss = 0 with torch.no_grad(): batch_idx = 0 for inputs in tqdm(loader): for i, input in enumerate(inputs): if type(inputs[i]) is list: for ind, inp in enumerate(inputs[i]): inputs[i][ind] = make_input(inp) else: inputs[i] = make_input(inputs[i]) combined_preds = net(inputs[0]) combined_loss, labels_loss = config['inference']["lossLayers"]( combined_preds, **{ 'heatmaps': inputs[1], 'labels': inputs[2] }) loss = labels_loss[:, -1].sum() + combined_loss[:, -1].sum() toprint = '\n{} {}: '.format(epoch, batch_idx) heatmap_loss = torch.sum(combined_loss.cpu().mul( torch.Tensor(config['train']['stack_loss']))) label_loss = torch.sum(labels_loss.cpu().mul( torch.Tensor(config['train']['stack_loss']))) if batch_idx % 100 == 0: toprint += ' \n{}'.format(str(combined_loss.cpu())) toprint += ' \n{}'.format(str(labels_loss.cpu())) else: toprint += ' {},{}'.format(heatmap_loss, label_loss) logger.write(toprint) logger.flush() batch_idx += 1 run_loss += loss.float().item() return run_loss
def make_train(batch_id, config, phase, **inputs): # get the gradient of the input and make it cuda data type for i in inputs: inputs[i] = make_input(inputs[i]) net = config['inference']['net'] config['batch_id'] = batch_id # check current phase, set it train or evaluation if phase == 'train': net = net.train() else: net = net.eval() # check if current stage is inference or not, it relate to 'train' and 'evaluation' if phase != 'inference': # if it is 'train'. # {i: inputs[i] for i in inputs if i!='imgs'} is to separate inputs from images result = net(inputs['imgs'], **{i: inputs[i] for i in inputs if i!='imgs'}) num_loss = len(config['train']['loss']) "I use the last outputs as the loss" "the weights of the loss are controlled by config['train']['loss'] " losses = {i[0]: result[-num_loss + idx]*i[1] for idx, i in enumerate(config['train']['loss'])} loss = 0 # this is to write the log of training process toprint = '\n{}: '.format(batch_id) for i in losses: loss = loss + torch.mean(losses[i]) my_loss = make_output( losses[i] ) my_loss = my_loss.mean(axis = 0) if my_loss.size == 1: toprint += ' {}: {}'.format(i, format(my_loss.mean(), '.8f')) else: toprint += '\n{}'.format(i) for j in my_loss: toprint += ' {}'.format(format(j.mean(), '.8f')) logger.write(toprint) logger.flush() if batch_id == 200000: ## decrease the learning rate after 200000 iterations for param_group in optimizer.param_groups: param_group['lr'] = 1e-5 if phase == 'train': optimizer = train_cfg['optimizer'] # set the gradient to zero, before backpropragation. The PyTorch accumulates the gradients # on subsequent backward pass, it suitable for RNN rather than CNN optimizer.zero_grad() loss.backward() optimizer.step() return None else: # if it is not 'train'. used when it is test.py. need it to return the predictions # return matrix cpu data type out = {} net = net.eval() result = net(**inputs) if type(result)!=list and type(result)!=tuple: result = [result] out['preds'] = [make_output(i) for i in result] return out
def test_tag_loss(): t = make_input( torch.Tensor((1, 2)), requires_grad=True ) t.register_hook(lambda x: print('t', x)) loss = singleTagLoss((t, [[[0,1]], [[1,1]]]))[0] loss.backward() print(loss)