def val(self): """ Validation function during the train phase. """ self.pose_net.eval() start_time = time.time() with torch.no_grad(): for j, data_dict in enumerate(self.val_loader): # Forward pass. out_dict = self.pose_net(data_dict) # Compute the loss of the val batch. loss = self.cpm_loss(out_dict, data_dict, gathered=self.configer.get('network', 'gathered')) self.val_losses.update(loss.item(), len(DCHelper.tolist(data_dict['meta']))) # Update the vars of the val phase. self.batch_time.update(time.time() - start_time) start_time = time.time() RunnerHelper.save_net(self, self.pose_net, iters=self.runner_state['iters']) # Print the log info & reset the states. Log.info( 'Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t' 'Loss {loss.avg:.8f}\n'.format( batch_time=self.batch_time, loss=self.val_losses)) self.batch_time.reset() self.val_losses.reset() self.pose_net.train()
def __test_img(self, image_path, json_path, raw_path, vis_path): Log.info('Image Path: {}'.format(image_path)) ori_img_rgb = ImageHelper.img2np(ImageHelper.pil_open_rgb(image_path)) ori_img_bgr = ImageHelper.rgb2bgr(ori_img_rgb) inputs = ImageHelper.resize(ori_img_rgb, tuple(self.configer.get('data', 'input_size')), Image.CUBIC) inputs = ToTensor()(inputs) inputs = Normalize(mean=self.configer.get('trans_params', 'mean'), std=self.configer.get('trans_params', 'std'))(inputs) with torch.no_grad(): inputs = inputs.unsqueeze(0).to(self.device) bbox, cls = self.det_net(inputs) bbox = bbox.cpu().data.squeeze(0) cls = F.softmax(cls.cpu().squeeze(0), dim=-1).data boxes, lbls, scores = self.__decode(bbox, cls) json_dict = self.__get_info_tree(boxes, lbls, scores, ori_img_rgb) image_canvas = self.det_parser.draw_bboxes(ori_img_bgr.copy(), json_dict, conf_threshold=self.configer.get('vis', 'conf_threshold')) cv2.imwrite(vis_path, image_canvas) cv2.imwrite(raw_path, ori_img_bgr) Log.info('Json Path: {}'.format(json_path)) JsonHelper.save_file(json_dict, json_path) return json_dict
def val(self): """ Validation function during the train phase. """ self.gan_net.eval() start_time = time.time() for j, data_dict in enumerate(self.val_loader): with torch.no_grad(): # Forward pass. out_dict = self.gan_net(data_dict) # Compute the loss of the val batch. self.val_losses.update( out_dict['loss_G'].mean().item() + out_dict['loss_D'].mean().item(), len(DCHelper.tolist(data_dict['meta']))) # Update the vars of the val phase. self.batch_time.update(time.time() - start_time) start_time = time.time() RunnerHelper.save_net(self, self.gan_net, val_loss=self.val_losses.avg) # Print the log info & reset the states. Log.info('Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t' 'Loss {loss.avg:.8f}\n'.format(batch_time=self.batch_time, loss=self.val_losses)) self.batch_time.reset() self.val_losses.reset() self.gan_net.train()
def val(self, data_loader=None): """ Validation function during the train phase. """ self.gan_net.eval() start_time = time.time() data_loader = self.val_loader if data_loader is None else data_loader for j, data_dict in enumerate(data_loader): inputs = data_dict['img'] with torch.no_grad(): # Forward pass. out_dict = self.gan_net(data_dict) # Compute the loss of the val batch. self.val_losses.update(out_dict['loss'].mean().item(), inputs.size(0)) # Update the vars of the val phase. self.batch_time.update(time.time() - start_time) start_time = time.time() RunnerHelper.save_net(self, self.gan_net, val_loss=self.val_losses.avg) # Print the log info & reset the states. Log.info('Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t' 'Loss {loss.avg:.8f}\n'.format(batch_time=self.batch_time, loss=self.val_losses)) self.batch_time.reset() self.val_losses.reset() self.gan_net.train()
def __val(self): """ Validation function during the train phase. """ self.det_net.eval() start_time = time.time() with torch.no_grad(): for j, (inputs, bboxes, labels) in enumerate(self.val_loader): # Change the data type. inputs, bboxes, labels = self.module_utilizer.to_device( inputs, bboxes, labels) # Forward pass. loc, cls = self.det_net(inputs) # Compute the loss of the val batch. loss = self.det_loss(loc, bboxes, cls, labels) self.val_losses.update(loss.item(), inputs.size(0)) # Update the vars of the val phase. self.batch_time.update(time.time() - start_time) start_time = time.time() self.module_utilizer.save_net(self.det_net, metric='iters') # Print the log info & reset the states. Log.info( 'Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t' 'Loss {loss.avg:.8f}\n'.format(batch_time=self.batch_time, loss=self.val_losses)) self.batch_time.reset() self.val_losses.reset() self.det_net.train()
def densenet169(self, **kwargs): r"""Densenet-169 model from `"Densely Connected Convolutional Networks" <https://arxiv.org/pdf/1608.06993.pdf>`_ Args: pretrained (bool): If True, returns a model pre-trained on ImageNet """ model = DenseNet(num_init_features=64, growth_rate=32, block_config=(6, 12, 32, 32), bn_type=self.configer.get('network', 'bn_type'), **kwargs) if self.configer.get('network', 'pretrained') or self.configer.get('network', 'pretrained_model') is not None: # '.'s are no longer allowed in module names, but pervious _DenseLayer # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'. # They are also in the checkpoints in model_urls. This pattern is used # to find such keys. pattern = re.compile( r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$') if self.configer.get('network', 'pretrained_model') is not None: Log.info('Loading pretrained model:{}'.format(self.configer.get('network', 'pretrained_model'))) pretrained_dict = torch.load(self.configer.get('network', 'pretrained_model')) else: pretrained_dict = self.load_url(model_urls['densenet169']) for key in list(pretrained_dict.keys()): res = pattern.match(key) if res: new_key = res.group(1) + res.group(2) pretrained_dict[new_key] = pretrained_dict[key] del pretrained_dict[key] model.load_state_dict(pretrained_dict) return model
def init_weights(net, init_type='normal', init_gain=0.02): """Initialize network weights. Parameters: net (network) -- network to be initialized init_type (str) -- the name of an initialization method: normal | xavier | kaiming | orthogonal init_gain (float) -- scaling factor for normal, xavier and orthogonal. We use 'normal' in the original pix2pix and CycleGAN paper. But xavier and kaiming might work better for some applications. Feel free to try yourself. """ def init_func(m): # define the initialization function classname = m.__class__.__name__ if hasattr(m, 'weight') and (classname.find('Conv') != -1 or classname.find('Linear') != -1): if init_type == 'normal': init.normal_(m.weight.data, 0.0, init_gain) elif init_type == 'xavier': init.xavier_normal_(m.weight.data, gain=init_gain) elif init_type == 'kaiming': init.kaiming_normal_(m.weight.data, a=0, mode='fan_in') elif init_type == 'orthogonal': init.orthogonal_(m.weight.data, gain=init_gain) else: raise NotImplementedError('initialization method [%s] is not implemented' % init_type) if hasattr(m, 'bias') and m.bias is not None: init.constant_(m.bias.data, 0.0) elif classname.find('BatchNorm2d') != -1: # BatchNorm Layer's weight is not a matrix; only normal distribution applies. init.normal_(m.weight.data, 1.0, init_gain) init.constant_(m.bias.data, 0.0) Log.info('initialize network with {}'.format(init_type)) net.apply(init_func) # apply the initialization function <init_func>
def parse_dir_pose(self, image_dir, json_dir, mask_dir=None): if image_dir is None or not os.path.exists(image_dir): Log.error('Image Dir: {} not existed.'.format(image_dir)) return if json_dir is None or not os.path.exists(json_dir): Log.error('Json Dir: {} not existed.'.format(json_dir)) return for image_file in os.listdir(image_dir): shotname, extension = os.path.splitext(image_file) Log.info(image_file) image_canvas = cv2.imread(os.path.join( image_dir, image_file)) # B, G, R order. with open(os.path.join(json_dir, '{}.json'.format(shotname)), 'r') as json_stream: info_tree = json.load(json_stream) image_canvas = self.draw_points(image_canvas, info_tree) if self.configer.exists('details', 'limb_seq'): image_canvas = self.link_points(image_canvas, info_tree) if mask_dir is not None: mask_file = os.path.join(mask_dir, '{}_vis.png'.format(shotname)) mask_canvas = cv2.imread(mask_file) image_canvas = cv2.addWeighted(image_canvas, 0.6, mask_canvas, 0.4, 0) cv2.imshow('main', image_canvas) cv2.waitKey()
def __call__(self): # the 30th layer of features is relu of conv5_3 model = vgg16(pretrained=False) if self.configer.get('network', 'pretrained') is not None: Log.info('Loading pretrained model: {}'.format( self.configer.get('network', 'pretrained'))) model.load_state_dict( torch.load(self.configer.get('network', 'pretrained'))) features = list(model.features)[:30] classifier = model.classifier classifier = list(classifier) del classifier[6] if not self.configer.get('network', 'use_drop'): del classifier[5] del classifier[2] classifier = nn.Sequential(*classifier) # freeze top4 conv for layer in features[:10]: for p in layer.parameters(): p.requires_grad = False return nn.Sequential(*features), classifier
def __test_img(self, image_path, json_path, raw_path, vis_path): Log.info('Image Path: {}'.format(image_path)) img = ImageHelper.read_image( image_path, tool=self.configer.get('data', 'image_tool'), mode=self.configer.get('data', 'input_mode')) ori_img_bgr = ImageHelper.get_cv2_bgr(img, mode=self.configer.get( 'data', 'input_mode')) inputs = self.blob_helper.make_input(img, input_size=self.configer.get( 'data', 'input_size'), scale=1.0) with torch.no_grad(): inputs = inputs.unsqueeze(0).to(self.device) _, _, detections = self.det_net(inputs) batch_detections = self.decode(detections, self.configer) json_dict = self.__get_info_tree(batch_detections[0], ori_img_bgr) image_canvas = self.det_parser.draw_bboxes( ori_img_bgr.copy(), json_dict, conf_threshold=self.configer.get('vis', 'conf_threshold')) ImageHelper.save(ori_img_bgr, raw_path) ImageHelper.save(image_canvas, vis_path) Log.info('Json Path: {}'.format(json_path)) JsonHelper.save_file(json_dict, json_path) return json_dict
def __val(self): """ Validation function during the train phase. """ self.det_net.eval() start_time = time.time() with torch.no_grad(): for j, data_dict in enumerate(self.val_loader): inputs = data_dict['img'] img_scale = data_dict['imgscale'] batch_gt_bboxes = data_dict['bboxes'] batch_gt_labels = data_dict['labels'] # Change the data type. gt_bboxes, gt_nums, gt_labels = self.__make_tensor( batch_gt_bboxes, batch_gt_labels) gt_bboxes, gt_num, gt_labels = self.module_utilizer.to_device( gt_bboxes, gt_nums, gt_labels) inputs = self.module_utilizer.to_device(inputs) # Forward pass. feat_list, train_group, test_group = self.det_net( inputs, gt_bboxes, gt_nums, gt_labels, img_scale) rpn_locs, rpn_scores, sample_roi_locs, sample_roi_scores, gt_roi_bboxes, gt_roi_labels = train_group gt_rpn_locs, gt_rpn_labels = self.rpn_target_generator( feat_list, batch_gt_bboxes, [inputs.size(3), inputs.size(2)]) gt_rpn_locs, gt_rpn_labels = self.module_utilizer.to_device( gt_rpn_locs, gt_rpn_labels) # Compute the loss of the train batch & backward. loss = self.fr_loss( [rpn_locs, rpn_scores, sample_roi_locs, sample_roi_scores], [gt_rpn_locs, gt_rpn_labels, gt_roi_bboxes, gt_roi_labels]) self.val_losses.update(loss.item(), inputs.size(0)) test_indices_and_rois, test_roi_locs, test_roi_scores, test_rois_num = test_group batch_detections = FastRCNNTest.decode( test_roi_locs, test_roi_scores, test_indices_and_rois, test_rois_num, self.configer, [inputs.size(3), inputs.size(2)]) batch_pred_bboxes = self.__get_object_list(batch_detections) self.det_running_score.update(batch_pred_bboxes, batch_gt_bboxes, batch_gt_labels) # Update the vars of the val phase. self.batch_time.update(time.time() - start_time) start_time = time.time() self.module_utilizer.save_net(self.det_net, save_mode='iters') # Print the log info & reset the states. Log.info( 'Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t' 'Loss {loss.avg:.8f}\n'.format(batch_time=self.batch_time, loss=self.val_losses)) Log.info('Val mAP: {}\n'.format(self.det_running_score.get_mAP())) self.det_running_score.reset() self.batch_time.reset() self.val_losses.reset() self.det_net.train()
def __test_img(self, image_path, save_path): Log.info('Image Path: {}'.format(image_path)) ori_image = ImageHelper.read_image( image_path, tool=self.configer.get('data', 'image_tool'), mode=self.configer.get('data', 'input_mode')) ori_width, ori_height = ImageHelper.get_size(ori_image) ori_img_bgr = ImageHelper.get_cv2_bgr(ori_image, mode=self.configer.get( 'data', 'input_mode')) heatmap_avg = np.zeros( (ori_height, ori_width, self.configer.get('network', 'heatmap_out'))) for i, scale in enumerate(self.configer.get('test', 'scale_search')): image = self.blob_helper.make_input(ori_image, input_size=self.configer.get( 'test', 'input_size'), scale=scale) with torch.no_grad(): heatmap_out_list = self.pose_net(image) heatmap_out = heatmap_out_list[-1] # extract outputs, resize, and remove padding heatmap = heatmap_out.squeeze(0).cpu().numpy().transpose( 1, 2, 0) heatmap = cv2.resize(heatmap, (ori_width, ori_height), interpolation=cv2.INTER_CUBIC) heatmap_avg = heatmap_avg + heatmap / len( self.configer.get('test', 'scale_search')) all_peaks = self.__extract_heatmap_info(heatmap_avg) image_canvas = self.__draw_key_point(all_peaks, ori_img_bgr) ImageHelper.save(image_canvas, save_path)
def __val(self): """ Validation function during the train phase. """ self.cls_net.eval() index = 0 acc_count = 0 while index < self.val_batch_loader.num_data(): # Change the data type. inputs, labels = self.val_batch_loader.next_batch() # Forward pass. outputs = self.cls_net(inputs) pred = outputs.data.max(1)[1] pred = pred.squeeze() if pred.item() == labels.item(): acc_count += 1 index += 1 Log.info('Top1 ACC = {}\n'.format(acc_count / index)) self.log_stream.write('{}, {}\n'.format(self.batch, acc_count / index)) self.log_stream.flush() self.cls_net.train()
def train(self): """ Train function of every epoch during train phase. """ self.pose_net.train() start_time = time.time() # Adjust the learning rate after every epoch. self.runner_state['epoch'] += 1 # data_tuple: (inputs, heatmap, maskmap, tagmap, num_objects) for i, data_dict in enumerate(self.train_loader): Trainer.update(self) inputs = data_dict['img'] heatmap = data_dict['heatmap'] self.data_time.update(time.time() - start_time) # Change the data type. inputs, heatmap = RunnerHelper.to_device(self, inputs, heatmap) # self.pose_visualizer.vis_peaks(heatmap[0], inputs[0], name='cpm') # Forward pass. outputs = self.pose_net(inputs) # Compute the loss of the train batch & backward. loss = self.mse_loss(outputs, heatmap) self.train_losses.update(loss.item(), inputs.size(0)) self.optimizer.zero_grad() loss.backward() self.optimizer.step() # Update the vars of the train phase. self.batch_time.update(time.time() - start_time) start_time = time.time() self.runner_state['iters'] += 1 # Print the log info & reset the states. if self.runner_state['iters'] % self.configer.get('solver', 'display_iter') == 0: Log.info('Train Epoch: {0}\tTrain Iteration: {1}\t' 'Time {batch_time.sum:.3f}s / {2}iters, ({batch_time.avg:.3f})\t' 'Data load {data_time.sum:.3f}s / {2}iters, ({data_time.avg:3f})\n' 'Learning rate = {3}\tLoss = {loss.val:.8f} (ave = {loss.avg:.8f})\n'.format( self.runner_state['epoch'], self.runner_state['iters'], self.configer.get('solver', 'display_iter'), RunnerHelper.get_lr(self.optimizer), batch_time=self.batch_time, data_time=self.data_time, loss=self.train_losses)) self.batch_time.reset() self.data_time.reset() self.train_losses.reset() if self.configer.get('lr', 'metric') == 'iters' \ and self.runner_state['iters'] == self.configer.get('solver', 'max_iters'): break # Check to val the current model. if self.runner_state['iters'] % self.configer.get('solver', 'test_interval') == 0: self.val()
def val(self): """ Validation function during the train phase. """ self.det_net.eval() start_time = time.time() with torch.no_grad(): for j, data_dict in enumerate(self.val_loader): inputs = data_dict['img'] batch_gt_bboxes = data_dict['bboxes'] batch_gt_labels = data_dict['labels'] metas = data_dict['meta'] data_dict['bboxes'] = DCHelper.todc( batch_gt_bboxes, gpu_list=self.configer.get('gpu'), cpu_only=True) data_dict['labels'] = DCHelper.todc( batch_gt_labels, gpu_list=self.configer.get('gpu'), cpu_only=True) data_dict['meta'] = DCHelper.todc( metas, gpu_list=self.configer.get('gpu'), cpu_only=True) # Forward pass. inputs = RunnerHelper.to_device(self, inputs) loss, test_group = self.det_net(data_dict) # Compute the loss of the train batch & backward. loss = loss.mean() self.val_losses.update(loss.item(), inputs.size(0)) test_indices_and_rois, test_roi_locs, test_roi_scores, test_rois_num = test_group batch_detections = FastRCNNTest.decode(test_roi_locs, test_roi_scores, test_indices_and_rois, test_rois_num, self.configer, metas) batch_pred_bboxes = self.__get_object_list(batch_detections) self.det_running_score.update(batch_pred_bboxes, batch_gt_bboxes, batch_gt_labels) # Update the vars of the val phase. self.batch_time.update(time.time() - start_time) start_time = time.time() RunnerHelper.save_net(self, self.det_net, iters=self.runner_state['iters']) # Print the log info & reset the states. Log.info( 'Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t' 'Loss {loss.avg:.8f}\n'.format(batch_time=self.batch_time, loss=self.val_losses)) Log.info('Val mAP: {}\n'.format(self.det_running_score.get_mAP())) self.det_running_score.reset() self.batch_time.reset() self.val_losses.reset() self.det_net.train()
def __test_img(self, image_path, save_path): Log.info('Image Path: {}'.format(image_path)) image_raw = ImageHelper.cv2_open_bgr(image_path) inputs = ImageHelper.bgr2rgb(image_raw) paf_avg, heatmap_avg = self.__get_paf_and_heatmap(inputs) all_peaks = self.__extract_heatmap_info(heatmap_avg) special_k, connection_all = self.__extract_paf_info(image_raw, paf_avg, all_peaks) subset, candidate = self.__get_subsets(connection_all, special_k, all_peaks) subset, img_canvas = self.__draw_key_point(subset, all_peaks, image_raw) img_canvas = self.__link_key_point(img_canvas, candidate, subset) cv2.imwrite(save_path, img_canvas)