def __init__(self, csv_path, mode='train', target_only=False): self.mode = mode # Read data into numpy arrays with open(csv_path, 'r') as fp: data = list(csv.reader(fp)) data = np.array(data[1:])[:, 1:].astype(float) if not target_only: feats = list(range(93)) else: # TODO: Using 40 states & 2 tested_positive features (indeices = 57 & 75) feats = list(range(40)) + [57, 75] if mode == 'test': # Testing data # data: 893 x 93 (40 states + day 1 (18) + day 2 (18) + day 3 (17)) data = data[:, feats] # Convert data into Pytorch tensors self.data = torch.FlaotTensor(data) else: # Training data # data: 2700 x 94 (40 states + day 1 (18) + day 2 (18) + day3 (18)) target = data[:, -1] data = data[:, feats] # Spliting training data into train & dev sets if mode == 'train': indices = [i for i in range(len(data)) if i % 10 != 0] elif mode == 'dev': indices = [i for i in range(len(data)) if 1 % 10 == 0] # Convert data into Pytorch tensors self.data = torch.FloatTensor(data[indices]) self.target = torch.FloatTensor(target[indices]) # Normalize features (you may remove this part to see what will happen) self.data[:, 40:] = \ (self.data[:, 40:] - self.data[:, 40:].mean(dim=0, keepdim=True)) \ / self.data[:, 40:].std(dim=0, keepdim=True) self.dim = self.data.shape[1] print( f'Finished reading the {self.mode} set of COVID19 Dataset ({len(data)} samples found, each dim = {self.dim})' )
def forward(self, im_data, im_info, gt_boxes, num_boxes, opt_frcn=None, vis_flag=False): batch_size = im_data.size(0) im_info = im_info.data gt_boxes = gt_boxes.data num_boxes = num_boxes.data #self.FRCN.eval() #self.FRCN.RCNN_base.eval() #for param in self.FRCN.parameters(): # param.requires_grad = False gt_boxes_frcn = gt_boxes.cpu().data.numpy()[0] for gt in gt_boxes_frcn: if gt[..., -1] < 4 and gt[..., -1] > 0: gt[..., -1] = 1 elif gt[..., -1] >= 4: gt[..., -1] = 2 gt_boxes_frcn = gt_boxes_frcn[None, ...] gt_boxes_frcn = torch.Tensor(gt_boxes_frcn) if self.use_cuda: gt_boxes_frcn = gt_boxes_frcn.cuda() rois, cls_prob, bbox_pred, rpn_loss_cls, rpn_loss_bbox, \ RCNN_loss_cls, RCNN_loss_bbox, rois_label \ = self.FRCN(im_data, im_info, gt_boxes_frcn, num_boxes) if opt_frcn is not None: opt.zero_grad() # get global and local region from Faster R-CNN base_feat = self.FRCN.RCNN_base(im_data) #print(rois.data.cpu().numpy()) scores = cls_prob.data boxes = rois.data[:, :, 1:5] box_deltas = self.FRCN._bbox_pred.data if cfg.TRAIN.BBOX_NORMALIZE_TARGETS_PRECOMPUTED: if self.class_agnostic: if self.use_cuda > 0: box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor( cfg.TRAIN.BBOX_NORMALIZE_STDS).cuda( ) + torch.FloatTensor( cfg.TRAIN.BBOX_NORMALIZE_MEANS).cuda() else: box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor( cfg.TRAIN.BBOX_NORMALIZE_STDS) * torch.FlaotTensor( cfg.TRAIN.BBOX_NORMALIZE_MEANS) box_deltas = box_deltas.view(1, -1, 4) else: if self.use_cuda > 0: box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor( cfg.TRAIN.BBOX_NORMALIZE_STDS).cuda( ) + torch.FloatTensor( cfg.TRAIN.BBOX_NORMALIZE_MEANS).cuda() else: box_deltas = box_deltas.view(-1, 4) * torhc.FlaotTensor( cfg.TRAIN.BBOX_NORMALIZE_STDS) + torch.FloatTensor( cfg.TRAIN.BBOX_NORMALIZE_MEANS) box_deltas = box_deltas.view(1, -1, 4 * 3) pred_boxes = bbox_transform_inv(boxes, box_deltas, 1) pred_boxes = clip_boxes(pred_boxes, im_info.data, 1) scores = scores.squeeze() pred_boxes = pred_boxes.squeeze() #print(scores) # get global region thresh = 0.0 region_g = np.ndarray((0, 5)) region_l = np.ndarray((0, 5)) # get glocal region inds = torch.nonzero(scores[:, 1] >= thresh).view(-1) if inds.numel() > 0: cls_scores = scores[:, 1][inds] _, order = torch.sort(cls_scores, 0, True) if self.class_agnostic: cls_boxes = pred_boxes[inds] else: cls_boxes = pred_boxes[inds][:, 4:8] cls_dets = torch.cat((cls_boxes, cls_scores.unsqueeze(1)), 1) cls_dets = cls_dets[order] region_g = cls_dets # get local region inds = torch.nonzero(scores[:, 2] >= thresh).view(-1) if inds.numel() > 0: cls_scores = scores[:, 2][inds] _, order = torch.sort(cls_scores, 0, True) if self.class_agnostic: cls_boxes = pred_boxes[inds] else: cls_boxes = pred_boxes[inds][:, 8:12] cls_dets = torch.cat((cls_boxes, cls_scores.unsqueeze(1)), 1) cls_dets = cls_dets[order] region_l = cls_dets # select region for recognition if not self.training: self.minibatch = 1 if self.training: if self.minibatch % 2 == 0: high_ind = self.minibatch // 2 low_ind = self.minibatch // 2 elif self.minibatch == 1: high_ind = 1 low_ind = 0 else: high_ind = self.minibatch // 2 + 1 low_ind = self.minibatch // 2 keep = nms(torch.tensor(region_g).cuda(), self.nms_iou, force_cpu=not cfg.USE_GPU_NMS) if type(keep) is not list: keep = keep.view(-1).long() region_g = region_g[keep] sort_ind = np.argsort(region_g[..., -1]) high_ind_g = sort_ind[-high_ind:] low_ind_g = sort_ind[:low_ind] keep = nms(torch.tensor(region_l).cuda(), self.nms_iou, force_cpu=not cfg.USE_GPU_NMS) if type(keep) is not list: keep = keep.view(-1).long() region_l = region_l[keep] sort_ind = np.argsort(region_l[..., -1]) high_ind_l = sort_ind[-high_ind:] low_ind_l = sort_ind[:low_ind] high_num = min(len(high_ind_g), len(high_ind_l)) high_ind_g = high_ind_g[:high_num] high_ind_l = high_ind_l[:high_num] low_num = min(len(low_ind_g), len(low_ind_l)) low_ind_g = low_ind_g[:low_num] low_ind_l = low_ind_l[:low_num] if len(high_ind_g.data) < 1: region_g_high = torch.tensor(np.ndarray((5))[None, ...]) else: region_g_high = region_g[high_ind_g] if len(low_ind_g.data) < 1: region_g_low = torch.tensor(np.ndarray((5))[None, ...]) else: region_g_low = region_g[low_ind_g] if len(high_ind_l.data) < 1: region_l_high = torch.tensor(np.ndarray((5))[None, ...]) else: region_l_high = region_l[high_ind_l] if len(low_ind_l.data) < 1: region_l_low = torch.tensor(np.ndarray((5))[None, ...]) else: region_l_low = region_l[low_ind_l] proposal_g = np.vstack((region_g_high, region_g_low)) proposal_l = np.vstack((region_l_high, region_l_low)) #proposal_g = np.vstack((region_g[high_ind_g], region_g[low_ind_g])) #proposal_l = np.vstack((region_l[high_ind_l], region_l[low_ind_l])) #self.proposal_g.data.resize_(proposal_g.size()).copy_(proposal_g) #self.proposal_l.data.resize_(proposal_l.size()).copy_(proposal_l) gt_boxes = gt_boxes.cpu().numpy()[0, :2] gt_g = gt_boxes[np.where(gt_boxes[..., -1] < 4)[0]] gt_l = gt_boxes[np.where(gt_boxes[..., -1] >= 4)[0]] # compute pare ground truth def compute_iou(ps, gt): iou_x1 = np.maximum(ps[..., 0], gt[0]) iou_y1 = np.maximum(ps[..., 1], gt[1]) iou_x2 = np.minimum(ps[..., 2], gt[2]) iou_y2 = np.minimum(ps[..., 3], gt[3]) iou_w = np.maximum(iou_x2 - iou_x1, 0) iou_h = np.maximum(iou_y2 - iou_y1, 0) iou_area = iou_w * iou_h gt_area = (gt[2] - gt[0]) * (gt[3] - gt[1]) p_area = (ps[..., 2] - ps[..., 0]) * (ps[..., 3] - ps[..., 1]) overlap = iou_area / (gt_area + p_area - iou_area) count = np.zeros((ps.shape[0]), dtype=int) count[overlap >= self.gt_iou] += 1 return count cou = compute_iou(proposal_g, gt_g[0]) + compute_iou( proposal_l, gt_l[0]) ## 2019.2.13 glcc_gt = np.zeros((proposal_g.shape[0]), dtype=int) glcc_gt[cou >= 1] = gt_g[0, -1] #glcc_gt[:] = gt_g[0, -1] glcc_gt = torch.tensor(glcc_gt, dtype=torch.long).cuda() #print(glcc_gt) #self.glcc_gt.data.resize_(glcc_gt.size()).copy_(glcc_gt) else: # test phase proposal_g = region_g[np.argmax(region_g[..., -1])][None, ...] proposal_l = region_l[np.argmax(region_l[..., -1])][None, ...] #self.proposal_g.data.resize_(proposal_g.size()).copy_(proposal_g.size()) #self.proposal_l.data.resize_(proposal_l.size()).copy_(proposal_l.size()) # if true, then show detection global and local region if vis_flag: gt_boxes = gt_boxes.astype(np.int) im = im_data.cpu().numpy()[0] im = np.transpose(im, (1, 2, 0))[..., ::-1] im -= im.min() im /= im.max() plt.imshow(im.astype(np.float)) ax = plt.axes() ax.add_patch( plt.Rectangle((region_g[0, 0], region_g[0, 1]), region_g[0, 2] - region_g[0, 0], region_g[0, 3] - region_g[0, 1], fill=False, edgecolor='red', linewidth=1)) ax.add_patch( plt.Rectangle((region_l[0, 0], region_l[0, 1]), region_l[0, 2] - region_l[0, 0], region_l[0, 3] - region_l[0, 1], fill=False, edgecolor='yellow', linewidth=1)) ax.add_patch( plt.Rectangle((gt_boxes[0, 0], gt_boxes[0, 1]), gt_boxes[0, 2] - gt_boxes[0, 0], gt_boxes[0, 3] - gt_boxes[0, 1], fill=False, edgecolor='green', linewidth=1)) ax.add_patch( plt.Rectangle((gt_boxes[1, 0], gt_boxes[1, 1]), gt_boxes[1, 2] - gt_boxes[1, 0], gt_boxes[1, 3] - gt_boxes[1, 1], fill=False, edgecolor='white', linewidth=1)) plt.show() rois_g = np.zeros((1, proposal_g.shape[0], 5), dtype=np.float32) rois_g[0, :, 1:5] = proposal_g[:, :4] #rois_g /= 16. rois_l = np.zeros((1, proposal_l.shape[0], 5), dtype=np.float32) rois_l[0, :, 1:5] = proposal_l[:, :4] #rois_l /= 16. rois_g = torch.tensor(rois_g, dtype=torch.float).cuda() rois_l = torch.tensor(rois_l, dtype=torch.float).cuda() self.rois_g.data.resize_(rois_g.size()).copy_(rois_g) self.rois_l.data.resize_(rois_l.size()).copy_(rois_l) # global region if cfg.POOLING_MODE == 'crop': grid_xy = _affine_grid_gen(self.rois_g.view(-1, 5), base_feat.size()[2:], self.FRCN.grid_size) grid_yx = torch.stack([grid_xy.data[..., 1], grid_xy.data[..., 0]], 3).contiguous() pooled_feat_g = self.FRCN.RCNN_roi_crop(base_feat, Variable(grid_yx).detach()) if cfg.CROP_RESIZE_WITH_MAX_POOL: pooled_feat_g = F.max_pool2d(pooled_feat_g, 2, 2) elif cfg.POOLING_MODE == 'align': pooled_feat_g = self.FRCN.RCNN_roi_align(base_feat, self.rois_g.view(-1, 5)) elif cfg.POOLING_MODE == 'pool': pooled_feat_g = self.FRCN.RCNN_roi_pool(base_feat, self.rois_g.view(-1, 5)) # local region if cfg.POOLING_MODE == 'crop': grid_xy = _affine_grid_gen(self.rois_l.view(-1, 5), base_feat.size()[2:], self.FRCN.grid_size) grid_yx = torch.stack([grid_xy.data[..., 1], grid_xy.data[..., 0]], 3).contiguous() pooled_feat_l = self.FRCN.RCNN_roi_crop(base_feat, Variable(grid_yx).detach()) if cfg.CROP_RESIZE_WITH_MAX_POOL: pooled_feat_l = F.max_pool2d(pooled_feat_l, 2, 2) elif cfg.POOLING_MODE == 'align': pooled_feat_l = self.FRCN.RCNN_roi_align(base_feat, self.rois_l.view(-1, 5)) elif cfg.POOLING_MODE == 'pool': pooled_feat_l = self.FRCN.RCNN_roi_pool(base_feat, self.rois_l.view(-1, 5)) #print(pooled_feat_g.cpu().detach().numpy().shape) x = torch.cat((pooled_feat_g, pooled_feat_l), dim=1) #print(x.cpu().detach().numpy().shape) x = self.glcc_conv1(x) x = F.relu(x) x = x.view(-1, self.roipool * self.roipool * 512) x = self.glcc_fc1(x) x = F.relu(x) x = nn.Dropout()(x) x = self.glcc_fc2(x) x = F.relu(x) x = nn.Dropout()(x) glcc_out = self.glcc_fc_out(x) if self.training: glcc_gt = torch.tensor(glcc_gt, dtype=torch.long).cuda() glcc_loss = F.cross_entropy(glcc_out, glcc_gt) else: glcc_out = F.softmax(glcc_out, dim=1) glcc_loss = 0. glcc_gt = None return rois, cls_prob, bbox_pred, rpn_loss_cls, rpn_loss_bbox, RCNN_loss_cls, RCNN_loss_bbox, rois_label, glcc_out, glcc_loss, glcc_gt
def forward(self, im_data, im_info, gt_boxes, num_boxes): batch_size = im_data.size(0) im_info = im_info.data gt_boxes = gt_boxes.data num_boxes = num_boxes.data rois, cls_prob, bbox_pred, rpn_loss_cls, rpn_loss_bbox, \ RCNN_loss_cls, RCNN_loss_bbox, rois_label \ = self.FRCN(im_data, im_info, gt_boxes, num_boxes) # get global and local region from Faster R-CNN base_feat = self.FRCN.RCNN_base(im_data) #print(rois.data.cpu().numpy()) scores = cls_prob.data boxes = rois.data[:, :, 1:5] box_deltas = self.FRCN._bbox_pred.data if cfg.TRAIN.BBOX_NORMALIZE_TARGETS_PRECOMPUTED: if self.class_agnostic: if self.use_cuda > 0: box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor( cfg.TRAIN.BBOX_NORMALIZE_STDS).cuda( ) + torch.FloatTensor( cfg.TRAIN.BBOX_NORMALIZE_MEANS).cuda() else: box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor( cfg.TRAIN.BBOX_NORMALIZE_STDS) * torch.FlaotTensor( cfg.TRAIN.BBOX_NORMALIZE_MEANS) box_deltas = box_deltas.view(1, -1, 4) else: if self.use_cuda > 0: box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor( cfg.TRAIN.BBOX_NORMALIZE_STDS).cuda( ) + torch.FloatTensor( cfg.TRAIN.BBOX_NORMALIZE_MEANS).cuda() else: box_deltas = box_deltas.view(-1, 4) * torhc.FlaotTensor( cfg.TRAIN.BBOX_NORMALIZE_STDS) + torch.FloatTensor( cfg.TRAIN.BBOX_NORMALIZE_MEANS) box_deltas = box_deltas.view(1, -1, 4 * len(self.classes)) pred_boxes = bbox_transform_inv(boxes, box_deltas, 1) pred_boxes = clip_boxes(pred_boxes, im_info.data, 1) scores = scores.squeeze() pred_boxes = pred_boxes.squeeze() # get global region thresh = 0.00 region_g = np.ndarray((0, 5)) region_l = np.ndarray((0, 5)) for j in range(1, 4): inds = torch.nonzero(scores[:, j] >= thresh).view(-1) inds_l = torch.nonzero(scores[:, j + 3] >= thresh).view(-1) #print(inds) if inds.numel() > 0 and inds_l.numel() > 0: cls_scores = scores[:, j][inds] cls_scores_l = scores[:, j + 3][inds_l] #print(cls_scores) #print(cls_scores_l) _, order = torch.sort(cls_scores, 0, True) _, order_l = torch.sort(cls_scores_l, 0, True) if self.class_agnostic: cls_boxes = pred_boxes[inds] cls_boxes_l = pred_boxes[inds_l] else: cls_boxes = pred_boxes[inds][:, j * 4:(j + 1) * 4] cls_boxes_l = pred_boxes[inds_l][:, (j + 3) * 4:(j + 4) * 4] cls_dets = torch.cat((cls_boxes, cls_scores.unsqueeze(1)), 1) cls_dets_l = torch.cat( (cls_boxes_l, cls_scores_l.unsqueeze(1)), 1) cls_dets = cls_dets[order] cls_dets_l = cls_dets_l[order_l] region_g = np.vstack( (region_g, cls_dets[np.argmax(cls_dets[..., -1])])) region_l = np.vstack( (region_l, cls_dets_l[np.argmax(cls_dets_l[..., -1])])) #print(cls_dets) #print(pred_boxes) # if true, then show detection global and local region if True: print(region_g) print(region_l) im = im_data.cpu().numpy()[0] im = np.transpose(im, (1, 2, 0))[..., ::-1] im -= im.min() im /= im.max() plt.imshow(im.astype(np.float)) ax = plt.axes() ax.add_patch( plt.Rectangle((region_g[0, 0], region_g[0, 1]), region_g[0, 2] - region_g[0, 0], region_g[0, 3] - region_g[0, 1], fill=False, edgecolor='red', linewidth=1)) ax.add_patch( plt.Rectangle((region_l[0, 0], region_l[0, 1]), region_l[0, 2] - region_l[0, 0], region_l[0, 3] - region_l[0, 1], fill=False, edgecolor='yellow', linewidth=1)) plt.show() rois_g = np.zeros((1, 1, 5), dtype=np.float32) rois_g[0, 0, 1:5] = region_g[0, :4] / 16. rois_l = np.zeros((1, 1, 5), dtype=np.float32) rois_l[0, 0, 1:5] = region_l[0, :4] / 16. GPU = 0 rois_g = torch.tensor(rois_g, dtype=torch.float).to(GPU) rois_l = torch.tensor(rois_l, dtype=torch.float).to(GPU) # global region if cfg.POOLING_MODE == 'crop': grid_xy = _affine_grid_gen(rois_g.view(-1, 5), base_feat.size()[2:], self.FRCN.grid_size) grid_yx = torch.stack([grid_xy.data[..., 1], grid_xy.data[..., 0]], 3).contiguous() pooled_feat_g = self.FRCN.RCNN_roi_crop(base_feat, Variable(grid_yx).detach()) if cfg.CROP_RESIZE_WITH_MAX_POOL: pooled_feat_g = F.max_pool2d(pooled_feat_g, 2, 2) elif cfg.POOLING_MODE == 'align': pooled_feat_g = self.RCNN_roi_align(base_feat, rois_g.view(-1, 5)) elif cfg.POOLING_MODE == 'pool': pooled_feat_g = self.RCNN_roi_pool(base_feat, rois_g.view(-1, 5)) # local region if cfg.POOLING_MODE == 'crop': grid_xy = _affine_grid_gen(rois_l.view(-1, 5), base_feat.size()[2:], self.FRCN.grid_size) grid_yx = torch.stack([grid_xy.data[..., 1], grid_xy.data[..., 0]], 3).contiguous() pooled_feat_l = self.FRCN.RCNN_roi_crop(base_feat, Variable(grid_yx).detach()) if cfg.CROP_RESIZE_WITH_MAX_POOL: pooled_feat_l = F.max_pool2d(pooled_feat_l, 2, 2) elif cfg.POOLING_MODE == 'align': pooled_feat_l = self.RCNN_roi_align(base_feat, rois_l.view(-1, 5)) elif cfg.POOLING_MODE == 'pool': pooled_feat_l = self.RCNN_roi_pool(base_feat, rois_l.view(-1, 5)) #print(pooled_feat_g.cpu().detach().numpy().shape) x = torch.cat((pooled_feat_g, pooled_feat_l), dim=1) #print(x.cpu().detach().numpy().shape) x = self.glcc_conv1(x) x = F.relu(x) x = x.view(-1, self.roipool * self.roipool * 512) x = self.glcc_fc1(x) s = F.relu(x) x = nn.Dropout2d()(x) x = self.glcc_fc2(x) x = F.relu(x) x = nn.Dropout2d()(x) x = self.glcc_fc_out(x) return rois, cls_prob, bbox_pred, rpn_loss_cls, rpn_loss_bbox, RCNN_loss_cls, RCNN_loss_bbox, rois_label, x