def __init__(self, csv_path, mode='train', target_only=False):
self.mode = mode
# Read data into numpy arrays
with open(csv_path, 'r') as fp:
data = list(csv.reader(fp))
data = np.array(data[1:])[:, 1:].astype(float)
if not target_only:
feats = list(range(93))
else:
# TODO: Using 40 states & 2 tested_positive features (indeices = 57 & 75)
feats = list(range(40)) + [57, 75]
if mode == 'test':
# Testing data
# data: 893 x 93 (40 states + day 1 (18) + day 2 (18) + day 3 (17))
data = data[:, feats]
# Convert data into Pytorch tensors
self.data = torch.FlaotTensor(data)
else:
# Training data
# data: 2700 x 94 (40 states + day 1 (18) + day 2 (18) + day3 (18))
target = data[:, -1]
data = data[:, feats]
# Spliting training data into train & dev sets
if mode == 'train':
indices = [i for i in range(len(data)) if i % 10 != 0]
elif mode == 'dev':
indices = [i for i in range(len(data)) if 1 % 10 == 0]
# Convert data into Pytorch tensors
self.data = torch.FloatTensor(data[indices])
self.target = torch.FloatTensor(target[indices])
# Normalize features (you may remove this part to see what will happen)
self.data[:, 40:] = \
(self.data[:, 40:] - self.data[:, 40:].mean(dim=0, keepdim=True)) \
/ self.data[:, 40:].std(dim=0, keepdim=True)
self.dim = self.data.shape[1]
print(
f'Finished reading the {self.mode} set of COVID19 Dataset ({len(data)} samples found, each dim = {self.dim})'
)
def forward(self,
im_data,
im_info,
gt_boxes,
num_boxes,
opt_frcn=None,
vis_flag=False):
batch_size = im_data.size(0)
im_info = im_info.data
gt_boxes = gt_boxes.data
num_boxes = num_boxes.data
#self.FRCN.eval()
#self.FRCN.RCNN_base.eval()
#for param in self.FRCN.parameters():
# param.requires_grad = False
gt_boxes_frcn = gt_boxes.cpu().data.numpy()[0]
for gt in gt_boxes_frcn:
if gt[..., -1] < 4 and gt[..., -1] > 0:
gt[..., -1] = 1
elif gt[..., -1] >= 4:
gt[..., -1] = 2
gt_boxes_frcn = gt_boxes_frcn[None, ...]
gt_boxes_frcn = torch.Tensor(gt_boxes_frcn)
if self.use_cuda:
gt_boxes_frcn = gt_boxes_frcn.cuda()
rois, cls_prob, bbox_pred, rpn_loss_cls, rpn_loss_bbox, \
RCNN_loss_cls, RCNN_loss_bbox, rois_label \
= self.FRCN(im_data, im_info, gt_boxes_frcn, num_boxes)
if opt_frcn is not None:
opt.zero_grad()
# get global and local region from Faster R-CNN
base_feat = self.FRCN.RCNN_base(im_data)
#print(rois.data.cpu().numpy())
scores = cls_prob.data
boxes = rois.data[:, :, 1:5]
box_deltas = self.FRCN._bbox_pred.data
if cfg.TRAIN.BBOX_NORMALIZE_TARGETS_PRECOMPUTED:
if self.class_agnostic:
if self.use_cuda > 0:
box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor(
cfg.TRAIN.BBOX_NORMALIZE_STDS).cuda(
) + torch.FloatTensor(
cfg.TRAIN.BBOX_NORMALIZE_MEANS).cuda()
else:
box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor(
cfg.TRAIN.BBOX_NORMALIZE_STDS) * torch.FlaotTensor(
cfg.TRAIN.BBOX_NORMALIZE_MEANS)
box_deltas = box_deltas.view(1, -1, 4)
else:
if self.use_cuda > 0:
box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor(
cfg.TRAIN.BBOX_NORMALIZE_STDS).cuda(
) + torch.FloatTensor(
cfg.TRAIN.BBOX_NORMALIZE_MEANS).cuda()
else:
box_deltas = box_deltas.view(-1, 4) * torhc.FlaotTensor(
cfg.TRAIN.BBOX_NORMALIZE_STDS) + torch.FloatTensor(
cfg.TRAIN.BBOX_NORMALIZE_MEANS)
box_deltas = box_deltas.view(1, -1, 4 * 3)
pred_boxes = bbox_transform_inv(boxes, box_deltas, 1)
pred_boxes = clip_boxes(pred_boxes, im_info.data, 1)
scores = scores.squeeze()
pred_boxes = pred_boxes.squeeze()
#print(scores)
# get global region
thresh = 0.0
region_g = np.ndarray((0, 5))
region_l = np.ndarray((0, 5))
# get glocal region
inds = torch.nonzero(scores[:, 1] >= thresh).view(-1)
if inds.numel() > 0:
cls_scores = scores[:, 1][inds]
_, order = torch.sort(cls_scores, 0, True)
if self.class_agnostic:
cls_boxes = pred_boxes[inds]
else:
cls_boxes = pred_boxes[inds][:, 4:8]
cls_dets = torch.cat((cls_boxes, cls_scores.unsqueeze(1)), 1)
cls_dets = cls_dets[order]
region_g = cls_dets
# get local region
inds = torch.nonzero(scores[:, 2] >= thresh).view(-1)
if inds.numel() > 0:
cls_scores = scores[:, 2][inds]
_, order = torch.sort(cls_scores, 0, True)
if self.class_agnostic:
cls_boxes = pred_boxes[inds]
else:
cls_boxes = pred_boxes[inds][:, 8:12]
cls_dets = torch.cat((cls_boxes, cls_scores.unsqueeze(1)), 1)
cls_dets = cls_dets[order]
region_l = cls_dets
# select region for recognition
if not self.training:
self.minibatch = 1
if self.training:
if self.minibatch % 2 == 0:
high_ind = self.minibatch // 2
low_ind = self.minibatch // 2
elif self.minibatch == 1:
high_ind = 1
low_ind = 0
else:
high_ind = self.minibatch // 2 + 1
low_ind = self.minibatch // 2
keep = nms(torch.tensor(region_g).cuda(),
self.nms_iou,
force_cpu=not cfg.USE_GPU_NMS)
if type(keep) is not list:
keep = keep.view(-1).long()
region_g = region_g[keep]
sort_ind = np.argsort(region_g[..., -1])
high_ind_g = sort_ind[-high_ind:]
low_ind_g = sort_ind[:low_ind]
keep = nms(torch.tensor(region_l).cuda(),
self.nms_iou,
force_cpu=not cfg.USE_GPU_NMS)
if type(keep) is not list:
keep = keep.view(-1).long()
region_l = region_l[keep]
sort_ind = np.argsort(region_l[..., -1])
high_ind_l = sort_ind[-high_ind:]
low_ind_l = sort_ind[:low_ind]
high_num = min(len(high_ind_g), len(high_ind_l))
high_ind_g = high_ind_g[:high_num]
high_ind_l = high_ind_l[:high_num]
low_num = min(len(low_ind_g), len(low_ind_l))
low_ind_g = low_ind_g[:low_num]
low_ind_l = low_ind_l[:low_num]
if len(high_ind_g.data) < 1:
region_g_high = torch.tensor(np.ndarray((5))[None, ...])
else:
region_g_high = region_g[high_ind_g]
if len(low_ind_g.data) < 1:
region_g_low = torch.tensor(np.ndarray((5))[None, ...])
else:
region_g_low = region_g[low_ind_g]
if len(high_ind_l.data) < 1:
region_l_high = torch.tensor(np.ndarray((5))[None, ...])
else:
region_l_high = region_l[high_ind_l]
if len(low_ind_l.data) < 1:
region_l_low = torch.tensor(np.ndarray((5))[None, ...])
else:
region_l_low = region_l[low_ind_l]
proposal_g = np.vstack((region_g_high, region_g_low))
proposal_l = np.vstack((region_l_high, region_l_low))
#proposal_g = np.vstack((region_g[high_ind_g], region_g[low_ind_g]))
#proposal_l = np.vstack((region_l[high_ind_l], region_l[low_ind_l]))
#self.proposal_g.data.resize_(proposal_g.size()).copy_(proposal_g)
#self.proposal_l.data.resize_(proposal_l.size()).copy_(proposal_l)
gt_boxes = gt_boxes.cpu().numpy()[0, :2]
gt_g = gt_boxes[np.where(gt_boxes[..., -1] < 4)[0]]
gt_l = gt_boxes[np.where(gt_boxes[..., -1] >= 4)[0]]
# compute pare ground truth
def compute_iou(ps, gt):
iou_x1 = np.maximum(ps[..., 0], gt[0])
iou_y1 = np.maximum(ps[..., 1], gt[1])
iou_x2 = np.minimum(ps[..., 2], gt[2])
iou_y2 = np.minimum(ps[..., 3], gt[3])
iou_w = np.maximum(iou_x2 - iou_x1, 0)
iou_h = np.maximum(iou_y2 - iou_y1, 0)
iou_area = iou_w * iou_h
gt_area = (gt[2] - gt[0]) * (gt[3] - gt[1])
p_area = (ps[..., 2] - ps[..., 0]) * (ps[..., 3] - ps[..., 1])
overlap = iou_area / (gt_area + p_area - iou_area)
count = np.zeros((ps.shape[0]), dtype=int)
count[overlap >= self.gt_iou] += 1
return count
cou = compute_iou(proposal_g, gt_g[0]) + compute_iou(
proposal_l, gt_l[0])
## 2019.2.13
glcc_gt = np.zeros((proposal_g.shape[0]), dtype=int)
glcc_gt[cou >= 1] = gt_g[0, -1]
#glcc_gt[:] = gt_g[0, -1]
glcc_gt = torch.tensor(glcc_gt, dtype=torch.long).cuda()
#print(glcc_gt)
#self.glcc_gt.data.resize_(glcc_gt.size()).copy_(glcc_gt)
else:
# test phase
proposal_g = region_g[np.argmax(region_g[..., -1])][None, ...]
proposal_l = region_l[np.argmax(region_l[..., -1])][None, ...]
#self.proposal_g.data.resize_(proposal_g.size()).copy_(proposal_g.size())
#self.proposal_l.data.resize_(proposal_l.size()).copy_(proposal_l.size())
# if true, then show detection global and local region
if vis_flag:
gt_boxes = gt_boxes.astype(np.int)
im = im_data.cpu().numpy()[0]
im = np.transpose(im, (1, 2, 0))[..., ::-1]
im -= im.min()
im /= im.max()
plt.imshow(im.astype(np.float))
ax = plt.axes()
ax.add_patch(
plt.Rectangle((region_g[0, 0], region_g[0, 1]),
region_g[0, 2] - region_g[0, 0],
region_g[0, 3] - region_g[0, 1],
fill=False,
edgecolor='red',
linewidth=1))
ax.add_patch(
plt.Rectangle((region_l[0, 0], region_l[0, 1]),
region_l[0, 2] - region_l[0, 0],
region_l[0, 3] - region_l[0, 1],
fill=False,
edgecolor='yellow',
linewidth=1))
ax.add_patch(
plt.Rectangle((gt_boxes[0, 0], gt_boxes[0, 1]),
gt_boxes[0, 2] - gt_boxes[0, 0],
gt_boxes[0, 3] - gt_boxes[0, 1],
fill=False,
edgecolor='green',
linewidth=1))
ax.add_patch(
plt.Rectangle((gt_boxes[1, 0], gt_boxes[1, 1]),
gt_boxes[1, 2] - gt_boxes[1, 0],
gt_boxes[1, 3] - gt_boxes[1, 1],
fill=False,
edgecolor='white',
linewidth=1))
plt.show()
rois_g = np.zeros((1, proposal_g.shape[0], 5), dtype=np.float32)
rois_g[0, :, 1:5] = proposal_g[:, :4]
#rois_g /= 16.
rois_l = np.zeros((1, proposal_l.shape[0], 5), dtype=np.float32)
rois_l[0, :, 1:5] = proposal_l[:, :4]
#rois_l /= 16.
rois_g = torch.tensor(rois_g, dtype=torch.float).cuda()
rois_l = torch.tensor(rois_l, dtype=torch.float).cuda()
self.rois_g.data.resize_(rois_g.size()).copy_(rois_g)
self.rois_l.data.resize_(rois_l.size()).copy_(rois_l)
# global region
if cfg.POOLING_MODE == 'crop':
grid_xy = _affine_grid_gen(self.rois_g.view(-1, 5),
base_feat.size()[2:],
self.FRCN.grid_size)
grid_yx = torch.stack([grid_xy.data[..., 1], grid_xy.data[..., 0]],
3).contiguous()
pooled_feat_g = self.FRCN.RCNN_roi_crop(base_feat,
Variable(grid_yx).detach())
if cfg.CROP_RESIZE_WITH_MAX_POOL:
pooled_feat_g = F.max_pool2d(pooled_feat_g, 2, 2)
elif cfg.POOLING_MODE == 'align':
pooled_feat_g = self.FRCN.RCNN_roi_align(base_feat,
self.rois_g.view(-1, 5))
elif cfg.POOLING_MODE == 'pool':
pooled_feat_g = self.FRCN.RCNN_roi_pool(base_feat,
self.rois_g.view(-1, 5))
# local region
if cfg.POOLING_MODE == 'crop':
grid_xy = _affine_grid_gen(self.rois_l.view(-1, 5),
base_feat.size()[2:],
self.FRCN.grid_size)
grid_yx = torch.stack([grid_xy.data[..., 1], grid_xy.data[..., 0]],
3).contiguous()
pooled_feat_l = self.FRCN.RCNN_roi_crop(base_feat,
Variable(grid_yx).detach())
if cfg.CROP_RESIZE_WITH_MAX_POOL:
pooled_feat_l = F.max_pool2d(pooled_feat_l, 2, 2)
elif cfg.POOLING_MODE == 'align':
pooled_feat_l = self.FRCN.RCNN_roi_align(base_feat,
self.rois_l.view(-1, 5))
elif cfg.POOLING_MODE == 'pool':
pooled_feat_l = self.FRCN.RCNN_roi_pool(base_feat,
self.rois_l.view(-1, 5))
#print(pooled_feat_g.cpu().detach().numpy().shape)
x = torch.cat((pooled_feat_g, pooled_feat_l), dim=1)
#print(x.cpu().detach().numpy().shape)
x = self.glcc_conv1(x)
x = F.relu(x)
x = x.view(-1, self.roipool * self.roipool * 512)
x = self.glcc_fc1(x)
x = F.relu(x)
x = nn.Dropout()(x)
x = self.glcc_fc2(x)
x = F.relu(x)
x = nn.Dropout()(x)
glcc_out = self.glcc_fc_out(x)
if self.training:
glcc_gt = torch.tensor(glcc_gt, dtype=torch.long).cuda()
glcc_loss = F.cross_entropy(glcc_out, glcc_gt)
else:
glcc_out = F.softmax(glcc_out, dim=1)
glcc_loss = 0.
glcc_gt = None
return rois, cls_prob, bbox_pred, rpn_loss_cls, rpn_loss_bbox, RCNN_loss_cls, RCNN_loss_bbox, rois_label, glcc_out, glcc_loss, glcc_gt
def forward(self, im_data, im_info, gt_boxes, num_boxes):
batch_size = im_data.size(0)
im_info = im_info.data
gt_boxes = gt_boxes.data
num_boxes = num_boxes.data
rois, cls_prob, bbox_pred, rpn_loss_cls, rpn_loss_bbox, \
RCNN_loss_cls, RCNN_loss_bbox, rois_label \
= self.FRCN(im_data, im_info, gt_boxes, num_boxes)
# get global and local region from Faster R-CNN
base_feat = self.FRCN.RCNN_base(im_data)
#print(rois.data.cpu().numpy())
scores = cls_prob.data
boxes = rois.data[:, :, 1:5]
box_deltas = self.FRCN._bbox_pred.data
if cfg.TRAIN.BBOX_NORMALIZE_TARGETS_PRECOMPUTED:
if self.class_agnostic:
if self.use_cuda > 0:
box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor(
cfg.TRAIN.BBOX_NORMALIZE_STDS).cuda(
) + torch.FloatTensor(
cfg.TRAIN.BBOX_NORMALIZE_MEANS).cuda()
else:
box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor(
cfg.TRAIN.BBOX_NORMALIZE_STDS) * torch.FlaotTensor(
cfg.TRAIN.BBOX_NORMALIZE_MEANS)
box_deltas = box_deltas.view(1, -1, 4)
else:
if self.use_cuda > 0:
box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor(
cfg.TRAIN.BBOX_NORMALIZE_STDS).cuda(
) + torch.FloatTensor(
cfg.TRAIN.BBOX_NORMALIZE_MEANS).cuda()
else:
box_deltas = box_deltas.view(-1, 4) * torhc.FlaotTensor(
cfg.TRAIN.BBOX_NORMALIZE_STDS) + torch.FloatTensor(
cfg.TRAIN.BBOX_NORMALIZE_MEANS)
box_deltas = box_deltas.view(1, -1, 4 * len(self.classes))
pred_boxes = bbox_transform_inv(boxes, box_deltas, 1)
pred_boxes = clip_boxes(pred_boxes, im_info.data, 1)
scores = scores.squeeze()
pred_boxes = pred_boxes.squeeze()
# get global region
thresh = 0.00
region_g = np.ndarray((0, 5))
region_l = np.ndarray((0, 5))
for j in range(1, 4):
inds = torch.nonzero(scores[:, j] >= thresh).view(-1)
inds_l = torch.nonzero(scores[:, j + 3] >= thresh).view(-1)
#print(inds)
if inds.numel() > 0 and inds_l.numel() > 0:
cls_scores = scores[:, j][inds]
cls_scores_l = scores[:, j + 3][inds_l]
#print(cls_scores)
#print(cls_scores_l)
_, order = torch.sort(cls_scores, 0, True)
_, order_l = torch.sort(cls_scores_l, 0, True)
if self.class_agnostic:
cls_boxes = pred_boxes[inds]
cls_boxes_l = pred_boxes[inds_l]
else:
cls_boxes = pred_boxes[inds][:, j * 4:(j + 1) * 4]
cls_boxes_l = pred_boxes[inds_l][:,
(j + 3) * 4:(j + 4) * 4]
cls_dets = torch.cat((cls_boxes, cls_scores.unsqueeze(1)), 1)
cls_dets_l = torch.cat(
(cls_boxes_l, cls_scores_l.unsqueeze(1)), 1)
cls_dets = cls_dets[order]
cls_dets_l = cls_dets_l[order_l]
region_g = np.vstack(
(region_g, cls_dets[np.argmax(cls_dets[..., -1])]))
region_l = np.vstack(
(region_l, cls_dets_l[np.argmax(cls_dets_l[..., -1])]))
#print(cls_dets)
#print(pred_boxes)
# if true, then show detection global and local region
if True:
print(region_g)
print(region_l)
im = im_data.cpu().numpy()[0]
im = np.transpose(im, (1, 2, 0))[..., ::-1]
im -= im.min()
im /= im.max()
plt.imshow(im.astype(np.float))
ax = plt.axes()
ax.add_patch(
plt.Rectangle((region_g[0, 0], region_g[0, 1]),
region_g[0, 2] - region_g[0, 0],
region_g[0, 3] - region_g[0, 1],
fill=False,
edgecolor='red',
linewidth=1))
ax.add_patch(
plt.Rectangle((region_l[0, 0], region_l[0, 1]),
region_l[0, 2] - region_l[0, 0],
region_l[0, 3] - region_l[0, 1],
fill=False,
edgecolor='yellow',
linewidth=1))
plt.show()
rois_g = np.zeros((1, 1, 5), dtype=np.float32)
rois_g[0, 0, 1:5] = region_g[0, :4] / 16.
rois_l = np.zeros((1, 1, 5), dtype=np.float32)
rois_l[0, 0, 1:5] = region_l[0, :4] / 16.
GPU = 0
rois_g = torch.tensor(rois_g, dtype=torch.float).to(GPU)
rois_l = torch.tensor(rois_l, dtype=torch.float).to(GPU)
# global region
if cfg.POOLING_MODE == 'crop':
grid_xy = _affine_grid_gen(rois_g.view(-1, 5),
base_feat.size()[2:],
self.FRCN.grid_size)
grid_yx = torch.stack([grid_xy.data[..., 1], grid_xy.data[..., 0]],
3).contiguous()
pooled_feat_g = self.FRCN.RCNN_roi_crop(base_feat,
Variable(grid_yx).detach())
if cfg.CROP_RESIZE_WITH_MAX_POOL:
pooled_feat_g = F.max_pool2d(pooled_feat_g, 2, 2)
elif cfg.POOLING_MODE == 'align':
pooled_feat_g = self.RCNN_roi_align(base_feat, rois_g.view(-1, 5))
elif cfg.POOLING_MODE == 'pool':
pooled_feat_g = self.RCNN_roi_pool(base_feat, rois_g.view(-1, 5))
# local region
if cfg.POOLING_MODE == 'crop':
grid_xy = _affine_grid_gen(rois_l.view(-1, 5),
base_feat.size()[2:],
self.FRCN.grid_size)
grid_yx = torch.stack([grid_xy.data[..., 1], grid_xy.data[..., 0]],
3).contiguous()
pooled_feat_l = self.FRCN.RCNN_roi_crop(base_feat,
Variable(grid_yx).detach())
if cfg.CROP_RESIZE_WITH_MAX_POOL:
pooled_feat_l = F.max_pool2d(pooled_feat_l, 2, 2)
elif cfg.POOLING_MODE == 'align':
pooled_feat_l = self.RCNN_roi_align(base_feat, rois_l.view(-1, 5))
elif cfg.POOLING_MODE == 'pool':
pooled_feat_l = self.RCNN_roi_pool(base_feat, rois_l.view(-1, 5))
#print(pooled_feat_g.cpu().detach().numpy().shape)
x = torch.cat((pooled_feat_g, pooled_feat_l), dim=1)
#print(x.cpu().detach().numpy().shape)
x = self.glcc_conv1(x)
x = F.relu(x)
x = x.view(-1, self.roipool * self.roipool * 512)
x = self.glcc_fc1(x)
s = F.relu(x)
x = nn.Dropout2d()(x)
x = self.glcc_fc2(x)
x = F.relu(x)
x = nn.Dropout2d()(x)
x = self.glcc_fc_out(x)
return rois, cls_prob, bbox_pred, rpn_loss_cls, rpn_loss_bbox, RCNN_loss_cls, RCNN_loss_bbox, rois_label, x