def predict(st, en, step, instance_list):
global pred_prob
global bbox_cnt
for th in range(st, en, step):
if (step == 1):
warp_prob = flo.get_warp_label(flow1[th - 1], flow2[th],
combine_prob(pred_prob[th - 1]))
elif (step == -1):
warp_prob = flo.get_warp_label(flow2[th + 1], flow1[th],
combine_prob(pred_prob[th + 1]))
bbox = gen_bbox(prob_to_label(warp_prob), range(instance_num), True)
new_instance_list = []
abort = True
temp = prob_to_label(combine_prob(pred_prob[th - step]))
for i in range(instance_num):
if (th == st) or (np.count_nonzero(orig_mask[i]) <=
np.count_nonzero(temp == (i + 1)) * 10):
if np.abs(bbox_cnt[th][i] - th) <= np.abs((st - step) - th):
continue
if i in instance_list:
abort = False
bbox_cnt[th][i] = (st - step)
new_instance_list.append(i)
else:
for j in instance_list:
if IoU(bbox[i, :], bbox[j, :]) > 1e-6:
new_instance_list.append(i)
break
if abort:
break
new_instance_list = sorted(new_instance_list)
temp_image = frames[th].astype(float)
f_prob = [
np.zeros(
[bbox[idx, 3] - bbox[idx, 1], bbox[idx, 2] - bbox[idx, 0], 2])
for idx in new_instance_list
]
image_patch = np.zeros(
(len(new_instance_list), patch_shape[1], patch_shape[0], 3), float)
flow_patch = np.zeros(
(len(new_instance_list), patch_shape[1], patch_shape[0], 2), float)
warp_label_patch = np.zeros(
(len(new_instance_list), patch_shape[1], patch_shape[0], 1), float)
for i in range(len(new_instance_list)):
idx = new_instance_list[i]
warp_label_patch[i, ..., 0] = cv2.resize(
warp_prob[bbox[idx, 1]:bbox[idx, 3], bbox[idx, 0]:bbox[idx, 2],
idx + 1], patch_shape).astype(float)
image_patch[i, ...] = cv2.resize(
temp_image[
int(0.5 +
bbox[idx, 1] * fr_h_r):int(0.5 +
bbox[idx, 3] * fr_h_r),
int(0.5 +
bbox[idx, 0] * fr_w_r):int(0.5 +
bbox[idx, 2] * fr_w_r), :],
patch_shape).astype(float)
if (step == 1):
flow_patch[i, ...] = cv2.resize(
flow2[th][bbox[idx, 1]:bbox[idx, 3],
bbox[idx, 0]:bbox[idx, 2], :],
patch_shape).astype(float)
else:
flow_patch[i, ...] = cv2.resize(
flow1[th][bbox[idx, 1]:bbox[idx, 3],
bbox[idx, 0]:bbox[idx, 2], :],
patch_shape).astype(float)
image_patch = torch.from_numpy(image_patch.transpose(
0, 3, 1, 2)).contiguous().float().cuda()
warp_label_patch = torch.from_numpy(
warp_label_patch.transpose(0, 3, 1,
2)).contiguous().float().cuda()
flow_patch = torch.from_numpy(flow_patch.transpose(
0, 3, 1, 2)).contiguous().float().cuda()
with torch.no_grad():
prob = model(image_patch, flow_patch, warp_label_patch)
prob = torch.nn.functional.softmax(prob, dim=1)
if use_flip:
image_patch = flip(image_patch, 3)
warp_label_patch = flip(warp_label_patch, 3)
flow_patch = flip(flow_patch, 3)
flow_patch[:, 0, ...] = -flow_patch[:, 0, ...]
with torch.no_grad():
prob_f = model(image_patch, flow_patch, warp_label_patch)
prob_f = torch.nn.functional.softmax(prob_f, dim=1)
prob_f = flip(prob_f, 3)
prob = (prob + prob_f) / 2.0
prob = prob.data.cpu().numpy().transpose(0, 2, 3, 1)
for i in range(len(new_instance_list)):
idx = new_instance_list[i]
f_prob[i] += cv2.resize(
prob[i, ...],
(bbox[idx, 2] - bbox[idx, 0], bbox[idx, 3] - bbox[idx, 1]))
for i in range(len(new_instance_list)):
idx = new_instance_list[i]
pred_prob[th][..., idx * 2] = 1
pred_prob[th][..., idx * 2 + 1] = 0
pred_prob[th][bbox[idx, 1]:bbox[idx, 3], bbox[idx, 0]:bbox[idx, 2],
idx * 2] = f_prob[i][..., 0]
pred_prob[th][bbox[idx, 1]:bbox[idx, 3], bbox[idx, 0]:bbox[idx, 2],
idx * 2 + 1] = f_prob[i][..., 1]
def predict(st, en, step, instance_list):
# st = 1, step =1, instance_list: obj number id.
global pred_prob
global bbox_cnt
global count
for th in range(st, en, step):
if (step == 1):
temp_prob_ = combine_prob(pred_prob[th - 1])
# print (temp_prob_.shape) # (720, 1280, 3)
# temp_prob_[temp_prob_ > mask_th] = 1
warp_prob = flo.get_warp_label(flow1[th - 1], flow2[th],
temp_prob_)
elif (step == -1):
temp_prob_ = combine_prob(pred_prob[th + 1])
# temp_prob_[temp_prob_ > mask_th] = 1
warp_prob = flo.get_warp_label(flow2[th + 1], flow1[th],
temp_prob_)
bbox = gen_bbox(prob_to_label(warp_prob), range(instance_num), True)
new_instance_list = []
abort = True
temp = prob_to_label(combine_prob(pred_prob[th - step]))
for i in range(instance_num):
if (th == st) or (np.count_nonzero(orig_mask[i]) <=
np.count_nonzero(temp == (i + 1)) * 10):
if np.abs(bbox_cnt[th][i] - th) <= np.abs((st - step) - th):
continue
if i in instance_list:
abort = False
bbox_cnt[th][i] = (st - step)
new_instance_list.append(i)
else:
for j in instance_list:
if IoU(bbox[i, :], bbox[j, :]) > 1e-6:
new_instance_list.append(i)
break
if abort:
break
new_instance_list = sorted(new_instance_list)
temp_image = frames[th].astype(float)
f_prob = [
np.zeros(
[bbox[idx, 3] - bbox[idx, 1], bbox[idx, 2] - bbox[idx, 0], 2])
for idx in new_instance_list
]
image_patch = np.zeros(
(len(new_instance_list), patch_shape[1], patch_shape[0], 3), float)
flow_patch = np.zeros(
(len(new_instance_list), patch_shape[1], patch_shape[0], 2), float)
warp_label_patch = np.zeros(
(len(new_instance_list), patch_shape[1], patch_shape[0], 1), float)
for i in range(len(new_instance_list)):
idx = new_instance_list[i]
warp_label_patch_temp = cv2.resize(
warp_prob[bbox[idx, 1]:bbox[idx, 3], bbox[idx, 0]:bbox[idx, 2],
idx + 1], patch_shape).astype(float)
# print (warp_label_patch_temp.shape) # (433, 433)
warp_label_patch_temp[warp_label_patch_temp >= mask_th] = 1
warp_label_patch_temp[warp_label_patch_temp < mask_th] = 0
warp_label_patch[i, ..., 0] = warp_label_patch_temp
image_patch[i, ...] = cv2.resize(
temp_image[
int(0.5 +
bbox[idx, 1] * fr_h_r):int(0.5 +
bbox[idx, 3] * fr_h_r),
int(0.5 +
bbox[idx, 0] * fr_w_r):int(0.5 +
bbox[idx, 2] * fr_w_r), :],
patch_shape).astype(float)
if (step == 1):
flow_patch[i, ...] = cv2.resize(
flow2[th][bbox[idx, 1]:bbox[idx, 3],
bbox[idx, 0]:bbox[idx, 2], :],
patch_shape).astype(float)
else:
flow_patch[i, ...] = cv2.resize(
flow1[th][bbox[idx, 1]:bbox[idx, 3],
bbox[idx, 0]:bbox[idx, 2], :],
patch_shape).astype(float)
image_patch = torch.from_numpy(image_patch.transpose(0, 3, 1, 2))
warp_label_patch = torch.from_numpy(
warp_label_patch.transpose(0, 3, 1,
2)).contiguous().float().cuda()
flow_patch = torch.from_numpy(flow_patch.transpose(
0, 3, 1, 2)).contiguous().float().cuda()
print(image_patch.size(), 'image_patch')
print(warp_label_patch.size(), 'warp_label_patch')
print(flow_patch.size(), 'flow_patch')
#notice here!!!!!!! if annotate it, you will be mad.
print((image_patch[0]).cpu().numpy().transpose(1, 2, 0).shape)
cv2.imwrite('output/%3d_prob.png' % count,
(image_patch[0]).cpu().numpy().transpose(1, 2, 0))
count += 1
cv2.imwrite('output/%3d_prob.png' % count,
(warp_label_patch[0][0] * 255).cpu().numpy())
count += 1
value_scale = 255
mean = [0.485, 0.456, 0.406]
mean = [item * value_scale for item in mean]
std = [0.229, 0.224, 0.225]
std = [item * value_scale for item in std]
for i in range(len(new_instance_list)):
for t, m, s in zip(image_patch[i], mean, std):
t.sub_(m).div_(s)
# notice here!!!!!!! if annotate it, you will be mad.
image_patch = image_patch.contiguous().float().cuda()
with torch.no_grad():
prob = model(
image_patch, warp_label_patch,
flow_patch) #model = torch.nn.DataParallel(model).cuda()
prob = torch.nn.functional.softmax(prob, dim=1)
cv2.imwrite(
'output/%3d_prob.png' % count,
(torch.argmax(prob[0], dim=0, keepdim=False)).cpu().numpy() *
100)
count += 1
# warp_label_patch debug1
# probxxx = prob.data.cpu().numpy().transpose(0, 2, 3, 1)
# print (probxxx[0].shape)
# out = np.argmax(probxxx[0], axis=2)
# print (out.shape)
# cv2.imwrite("debugs/"+str(th)+"_out.png", (out * 255))
if use_flip:
image_patch = flip(image_patch, 3)
warp_label_patch = flip(warp_label_patch, 3)
flow_patch = flip(flow_patch, 3)
flow_patch[:, 0, ...] = -flow_patch[:, 0, ...]
with torch.no_grad():
prob_f = model(image_patch, warp_label_patch, flow_patch)
#cv2.imwrite('output/%d_probf.png' % count, (prob_f[0][0]*255).cpu().numpy())
#count += 1
prob_f = torch.nn.functional.softmax(prob_f, dim=1)
prob_f = flip(prob_f, 3)
prob = (prob + prob_f) / 2.0
prob = prob.data.cpu().numpy().transpose(0, 2, 3, 1)
for i in range(len(new_instance_list)):
idx = new_instance_list[i]
f_prob[i] += cv2.resize(
prob[i, ...],
(bbox[idx, 2] - bbox[idx, 0], bbox[idx, 3] - bbox[idx, 1]))
for i in range(len(new_instance_list)):
idx = new_instance_list[i]
pred_prob[th][..., idx * 2] = 1
pred_prob[th][..., idx * 2 + 1] = 0
pred_prob[th][bbox[idx, 1]:bbox[idx, 3], bbox[idx, 0]:bbox[idx, 2],
idx * 2] = f_prob[i][..., 0]
pred_prob[th][bbox[idx, 1]:bbox[idx, 3], bbox[idx, 0]:bbox[idx, 2],
idx * 2 + 1] = f_prob[i][..., 1]