def get_frames(video_dirs):
if isinstance(video_dirs, str):
video_dirs = [video_dirs]
for video_dir in tqdm(video_dirs):
img_dir = video_dir
label_dir = join(video_dir, 'Label')
result_dir = join('./result', 'epoch1', video_dir.split('/')[-1])
save_path = join(result_dir, 'vis')
if not os.path.exists(save_path):
os.makedirs(save_path)
img_list = sorted(glob(join(img_dir, '*.jpg')))
label_list = sorted(glob(join(label_dir, '*.pfm')))
result_list = sorted(glob(join(result_dir, '*.png')))
# assert len(img_list) == len(label_list) == len(result_list)
l = len(label_list)
img_list, result_list = img_list[:l], result_list[:l]
for i, (img, label, result) in enumerate(
tqdm(zip(img_list, label_list, result_list),
total=len(img_list))):
img = cv2.imread(img)
label = read(label)
result = cv2.imread(result)
masked_img = (result > 0) * result + (result == 0) * img
# img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
# result = cv2.cvtColor(result, cv2.COLOR_BGR2RGB)
fig = plt.figure(figsize=(10, 8))
ax = fig.add_subplot(221)
plt.imshow(img[:, :, ::-1])
bx = fig.add_subplot(222)
plt.imshow(masked_img[:, :, ::-1])
cx = fig.add_subplot(223)
plt.imshow(result[..., ::-1])
dx = fig.add_subplot(224)
plt.imshow(visualize_label(label).astype(np.uint8))
plt.savefig(join(save_path, f'{i:03d}.jpg'))
plt.close()
def evaluate(result_path):
data_path = '/home/qzy/data/FBMS/Testset'
data_dirs = sorted(glob(join(data_path, '*', 'GroundTruth')))
result_dirs = sorted(glob(join(result_path, '*', 'label')))
assert len(data_dirs) == len(result_dirs)
avg_p, avg_r, avg_f, avg_delta_obj = [], [], [], []
for data_dir, result_dir in zip(data_dirs, result_dirs):
gt_img_list = sorted(glob(join(data_dir, '*.png')))
gt_idx_list = list(
map(lambda x: int(x.split('/')[-1][:3]) - 1, gt_img_list))
if gt_idx_list[0] > 0:
x = gt_idx_list[0]
gt_idx_list = [y - x for y in gt_idx_list]
label_list = sorted(glob(join(result_dir, '*.pfm')))
gts, labels = [], []
delta_obj = 0.
for i, idx in enumerate(gt_idx_list[:-1]):
gt = read(gt_img_list[i])
if gt.ndim == 3:
gt = gt[..., 0]
label = read(label_list[idx]).astype(np.uint8)
# embed()
delta_obj += abs(len(np.unique(gt)) - len(np.unique(label)))
gts.append(gt)
labels.append(label)
gts = np.stack(gts)
labels = np.stack(labels)
delta_obj /= (len(gt_idx_list) - 1)
# objs = np.unique(gts)[1:]
objs = np.unique(gts)
n_obj = len(objs)
regions = np.zeros(n_obj)
for i, obj in enumerate(objs):
regions[i] = np.sum(gts == obj)
_labels = np.unique(labels)
n_label = len(_labels)
clusters = np.zeros(n_label)
for j, label in enumerate(_labels):
clusters[j] = np.sum(labels == label)
# embed()
overlap = np.zeros((n_obj, n_label))
for i, obj in enumerate(objs):
for j, label in enumerate(_labels):
# embed()
overlap[i, j] = np.sum((gts == obj) * (labels == label))
P = overlap / clusters[None, :]
R = overlap / regions[:, None]
F = np.zeros_like(overlap)
for i in range(n_obj):
for j in range(n_label):
if overlap[i, j] > 0.:
F[i, j] = (2 * P[i, j] * R[i, j]) / (P[i, j] + R[i, j])
row_indices, col_indices = linear_sum_assignment(-F)
precision, recall = 0., 0.
for row, col in zip(row_indices, col_indices):
precision += P[row, col]
recall += R[row, col]
for row in range(n_obj):
if row not in row_indices:
precision += 1.
precision /= n_obj
recall /= n_obj
f_value = (2 * precision * recall) / (precision + recall)
avg_p.append(precision)
avg_r.append(recall)
avg_f.append(f_value)
avg_delta_obj.append(abs(n_obj - n_label))
# avg_delta_obj.append(delta_obj)
avg_p = np.mean(avg_p)
avg_r = np.mean(avg_r)
avg_f = np.mean(avg_f)
avg_delta_obj = np.mean(avg_delta_obj)
print(f"Model: {result_path.split('/')[-1]}")
print("Precision: %f\tRecall: %f\tF-Score: %f\tDelta_Obj: %f" %
(avg_p, avg_r, avg_f, avg_delta_obj))
def main():
root = '/home/qzy/data/FBMS/Trainingset'
video_dirs = sorted(glob(join(root, '*')))
for video_dir in tqdm(video_dirs):
gt_root = join(video_dir, 'GroundTruth')
save_root = join(video_dir, 'Label')
if not os.path.exists(save_root):
os.makedirs(save_root)
gt_img_list = sorted(glob(join(gt_root, '*.png')))
img_list = sorted(glob(join(video_dir, '*.jpg')))
flow_list = sorted(glob(join(video_dir, 'Flow', '*.flo')))
inv_flow_list = sorted(glob(join(video_dir, 'Inv_Flow', '*.flo')))
gt_idx_list = list(
map(lambda x: int(x.split('/')[-1][:3]) - 1, gt_img_list))
if gt_idx_list[0] > 0:
x = gt_idx_list[0]
gt_idx_list = [y - x for y in gt_idx_list]
n_clip = len(gt_idx_list) - 1
gt = []
for gt_img in gt_img_list:
gt_img = cv2.imread(gt_img)[..., 0]
gt_img = torch.tensor(gt_img).cuda()
gt.append(gt_img)
gt = torch.stack(gt)
objs = torch.unique(gt, sorted=True)
for i in range(n_clip):
start = gt_idx_list[i]
end = gt_idx_list[i + 1]
start_gt = cv2.imread(gt_img_list[i])[..., 0]
start_gt = torch.tensor(start_gt).cuda()
end_gt = cv2.imread(gt_img_list[i + 1])[..., 0]
end_gt = torch.tensor(end_gt).cuda()
# objs = torch.unique(torch.stack([start_gt, end_gt]), sorted=True)
for idx, obj in enumerate(objs):
start_gt[start_gt == obj] = idx
end_gt[end_gt == obj] = idx
start_gt = F.one_hot(start_gt.long(),
num_classes=objs.shape[0]).permute(
2, 0, 1).unsqueeze(0).float()
end_gt = F.one_hot(end_gt.long(),
num_classes=objs.shape[0]).permute(
2, 0, 1).unsqueeze(0).float()
imgs, flows, inv_flows = [], [], []
for t in range(start, end + 1):
img = cv2.imread(img_list[t])
# print(img_list[t])
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = torch.tensor(img).permute(2, 0,
1).float().unsqueeze(0).cuda()
imgs.append(img)
if t < end:
flow = read(flow_list[t])
# print(flow_list[t])
flow = torch.tensor(flow).permute(2, 0,
1).unsqueeze(0).cuda()
flows.append(flow)
inv_flow = read(inv_flow_list[t + 1])
# print(inv_flow_list[t])
inv_flow = torch.tensor(inv_flow).permute(
2, 0, 1).unsqueeze(0).cuda()
inv_flows.append(inv_flow)
H, W = start_gt.shape[-2:]
xx = torch.arange(0, W).view(1, -1).repeat(H, 1)
yy = torch.arange(0, H).view(-1, 1).repeat(1, W)
xx = xx.view(1, 1, H, W)
yy = yy.view(1, 1, H, W)
grid = torch.cat((xx, yy), 1).float().cuda()
def get_vgrid(f):
vgrid = grid + f
vgrid[:, 0, :, :] = 2.0 * vgrid[:, 0, :, :].clone() / max(
W - 1, 1) - 1.0
vgrid[:, 1, :, :] = 2.0 * vgrid[:, 1, :, :].clone() / max(
H - 1, 1) - 1.0
vgrid = vgrid.permute(0, 2, 3, 1)
return vgrid
# forward propagate
fwd_labels = [
start_gt,
]
for t in range(len(flows) - 1):
fwd_flow = flows[t]
bwd_flow = inv_flows[t]
vgrid = get_vgrid(bwd_flow)
warped_fwd_flow = F.grid_sample(fwd_flow, vgrid)
mask = torch.sum((warped_fwd_flow + bwd_flow) ** 2, dim=1, keepdim=True) <= \
torch.sum(0.01 * (warped_fwd_flow ** 2 + bwd_flow ** 2), dim=1, keepdim=True) + 0.5
warped_label = F.grid_sample(fwd_labels[-1],
vgrid) * mask.float()
fwd_labels.append(warped_label)
fwd_labels.append(torch.zeros_like(start_gt).cuda())
# backward propagate
bwd_labels = [
end_gt,
]
for t in reversed(range(1, len(flows))):
fwd_flow = flows[t]
bwd_flow = inv_flows[t]
vgrid = get_vgrid(fwd_flow)
warped_bwd_flow = F.grid_sample(bwd_flow, vgrid)
mask = torch.sum((warped_bwd_flow + fwd_flow) ** 2, dim=1, keepdim=True) <= \
torch.sum(0.01 * (warped_bwd_flow ** 2 + fwd_flow ** 2), dim=1, keepdim=True) + 0.5
warped_label = F.grid_sample(bwd_labels[-1],
vgrid) * mask.float()
bwd_labels.append(warped_label)
bwd_labels.append(torch.zeros_like(end_gt).cuda())
labels = []
for fwd_label, bwd_label in zip(fwd_labels, reversed(bwd_labels)):
label = fwd_label + bwd_label
label = torch.argmax(label,
dim=1).squeeze().float().cpu().numpy()
labels.append(label)
for idx, label in enumerate(labels):
save_name = join(save_root, f'{start + idx + 1:03d}.pfm')
write(save_name, label)
print([x.sum() for x in labels])
def __getitem__(self, idx):
T = self.T
idx = idx // (10 - T + 1)
start = idx % (10 - T + 1)
idir, fdir, ifdir, oidir, mdir = self.dir_list[idx]
img_names = sorted(glob(join(idir, '*.png')))[start:start + T]
flow_names = sorted(glob(join(fdir, '*.pfm')))[start:start + T]
inv_flow_names = sorted(glob(join(ifdir, '*.pfm')))[start:start + T]
object_id_names = sorted(glob(join(oidir, '*.pfm')))[start:start + T]
mask_names = sorted(glob(join(mdir, '*.png')))[start:start + T]
try:
# get image
imgs = []
for img_name in img_names:
img = cv2.imread(img_name)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
# transform = tf.Compose([
# tf.ToTensor(),
# tf.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),
# ])
transform = lambda x: torch.tensor(x).permute(2, 0, 1)
img = transform(img).unsqueeze(0).float()
imgs.append(img)
imgs = torch.cat(imgs, dim=0)
H, W = imgs.shape[-2:]
if self.resize is not None:
imgs = F.interpolate(imgs, size=self.resize,
mode='bilinear').permute(1, 0, 2, 3)
else:
imgs = imgs.permute(1, 0, 2, 3)
# get flow
if self.gt_flow:
flows = []
for flow_name in flow_names:
flow = read(flow_name)
flow = torch.tensor(flow[..., :2].copy()).permute(
2, 0, 1).unsqueeze(0)
flows.append(flow)
# flows.append(torch.zeros_like(flows[0]))
flows = torch.cat(flows, dim=0)
inv_flows = []
# inv_flows.append(torch.zeros_like(flows[:1]))
for inv_flow_name in inv_flow_names:
inv_flow = read(inv_flow_name)
inv_flow = torch.tensor(inv_flow[..., :2].copy()).permute(
2, 0, 1).unsqueeze(0)
inv_flows.append(inv_flow)
inv_flows = torch.cat(inv_flows, dim=0)
if self.resize is not None:
H, W = flows.shape[-2:]
flows = F.interpolate(flows,
size=self.resize,
mode='bilinear')
flows = flows.permute(1, 0, 2, 3)
flows[0] = flows[0] / W * self.resize[1]
flows[1] = flows[1] / H * self.resize[0]
inv_flows = F.interpolate(inv_flows,
size=self.resize,
mode='bilinear')
inv_flows = inv_flows.permute(1, 0, 2, 3)
inv_flows[0] = inv_flows[0] / W * self.resize[1]
inv_flows[1] = inv_flows[1] / H * self.resize[0]
else:
flows = flows.permute(1, 0, 2, 3)
inv_flows = inv_flows.permute(1, 0, 2, 3)
else:
flows, inv_flows = generate_flow(self.model, imgs)
# get mask
masks = []
for mask_name in mask_names:
mask = cv2.imread(mask_name)[..., 0]
mask = torch.tensor(mask) > 0
mask = mask.float()[None, None, ...]
masks.append(mask)
masks = torch.cat(masks, dim=0)
if self.resize is not None:
masks = F.interpolate(masks,
size=self.resize).permute(1, 0, 2, 3)
else:
masks = masks.permute(1, 0, 2, 3)
# get label
labels = []
for label_name in object_id_names:
label = read(label_name)
label = torch.tensor(label.copy())[None, None, ...]
labels.append(label)
labels = torch.cat(labels, dim=0)
# print(labels.shape)
if self.resize is not None:
labels = F.interpolate(labels,
size=self.resize).permute(1, 0, 2, 3)
else:
labels = labels.permute(1, 0, 2, 3)
labels = masks * labels + (1 - masks) * -1
objs = np.sort(np.unique(labels))
# print(objs)
n_clusters = len(objs) - 1
for i, obj in enumerate(objs):
labels[labels == obj] = i
one_hot_labels = F.one_hot(labels.squeeze().long())
one_hot_labels = one_hot_labels.permute(3, 0, 1, 2)[1:]
assert not torch.isnan(imgs).any()
assert not torch.isnan(flows).any()
assert not torch.isnan(inv_flows).any()
assert not torch.isnan(masks).any()
assert not torch.isnan(one_hot_labels).any()
except:
with open('bad_data.txt', 'a+') as f:
f.write(idir + '\n')
return 0, 0, 0, 0, 0, 0, 0
# print(one_hot_labels.shape)
return imgs, flows, inv_flows, masks, one_hot_labels, n_clusters, idir
def __getitem__(self, idx):
T = self.T
i = 0
while idx >= self.video_len[i]:
idx -= self.video_len[i]
i += 1
img_dir, flow_dir, inv_flow_dir, label_dir = self.dir_list[i]
img_names = sorted(glob(join(img_dir, '*.jpg')))[idx:idx + T]
flow_names = sorted(glob(join(flow_dir, '*.flo')))[idx:idx + T]
inv_flow_names = sorted(glob(join(inv_flow_dir, '*.flo')))[idx:idx + T]
label_names = sorted(glob(join(label_dir, '*.png')))[idx:idx + T]
try:
# get image
imgs = [Image.open(img_name) for img_name in img_names]
imgs = self.aug.color_warp(imgs)
transform = tf.Compose([
tf.ToTensor(),
tf.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
])
imgs = [transform(img) for img in imgs]
imgs = torch.stack(imgs)
# imgs = []
# for img in _imgs:
# img = np.asarray(img)
# transform = lambda x: torch.tensor(x).permute(2, 0, 1)
# img = transform(img).unsqueeze(0).float()
# imgs.append(img)
# imgs = torch.cat(imgs, dim=0)
# get flow
flows = []
for flow_name in flow_names:
flow = read(flow_name)
flow = torch.tensor(flow).permute(2, 0, 1).unsqueeze(0)
flows.append(flow)
# flows.append(torch.zeros_like(flows[0]))
flows = torch.cat(flows, dim=0)
inv_flows = []
# inv_flows.append(torch.zeros_like(flows[:1]))
for inv_flow_name in inv_flow_names:
inv_flow = read(inv_flow_name)
inv_flow = torch.tensor(inv_flow).permute(2, 0, 1).unsqueeze(0)
inv_flows.append(inv_flow)
inv_flows = torch.cat(inv_flows, dim=0)
# get label
labels = []
for label_name in label_names:
label = cv2.imread(label_name)
label = torch.tensor(label).float()
label = label[
..., 0] + label[..., 1] * 255 + label[..., 2] * 255 * 255
label = label[None, None, ...]
labels.append(label)
labels = torch.cat(labels, dim=0)
objs = torch.unique(labels)
for i, obj in enumerate(objs):
labels[labels == obj] = i
# print(labels.shape)
masks = (labels > 0).float()
imgs, flows, inv_flows, masks, labels = resize(
self.aug((imgs, flows, inv_flows, masks, labels)), self.resize)
one_hot_labels = F.one_hot(labels.squeeze().long())
one_hot_labels = one_hot_labels.permute(3, 0, 1, 2)[1:]
n_clusters = one_hot_labels.size(0)
assert not torch.isnan(imgs).any()
assert not torch.isnan(flows).any()
assert not torch.isnan(inv_flows).any()
assert not torch.isnan(one_hot_labels).any()
assert n_clusters > 0
except:
with open('bad_data.txt', 'a+') as f:
f.write(img_dir + '\n')
return 0, 0, 0, 0, 0, 0, 0
return imgs, flows, inv_flows, masks, one_hot_labels, n_clusters, img_dir