def _load_videos(path):
tags = SEQUENCES
folders = [path + "GroundTruth/" + s + "/" for s in SEQUENCES]
img_sets = [path + "ImageSets/" + s.split("/")[0] + ".txt" for s in SEQUENCES]
videos = []
for seq, folder, img_set in zip(SEQUENCES, folders, img_sets):
if seq in GT_BMP_SEQUENCES:
ending = ".bmp"
else:
ending = ".png"
fns = [l.strip() for l in open(img_set).readlines()[1:]]
#TODO
label_files = [folder + fn + ending for fn in fns]
assert len(label_files) > 0
img_files = [f.replace("/GroundTruth/", "/JPEGImages/").replace(seq, seq.split("/")[0]) for f in label_files]
if seq.split("/")[0] in BMP_SEQUENCES:
img_files = [f.replace(".png", ".bmp") for f in img_files]
video = []
for img_file, label_file in zip(img_files, label_files):
img_val = scipy.ndimage.imread(img_file) / 255.0
label_val = scipy.ndimage.imread(label_file) / 255
label_val = label_val[..., :1]
assert label_val.ndim == 3, label_val.ndim
#label_val = numpy.expand_dims(label_val, 2)
tag = seq.replace("/", "_") + "/" + label_file.split("/")[-1]
tensors = create_tensor_dict(unnormalized_img=img_val, label=label_val, tag=tag)
video.append(tensors)
videos.append(video)
return tags, videos
def _load_frame(idx, im, an, imgs, flow_dir, flow_into_past, flow_into_future,
flow_as_angle):
im_val = scipy.ndimage.imread(im) / 255.0
flow_past, flow_future = _load_flows(idx, imgs, im_val.shape, flow_dir,
flow_into_past, flow_into_future,
flow_as_angle)
# try:
an_raw = scipy.ndimage.imread(an)
# except:
# print('Empty ann ', an)
# import numpy as np
# an_raw = np.zeros_like(im_val)
if "adaptation" in an.split("/")[-1]:
an_postproc = an_raw
an_postproc[an_raw == 128] = 1
else:
an_postproc = an_raw / 255
an_val = numpy.expand_dims(an_postproc, 2)
tag_val = im
tensors = create_tensor_dict(unnormalized_img=im_val,
label=an_val,
tag=tag_val,
flow_past=flow_past,
flow_future=flow_future)
return tensors
def _load_frame(im, an):
sp = im.split("/")
if "__" in sp[-2]:
seq_full = sp[-2]
seq_base = seq_full.split("__")[0]
im_val = scipy.ndimage.imread(im.replace(seq_full, seq_base)) / 255.0
if an is None:
an_postproc = numpy.zeros(im_val.shape[:2], numpy.uint8)
else: # STEFANO EDIT --> IT WON?T WORK ON MULTI LABEL MASKS; THIS WAY!!!!
# an_raw = scipy.ndimage.imread(an.replace(seq_full, seq_base))
# an_postproc = (an_raw == color).astype(numpy.uint8).min(axis=2)
id_ = numpy.array(int(im.split("__")[1].split("/")[-2]))
# load like this to prevent scipy from converting the palette indices to rgbº
# an_raw = numpy.array(Image.open(an.replace(seq_full, seq_base)))
# an_postproc = (an_raw == id_).astype(numpy.uint8)
an_raw = cv2.imread(an.replace(seq_full, seq_base), cv2.IMREAD_GRAYSCALE)
an_postproc = an_raw
an_postproc[an_raw>0] = 1
else:
im_val = scipy.ndimage.imread(im) / 255.0
if an is None:
an_postproc = numpy.zeros(im_val.shape[:2], numpy.uint8)
else:
an_postproc = (numpy.array(Image.open(an)) > 0).astype("uint8")
an_val = numpy.expand_dims(an_postproc, 2)
tag_val = im
tensors = create_tensor_dict(unnormalized_img=im_val, label=an_val, tag=tag_val)
return tensors
def create_feed_dict(self, img, label, tag, old_label=None, flow_past=None, flow_future=None,
u0=None, u1=None):
if old_label is not None and self.old_label_as_distance_transform:
old_label = create_old_mask(old_label[:, :, 0])
old_label = old_label[:, :, np.newaxis]
tensors = create_tensor_dict(unnormalized_img=img, label=label, tag=tag, old_label=old_label,
flow_past=flow_past, flow_future=flow_future,
u0=u0, u1=u1)
self.feed_dict = {self.img_placeholder: tensors["unnormalized_img"],
self.label_placeholder: tensors["label"],
self.tag_placeholder: tensors["tag"]}
if "old_label" in tensors:
self.feed_dict[self.old_label_placeholder] = tensors["old_label"]
if "flow_past" in tensors:
self.feed_dict[self.flow_into_past_placeholder] = tensors["flow_past"]
if "flow_future" in tensors:
self.feed_dict[self.flow_into_future_placeholder] = tensors["flow_future"]
if Constants.DT_NEG in tensors:
self.feed_dict[self.u0_placeholder] = tensors[Constants.DT_NEG]
if Constants.DT_POS in tensors:
self.feed_dict[self.u1_placeholder] = tensors[Constants.DT_POS]
return self.feed_dict
def _load_lucid_data_for_seq(self, video_idx):
seq = self.video_tag(video_idx)
folder = self.lucid_data_dir + seq
imgs = glob.glob(folder + "/*.jpg")
tags = imgs
masks = [x.replace(".jpg", ".png") for x in imgs]
imgs = [scipy.ndimage.imread(im) / 255.0 for im in imgs]
masks = [numpy.expand_dims(scipy.ndimage.imread(mask) / 255, axis=2) for mask in masks]
tensors = [create_tensor_dict(unnormalized_img=im, label=mask, tag=tag) for im, mask, tag in zip(imgs, masks, tags)]
return tensors
def _load_frame(idx, im, an, imgs, subset, flow_dir, flow_into_past,
flow_into_future, flow_as_angle, use_old_label):
im_val = scipy.ndimage.imread(im) / 255.0
flow_past, flow_future = _load_flows(idx, imgs, im_val.shape, flow_dir,
flow_into_past, flow_into_future,
flow_as_angle)
if an != "":
an_raw = scipy.ndimage.imread(an)
if "adaptation" in an.split("/")[-1]:
an_postproc = an_raw
an_postproc[an_raw == 128] = 1
else:
an_postproc = an_raw / 255
# if subset == "train":
# Create a bounding box image
else:
an_postproc = np.zeros_like(im_val[:, :, 0])
an_bbox = get_bounding_box(an_postproc, 1)
an_bbox = np.expand_dims(an_bbox, 2)
an_val = np.expand_dims(an_postproc, 2)
tag_val = im
if use_old_label:
tensors = create_tensor_dict(unnormalized_img=im_val,
label=an_val,
tag=tag_val,
flow_past=flow_past,
flow_future=flow_future,
old_label=an_bbox)
else:
tensors = create_tensor_dict(unnormalized_img=im_val,
label=an_val,
tag=tag_val,
flow_past=flow_past,
flow_future=flow_future)
return tensors
def _read_inputfiles(self, queue, resize_mode, input_size, augmentors):
im_path = queue[0]
label_path = queue[1]
old_label_path = queue[2]
img = load_image_tensorflow(im_path, jpg=True)
old_label = load_png_mask_tensorflow(old_label_path)
def my_damage_mask(mask):
return damage_mask(mask, self.old_mask_scale_factor,
self.old_mask_shift_absolute,
self.old_mask_shift_factor)
old_label, = tf.py_func(my_damage_mask, [old_label], [tf.float32])
label = load_png_mask_tensorflow(label_path)
flow_past = flow_future = None
if self.flow_into_past:
flow_past_path = queue[3]
flow_past = load_flow_from_flo_tensorflow(flow_past_path,
self.flow_as_angle)
if self.flow_into_future:
flow_future_path = queue[4] if self.flow_into_past else queue[3]
flow_future = load_flow_from_flo_tensorflow(
flow_future_path, self.flow_as_angle)
tensors = create_tensor_dict(unnormalized_img=img,
label=label,
tag=im_path,
raw_label=label,
old_label=old_label,
flow_past=flow_past,
flow_future=flow_future)
resize_mode, input_size = self._get_resize_params(
self.subset, self.image_size)
tensors = resize(tensors, resize_mode, input_size)
for augmentor in augmentors:
tensors = augmentor.apply(tensors)
tensors = assemble_input_tensors(tensors, DAVIS_IMAGE_SIZE)
label.set_shape(list(DAVIS_IMAGE_SIZE) + [1])
summaries = []
return tensors, summaries
def _load_lucid_data_for_seq(self, video_idx):
seq = self.video_tag(video_idx)
assert "__" in seq, "for merged case not implemented yet"
folder = self.lucid_data_dir + seq
imgs = glob.glob(folder + "/*.jpg")
tags = imgs
masks = [x.replace(".jpg", ".png") for x in imgs]
imgs = [scipy.ndimage.imread(im) / 255.0 for im in imgs]
masks = [numpy.expand_dims(scipy.ndimage.imread(mask) / 255, axis=2) for mask in masks]
# id_ = numpy.array(int(seq.split("__")[1]))
# masks = [numpy.array(Image.open(mask)) for mask in masks]
# masks = [(mask == id_).astype(numpy.uint8) for mask in masks]
# masks = [numpy.expand_dims(mask, 2) for mask in masks]
tensors = [create_tensor_dict(unnormalized_img=im, label=mask, tag=tag) for im, mask, tag in zip(imgs, masks, tags)]
return tensors
def create_input_tensors_dict(self, batch_size):
use_index_img = self.subset != "train"
tensors = create_tensor_dict(
unnormalized_img=self.img_placeholder,
label=self.label_placeholder,
tag=self.tag_placeholder,
raw_label=self.label_placeholder,
old_label=self.old_label_placeholder,
flow_past=self.flow_into_past_placeholder,
flow_future=self.flow_into_future_placeholder,
use_index_img=use_index_img,
u0=self.u0_placeholder,
u1=self.u1_placeholder)
# TODO: need to set shape here?
resize_mode, input_size = self._get_resize_params(
self.subset, self.image_size)
tensors = resize(tensors, resize_mode, input_size)
if len(input_size) == 3:
input_size = input_size[1:]
tensors = self._prepare_augmented_batch(tensors,
batch_size,
image_size=input_size)
if self.use_summaries:
inputs = tensors["inputs"]
summ0 = tf.summary.image("inputs",
unnormalize(inputs[:, :, :, :3]))
summ1 = tf.summary.image(
"labels", tensors["labels"] *
255) # will only work well for binary segmentation
self.summaries.append(summ0)
self.summaries.append(summ1)
# Old label would be present either as a single channel, or along with 2 other distance transform channels.
if inputs.get_shape()[-1] == 4 or inputs.get_shape == 6:
summ2 = tf.summary.image("old_labels",
inputs[:, :, :, 3:4] * 255)
self.summaries.append(summ2)
return tensors
def create_feed_dict(self,
img,
label,
tag,
old_label=None,
flow_past=None,
flow_future=None,
u0=None,
u1=None,
flow=None):
tensors = create_tensor_dict(unnormalized_img=img,
label=label,
tag=tag,
old_label=old_label,
flow_past=flow_past,
flow_future=flow_future,
u0=u0,
u1=u1,
flow=flow)
self.feed_dict = {
self.img_placeholder: tensors["unnormalized_img"],
self.flow_placeholder: tensors["flow"],
self.label_placeholder: tensors["label"],
self.tag_placeholder: tensors["tag"]
}
if "old_label" in tensors:
self.feed_dict[self.old_label_placeholder] = tensors["old_label"]
if "flow_past" in tensors:
self.feed_dict[
self.flow_into_past_placeholder] = tensors["flow_past"]
if "flow_future" in tensors:
self.feed_dict[
self.flow_into_future_placeholder] = tensors["flow_future"]
if Constants.DT_NEG in tensors:
self.feed_dict[self.u0_placeholder] = tensors[Constants.DT_NEG]
if Constants.DT_POS in tensors:
self.feed_dict[self.u1_placeholder] = tensors[Constants.DT_POS]
return self.feed_dict
def _load_video(folder):
label_files = sorted(glob.glob(folder + "*.jpg"))
assert len(label_files) > 0
indices = [int(f.split("/")[-1][:-4]) for f in label_files]
min_idx = indices[0]
assert min_idx == min(indices)
max_idx = max(indices)
img_folder = folder.replace("/labels/", "/images/")
video = []
shape = None
for idx in xrange(min_idx, max_idx + 1):
img_file = img_folder + ("frame%04d.jpg" % idx)
if idx in indices:
label_file = label_files[indices.index(idx)]
label_val = scipy.ndimage.imread(label_file) / 255
label_val = numpy.expand_dims(label_val, 2)
shape = label_val.shape
else:
assert shape is not None
label_val = numpy.zeros(shape, dtype=numpy.uint8)
img_val = scipy.ndimage.imread(img_file) #/ 255.0
#scale down to match label here!
assert shape is not None
#assert img_val.shape[0] == 2 * shape[0]
#assert img_val.shape[1] == 2 * shape[1]
img_val = imresize(img_val, shape[:2])
img_val = img_val.astype("float32") / 255.0
tag = img_file.split("/")[-7] + "_" + img_file.split(
"/")[-5] + "/" + img_file.split("/")[-1]
if idx not in indices:
tag += "_DO_NOT_STORE_LOGITS"
tensors = create_tensor_dict(unnormalized_img=img_val,
label=label_val,
tag=tag)
video.append(tensors)
return video
def _load_frame(idx, im, an, imgs, flow_dir, flow_into_past, flow_into_future, flow_as_angle):
im_val = scipy.ndimage.imread(im) / 255.0
flow_past, flow_future = _load_flows(idx, imgs, im_val.shape, flow_dir, flow_into_past, flow_into_future,
flow_as_angle)
if an != "":
an_raw = scipy.ndimage.imread(an)
if "adaptation" in an.split("/")[-1]:
an_postproc = an_raw
an_postproc[an_raw == 128] = 1
else:
an_postproc = an_raw / 255
else:
an_postproc = numpy.zeros_like(im_val[:, :, 0])
an_val = numpy.expand_dims(an_postproc, 2)
tag_val = im
tensors = create_tensor_dict(unnormalized_img=im_val, label=an_val, tag=tag_val, flow_past=flow_past,
flow_future=flow_future)
return tensors
def _load_videos(path):
pattern = path + "*/data/*/*/*/images/"
folders = sorted(glob.glob(pattern))
#filter out the 2 sequences, which only have a single annotated frame
folders = [f for f in folders if "motorbike/data/0007/shots/001" not in f and "cow/data/0019/shots/001" not in f]
tags = [f.split("/")[-7] + "_" + f.split("/")[-5] for f in folders]
videos = []
for folder in folders:
img_files = sorted(glob.glob(folder + "*.png"))
label_files = [f.replace("/images/", "/labels/").replace(".png", ".jpg") for f in img_files]
video = []
for img_file, label_file in zip(img_files, label_files):
img_val = scipy.ndimage.imread(img_file) / 255.0
label_val = scipy.ndimage.imread(label_file) / 255
label_val = numpy.expand_dims(label_val, 2)
tag = img_file.split("/")[-7] + "_" + img_file.split("/")[-5] + "/" + img_file.split("/")[-1]
tensors = create_tensor_dict(unnormalized_img=img_val, label=label_val, tag=tag)
video.append(tensors)
videos.append(video)
return tags, videos
def create_input_tensors_dict(self, batch_size):
use_index_img = self.subset != "train"
tensors = create_tensor_dict(
unnormalized_img=self.img_placeholder,
label=self.label_placeholder,
tag=self.tag_placeholder,
raw_label=self.label_placeholder,
old_label=self.old_label_placeholder,
flow_past=self.flow_into_past_placeholder,
flow_future=self.flow_into_future_placeholder,
use_index_img=use_index_img,
u0=self.u0_placeholder,
u1=self.u1_placeholder,
flow=self.flow_placeholder)
# TODO: need to set shape here?
resize_mode, input_size = self._get_resize_params(
self.subset, self.image_size)
tensors = resize(tensors, resize_mode, input_size)
if len(input_size) == 3:
input_size = input_size[1:]
tensors = self._prepare_augmented_batch(tensors,
batch_size,
image_size=input_size)
if self.use_summaries:
inputs = tensors["inputs"]
summ0 = tf.summary.image("inputs",
unnormalize(inputs[:, :, :, :3]))
summ1 = tf.summary.image(
"labels", tensors["labels"] *
255) # will only work well for binary segmentation
self.summaries.append(summ0)
self.summaries.append(summ1)
return tensors
