def _load_videos(path): tags = SEQUENCES folders = [path + "GroundTruth/" + s + "/" for s in SEQUENCES] img_sets = [path + "ImageSets/" + s.split("/")[0] + ".txt" for s in SEQUENCES] videos = [] for seq, folder, img_set in zip(SEQUENCES, folders, img_sets): if seq in GT_BMP_SEQUENCES: ending = ".bmp" else: ending = ".png" fns = [l.strip() for l in open(img_set).readlines()[1:]] #TODO label_files = [folder + fn + ending for fn in fns] assert len(label_files) > 0 img_files = [f.replace("/GroundTruth/", "/JPEGImages/").replace(seq, seq.split("/")[0]) for f in label_files] if seq.split("/")[0] in BMP_SEQUENCES: img_files = [f.replace(".png", ".bmp") for f in img_files] video = [] for img_file, label_file in zip(img_files, label_files): img_val = scipy.ndimage.imread(img_file) / 255.0 label_val = scipy.ndimage.imread(label_file) / 255 label_val = label_val[..., :1] assert label_val.ndim == 3, label_val.ndim #label_val = numpy.expand_dims(label_val, 2) tag = seq.replace("/", "_") + "/" + label_file.split("/")[-1] tensors = create_tensor_dict(unnormalized_img=img_val, label=label_val, tag=tag) video.append(tensors) videos.append(video) return tags, videos
def _load_frame(idx, im, an, imgs, flow_dir, flow_into_past, flow_into_future, flow_as_angle): im_val = scipy.ndimage.imread(im) / 255.0 flow_past, flow_future = _load_flows(idx, imgs, im_val.shape, flow_dir, flow_into_past, flow_into_future, flow_as_angle) # try: an_raw = scipy.ndimage.imread(an) # except: # print('Empty ann ', an) # import numpy as np # an_raw = np.zeros_like(im_val) if "adaptation" in an.split("/")[-1]: an_postproc = an_raw an_postproc[an_raw == 128] = 1 else: an_postproc = an_raw / 255 an_val = numpy.expand_dims(an_postproc, 2) tag_val = im tensors = create_tensor_dict(unnormalized_img=im_val, label=an_val, tag=tag_val, flow_past=flow_past, flow_future=flow_future) return tensors
def _load_frame(im, an): sp = im.split("/") if "__" in sp[-2]: seq_full = sp[-2] seq_base = seq_full.split("__")[0] im_val = scipy.ndimage.imread(im.replace(seq_full, seq_base)) / 255.0 if an is None: an_postproc = numpy.zeros(im_val.shape[:2], numpy.uint8) else: # STEFANO EDIT --> IT WON?T WORK ON MULTI LABEL MASKS; THIS WAY!!!! # an_raw = scipy.ndimage.imread(an.replace(seq_full, seq_base)) # an_postproc = (an_raw == color).astype(numpy.uint8).min(axis=2) id_ = numpy.array(int(im.split("__")[1].split("/")[-2])) # load like this to prevent scipy from converting the palette indices to rgbº # an_raw = numpy.array(Image.open(an.replace(seq_full, seq_base))) # an_postproc = (an_raw == id_).astype(numpy.uint8) an_raw = cv2.imread(an.replace(seq_full, seq_base), cv2.IMREAD_GRAYSCALE) an_postproc = an_raw an_postproc[an_raw>0] = 1 else: im_val = scipy.ndimage.imread(im) / 255.0 if an is None: an_postproc = numpy.zeros(im_val.shape[:2], numpy.uint8) else: an_postproc = (numpy.array(Image.open(an)) > 0).astype("uint8") an_val = numpy.expand_dims(an_postproc, 2) tag_val = im tensors = create_tensor_dict(unnormalized_img=im_val, label=an_val, tag=tag_val) return tensors
def create_feed_dict(self, img, label, tag, old_label=None, flow_past=None, flow_future=None, u0=None, u1=None): if old_label is not None and self.old_label_as_distance_transform: old_label = create_old_mask(old_label[:, :, 0]) old_label = old_label[:, :, np.newaxis] tensors = create_tensor_dict(unnormalized_img=img, label=label, tag=tag, old_label=old_label, flow_past=flow_past, flow_future=flow_future, u0=u0, u1=u1) self.feed_dict = {self.img_placeholder: tensors["unnormalized_img"], self.label_placeholder: tensors["label"], self.tag_placeholder: tensors["tag"]} if "old_label" in tensors: self.feed_dict[self.old_label_placeholder] = tensors["old_label"] if "flow_past" in tensors: self.feed_dict[self.flow_into_past_placeholder] = tensors["flow_past"] if "flow_future" in tensors: self.feed_dict[self.flow_into_future_placeholder] = tensors["flow_future"] if Constants.DT_NEG in tensors: self.feed_dict[self.u0_placeholder] = tensors[Constants.DT_NEG] if Constants.DT_POS in tensors: self.feed_dict[self.u1_placeholder] = tensors[Constants.DT_POS] return self.feed_dict
def _load_lucid_data_for_seq(self, video_idx): seq = self.video_tag(video_idx) folder = self.lucid_data_dir + seq imgs = glob.glob(folder + "/*.jpg") tags = imgs masks = [x.replace(".jpg", ".png") for x in imgs] imgs = [scipy.ndimage.imread(im) / 255.0 for im in imgs] masks = [numpy.expand_dims(scipy.ndimage.imread(mask) / 255, axis=2) for mask in masks] tensors = [create_tensor_dict(unnormalized_img=im, label=mask, tag=tag) for im, mask, tag in zip(imgs, masks, tags)] return tensors
def _load_frame(idx, im, an, imgs, subset, flow_dir, flow_into_past, flow_into_future, flow_as_angle, use_old_label): im_val = scipy.ndimage.imread(im) / 255.0 flow_past, flow_future = _load_flows(idx, imgs, im_val.shape, flow_dir, flow_into_past, flow_into_future, flow_as_angle) if an != "": an_raw = scipy.ndimage.imread(an) if "adaptation" in an.split("/")[-1]: an_postproc = an_raw an_postproc[an_raw == 128] = 1 else: an_postproc = an_raw / 255 # if subset == "train": # Create a bounding box image else: an_postproc = np.zeros_like(im_val[:, :, 0]) an_bbox = get_bounding_box(an_postproc, 1) an_bbox = np.expand_dims(an_bbox, 2) an_val = np.expand_dims(an_postproc, 2) tag_val = im if use_old_label: tensors = create_tensor_dict(unnormalized_img=im_val, label=an_val, tag=tag_val, flow_past=flow_past, flow_future=flow_future, old_label=an_bbox) else: tensors = create_tensor_dict(unnormalized_img=im_val, label=an_val, tag=tag_val, flow_past=flow_past, flow_future=flow_future) return tensors
def _read_inputfiles(self, queue, resize_mode, input_size, augmentors): im_path = queue[0] label_path = queue[1] old_label_path = queue[2] img = load_image_tensorflow(im_path, jpg=True) old_label = load_png_mask_tensorflow(old_label_path) def my_damage_mask(mask): return damage_mask(mask, self.old_mask_scale_factor, self.old_mask_shift_absolute, self.old_mask_shift_factor) old_label, = tf.py_func(my_damage_mask, [old_label], [tf.float32]) label = load_png_mask_tensorflow(label_path) flow_past = flow_future = None if self.flow_into_past: flow_past_path = queue[3] flow_past = load_flow_from_flo_tensorflow(flow_past_path, self.flow_as_angle) if self.flow_into_future: flow_future_path = queue[4] if self.flow_into_past else queue[3] flow_future = load_flow_from_flo_tensorflow( flow_future_path, self.flow_as_angle) tensors = create_tensor_dict(unnormalized_img=img, label=label, tag=im_path, raw_label=label, old_label=old_label, flow_past=flow_past, flow_future=flow_future) resize_mode, input_size = self._get_resize_params( self.subset, self.image_size) tensors = resize(tensors, resize_mode, input_size) for augmentor in augmentors: tensors = augmentor.apply(tensors) tensors = assemble_input_tensors(tensors, DAVIS_IMAGE_SIZE) label.set_shape(list(DAVIS_IMAGE_SIZE) + [1]) summaries = [] return tensors, summaries
def _load_lucid_data_for_seq(self, video_idx): seq = self.video_tag(video_idx) assert "__" in seq, "for merged case not implemented yet" folder = self.lucid_data_dir + seq imgs = glob.glob(folder + "/*.jpg") tags = imgs masks = [x.replace(".jpg", ".png") for x in imgs] imgs = [scipy.ndimage.imread(im) / 255.0 for im in imgs] masks = [numpy.expand_dims(scipy.ndimage.imread(mask) / 255, axis=2) for mask in masks] # id_ = numpy.array(int(seq.split("__")[1])) # masks = [numpy.array(Image.open(mask)) for mask in masks] # masks = [(mask == id_).astype(numpy.uint8) for mask in masks] # masks = [numpy.expand_dims(mask, 2) for mask in masks] tensors = [create_tensor_dict(unnormalized_img=im, label=mask, tag=tag) for im, mask, tag in zip(imgs, masks, tags)] return tensors
def create_input_tensors_dict(self, batch_size): use_index_img = self.subset != "train" tensors = create_tensor_dict( unnormalized_img=self.img_placeholder, label=self.label_placeholder, tag=self.tag_placeholder, raw_label=self.label_placeholder, old_label=self.old_label_placeholder, flow_past=self.flow_into_past_placeholder, flow_future=self.flow_into_future_placeholder, use_index_img=use_index_img, u0=self.u0_placeholder, u1=self.u1_placeholder) # TODO: need to set shape here? resize_mode, input_size = self._get_resize_params( self.subset, self.image_size) tensors = resize(tensors, resize_mode, input_size) if len(input_size) == 3: input_size = input_size[1:] tensors = self._prepare_augmented_batch(tensors, batch_size, image_size=input_size) if self.use_summaries: inputs = tensors["inputs"] summ0 = tf.summary.image("inputs", unnormalize(inputs[:, :, :, :3])) summ1 = tf.summary.image( "labels", tensors["labels"] * 255) # will only work well for binary segmentation self.summaries.append(summ0) self.summaries.append(summ1) # Old label would be present either as a single channel, or along with 2 other distance transform channels. if inputs.get_shape()[-1] == 4 or inputs.get_shape == 6: summ2 = tf.summary.image("old_labels", inputs[:, :, :, 3:4] * 255) self.summaries.append(summ2) return tensors
def create_feed_dict(self, img, label, tag, old_label=None, flow_past=None, flow_future=None, u0=None, u1=None, flow=None): tensors = create_tensor_dict(unnormalized_img=img, label=label, tag=tag, old_label=old_label, flow_past=flow_past, flow_future=flow_future, u0=u0, u1=u1, flow=flow) self.feed_dict = { self.img_placeholder: tensors["unnormalized_img"], self.flow_placeholder: tensors["flow"], self.label_placeholder: tensors["label"], self.tag_placeholder: tensors["tag"] } if "old_label" in tensors: self.feed_dict[self.old_label_placeholder] = tensors["old_label"] if "flow_past" in tensors: self.feed_dict[ self.flow_into_past_placeholder] = tensors["flow_past"] if "flow_future" in tensors: self.feed_dict[ self.flow_into_future_placeholder] = tensors["flow_future"] if Constants.DT_NEG in tensors: self.feed_dict[self.u0_placeholder] = tensors[Constants.DT_NEG] if Constants.DT_POS in tensors: self.feed_dict[self.u1_placeholder] = tensors[Constants.DT_POS] return self.feed_dict
def _load_video(folder): label_files = sorted(glob.glob(folder + "*.jpg")) assert len(label_files) > 0 indices = [int(f.split("/")[-1][:-4]) for f in label_files] min_idx = indices[0] assert min_idx == min(indices) max_idx = max(indices) img_folder = folder.replace("/labels/", "/images/") video = [] shape = None for idx in xrange(min_idx, max_idx + 1): img_file = img_folder + ("frame%04d.jpg" % idx) if idx in indices: label_file = label_files[indices.index(idx)] label_val = scipy.ndimage.imread(label_file) / 255 label_val = numpy.expand_dims(label_val, 2) shape = label_val.shape else: assert shape is not None label_val = numpy.zeros(shape, dtype=numpy.uint8) img_val = scipy.ndimage.imread(img_file) #/ 255.0 #scale down to match label here! assert shape is not None #assert img_val.shape[0] == 2 * shape[0] #assert img_val.shape[1] == 2 * shape[1] img_val = imresize(img_val, shape[:2]) img_val = img_val.astype("float32") / 255.0 tag = img_file.split("/")[-7] + "_" + img_file.split( "/")[-5] + "/" + img_file.split("/")[-1] if idx not in indices: tag += "_DO_NOT_STORE_LOGITS" tensors = create_tensor_dict(unnormalized_img=img_val, label=label_val, tag=tag) video.append(tensors) return video
def _load_frame(idx, im, an, imgs, flow_dir, flow_into_past, flow_into_future, flow_as_angle): im_val = scipy.ndimage.imread(im) / 255.0 flow_past, flow_future = _load_flows(idx, imgs, im_val.shape, flow_dir, flow_into_past, flow_into_future, flow_as_angle) if an != "": an_raw = scipy.ndimage.imread(an) if "adaptation" in an.split("/")[-1]: an_postproc = an_raw an_postproc[an_raw == 128] = 1 else: an_postproc = an_raw / 255 else: an_postproc = numpy.zeros_like(im_val[:, :, 0]) an_val = numpy.expand_dims(an_postproc, 2) tag_val = im tensors = create_tensor_dict(unnormalized_img=im_val, label=an_val, tag=tag_val, flow_past=flow_past, flow_future=flow_future) return tensors
def _load_videos(path): pattern = path + "*/data/*/*/*/images/" folders = sorted(glob.glob(pattern)) #filter out the 2 sequences, which only have a single annotated frame folders = [f for f in folders if "motorbike/data/0007/shots/001" not in f and "cow/data/0019/shots/001" not in f] tags = [f.split("/")[-7] + "_" + f.split("/")[-5] for f in folders] videos = [] for folder in folders: img_files = sorted(glob.glob(folder + "*.png")) label_files = [f.replace("/images/", "/labels/").replace(".png", ".jpg") for f in img_files] video = [] for img_file, label_file in zip(img_files, label_files): img_val = scipy.ndimage.imread(img_file) / 255.0 label_val = scipy.ndimage.imread(label_file) / 255 label_val = numpy.expand_dims(label_val, 2) tag = img_file.split("/")[-7] + "_" + img_file.split("/")[-5] + "/" + img_file.split("/")[-1] tensors = create_tensor_dict(unnormalized_img=img_val, label=label_val, tag=tag) video.append(tensors) videos.append(video) return tags, videos
def create_input_tensors_dict(self, batch_size): use_index_img = self.subset != "train" tensors = create_tensor_dict( unnormalized_img=self.img_placeholder, label=self.label_placeholder, tag=self.tag_placeholder, raw_label=self.label_placeholder, old_label=self.old_label_placeholder, flow_past=self.flow_into_past_placeholder, flow_future=self.flow_into_future_placeholder, use_index_img=use_index_img, u0=self.u0_placeholder, u1=self.u1_placeholder, flow=self.flow_placeholder) # TODO: need to set shape here? resize_mode, input_size = self._get_resize_params( self.subset, self.image_size) tensors = resize(tensors, resize_mode, input_size) if len(input_size) == 3: input_size = input_size[1:] tensors = self._prepare_augmented_batch(tensors, batch_size, image_size=input_size) if self.use_summaries: inputs = tensors["inputs"] summ0 = tf.summary.image("inputs", unnormalize(inputs[:, :, :, :3])) summ1 = tf.summary.image( "labels", tensors["labels"] * 255) # will only work well for binary segmentation self.summaries.append(summ0) self.summaries.append(summ1) return tensors