def test_sintel(self): for dataset in ['sintel-test-clean', 'sintel-test-final', 'sintel-train-clean', 'sintel-train-final']: size = DATASETS_AND_SIZE[dataset] path = dataset_locations[dataset] ds = sintel.make_dataset(path, mode='test') self._check_size(ds, size)
def test_sintel_eval(self): dataset = sintel.make_dataset( path=dataset_locations['sintel-train-clean'], mode='eval-occlusion') dataset = dataset.take(1) results = sintel.evaluate(mock_inference_fn, dataset, height=200, width=400, num_plots=0, plot_dir='/tmp/sintel') expected_keys = sintel.list_eval_keys() self.assertEqual(set(expected_keys), set(results.keys()))
def test_sintel(self): dataset = sintel.make_dataset( path=dataset_locations['sintel-train-clean'], mode='eval-occlusion') dataset = dataset.prefetch(1) dataset = dataset.batch(1) data_it = tf.compat.v1.data.make_one_shot_iterator(dataset) data = data_it.next() image1 = data[0][:, 0] image2 = data[0][:, 1] flow = data[1] self._check_images_and_flow( image1, image2, flow, save_images=False, plot_dir='/tmp/sintel')
def make_train_iterator( train_on, height, width, shuffle_buffer_size, batch_size, seq_len, crop_instead_of_resize=False, apply_augmentation=True, include_ground_truth=False, resize_gt_flow=True, include_occlusions=False, seed=41, mode='train', ): """Build joint training iterator for all data in train_on. Args: train_on: string of the format 'format0:path0;format1:path1', e.g. 'kitti:/usr/local/home/...'. height: int, height to which the images will be resized or cropped. width: int, width to which the images will be resized or cropped. shuffle_buffer_size: int, size that will be used for the shuffle buffer. batch_size: int, batch size for the iterator. seq_len: int, number of frames per sequences (at the moment this should always be 2) crop_instead_of_resize: bool, indicates if cropping should be used instead of resizing apply_augmentation: bool, indicates if geometric and photometric data augmentation shall be activated (paramaters are gin configurable) include_ground_truth: bool, if True, return ground truth optical flow with the training images. This only exists for some datasets (Kitti, Sintel). resize_gt_flow: bool, indicates if ground truth flow should be resized (only important if resizing and supervised training is used) include_occlusions: bool, indicates if ground truth occlusions should be loaded (currently not supported in combination with augmentation) seed: A seed for a random number generator, controls shuffling of data. mode: str, will be passed on to the data iterator class. Can be used to specify different settings within the data iterator. Returns: A tf.data.Iterator that produces batches of images of shape [batch size, sequence length=3, height, width, channels=3] """ train_datasets = [] # Split strings according to pattern "format0:path0;format1:path1". for format_and_path in train_on.split(';'): data_format, path = format_and_path.split(':') if include_occlusions: mode += '-include-occlusions' if include_ground_truth: mode += '-supervised' if include_occlusions and 'sintel' not in data_format: raise ValueError( 'The parameter include_occlusions is only supported for' 'sintel data.') if include_ground_truth and ('chairs' not in data_format and 'sintel' not in data_format and 'kitti' not in data_format): raise NotImplementedError( 'The parameter include_ground_truth is only' 'supported for flying_chairs, sintel, kitti and' 'wod data at the moment.') # Add a dataset based on format and path. if 'kitti' in data_format: dataset = kitti.make_dataset( path, mode=mode, seq_len=seq_len, shuffle_buffer_size=shuffle_buffer_size, height=None if crop_instead_of_resize else height, width=None if crop_instead_of_resize else width, resize_gt_flow=resize_gt_flow, seed=seed, ) elif 'chairs' in data_format: dataset = flow_dataset.make_dataset( path, mode=mode, seq_len=seq_len, shuffle_buffer_size=shuffle_buffer_size, height=None if crop_instead_of_resize else height, width=None if crop_instead_of_resize else width, resize_gt_flow=resize_gt_flow, gt_flow_shape=[384, 512, 2], seed=seed, ) elif 'sintel' in data_format: dataset = sintel.make_dataset( path, mode=mode, seq_len=seq_len, shuffle_buffer_size=shuffle_buffer_size, height=None if crop_instead_of_resize else height, width=None if crop_instead_of_resize else width, resize_gt_flow=resize_gt_flow, seed=seed, ) else: print('Unknown data format "{}"'.format(data_format)) continue train_datasets.append(dataset) # prepare augmentation function # in case no crop is desired set it to the size images have been resized to # This will fail if none or both are specified. augmentation_fn = partial(uflow_augmentation.apply_augmentation, crop_height=height, crop_width=width) # returns a function to apply ensure_shape on all the available data def _ensure_shapes(): # shape of the data imgs_shape = (batch_size, seq_len, height, width, 3) if resize_gt_flow: flow_shape = (batch_size, height, width, 2) valid_shape = (batch_size, height, width, 1) else: flow_shape = (batch_size, None, None, 2) valid_shape = (batch_size, None, None, 1) occ_shape = (batch_size, height, width, 1) # different cases of data combinations if include_ground_truth and apply_augmentation: return lambda imgs, imgs_na, flow, valid: (tf.ensure_shape( imgs, imgs_shape), { 'images_without_photo_aug': tf.ensure_shape(imgs_na, imgs_shape), 'flow_uv': tf.ensure_shape(flow, flow_shape), 'flow_valid': tf.ensure_shape(valid, valid_shape) }) elif include_ground_truth and include_occlusions: return lambda imgs, flow, valid, occ: ( tf.ensure_shape(imgs, imgs_shape), { 'flow_uv': tf.ensure_shape(flow, flow_shape), 'flow_valid': tf.ensure_shape(valid, valid_shape), 'occlusions': tf.ensure_shape(occ, occ_shape) }) elif include_ground_truth: return lambda imgs, flow, valid: ( tf.ensure_shape(imgs, imgs_shape), { 'flow_uv': tf.ensure_shape(flow, flow_shape), 'flow_valid': tf.ensure_shape(valid, valid_shape) }) elif include_occlusions: return lambda imgs, occ: (tf.ensure_shape(imgs, imgs_shape), { 'occlusions': tf.ensure_shape(occ, occ_shape) }) elif apply_augmentation: return lambda imgs, imgs_na: (tf.ensure_shape(imgs, imgs_shape), { 'images_without_photo_aug': tf.ensure_shape(imgs_na, imgs_shape) }) else: return lambda imgs: (tf.ensure_shape(imgs, imgs_shape), {}) train_ds = train_datasets[0] # Perform data augmentation # This cannot handle occlusions at the moment. if apply_augmentation: train_ds = train_ds.map(augmentation_fn) train_ds = train_ds.batch(batch_size) train_ds = train_ds.prefetch(1) train_ds = train_ds.map(_ensure_shapes()) train_it = tf.compat.v1.data.make_one_shot_iterator(train_ds) return train_it
def make_eval_function(eval_on, height, width, progress_bar, plot_dir, num_plots): """Build an evaluation function for uflow. Args: eval_on: string of the format 'format0:path0;format1:path1', e.g. 'kitti:/usr/local/home/...'. height: int, the height to which the images should be resized for inference. width: int, the width to which the images should be resized for inference. progress_bar: boolean, flag to indicate whether the function should print a progress_bar during evaluaton. plot_dir: string, optional path to a directory in which plots are saved (if num_plots > 0). num_plots: int, maximum number of qualitative results to plot for the evaluation. Returns: A pair consisting of an evaluation function and a list of strings that holds the keys of the evaluation result. """ eval_functions_and_datasets = [] eval_keys = [] # Split strings according to pattern "format0:path0;format1:path1". for format_and_path in eval_on.split(';'): data_format, path = format_and_path.split(':') # Add a dataset based on format and path. if 'kitti' in data_format: if 'benchmark' in data_format: dataset = kitti.make_dataset(path, mode='test') eval_fn = kitti.benchmark else: dataset = kitti.make_dataset(path, mode='eval') eval_fn = partial(kitti.evaluate, prefix=data_format) eval_keys += kitti.list_eval_keys(prefix=data_format) elif 'chairs' in data_format or 'custom' in data_format: dataset = flow_dataset.make_dataset(path, mode='eval') eval_fn = partial( flow_dataset.evaluate, prefix=data_format, max_num_evals= 1000, # We do this to avoid evaluating on 22k samples. has_occlusion=False) eval_keys += flow_dataset.list_eval_keys(prefix=data_format) elif 'sintel' in data_format: if 'benchmark' in data_format: # pylint:disable=g-long-lambda # pylint:disable=cell-var-from-loop eval_fn = lambda uflow: sintel.benchmark(inference_fn=uflow. infer, height=height, width=width, sintel_path=path, plot_dir=plot_dir, num_plots=num_plots) if len(eval_on.split(';')) != 1: raise ValueError( 'Sintel benchmark should be done in isolation.') return eval_fn, [] dataset = sintel.make_dataset(path, mode='eval-occlusion') eval_fn = partial(sintel.evaluate, prefix=data_format) eval_keys += sintel.list_eval_keys(prefix=data_format) else: print('Unknown data format "{}"'.format(data_format)) continue dataset = dataset.prefetch(1) eval_functions_and_datasets.append((eval_fn, dataset)) # Make an eval function that aggregates all evaluations. def eval_function(uflow): result = dict() for eval_fn, ds in eval_functions_and_datasets: results = eval_fn(uflow.infer, ds, height, width, progress_bar, plot_dir, num_plots) for k, v in results.items(): result[k] = v return result return eval_function, eval_keys