def test_process_image_and_label(self): """Test DataPreprocessor.process_image_and_label. """ dataset = tf.data.Dataset.from_tensor_slices({ 'image': tf.constant(np.zeros((3, 3, 3))), 'label': tf.constant(np.zeros((3, 1))) }) dut = DataPreprocessor(dataset) # Lambda function with the external parameter @p a. dut.process_image_and_label(lambda image, label, a: (image - a - 1, label + a + 1), a=1) batch = dut.dataset.make_one_shot_iterator().get_next() image_equal_op = tf.equal(batch['image'], tf.constant(-np.ones((3, 3)) * 2)) label_equal_op = tf.equal(batch['label'], tf.constant(np.ones((1, )) * 2)) with self.test_session() as sess: for _ in range(3): for op in sess.run((image_equal_op, label_equal_op)): self.assertTrue(np.all(op))
def prepare_dataset() -> None: """Download dataset and split into training/dev/test data for experiments :return: None :rtype: None """ logger = logging.getLogger(__name__) logger.info('Running {}'.format(prepare_dataset.__name__)) cfg = ConfigLoader.load_config() random_seed = cfg['preprocessing']['random_seed'] RandUtils.set_random_seed(random_seed) logger.info('Downloading dataset and preparing splits') dp = DataPreprocessor(cfg['paths']['data_dir']) dataset_path = dp.download_dataset() dp.split_dataset(dataset_path) logger.info('Done')
resume_fullpath = None if options['resume_path']: resume_fullpath = os.path.expanduser(options['resume_path']) finetune_from = None if options['finetune_from']: finetune_from = options['finetune_from'] # Data part should live in cpu. with tf.device('/cpu:0'): # Read from tfrecord format data made by src.data.tfrecord.TFRecordWriter. train_dataset = read_tfrecord( os.path.join(options['tfdata_dir'], 'train_*.tfrecord')) train_data_processor = DataPreprocessor( dataset=train_dataset, num_parallel_calls=options['num_parallel_calls'], batch_size=options['batch_size'], shuffle_buffer_size=options['shuffle_buffer_size'], prefetch_buffer_size=options['prefetch_buffer_size']) val_dataset = read_tfrecord( os.path.join(options['tfdata_dir'], 'val_*.tfrecord')) val_data_processor = DataPreprocessor( dataset=val_dataset, num_parallel_calls=options['num_parallel_calls'], batch_size=1, shuffle_buffer_size=1, prefetch_buffer_size=1) # Pre-process training data. # Add more pre-procesing blocks. logging.info('Preprocess train data')
if not os.path.isdir(save_dir_fullpath): raise AttributeError('--save_dir should be existing directory path.') save_dir_fullpath = os.path.join(save_dir_fullpath, str(datetime.now())) os.makedirs(save_dir_fullpath) logging.info('Created save directory to {}'.format(save_dir_fullpath)) ckpt_fullpath = os.path.expanduser(options['ckpt_path']) # Data part should live in cpu. with tf.device('/cpu:0'): # Read from tfrecord format data made by src.data.tfrecord.TFRecordWriter. test_dataset = read_tfrecord( os.path.join(options['tfdata_dir'], 'test_*.tfrecord')) test_data_processor = DataPreprocessor( dataset=test_dataset, num_parallel_calls=options['num_parallel_calls'], batch_size=options['batch_size'], shuffle_buffer_size=None, prefetch_buffer_size=1) # Pre-process test data. # Add more pre-procesing blocks. logging.info('Preprocess test data') if options['test_resized_height'] is not None and options[ 'test_resized_width'] is not None: logging.info('Resize image to ({}, {})'.format( options['test_resized_height'], options['test_resized_width'])) test_data_processor.process_image_and_label( resize_image_and_label, size=tf.constant((options['test_resized_height'], options['test_resized_width'])))
def main(): input_params = args_parser().parse_args() input_params = build_preprocessor_params(input_params) input_params = DataPreprocessorConfig(*input_params) DataPreprocessor(input_params).preprocess()