def test_tf_experimental_source_disabled(): pipe = Pipeline(10, 4, 0) with pipe: input = fn.external_source(source=lambda: np.full((4, 4), 0), batch=False) pipe.set_outputs(fn.pad(input)) dali_tf.DALIDataset(pipe, output_dtypes=tf.int32)
def dali_pipe_artificial_shape(shapes, tf_type, dali_type, batch): class TestPipeline(pipeline.Pipeline): def __init__(self, **kwargs): super(TestPipeline, self).__init__(**kwargs) self.constant = ops.Constant(dtype=dali_type, idata=[1, 1], shape=[1, 2, 1]) def define_graph(self): return self.constant().gpu() pipe = TestPipeline(batch_size=batch, seed=0) ds = dali_tf.DALIDataset(pipe, batch_size=batch, output_dtypes=tf_type, output_shapes=shapes) ds_iter = iter(ds) for _ in range(10): out, = ds_iter.next() if len(shapes) == 4: assert_equals(out.shape, (batch, 1, 2, 1)) if len(shapes) == 3: assert_equals(out.shape, (batch, 1, 2)) if len(shapes) == 2: assert_equals(out.shape, ( batch, 2, )) if len(shapes) == 1: assert_equals(out.shape, (2, ))
def test_tf_experimental_inputs_disabled(): pipeline = get_image_pipeline(4, 4, 'cpu', 0) dali_tf.DALIDataset(pipeline, input_datasets={ "test": tf.data.Dataset.from_tensors(np.int32([42, 42])) })
def dali_pipe_multiple_out(shapes, types, batch): class TestPipeline(pipeline.Pipeline): def __init__(self, **kwargs): super(TestPipeline, self).__init__(**kwargs) self.reader = ops.readers.File(file_root=data_path, file_list=file_list_path) self.decoder = ops.ImageDecoder(device='mixed') self.resize = ops.Resize(device="gpu", resize_x=200, resize_y=200) def define_graph(self): data, label = self.reader() image = self.decoder(data) resized = self.resize(image) return resized, label.gpu() pipe = TestPipeline(batch_size=batch, seed=0) ds = dali_tf.DALIDataset(pipe, batch_size=batch, output_dtypes=types, output_shapes=shapes) ds_iter = iter(ds) for _ in range(10): image, label = ds_iter.next() if shapes is None or shapes[0] is None or len(shapes[0]) == 4: assert_equals(image.shape, (batch, 200, 200, 3)) else: assert_equals(image.shape, (200, 200, 3)) if shapes is None or shapes[1] is None or len(shapes[1]) == 2: assert_equals(label.shape, (batch, 1)) else: assert_equals(label.shape, (batch, ))
def dali_pipe_batch_1(shapes, types, as_single_tuple=False): class TestPipeline(pipeline.Pipeline): def __init__(self, **kwargs): super(TestPipeline, self).__init__(**kwargs) self.reader = ops.readers.File(file_root=data_path, file_list=file_list_path) self.decoder = ops.ImageDecoder(device='mixed') def define_graph(self): data, _ = self.reader() image = self.decoder(data) return image pipe = TestPipeline(batch_size=1, seed=0) ds = dali_tf.DALIDataset(pipe, batch_size=1, output_dtypes=types, output_shapes=shapes) # for clarity, we could have used the previous `pipe` pipe_ref = TestPipeline(batch_size=1, seed=0, device_id=0, num_threads=4) pipe_ref.build() ds_iter = iter(ds) # See if the iteration over different images works if as_single_tuple: shapes = shapes[0] for _ in range(10): image, = ds_iter.next() image_ref, = pipe_ref.run() if shapes is None or len(shapes) == 4: assert_equals(image.shape, ([ 1, ] + image_ref[0].shape())) else: assert_equals(image.shape, image_ref[0].shape())
def dali_dataset(data_dir, batch_size=1, image_size=1216, device_id=0, num_gpus=1, subset='train', year='2017'): images = pathlib.Path(data_dir).joinpath('{}{}'.format(subset, year)).as_posix() annotations = pathlib.Path(data_dir).joinpath('annotations').joinpath('instances_{}{}.json'.format(subset, year)).as_posix() pipe = COCOPipeline(file_root=images, annotations_file=annotations, batch_size=batch_size, num_threads=4, device_id=device_id, num_gpus=num_gpus) shapes = [ (batch_size, 1216, 1216, 3), (batch_size, 100, 4), (batch_size, 100), (batch_size, 3), (batch_size, 3), (batch_size, 3), (batch_size)] dtypes = [ tf.float32, tf.float32, tf.int32, tf.float32, tf.float32, tf.float32, tf.float32] tf_pipe = dali_tf.DALIDataset( pipeline=pipe, batch_size=batch_size, shapes=shapes, dtypes=dtypes, device_id=device_id) return tf_pipe
def dali_pipe_deprecated(dataset_kwargs, shapes, tf_type, dali_type, batch, expected_warnings_count): class TestPipeline(pipeline.Pipeline): def __init__(self, **kwargs): super(TestPipeline, self).__init__(**kwargs) self.constant = ops.Constant(dtype=dali_type, idata=[1, 1], shape=[2]) def define_graph(self): return self.constant().gpu() pipe = TestPipeline(batch_size=batch, seed=0) with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") ds = dali_tf.DALIDataset(pipe, batch_size=batch, **dataset_kwargs) assert_equals(len(w), expected_warnings_count) ds_iter = iter(ds) for _ in range(10): out, = ds_iter.next() if isinstance(shapes, int) or len(shapes) == 1: assert_equals(out.shape, (2, )) else: assert_equals(out.shape, (batch, 2)) assert_equals(out.dtype, tf_type)
def main(device_id, batch_size): with tf.device('/gpu:{}'.format(str(device_id))): pipe=get_pipeline(device_id,batch_size) # Create dataset # Define shapes and types of the outputs shapes = [ (batch_size, 224, 224,3), (batch_size,1)] dtypes = [ tf.float32, tf.int32] out = dali_tf.DALIDataset( pipeline=pipe, batch_size=batch_size, shapes=shapes, dtypes=dtypes, device_id=0) #out = out.with_options(dataset_options()) m=get_reset50_model() m.compile( optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) # Train using DALI dataset m.fit( out, epochs=10, steps_per_epoch=batch_size*8, use_multiprocessing=False )
def build(self) -> tf.data.Dataset: """Construct a dataset end-to-end and return it. Args: input_context: An optional context provided by `tf.distribute` for cross-replica training. Returns: A TensorFlow dataset outputting batched images and labels. """ if self._use_dali: print("Using dali for {train} dataloading".format( train="training" if self.is_training else "validation")) tfrec_filenames = sorted( tf.io.gfile.glob( os.path.join(self._data_dir, '%s-*' % self._split))) tfrec_idx_filenames = sorted( tf.io.gfile.glob( os.path.join(self._index_file, '%s-*' % self._split))) # # Create pipeline dali_pipeline = Dali.DaliPipeline( tfrec_filenames=tfrec_filenames, tfrec_idx_filenames=tfrec_idx_filenames, height=self._image_size, width=self._image_size, batch_size=self.local_batch_size, num_threads=1, device_id=hvd.local_rank(), shard_id=hvd.rank(), num_gpus=hvd.size(), num_classes=self.num_classes, deterministic=False, dali_cpu=False, training=self.is_training) # Define shapes and types of the outputs shapes = ((self.local_batch_size, self._image_size, self._image_size, 3), (self.local_batch_size, self._num_classes)) dtypes = (tf.float32, tf.float32) # Create dataset dataset = dali_tf.DALIDataset(pipeline=dali_pipeline, batch_size=self.local_batch_size, output_shapes=shapes, output_dtypes=dtypes, device_id=hvd.local_rank()) # if self.is_training and self._augmenter: # print('Augmenting with {}'.format(self._augmenter)) # dataset.unbatch().map(self.augment_pipeline, num_parallel_calls=tf.data.experimental.AUTOTUNE).batch(self.local_batch_size) return dataset else: print("Using tf native pipeline for {train} dataloading".format( train="training" if self.is_training else "validation")) dataset = self.load_records() dataset = self.pipeline(dataset) return dataset
def dataset(self): output_shapes = ((self._batch_size, self._image_size[0], self._image_size[0], 3), (self._batch_size, None, 5)) output_dtypes = (tf.float32, tf.float32) return dali_tf.DALIDataset(pipeline=self._pipe, batch_size=self._batch_size, output_shapes=output_shapes, output_dtypes=output_dtypes, device_id=0)
def to_image_dataset(image_pipeline_desc, device_str): dataset_pipeline, shapes, dtypes = image_pipeline_desc with tf.device(device_str): dali_dataset = dali_tf.DALIDataset( pipeline=dataset_pipeline, batch_size=dataset_pipeline.batch_size, output_shapes=shapes, output_dtypes=dtypes, num_threads=dataset_pipeline.num_threads, device_id=dataset_pipeline.device_id) return dali_dataset
def dali_const_dataset(batch_size, sample_size, device_id): pipeline = Pipeline(batch_size, 4, device_id) const = fn.constant(device='gpu', fdata=sample_size * [1.]) pipeline.set_outputs(const) dali_dataset = dali_tf.DALIDataset( pipeline=pipeline, batch_size=batch_size, output_shapes=((batch_size, sample_size)), output_dtypes=(tf.float32), device_id=device_id) return dali_dataset
def test_python_operator_error(): dataset_pipeline = PythonOperatorPipeline() shapes = [(1, 3, 3, 3)] dtypes = [tf.float32] with tf.device('/cpu:0'): daliset = dali_tf.DALIDataset(pipeline=dataset_pipeline, batch_size=1, shapes=shapes, dtypes=dtypes, num_threads=1, device_id=0)
def _test_tf_dataset(device, device_id=0): skip_for_incompatible_tf() batch_size = 12 num_threads = 4 iterations = 10 dataset_pipeline = TestPipeline(batch_size, num_threads, device, device_id) shapes = [(batch_size, 3, 224, 224), (batch_size, 1), (batch_size, 1)] dtypes = [tf.float32, tf.int32, tf.int16] dataset_results = [] with tf.device('/{0}:{1}'.format(device, device_id)): daliset = dali_tf.DALIDataset(pipeline=dataset_pipeline, batch_size=batch_size, shapes=shapes, dtypes=dtypes, num_threads=num_threads, device_id=device_id) daliset = daliset.with_options(_dataset_options()) iterator = tf.compat.v1.data.make_initializable_iterator(daliset) next_element = iterator.get_next() with tf.compat.v1.Session() as sess: sess.run([ tf.compat.v1.global_variables_initializer(), iterator.initializer ]) for _ in range(iterations): dataset_results.append(sess.run(next_element)) standalone_pipeline = TestPipeline(batch_size, num_threads, device, device_id=0) standalone_pipeline.build() standalone_results = [] for _ in range(iterations): if device is 'gpu': standalone_results.append( tuple(result.as_cpu().as_array() for result in standalone_pipeline.run())) else: standalone_results.append( tuple(result.as_array() for result in standalone_pipeline.run())) for dataset_result, standalone_result in zip(dataset_results, standalone_results): for dataset_out, standalone_out in zip(dataset_result, standalone_result): assert np.array_equal(dataset_out, standalone_out)
def _get_mnist_dataset(device='cpu', device_id=0, shard_id=0, num_shards=1): mnist_pipeline = MnistPipeline(4, data_path, device, device_id, shard_id, num_shards) shapes = [(BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE), (BATCH_SIZE)] dtypes = [tf.float32, tf.int32] daliset = dali_tf.DALIDataset(pipeline=mnist_pipeline, batch_size=BATCH_SIZE, shapes=shapes, dtypes=dtypes, num_threads=4, device_id=device_id) return daliset.with_options(dataset_options())
def test_different_num_shapes_dtypes(): batch_size = 12 num_threads = 4 dataset_pipe, shapes, dtypes = get_image_pipeline(batch_size, num_threads, 'cpu') dtypes = tuple(dtypes[0:2]) with tf.device('/cpu:0'): dali_tf.DALIDataset(pipeline=dataset_pipe, batch_size=batch_size, output_shapes=shapes, output_dtypes=dtypes, num_threads=num_threads)
def test_different_num_shapes_dtypes(): batch_size = 12 num_threads = 4 dataset_pipeline = TestPipeline(batch_size, num_threads, 'cpu') shapes = [(batch_size, 3, 224, 224), (batch_size, 1), (batch_size, 1)] dtypes = [tf.float32, tf.float32] with tf.device('/cpu:0'): dali_tf.DALIDataset(pipeline=dataset_pipeline, batch_size=batch_size, shapes=shapes, dtypes=dtypes, num_threads=num_threads)
def dali_pipe_types(tf_type, dali_type): class TestPipeline(pipeline.Pipeline): def __init__(self, **kwargs): super(TestPipeline, self).__init__(**kwargs) self.constant = ops.Constant(dtype=dali_type, idata=[1, 1], shape=[2]) def define_graph(self): return self.constant().gpu() pipe = TestPipeline(batch_size=1, seed=0) ds = dali_tf.DALIDataset(pipe, batch_size=1, output_dtypes=tf_type) ds_iter = iter(ds) out, = ds_iter.next() assert_equals(out.dtype, tf_type)
def test_python_operator_not_allowed_in_tf_dataset_error(): pipeline = Pipeline(1, 1, 0, exec_pipelined=False, exec_async=False) with pipeline: output = fn.python_function(function=lambda: np.zeros((3, 3, 3))) pipeline.set_outputs(output) shapes = ((1, 3, 3, 3)) dtypes = (tf.float32) with tf.device('/cpu:0'): _ = dali_tf.DALIDataset(pipeline=pipeline, batch_size=1, output_shapes=shapes, output_dtypes=dtypes, num_threads=1, device_id=0)
def get_dali_dataset_from_pipeline(dataset_pipeline, batch_size, num_threads, device, device_id, num_devices=1): shapes = ((batch_size, 3, 224, 224), (batch_size, 1, 1), (batch_size, 1)) dtypes = (tf.float32, tf.int32, tf.int16) with tf.device('/{0}:{1}'.format(device, device_id)): dali_dataset = dali_tf.DALIDataset(pipeline=dataset_pipeline, batch_size=batch_size, output_shapes=shapes, output_dtypes=dtypes, num_threads=num_threads, device_id=device_id) return dali_dataset
def get_dataset(device='cpu', device_id=0, shard_id=0, num_shards=1, fail_on_device_mismatch=True): pipeline = mnist_pipeline(4, data_path, device, device_id, shard_id, num_shards) shapes = ((BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE), (BATCH_SIZE, )) dtypes = (tf.float32, tf.int32) daliset = dali_tf.DALIDataset( pipeline=pipeline, batch_size=BATCH_SIZE, output_shapes=shapes, output_dtypes=dtypes, num_threads=4, device_id=device_id, fail_on_device_mismatch=fail_on_device_mismatch) return daliset
def test_tf_dataset(): skip_for_incompatible_tf() batch_size = 12 num_threads = 4 epochs = 10 dataset_pipeline = TestPipeline(batch_size, num_threads) shapes = [ (batch_size, 3, 224, 224), (batch_size, 1), (batch_size, 1)] dtypes = [ tf.float32, tf.int32, tf.float16] dataset_results = [] with tf.device('/cpu:0'): daliset = dali_tf.DALIDataset( pipeline=dataset_pipeline, batch_size=batch_size, shapes=shapes, dtypes=dtypes, num_threads=num_threads) with tf.compat.v1.Session() as sess: iterator = tf.compat.v1.data.make_one_shot_iterator(daliset) next_element = iterator.get_next() for _ in range(epochs): dataset_results.append(sess.run(next_element)) standalone_pipeline = TestPipeline(batch_size, num_threads) standalone_pipeline.build() standalone_results = [] for _ in range(epochs): standalone_results.append( tuple(result.as_array() for result in standalone_pipeline.run())) for dataset_result, standalone_result in zip(dataset_results, standalone_results): for dataset_out, standalone_out in zip(dataset_result, standalone_result): assert np.array_equal(dataset_out, standalone_out)
def test_different_num_shapes_dtypes(): batch_size = 12 num_threads = 4 dataset_pipeline = get_pipeline(batch_size, num_threads, 'cpu') shapes = ( (batch_size, 3, 224, 224), (batch_size, 1, 1), (batch_size, 1)) dtypes = ( tf.float32, tf.float32) with tf.device('/cpu:0'): dali_tf.DALIDataset( pipeline=dataset_pipeline, batch_size=batch_size, output_shapes=shapes, output_dtypes=dtypes, num_threads=num_threads)
def __init__(self, filenames, idx_filenames, height, width, batch_size, num_threads, dtype=tf.uint8, dali_cpu=True, deterministic=False, training=False): device_id = hvd.local_rank() shard_id = hvd.rank() num_gpus = hvd.size() self.pipe = get_dali_pipeline( tfrec_filenames=filenames, tfrec_idx_filenames=idx_filenames, height=height, width=width, batch_size=batch_size, num_threads=num_threads, device_id=device_id, shard_id=shard_id, num_gpus=num_gpus, dali_cpu=dali_cpu, training=training, seed=7 * (1 + hvd.rank()) if deterministic else None) self.daliop = dali_tf.DALIIterator() self.batch_size = batch_size self.height = height self.width = width self.device_id = device_id self.dalidataset = dali_tf.DALIDataset( pipeline=self.pipe, output_shapes=((batch_size, height, width, 3), (batch_size)), batch_size=batch_size, output_dtypes=(tf.float32, tf.int64), device_id=device_id)
def test_dali_tf_dataset_cpu_only(): skip_for_incompatible_tf() try: tf.compat.v1.enable_eager_execution() except Exception: pass batch_size = 3 value = random.randint(0, 1000) pipe = get_dali_pipe(batch_size=batch_size, device_id=types.CPU_ONLY_DEVICE_ID, num_threads=1, value=value) with tf.device('/cpu'): ds = dali_tf.DALIDataset(pipe, device_id=types.CPU_ONLY_DEVICE_ID, batch_size=1, output_dtypes=tf.int32, output_shapes=[1]) ds = iter(ds) data = next(ds) assert data == np.array([value])
def get_dataset(self): output_shapes = [ (self._batch_size, self._image_size[0], self._image_size[1], 3), (self._batch_size,), (self._batch_size, None, 4), (self._batch_size, None), ] output_dtypes = [tf.float32, tf.float32, tf.float32, tf.int32] for level in range(self._anchors.min_level, self._anchors.max_level + 1): feat_size = self._anchors.feat_sizes[level] output_shapes.append( ( self._batch_size, feat_size["height"], feat_size["width"], self._anchors.get_anchors_per_location(), ) ) output_shapes.append( ( self._batch_size, feat_size["height"], feat_size["width"], self._anchors.get_anchors_per_location() * 4, ) ) output_dtypes.append(tf.int32) output_dtypes.append(tf.float32) dataset = dali_tf.DALIDataset( pipeline=self._pipe, batch_size=self._batch_size, output_shapes=tuple(output_shapes), output_dtypes=tuple(output_dtypes), ) return dataset
def load_tfrecord(directory, batch_size, training): tfrecord = [] tfrecord_idx = [] for f in os.listdir(directory): fullpath = os.path.join(directory, f) if not os.path.isfile(fullpath): continue if f.endswith(".tfrecord"): tfrecord.append(fullpath) if f.endswith(".idx"): tfrecord_idx.append(fullpath) tfrecord.sort() tfrecord_idx.sort() pipe = Pipeline(batch_size=batch_size, num_threads=32, device_id=0) with pipe: inputs = fn.tfrecord_reader( path=tfrecord, index_path=tfrecord_idx, features={ "frame_one": tfrec.FixedLenFeature((), tfrec.string, ""), "frame_two": tfrec.FixedLenFeature((), tfrec.string, ""), "frame_three": tfrec.FixedLenFeature((), tfrec.string, ""), "frame_four": tfrec.FixedLenFeature((), tfrec.string, ""), "plus_one_position": tfrec.FixedLenFeature([3], tfrec.float32, 0.0), "plus_one_orientation": tfrec.FixedLenFeature([3], tfrec.float32, 0.0), "plus_two_position": tfrec.FixedLenFeature([3], tfrec.float32, 0.0), "plus_two_orientation": tfrec.FixedLenFeature([3], tfrec.float32, 0.0), "plus_three_position": tfrec.FixedLenFeature([3], tfrec.float32, 0.0), "plus_three_orientation": tfrec.FixedLenFeature([3], tfrec.float32, 0.0), "speed": tfrec.FixedLenFeature([], tfrec.float32, 0.0), }) frame1 = inputs["frame_one"] frame1 = fn.image_decoder(frame1, device="mixed", output_type=types.RGB) # frame1 = fn.resize(frame1, device="gpu", resize_shorter=256.) frame1 = fn.crop_mirror_normalize(frame1, device="gpu", dtype=types.FLOAT, mean=[0., 0., 0.], std=[1., 1., 1.]) frame1 = fn.transpose(frame1, device="gpu", perm=[1, 2, 0]) frame2 = inputs["frame_two"] frame2 = fn.image_decoder(frame2, device="mixed", output_type=types.RGB) # frame2 = fn.resize(frame2, device="gpu", resize_shorter=256.) frame2 = fn.crop_mirror_normalize(frame2, device="gpu", dtype=types.FLOAT, mean=[0., 0., 0.], std=[1., 1., 1.]) frame2 = fn.transpose(frame2, device="gpu", perm=[1, 2, 0]) position = inputs["plus_one_position"].gpu() orientation = inputs["plus_one_orientation"].gpu() speed = inputs["speed"].gpu() image = fn.cat(frame1, frame2, device="gpu", axis=2) pose = fn.cat(position, orientation, device="gpu", axis=0) pipe.set_outputs(image, pose, speed) # Define shapes and types of the outputs shapes = ((batch_size, 480, 640), (batch_size, 6), (batch_size)) dtypes = (tf.float32, tf.float32) # Create dataset return dali_tf.DALIDataset(pipeline=pipe, batch_size=batch_size, output_shapes=shapes, output_dtypes=dtypes, device_id=0)
def _test_tf_dataset_multigpu(): num_devices = num_available_gpus() batch_size = 8 num_threads = 4 shapes = [(batch_size, 3, 224, 224), (batch_size, 1), (batch_size, 1)] dtypes = [tf.float32, tf.int32, tf.int16] dataset_results = [] initializers = [tf.compat.v1.global_variables_initializer()] ops_to_run = [] for device_id in range(num_devices): with tf.device('/gpu:{0}'.format(device_id)): dataset_pipeline = TestPipeline(batch_size, num_threads, 'gpu', device_id, device_id, num_devices) daliset = dali_tf.DALIDataset(pipeline=dataset_pipeline, batch_size=batch_size, shapes=shapes, dtypes=dtypes, num_threads=num_threads, device_id=device_id) daliset = daliset.with_options(dataset_options()) iterator = tf.compat.v1.data.make_initializable_iterator(daliset) initializers.append(iterator.initializer) ops_to_run.append(iterator.get_next()) standalone_pipeline = TestPipeline(batch_size, num_threads, device='gpu', device_id=0) standalone_pipeline.build() dataset_size = standalone_pipeline.epoch_size("Reader") iterations = dataset_size // batch_size with tf.compat.v1.Session() as sess: sess.run(initializers) for _ in range(iterations): dataset_results.append(sess.run(ops_to_run)) standalone_results = [] for _ in range(iterations): standalone_results.append( tuple(result.as_cpu().as_array() for result in standalone_pipeline.run())) assert len(dataset_results) == iterations for it in range(iterations): assert len(dataset_results[it]) == num_devices for device_id in range(num_devices): batch_id = iterations - ( (it + device_id * (iterations // num_devices)) % iterations) - 1 it_id = iterations - it - 1 assert np.array_equal(standalone_results[it_id][0], dataset_results[batch_id][device_id][0]) assert np.array_equal(standalone_results[it_id][1], dataset_results[batch_id][device_id][1]) assert np.array_equal(standalone_results[it_id][2], dataset_results[batch_id][device_id][2])
train_data = train_dataset_generator(dataset.take(train_sz), is_training=True, batch_sz=BATCH_SZ, columns=channels, data_size=train_sz) val_data = get_dataset(dataset.skip(train_sz).take(valid_sz), start=train_sz, end=train_sz+valid_sz, columns=channels) #test_data = test_dataset(dataset.skip(train_sz+valid_sz).take(test_sz), start=train_sz+valid_sz, end=train_sz+valid_sz+test_sz) ''' print("\n Timestamp: " + str(tf.cast(tf.timestamp(), tf.float64))) with tf.device('/gpu:0'): daliop = dali_tf.DALIIterator() shapes = [(BATCH_SZ, 125, 125, 8), (BATCH_SZ, 2)] dtypes = [tf.float32, tf.int32] # Create TF dataset out_dataset = dali_tf.DALIDataset(pipeline=pipe, batch_size=BATCH_SZ, shapes=shapes, dtypes=dtypes, device_id=0) opt = NovoGrad(learning_rate=lr_init * hvd.size()) #Wrap the optimizer in a Horovod distributed optimizer -> uses hvd.DistributedOptimizer() to compute gradients. opt = hvd.DistributedOptimizer(opt) resnet.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'], experimental_run_tf_function=False) # Train using DALI dataset history = resnet.fit( out_dataset, steps_per_epoch=train_sz // (BATCH_SZ * hvd.size()),
pipe3 = Opt3Pipeline("smallimgs_path.txt", 2, 0, batch_size, 8, 0) with tf.device('/gpu:0'): # Create dataset # Define shapes and types of the outputs shapes = [ (batch_size, 224, 224,3), (batch_size,1)] dtypes = [ tf.float32, tf.int32] out = dali_tf.DALIDataset( pipeline=pipe3, batch_size=batch_size, shapes=shapes, dtypes=dtypes, device_id=0) #out = out.with_options(dataset_options()) m=get_reset50_model() m.compile( optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) # Train using DALI dataset m.fit( out,