def test_tf_experimental_source_disabled():
pipe = Pipeline(10, 4, 0)
with pipe:
input = fn.external_source(source=lambda: np.full((4, 4), 0),
batch=False)
pipe.set_outputs(fn.pad(input))
dali_tf.DALIDataset(pipe, output_dtypes=tf.int32)
def dali_pipe_artificial_shape(shapes, tf_type, dali_type, batch):
class TestPipeline(pipeline.Pipeline):
def __init__(self, **kwargs):
super(TestPipeline, self).__init__(**kwargs)
self.constant = ops.Constant(dtype=dali_type,
idata=[1, 1],
shape=[1, 2, 1])
def define_graph(self):
return self.constant().gpu()
pipe = TestPipeline(batch_size=batch, seed=0)
ds = dali_tf.DALIDataset(pipe,
batch_size=batch,
output_dtypes=tf_type,
output_shapes=shapes)
ds_iter = iter(ds)
for _ in range(10):
out, = ds_iter.next()
if len(shapes) == 4:
assert_equals(out.shape, (batch, 1, 2, 1))
if len(shapes) == 3:
assert_equals(out.shape, (batch, 1, 2))
if len(shapes) == 2:
assert_equals(out.shape, (
batch,
2,
))
if len(shapes) == 1:
assert_equals(out.shape, (2, ))
def test_tf_experimental_inputs_disabled():
pipeline = get_image_pipeline(4, 4, 'cpu', 0)
dali_tf.DALIDataset(pipeline,
input_datasets={
"test":
tf.data.Dataset.from_tensors(np.int32([42, 42]))
})
def dali_pipe_multiple_out(shapes, types, batch):
class TestPipeline(pipeline.Pipeline):
def __init__(self, **kwargs):
super(TestPipeline, self).__init__(**kwargs)
self.reader = ops.readers.File(file_root=data_path,
file_list=file_list_path)
self.decoder = ops.ImageDecoder(device='mixed')
self.resize = ops.Resize(device="gpu", resize_x=200, resize_y=200)
def define_graph(self):
data, label = self.reader()
image = self.decoder(data)
resized = self.resize(image)
return resized, label.gpu()
pipe = TestPipeline(batch_size=batch, seed=0)
ds = dali_tf.DALIDataset(pipe,
batch_size=batch,
output_dtypes=types,
output_shapes=shapes)
ds_iter = iter(ds)
for _ in range(10):
image, label = ds_iter.next()
if shapes is None or shapes[0] is None or len(shapes[0]) == 4:
assert_equals(image.shape, (batch, 200, 200, 3))
else:
assert_equals(image.shape, (200, 200, 3))
if shapes is None or shapes[1] is None or len(shapes[1]) == 2:
assert_equals(label.shape, (batch, 1))
else:
assert_equals(label.shape, (batch, ))
def dali_pipe_batch_1(shapes, types, as_single_tuple=False):
class TestPipeline(pipeline.Pipeline):
def __init__(self, **kwargs):
super(TestPipeline, self).__init__(**kwargs)
self.reader = ops.readers.File(file_root=data_path,
file_list=file_list_path)
self.decoder = ops.ImageDecoder(device='mixed')
def define_graph(self):
data, _ = self.reader()
image = self.decoder(data)
return image
pipe = TestPipeline(batch_size=1, seed=0)
ds = dali_tf.DALIDataset(pipe,
batch_size=1,
output_dtypes=types,
output_shapes=shapes)
# for clarity, we could have used the previous `pipe`
pipe_ref = TestPipeline(batch_size=1, seed=0, device_id=0, num_threads=4)
pipe_ref.build()
ds_iter = iter(ds)
# See if the iteration over different images works
if as_single_tuple:
shapes = shapes[0]
for _ in range(10):
image, = ds_iter.next()
image_ref, = pipe_ref.run()
if shapes is None or len(shapes) == 4:
assert_equals(image.shape, ([
1,
] + image_ref[0].shape()))
else:
assert_equals(image.shape, image_ref[0].shape())
def dali_dataset(data_dir, batch_size=1, image_size=1216, device_id=0, num_gpus=1, subset='train', year='2017'):
images = pathlib.Path(data_dir).joinpath('{}{}'.format(subset, year)).as_posix()
annotations = pathlib.Path(data_dir).joinpath('annotations').joinpath('instances_{}{}.json'.format(subset, year)).as_posix()
pipe = COCOPipeline(file_root=images,
annotations_file=annotations,
batch_size=batch_size,
num_threads=4,
device_id=device_id,
num_gpus=num_gpus)
shapes = [
(batch_size, 1216, 1216, 3),
(batch_size, 100, 4),
(batch_size, 100),
(batch_size, 3),
(batch_size, 3),
(batch_size, 3),
(batch_size)]
dtypes = [
tf.float32,
tf.float32,
tf.int32,
tf.float32,
tf.float32,
tf.float32,
tf.float32]
tf_pipe = dali_tf.DALIDataset(
pipeline=pipe,
batch_size=batch_size,
shapes=shapes,
dtypes=dtypes,
device_id=device_id)
return tf_pipe
def dali_pipe_deprecated(dataset_kwargs, shapes, tf_type, dali_type, batch,
expected_warnings_count):
class TestPipeline(pipeline.Pipeline):
def __init__(self, **kwargs):
super(TestPipeline, self).__init__(**kwargs)
self.constant = ops.Constant(dtype=dali_type,
idata=[1, 1],
shape=[2])
def define_graph(self):
return self.constant().gpu()
pipe = TestPipeline(batch_size=batch, seed=0)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
ds = dali_tf.DALIDataset(pipe, batch_size=batch, **dataset_kwargs)
assert_equals(len(w), expected_warnings_count)
ds_iter = iter(ds)
for _ in range(10):
out, = ds_iter.next()
if isinstance(shapes, int) or len(shapes) == 1:
assert_equals(out.shape, (2, ))
else:
assert_equals(out.shape, (batch, 2))
assert_equals(out.dtype, tf_type)
def main(device_id, batch_size):
with tf.device('/gpu:{}'.format(str(device_id))):
pipe=get_pipeline(device_id,batch_size)
# Create dataset
# Define shapes and types of the outputs
shapes = [
(batch_size, 224, 224,3),
(batch_size,1)]
dtypes = [
tf.float32,
tf.int32]
out = dali_tf.DALIDataset(
pipeline=pipe,
batch_size=batch_size,
shapes=shapes,
dtypes=dtypes,
device_id=0)
#out = out.with_options(dataset_options())
m=get_reset50_model()
m.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
# Train using DALI dataset
m.fit(
out,
epochs=10,
steps_per_epoch=batch_size*8,
use_multiprocessing=False
)
def build(self) -> tf.data.Dataset:
"""Construct a dataset end-to-end and return it.
Args:
input_context: An optional context provided by `tf.distribute` for
cross-replica training.
Returns:
A TensorFlow dataset outputting batched images and labels.
"""
if self._use_dali:
print("Using dali for {train} dataloading".format(
train="training" if self.is_training else "validation"))
tfrec_filenames = sorted(
tf.io.gfile.glob(
os.path.join(self._data_dir, '%s-*' % self._split)))
tfrec_idx_filenames = sorted(
tf.io.gfile.glob(
os.path.join(self._index_file, '%s-*' % self._split)))
# # Create pipeline
dali_pipeline = Dali.DaliPipeline(
tfrec_filenames=tfrec_filenames,
tfrec_idx_filenames=tfrec_idx_filenames,
height=self._image_size,
width=self._image_size,
batch_size=self.local_batch_size,
num_threads=1,
device_id=hvd.local_rank(),
shard_id=hvd.rank(),
num_gpus=hvd.size(),
num_classes=self.num_classes,
deterministic=False,
dali_cpu=False,
training=self.is_training)
# Define shapes and types of the outputs
shapes = ((self.local_batch_size, self._image_size,
self._image_size, 3), (self.local_batch_size,
self._num_classes))
dtypes = (tf.float32, tf.float32)
# Create dataset
dataset = dali_tf.DALIDataset(pipeline=dali_pipeline,
batch_size=self.local_batch_size,
output_shapes=shapes,
output_dtypes=dtypes,
device_id=hvd.local_rank())
# if self.is_training and self._augmenter:
# print('Augmenting with {}'.format(self._augmenter))
# dataset.unbatch().map(self.augment_pipeline, num_parallel_calls=tf.data.experimental.AUTOTUNE).batch(self.local_batch_size)
return dataset
else:
print("Using tf native pipeline for {train} dataloading".format(
train="training" if self.is_training else "validation"))
dataset = self.load_records()
dataset = self.pipeline(dataset)
return dataset
def dataset(self):
output_shapes = ((self._batch_size, self._image_size[0],
self._image_size[0], 3), (self._batch_size, None, 5))
output_dtypes = (tf.float32, tf.float32)
return dali_tf.DALIDataset(pipeline=self._pipe,
batch_size=self._batch_size,
output_shapes=output_shapes,
output_dtypes=output_dtypes,
device_id=0)
def to_image_dataset(image_pipeline_desc, device_str):
dataset_pipeline, shapes, dtypes = image_pipeline_desc
with tf.device(device_str):
dali_dataset = dali_tf.DALIDataset(
pipeline=dataset_pipeline,
batch_size=dataset_pipeline.batch_size,
output_shapes=shapes,
output_dtypes=dtypes,
num_threads=dataset_pipeline.num_threads,
device_id=dataset_pipeline.device_id)
return dali_dataset
def dali_const_dataset(batch_size, sample_size, device_id):
pipeline = Pipeline(batch_size, 4, device_id)
const = fn.constant(device='gpu', fdata=sample_size * [1.])
pipeline.set_outputs(const)
dali_dataset = dali_tf.DALIDataset(
pipeline=pipeline,
batch_size=batch_size,
output_shapes=((batch_size, sample_size)),
output_dtypes=(tf.float32),
device_id=device_id)
return dali_dataset
def test_python_operator_error():
dataset_pipeline = PythonOperatorPipeline()
shapes = [(1, 3, 3, 3)]
dtypes = [tf.float32]
with tf.device('/cpu:0'):
daliset = dali_tf.DALIDataset(pipeline=dataset_pipeline,
batch_size=1,
shapes=shapes,
dtypes=dtypes,
num_threads=1,
device_id=0)
def _test_tf_dataset(device, device_id=0):
skip_for_incompatible_tf()
batch_size = 12
num_threads = 4
iterations = 10
dataset_pipeline = TestPipeline(batch_size, num_threads, device, device_id)
shapes = [(batch_size, 3, 224, 224), (batch_size, 1), (batch_size, 1)]
dtypes = [tf.float32, tf.int32, tf.int16]
dataset_results = []
with tf.device('/{0}:{1}'.format(device, device_id)):
daliset = dali_tf.DALIDataset(pipeline=dataset_pipeline,
batch_size=batch_size,
shapes=shapes,
dtypes=dtypes,
num_threads=num_threads,
device_id=device_id)
daliset = daliset.with_options(_dataset_options())
iterator = tf.compat.v1.data.make_initializable_iterator(daliset)
next_element = iterator.get_next()
with tf.compat.v1.Session() as sess:
sess.run([
tf.compat.v1.global_variables_initializer(), iterator.initializer
])
for _ in range(iterations):
dataset_results.append(sess.run(next_element))
standalone_pipeline = TestPipeline(batch_size,
num_threads,
device,
device_id=0)
standalone_pipeline.build()
standalone_results = []
for _ in range(iterations):
if device is 'gpu':
standalone_results.append(
tuple(result.as_cpu().as_array()
for result in standalone_pipeline.run()))
else:
standalone_results.append(
tuple(result.as_array()
for result in standalone_pipeline.run()))
for dataset_result, standalone_result in zip(dataset_results,
standalone_results):
for dataset_out, standalone_out in zip(dataset_result,
standalone_result):
assert np.array_equal(dataset_out, standalone_out)
def _get_mnist_dataset(device='cpu', device_id=0, shard_id=0, num_shards=1):
mnist_pipeline = MnistPipeline(4, data_path, device, device_id, shard_id,
num_shards)
shapes = [(BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE), (BATCH_SIZE)]
dtypes = [tf.float32, tf.int32]
daliset = dali_tf.DALIDataset(pipeline=mnist_pipeline,
batch_size=BATCH_SIZE,
shapes=shapes,
dtypes=dtypes,
num_threads=4,
device_id=device_id)
return daliset.with_options(dataset_options())
def test_different_num_shapes_dtypes():
batch_size = 12
num_threads = 4
dataset_pipe, shapes, dtypes = get_image_pipeline(batch_size, num_threads,
'cpu')
dtypes = tuple(dtypes[0:2])
with tf.device('/cpu:0'):
dali_tf.DALIDataset(pipeline=dataset_pipe,
batch_size=batch_size,
output_shapes=shapes,
output_dtypes=dtypes,
num_threads=num_threads)
def test_different_num_shapes_dtypes():
batch_size = 12
num_threads = 4
dataset_pipeline = TestPipeline(batch_size, num_threads, 'cpu')
shapes = [(batch_size, 3, 224, 224), (batch_size, 1), (batch_size, 1)]
dtypes = [tf.float32, tf.float32]
with tf.device('/cpu:0'):
dali_tf.DALIDataset(pipeline=dataset_pipeline,
batch_size=batch_size,
shapes=shapes,
dtypes=dtypes,
num_threads=num_threads)
def dali_pipe_types(tf_type, dali_type):
class TestPipeline(pipeline.Pipeline):
def __init__(self, **kwargs):
super(TestPipeline, self).__init__(**kwargs)
self.constant = ops.Constant(dtype=dali_type, idata=[1, 1], shape=[2])
def define_graph(self):
return self.constant().gpu()
pipe = TestPipeline(batch_size=1, seed=0)
ds = dali_tf.DALIDataset(pipe, batch_size=1, output_dtypes=tf_type)
ds_iter = iter(ds)
out, = ds_iter.next()
assert_equals(out.dtype, tf_type)
def test_python_operator_not_allowed_in_tf_dataset_error():
pipeline = Pipeline(1, 1, 0, exec_pipelined=False, exec_async=False)
with pipeline:
output = fn.python_function(function=lambda: np.zeros((3, 3, 3)))
pipeline.set_outputs(output)
shapes = ((1, 3, 3, 3))
dtypes = (tf.float32)
with tf.device('/cpu:0'):
_ = dali_tf.DALIDataset(pipeline=pipeline,
batch_size=1,
output_shapes=shapes,
output_dtypes=dtypes,
num_threads=1,
device_id=0)
def get_dali_dataset_from_pipeline(dataset_pipeline,
batch_size,
num_threads,
device,
device_id,
num_devices=1):
shapes = ((batch_size, 3, 224, 224), (batch_size, 1, 1), (batch_size, 1))
dtypes = (tf.float32, tf.int32, tf.int16)
with tf.device('/{0}:{1}'.format(device, device_id)):
dali_dataset = dali_tf.DALIDataset(pipeline=dataset_pipeline,
batch_size=batch_size,
output_shapes=shapes,
output_dtypes=dtypes,
num_threads=num_threads,
device_id=device_id)
return dali_dataset
def get_dataset(device='cpu',
device_id=0,
shard_id=0,
num_shards=1,
fail_on_device_mismatch=True):
pipeline = mnist_pipeline(4, data_path, device, device_id, shard_id,
num_shards)
shapes = ((BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE), (BATCH_SIZE, ))
dtypes = (tf.float32, tf.int32)
daliset = dali_tf.DALIDataset(
pipeline=pipeline,
batch_size=BATCH_SIZE,
output_shapes=shapes,
output_dtypes=dtypes,
num_threads=4,
device_id=device_id,
fail_on_device_mismatch=fail_on_device_mismatch)
return daliset
def test_tf_dataset():
skip_for_incompatible_tf()
batch_size = 12
num_threads = 4
epochs = 10
dataset_pipeline = TestPipeline(batch_size, num_threads)
shapes = [
(batch_size, 3, 224, 224),
(batch_size, 1),
(batch_size, 1)]
dtypes = [
tf.float32,
tf.int32,
tf.float16]
dataset_results = []
with tf.device('/cpu:0'):
daliset = dali_tf.DALIDataset(
pipeline=dataset_pipeline,
batch_size=batch_size,
shapes=shapes,
dtypes=dtypes,
num_threads=num_threads)
with tf.compat.v1.Session() as sess:
iterator = tf.compat.v1.data.make_one_shot_iterator(daliset)
next_element = iterator.get_next()
for _ in range(epochs):
dataset_results.append(sess.run(next_element))
standalone_pipeline = TestPipeline(batch_size, num_threads)
standalone_pipeline.build()
standalone_results = []
for _ in range(epochs):
standalone_results.append(
tuple(result.as_array() for result in standalone_pipeline.run()))
for dataset_result, standalone_result in zip(dataset_results, standalone_results):
for dataset_out, standalone_out in zip(dataset_result, standalone_result):
assert np.array_equal(dataset_out, standalone_out)
def test_different_num_shapes_dtypes():
batch_size = 12
num_threads = 4
dataset_pipeline = get_pipeline(batch_size, num_threads, 'cpu')
shapes = (
(batch_size, 3, 224, 224),
(batch_size, 1, 1),
(batch_size, 1))
dtypes = (
tf.float32,
tf.float32)
with tf.device('/cpu:0'):
dali_tf.DALIDataset(
pipeline=dataset_pipeline,
batch_size=batch_size,
output_shapes=shapes,
output_dtypes=dtypes,
num_threads=num_threads)
def __init__(self,
filenames,
idx_filenames,
height,
width,
batch_size,
num_threads,
dtype=tf.uint8,
dali_cpu=True,
deterministic=False,
training=False):
device_id = hvd.local_rank()
shard_id = hvd.rank()
num_gpus = hvd.size()
self.pipe = get_dali_pipeline(
tfrec_filenames=filenames,
tfrec_idx_filenames=idx_filenames,
height=height,
width=width,
batch_size=batch_size,
num_threads=num_threads,
device_id=device_id,
shard_id=shard_id,
num_gpus=num_gpus,
dali_cpu=dali_cpu,
training=training,
seed=7 * (1 + hvd.rank()) if deterministic else None)
self.daliop = dali_tf.DALIIterator()
self.batch_size = batch_size
self.height = height
self.width = width
self.device_id = device_id
self.dalidataset = dali_tf.DALIDataset(
pipeline=self.pipe,
output_shapes=((batch_size, height, width, 3), (batch_size)),
batch_size=batch_size,
output_dtypes=(tf.float32, tf.int64),
device_id=device_id)
def test_dali_tf_dataset_cpu_only():
skip_for_incompatible_tf()
try:
tf.compat.v1.enable_eager_execution()
except Exception:
pass
batch_size = 3
value = random.randint(0, 1000)
pipe = get_dali_pipe(batch_size=batch_size,
device_id=types.CPU_ONLY_DEVICE_ID,
num_threads=1,
value=value)
with tf.device('/cpu'):
ds = dali_tf.DALIDataset(pipe,
device_id=types.CPU_ONLY_DEVICE_ID,
batch_size=1,
output_dtypes=tf.int32,
output_shapes=[1])
ds = iter(ds)
data = next(ds)
assert data == np.array([value])
def get_dataset(self):
output_shapes = [
(self._batch_size, self._image_size[0], self._image_size[1], 3),
(self._batch_size,),
(self._batch_size, None, 4),
(self._batch_size, None),
]
output_dtypes = [tf.float32, tf.float32, tf.float32, tf.int32]
for level in range(self._anchors.min_level, self._anchors.max_level + 1):
feat_size = self._anchors.feat_sizes[level]
output_shapes.append(
(
self._batch_size,
feat_size["height"],
feat_size["width"],
self._anchors.get_anchors_per_location(),
)
)
output_shapes.append(
(
self._batch_size,
feat_size["height"],
feat_size["width"],
self._anchors.get_anchors_per_location() * 4,
)
)
output_dtypes.append(tf.int32)
output_dtypes.append(tf.float32)
dataset = dali_tf.DALIDataset(
pipeline=self._pipe,
batch_size=self._batch_size,
output_shapes=tuple(output_shapes),
output_dtypes=tuple(output_dtypes),
)
return dataset
def load_tfrecord(directory, batch_size, training):
tfrecord = []
tfrecord_idx = []
for f in os.listdir(directory):
fullpath = os.path.join(directory, f)
if not os.path.isfile(fullpath):
continue
if f.endswith(".tfrecord"):
tfrecord.append(fullpath)
if f.endswith(".idx"):
tfrecord_idx.append(fullpath)
tfrecord.sort()
tfrecord_idx.sort()
pipe = Pipeline(batch_size=batch_size, num_threads=32, device_id=0)
with pipe:
inputs = fn.tfrecord_reader(
path=tfrecord,
index_path=tfrecord_idx,
features={
"frame_one":
tfrec.FixedLenFeature((), tfrec.string, ""),
"frame_two":
tfrec.FixedLenFeature((), tfrec.string, ""),
"frame_three":
tfrec.FixedLenFeature((), tfrec.string, ""),
"frame_four":
tfrec.FixedLenFeature((), tfrec.string, ""),
"plus_one_position":
tfrec.FixedLenFeature([3], tfrec.float32, 0.0),
"plus_one_orientation":
tfrec.FixedLenFeature([3], tfrec.float32, 0.0),
"plus_two_position":
tfrec.FixedLenFeature([3], tfrec.float32, 0.0),
"plus_two_orientation":
tfrec.FixedLenFeature([3], tfrec.float32, 0.0),
"plus_three_position":
tfrec.FixedLenFeature([3], tfrec.float32, 0.0),
"plus_three_orientation":
tfrec.FixedLenFeature([3], tfrec.float32, 0.0),
"speed":
tfrec.FixedLenFeature([], tfrec.float32, 0.0),
})
frame1 = inputs["frame_one"]
frame1 = fn.image_decoder(frame1,
device="mixed",
output_type=types.RGB)
# frame1 = fn.resize(frame1, device="gpu", resize_shorter=256.)
frame1 = fn.crop_mirror_normalize(frame1,
device="gpu",
dtype=types.FLOAT,
mean=[0., 0., 0.],
std=[1., 1., 1.])
frame1 = fn.transpose(frame1, device="gpu", perm=[1, 2, 0])
frame2 = inputs["frame_two"]
frame2 = fn.image_decoder(frame2,
device="mixed",
output_type=types.RGB)
# frame2 = fn.resize(frame2, device="gpu", resize_shorter=256.)
frame2 = fn.crop_mirror_normalize(frame2,
device="gpu",
dtype=types.FLOAT,
mean=[0., 0., 0.],
std=[1., 1., 1.])
frame2 = fn.transpose(frame2, device="gpu", perm=[1, 2, 0])
position = inputs["plus_one_position"].gpu()
orientation = inputs["plus_one_orientation"].gpu()
speed = inputs["speed"].gpu()
image = fn.cat(frame1, frame2, device="gpu", axis=2)
pose = fn.cat(position, orientation, device="gpu", axis=0)
pipe.set_outputs(image, pose, speed)
# Define shapes and types of the outputs
shapes = ((batch_size, 480, 640), (batch_size, 6), (batch_size))
dtypes = (tf.float32, tf.float32)
# Create dataset
return dali_tf.DALIDataset(pipeline=pipe,
batch_size=batch_size,
output_shapes=shapes,
output_dtypes=dtypes,
device_id=0)
def _test_tf_dataset_multigpu():
num_devices = num_available_gpus()
batch_size = 8
num_threads = 4
shapes = [(batch_size, 3, 224, 224), (batch_size, 1), (batch_size, 1)]
dtypes = [tf.float32, tf.int32, tf.int16]
dataset_results = []
initializers = [tf.compat.v1.global_variables_initializer()]
ops_to_run = []
for device_id in range(num_devices):
with tf.device('/gpu:{0}'.format(device_id)):
dataset_pipeline = TestPipeline(batch_size, num_threads, 'gpu',
device_id, device_id, num_devices)
daliset = dali_tf.DALIDataset(pipeline=dataset_pipeline,
batch_size=batch_size,
shapes=shapes,
dtypes=dtypes,
num_threads=num_threads,
device_id=device_id)
daliset = daliset.with_options(dataset_options())
iterator = tf.compat.v1.data.make_initializable_iterator(daliset)
initializers.append(iterator.initializer)
ops_to_run.append(iterator.get_next())
standalone_pipeline = TestPipeline(batch_size,
num_threads,
device='gpu',
device_id=0)
standalone_pipeline.build()
dataset_size = standalone_pipeline.epoch_size("Reader")
iterations = dataset_size // batch_size
with tf.compat.v1.Session() as sess:
sess.run(initializers)
for _ in range(iterations):
dataset_results.append(sess.run(ops_to_run))
standalone_results = []
for _ in range(iterations):
standalone_results.append(
tuple(result.as_cpu().as_array()
for result in standalone_pipeline.run()))
assert len(dataset_results) == iterations
for it in range(iterations):
assert len(dataset_results[it]) == num_devices
for device_id in range(num_devices):
batch_id = iterations - (
(it + device_id *
(iterations // num_devices)) % iterations) - 1
it_id = iterations - it - 1
assert np.array_equal(standalone_results[it_id][0],
dataset_results[batch_id][device_id][0])
assert np.array_equal(standalone_results[it_id][1],
dataset_results[batch_id][device_id][1])
assert np.array_equal(standalone_results[it_id][2],
dataset_results[batch_id][device_id][2])
train_data = train_dataset_generator(dataset.take(train_sz), is_training=True, batch_sz=BATCH_SZ, columns=channels, data_size=train_sz)
val_data = get_dataset(dataset.skip(train_sz).take(valid_sz), start=train_sz, end=train_sz+valid_sz, columns=channels)
#test_data = test_dataset(dataset.skip(train_sz+valid_sz).take(test_sz), start=train_sz+valid_sz, end=train_sz+valid_sz+test_sz)
'''
print("\n Timestamp: " + str(tf.cast(tf.timestamp(), tf.float64)))
with tf.device('/gpu:0'):
daliop = dali_tf.DALIIterator()
shapes = [(BATCH_SZ, 125, 125, 8), (BATCH_SZ, 2)]
dtypes = [tf.float32, tf.int32]
# Create TF dataset
out_dataset = dali_tf.DALIDataset(pipeline=pipe,
batch_size=BATCH_SZ,
shapes=shapes,
dtypes=dtypes,
device_id=0)
opt = NovoGrad(learning_rate=lr_init * hvd.size())
#Wrap the optimizer in a Horovod distributed optimizer -> uses hvd.DistributedOptimizer() to compute gradients.
opt = hvd.DistributedOptimizer(opt)
resnet.compile(optimizer=opt,
loss='binary_crossentropy',
metrics=['accuracy'],
experimental_run_tf_function=False)
# Train using DALI dataset
history = resnet.fit(
out_dataset,
steps_per_epoch=train_sz // (BATCH_SZ * hvd.size()),
pipe3 = Opt3Pipeline("smallimgs_path.txt", 2, 0, batch_size, 8, 0)
with tf.device('/gpu:0'):
# Create dataset
# Define shapes and types of the outputs
shapes = [
(batch_size, 224, 224,3),
(batch_size,1)]
dtypes = [
tf.float32,
tf.int32]
out = dali_tf.DALIDataset(
pipeline=pipe3,
batch_size=batch_size,
shapes=shapes,
dtypes=dtypes,
device_id=0)
#out = out.with_options(dataset_options())
m=get_reset50_model()
m.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
# Train using DALI dataset
m.fit(
out,
