@pytest.mark.skipif(not tfds_loaded(), reason="requires tfds to be loaded")
def test_from_tfds_robonet():
import tensorflow_datasets as tfds
with tfds.testing.mock_data(num_examples=5):
ds = hub.Dataset.from_tfds("robonet", num=5)
res_ds = ds.store(
"./data/test_tfds/robonet",
length=5) # mock data doesn't have length, so explicitly provided
assert res_ds["video", 0, 0:2, 1, 2, 0].compute().tolist() == [59, 177]
assert res_ds["video", 2, 0:2, 1, 2, 0].compute().tolist() == [127, 62]
@pytest.mark.skipif(not tensorflow_loaded(),
reason="requires tensorflow to be loaded")
def test_from_tensorflow():
import tensorflow as tf
ds = tf.data.Dataset.from_tensor_slices(tf.range(10))
out_ds = hub.Dataset.from_tensorflow(ds)
res_ds = out_ds.store("./data/test_from_tf/ds1")
assert res_ds["data"].numpy().tolist() == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
ds = tf.data.Dataset.from_tensor_slices({"a": [1, 2], "b": [5, 6]})
out_ds = hub.Dataset.from_tensorflow(ds)
res_ds = out_ds.store("./data/test_from_tf/ds2")
assert res_ds["a"].numpy().tolist() == [1, 2]
assert res_ds["b"].numpy().tolist() == [5, 6]
ds = hub.Dataset.from_tfds("coco", num=5)
res_ds = ds.store(
"./data/test_tfds/coco", length=5
) # mock data doesn't have length, so explicitly provided
assert res_ds["image_id"].numpy().tolist() == [90, 38, 112, 194, 105]
assert res_ds["objects"].numpy()[0]["label"][0:5].tolist() == [
12,
15,
33,
23,
12,
]
@pytest.mark.skipif(not tensorflow_loaded(), reason="requires tensorflow to be loaded")
def test_from_tensorflow():
import tensorflow as tf
ds = tf.data.Dataset.from_tensor_slices(tf.range(10))
out_ds = hub.Dataset.from_tensorflow(ds)
res_ds = out_ds.store("./data/test_from_tf/ds1")
assert res_ds["data"].numpy().tolist() == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
ds = tf.data.Dataset.from_tensor_slices({"a": [1, 2], "b": [5, 6]})
out_ds = hub.Dataset.from_tensorflow(ds)
res_ds = out_ds.store("./data/test_from_tf/ds2")
assert res_ds["a"].numpy().tolist() == [1, 2]
assert res_ds["b"].numpy().tolist() == [5, 6]
torch.nn.ReLU(),
torch.nn.Linear(100, 10),
)
model_init = Model(model_arch)
model_init.store("./data/")
loaded_model = Model()
loaded_model.load(f"./data/{model_init._model.__class__.__name__}.pth")
assert repr(model_init._model) == repr(loaded_model._model)
for (p1, p2) in zip(model_init._model.named_parameters(),
loaded_model._model.named_parameters()):
assert p1[0] == p2[0]
assert torch.equal(p1[1].data, p2[1].data)
@pytest.mark.skipif(
not tensorflow_loaded(),
reason="requires tensorflow to be loaded",
)
def test_store_load_tf():
if tensorflow_spec is None:
raise ModuleNotFoundError("Module 'tensorflow' is not installed")
ext = ".h5" if int((tf.__version__)[0]) < 2 else ".tf"
def get_model():
inputs = tf.keras.Input(shape=(32, ))
outputs = tf.keras.layers.Dense(1)(inputs)
model = tf.keras.Model(inputs, outputs)
model.compile(optimizer="adam", loss="mean_squared_error")
return model
model_arch = get_model()
batch_size=1,
num_workers=0,
collate_fn=torch_ds.collate_fn)
data = list(train_loader)
assert len(data) == 2
for i in range(2):
assert "t1" in data[i]
assert "t2" in data[i]
assert data[0]["t1"][0].tolist() == [1, 2]
assert data[0]["t2"][0] == [5, 6, 7, 8]
assert data[1]["t1"][0].tolist() == [3, 4]
assert data[1]["t2"][0] == [7, 8, 9]
@pytest.mark.skipif(
True or not (tensorflow_loaded() and pytorch_loaded() and dask_loaded()),
reason="requires both pytorch and tensorflow to be loaded",
)
def test_to_backend_with_tf_and_pytorch():
import tensorflow as tf
import torch
tf.compat.v1.enable_eager_execution()
ds = dataset.load("mnist/mnist")
tfds = ds.to_tensorflow()
ptds = ds.to_pytorch()
ptds = torch.utils.data.DataLoader(
ptds,
batch_size=1,
num_workers=1,