def test_tf_loader_with_compressor(self):
from matorage.tensorflow import Dataset
data_config = DataConfig(**self.storage_config,
dataset_name="test_tf_loader_with_compressor",
additional={"framework": "tensorflow"},
compressor={
"complevel": 4,
"complib": "zlib"
},
attributes=[
DataAttribute("image",
"uint8", (2, 2),
itemsize=32),
DataAttribute("target",
"uint8", (1),
itemsize=32),
])
self.test_tf_saver(data_config=data_config)
self.dataset = Dataset(config=data_config,
cache_folder_path=self.cache_folder_path)
for batch_idx, (image, target) in enumerate(
tqdm(self.dataset.dataloader, total=2)):
pass
def test_datasaver_filetype(self):
from matorage.tensorflow import Dataset
self.data_config = DataConfig(
**self.storage_config,
dataset_name="test_datasaver_filetype",
attributes=[DataAttribute("x", "float64", (2), itemsize=32)],
)
self.data_saver = DataSaver(config=self.data_config)
x = np.asarray([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
self.assertEqual(x.shape, (3, 2))
self.data_saver({"x": x})
_file = open("test.txt", "w")
_file.write('this is test')
self.data_saver({"file": "test.txt"}, filetype=True)
_file.close()
self.data_saver.disconnect()
self.dataset = Dataset(config=self.data_config,
cache_folder_path=self.cache_folder_path)
self.assertEqual(self.dataset.get_filetype_list, ["file"])
_local_filepath = self.dataset.get_filetype_from_key("file")
with open(_local_filepath, 'r') as f:
self.assertEqual(f.read(), 'this is test')
def test_tf_loader(self):
from matorage.tensorflow import Dataset
self.test_tf_saver()
self.dataset = Dataset(config=self.data_config,
cache_folder_path=self.cache_folder_path)
for batch_idx, (image, target) in enumerate(
tqdm(self.dataset.dataloader, total=2)):
pass
def test_loader_from_json_file(self):
from matorage.tensorflow import Dataset
self.test_tf_saver(save_to_json_file=True)
self.data_config = None
self.data_config = DataConfig.from_json_file(self.data_config_file)
self.dataset = Dataset(config=self.data_config,
cache_folder_path=self.cache_folder_path)
for batch_idx, (image, target) in enumerate(
tqdm(self.dataset.dataloader, total=2)):
pass
def test_tf_index_with_compressor(self):
from matorage.tensorflow import Dataset
data_config = DataConfig(**self.storage_config,
dataset_name="test_tf_index_with_compressor",
additional={"framework": "tensorflow"},
compressor={
"complevel": 4,
"complib": "zlib"
},
attributes=[
DataAttribute("image",
"uint8", (2, 2),
itemsize=32),
DataAttribute("target",
"uint8", (1),
itemsize=32),
])
self.test_tf_saver(data_config=data_config)
dataset = Dataset(config=self.data_config, index=True)
assert tf.reduce_all(
tf.equal(dataset[0][0],
tf.constant([[1, 2], [3, 4]], dtype=tf.uint8)))
assert tf.reduce_all(
tf.equal(dataset[0][1], tf.constant([0], dtype=tf.uint8)))
def test_tf_not_clear(self):
from matorage.tensorflow import Dataset
self.test_tf_loader()
if os.path.exists(self.dataset.cache_path):
with open(self.dataset.cache_path) as f:
_pre_file_mapper = json.load(f)
self.dataset = Dataset(config=self.data_config,
clear=False,
cache_folder_path=self.cache_folder_path)
if os.path.exists(self.dataset.cache_path):
with open(self.dataset.cache_path) as f:
_next_file_mapper = json.load(f)
self.assertEqual(_pre_file_mapper, _next_file_mapper)
def test_tf_index(self):
from matorage.tensorflow import Dataset
self.test_tf_loader()
dataset = Dataset(config=self.data_config, index=True)
assert tf.reduce_all(
tf.equal(dataset[0][0],
tf.constant([[1, 2], [3, 4]], dtype=tf.uint8)))
assert tf.reduce_all(
tf.equal(dataset[0][1], tf.constant([0], dtype=tf.uint8)))
class TFDataTest(DataTest, unittest.TestCase):
def test_tf_saver(self, data_config=None, save_to_json_file=False):
if data_config is None:
self.data_config = DataConfig(**self.storage_config,
dataset_name="test_tf_saver",
additional={
"framework": "tensorflow"
},
attributes=[
DataAttribute("image",
"uint8", (2, 2),
itemsize=32),
DataAttribute("target",
"uint8", (1),
itemsize=32),
])
else:
self.data_config = data_config
if save_to_json_file:
self.data_config_file = "data_config_file.json"
self.data_config.to_json_file(self.data_config_file)
self.data_saver = DataSaver(config=self.data_config)
self.data_saver({
"image":
np.asarray([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]),
"target":
np.asarray([0, 1]),
})
self.data_saver.disconnect()
def test_tf_loader(self):
from matorage.tensorflow import Dataset
self.test_tf_saver()
self.dataset = Dataset(config=self.data_config,
cache_folder_path=self.cache_folder_path)
for batch_idx, (image, target) in enumerate(
tqdm(self.dataset.dataloader, total=2)):
pass
def test_tf_loader_with_compressor(self):
from matorage.tensorflow import Dataset
data_config = DataConfig(**self.storage_config,
dataset_name="test_tf_loader_with_compressor",
additional={"framework": "tensorflow"},
compressor={
"complevel": 4,
"complib": "zlib"
},
attributes=[
DataAttribute("image",
"uint8", (2, 2),
itemsize=32),
DataAttribute("target",
"uint8", (1),
itemsize=32),
])
self.test_tf_saver(data_config=data_config)
self.dataset = Dataset(config=data_config,
cache_folder_path=self.cache_folder_path)
for batch_idx, (image, target) in enumerate(
tqdm(self.dataset.dataloader, total=2)):
pass
def test_tf_index(self):
from matorage.tensorflow import Dataset
self.test_tf_loader()
dataset = Dataset(config=self.data_config,
index=True,
cache_folder_path=self.cache_folder_path)
assert tf.reduce_all(
tf.equal(dataset[0][0],
tf.constant([[1, 2], [3, 4]], dtype=tf.uint8)))
assert tf.reduce_all(
tf.equal(dataset[0][1], tf.constant([0], dtype=tf.uint8)))
def test_tf_index_with_compressor(self):
from matorage.tensorflow import Dataset
data_config = DataConfig(**self.storage_config,
dataset_name="test_tf_index_with_compressor",
additional={"framework": "tensorflow"},
compressor={
"complevel": 4,
"complib": "zlib"
},
attributes=[
DataAttribute("image",
"uint8", (2, 2),
itemsize=32),
DataAttribute("target",
"uint8", (1),
itemsize=32),
])
self.test_tf_saver(data_config=data_config)
dataset = Dataset(config=self.data_config,
index=True,
cache_folder_path=self.cache_folder_path)
assert tf.reduce_all(
tf.equal(dataset[0][0],
tf.constant([[1, 2], [3, 4]], dtype=tf.uint8)))
assert tf.reduce_all(
tf.equal(dataset[0][1], tf.constant([0], dtype=tf.uint8)))
def test_saver_from_json_file(self):
self.test_tf_saver(save_to_json_file=True)
self.data_config = None
self.data_saver = None
self.data_config = DataConfig.from_json_file(self.data_config_file)
self.data_saver = DataSaver(config=self.data_config)
self.data_saver({
"image":
np.asarray([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]),
"target":
np.asarray([0, 1]),
})
self.data_saver.disconnect()
def test_loader_from_json_file(self):
from matorage.tensorflow import Dataset
self.test_tf_saver(save_to_json_file=True)
self.data_config = None
self.data_config = DataConfig.from_json_file(self.data_config_file)
self.dataset = Dataset(config=self.data_config,
cache_folder_path=self.cache_folder_path)
for batch_idx, (image, target) in enumerate(
tqdm(self.dataset.dataloader, total=2)):
pass
def test_tf_not_clear(self):
from matorage.tensorflow import Dataset
self.test_tf_loader()
if os.path.exists(self.dataset.cache_path):
with open(self.dataset.cache_path) as f:
_pre_file_mapper = json.load(f)
self.dataset = Dataset(config=self.data_config,
clear=False,
cache_folder_path=self.cache_folder_path)
if os.path.exists(self.dataset.cache_path):
with open(self.dataset.cache_path) as f:
_next_file_mapper = json.load(f)
self.assertEqual(_pre_file_mapper, _next_file_mapper)
def test_datasaver_filetype(self):
from matorage.tensorflow import Dataset
self.data_config = DataConfig(
**self.storage_config,
dataset_name="test_datasaver_filetype",
attributes=[DataAttribute("x", "float64", (2), itemsize=32)],
)
self.data_saver = DataSaver(config=self.data_config)
x = np.asarray([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
self.assertEqual(x.shape, (3, 2))
self.data_saver({"x": x})
_file = open("test.txt", "w")
_file.write('this is test')
self.data_saver({"file": "test.txt"}, filetype=True)
_file.close()
self.data_saver.disconnect()
self.dataset = Dataset(config=self.data_config,
cache_folder_path=self.cache_folder_path)
self.assertEqual(self.dataset.get_filetype_list, ["file"])
_local_filepath = self.dataset.get_filetype_from_key("file")
with open(_local_filepath, 'r') as f:
self.assertEqual(f.read(), 'this is test')
def test_tf_saver_nas(self):
self.data_config = DataConfig(**self.nas_config,
dataset_name="test_tf_saver_nas",
additional={"framework": "tensorflow"},
attributes=[
DataAttribute("image",
"uint8", (2, 2),
itemsize=32),
DataAttribute("target",
"uint8", (1),
itemsize=32),
])
self.data_saver = DataSaver(config=self.data_config)
self.data_saver({
"image":
np.asarray([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]),
"target":
np.asarray([0, 1]),
})
self.data_saver.disconnect()
def test_tf_loader_nas(self):
from matorage.tensorflow import Dataset
self.test_tf_saver_nas()
self.dataset = Dataset(config=self.data_config,
cache_folder_path=self.cache_folder_path)
for batch_idx, (image, target) in enumerate(
tqdm(self.dataset.dataloader, total=2)):
pass
