def test_datasaver_filetype(self):
from matorage.torch import Dataset
self.data_config = DataConfig(
**self.storage_config,
dataset_name="test_datasaver_filetype",
attributes=[DataAttribute("x", "float64", (2), itemsize=32)],
)
self.data_saver = DataSaver(config=self.data_config)
x = np.asarray([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
self.assertEqual(x.shape, (3, 2))
self.data_saver({"x": x})
_file = open("test.txt", "w")
_file.write('this is test')
self.data_saver({"file": "test.txt"}, filetype=True)
_file.close()
self.data_saver.disconnect()
self.dataset = Dataset(config=self.data_config,
cache_folder_path=self.cache_folder_path)
self.assertEqual(self.dataset.get_filetype_list, ["file"])
_local_filepath = self.dataset.get_filetype_from_key("file")
with open(_local_filepath, 'r') as f:
self.assertEqual(f.read(), 'this is test')
def test_tf_saver(self, data_config=None, save_to_json_file=False):
if data_config is None:
self.data_config = DataConfig(**self.storage_config,
dataset_name="test_tf_saver",
additional={
"framework": "tensorflow"
},
attributes=[
DataAttribute("image",
"uint8", (2, 2),
itemsize=32),
DataAttribute("target",
"uint8", (1),
itemsize=32),
])
else:
self.data_config = data_config
if save_to_json_file:
self.data_config_file = "data_config_file.json"
self.data_config.to_json_file(self.data_config_file)
self.data_saver = DataSaver(config=self.data_config)
self.data_saver({
"image":
np.asarray([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]),
"target":
np.asarray([0, 1]),
})
self.data_saver.disconnect()
def test_datasaver_s3_filetype(self):
from matorage.torch import Dataset
self.storage_config = {
'endpoint': 's3.us-east-1.amazonaws.com',
'access_key': os.environ['access_key'],
'secret_key': os.environ['secret_key'],
'region': 'us-east-1',
'secure': False,
}
self.data_config = DataConfig(
**self.storage_config,
dataset_name="test_datasaver_s3_filetype",
attributes=[DataAttribute("x", "float64", (2), itemsize=32)],
)
self.data_saver = DataSaver(config=self.data_config)
x = np.asarray([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
self.assertEqual(x.shape, (3, 2))
self.data_saver({"x": x})
_file = open("test.txt", "w")
_file.write('this is test')
self.data_saver({"file": "test.txt"}, filetype=True)
_file.close()
self.data_saver.disconnect()
self.dataset = Dataset(config=self.data_config,
cache_folder_path=self.cache_folder_path)
self.assertEqual(self.dataset.get_filetype_list, ["file"])
_local_filepath = self.dataset.get_filetype_from_key("file")
with open(_local_filepath, 'r') as f:
self.assertEqual(f.read(), 'this is test')
def test_datasaver_float64_attribute(self):
self.data_config = DataConfig(
**self.storage_config,
dataset_name="test_datasaver_float64_attribute",
attributes=[DataAttribute("x", "float64", (2), itemsize=32)])
self.data_saver = DataSaver(config=self.data_config)
x = np.asarray([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
self.assertEqual(x.shape, (3, 2))
self.data_saver({"x": x})
def test_datasaver_uint32_attribute(self):
self.data_config = DataConfig(
**self.storage_config,
dataset_name="test_datasaver_uint32_attribute",
attributes=[DataAttribute("x", "uint32", (2), itemsize=32)])
self.data_saver = DataSaver(config=self.data_config)
x = np.asarray([[1, 2], [3, 4], [5, 6]])
self.assertEqual(x.shape, (3, 2))
self.data_saver({"x": x})
def test_datasaver_bool_attribute(self):
self.data_config = DataConfig(
**self.storage_config,
dataset_name="test_datasaver_bool_attribute",
attributes=[DataAttribute("x", "bool", (2), itemsize=32)])
self.data_saver = DataSaver(config=self.data_config)
x = np.asarray([[True, False], [False, True], [True, True]])
self.assertEqual(x.shape, (3, 2))
self.data_saver({"x": x})
def test_datasaver_string_attribute(self):
self.data_config = DataConfig(
**self.storage_config,
dataset_name="test_datasaver_string_attribute",
attributes=[DataAttribute("x", "string", (2), itemsize=32)])
self.data_saver = DataSaver(config=self.data_config)
x = np.asarray([["a", "b"], ["c", "d"], ["e", "f"]])
self.assertEqual(x.shape, (3, 2))
self.data_saver({"x": x})
def test_datasaver_nas(self):
self.data_config = DataConfig(
endpoint="/tmp",
dataset_name="test_datasaver_nas",
attributes=[DataAttribute("x", "float64", (2), itemsize=32)],
)
self.data_saver = DataSaver(config=self.data_config)
x = np.asarray([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
self.assertEqual(x.shape, (3, 2))
self.data_saver({"x": x})
def test_datasaver_refresh(self):
self.data_config = DataConfig(
**self.storage_config,
dataset_name="test_datasaver_refresh",
attributes=[DataAttribute("x", "float64", (2), itemsize=32)],
)
for refresh in [False, True]:
self.data_saver = DataSaver(config=self.data_config, refresh=refresh)
x = np.asarray([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
self.assertEqual(x.shape, (3, 2))
self.data_saver({"x": x})
self.data_saver.disconnect()
def test_datasaver_blosc(self):
for level in range(10):
self.data_config = DataConfig(
**self.storage_config,
dataset_name="test_datasaver_blosc",
attributes=[DataAttribute("x", "float64", (2), itemsize=32)],
compressor={"complevel": level, "complib": "blosc"},
)
self.data_saver = DataSaver(config=self.data_config)
x = np.asarray([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
self.assertEqual(x.shape, (3, 2))
self.data_saver({"x": x})
self.data_saver.disconnect()
def test_saver_from_json_file(self):
self.test_tf_saver(save_to_json_file=True)
self.data_config = None
self.data_saver = None
self.data_config = DataConfig.from_json_file(self.data_config_file)
self.data_saver = DataSaver(config=self.data_config)
self.data_saver({
"image":
np.asarray([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]),
"target":
np.asarray([0, 1]),
})
self.data_saver.disconnect()
def test_datasaver_s3(self):
self.storage_config = {
'endpoint': 's3.us-east-1.amazonaws.com',
'access_key': os.environ['access_key'],
'secret_key': os.environ['secret_key'],
'region': 'us-east-1',
'secure': False,
}
self.data_config = DataConfig(
**self.storage_config,
dataset_name="test_datasaver_s3",
attributes=[DataAttribute("x", "float64", (2), itemsize=32)],
)
self.data_saver = DataSaver(config=self.data_config)
x = np.asarray([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
self.assertEqual(x.shape, (3, 2))
self.data_saver({"x": x})
self.data_saver.disconnect()
def test_torch_saver_nas(self):
self.data_config = DataConfig(**self.nas_config,
dataset_name="test_torch_saver_nas",
additional={"framework": "pytorch"},
attributes=[
DataAttribute("image",
"uint8", (2, 2),
itemsize=32),
DataAttribute("target",
"uint8", (1),
itemsize=32),
])
self.data_saver = DataSaver(config=self.data_config)
self.data_saver({
"image":
np.asarray([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]),
"target":
np.asarray([0, 1]),
})
self.data_saver.disconnect()