def test_dataset_append_and_read():
dt = {"first": "float", "second": "float"}
os.makedirs("./data/test/test_dataset_append_and_read", exist_ok=True)
shutil.rmtree("./data/test/test_dataset_append_and_read")
ds = Dataset(
schema=dt,
shape=(2,),
url="./data/test/test_dataset_append_and_read",
mode="a",
)
ds["first"][0] = 2.3
ds.meta_information["description"] = "This is my description"
assert ds.meta_information["description"] == "This is my description"
assert ds["second"][0].numpy() != 2.3
ds.close()
ds = Dataset(
url="./data/test/test_dataset_append_and_read",
mode="r",
)
assert ds.meta_information["description"] == "This is my description"
ds.meta_information["hello"] = 5
ds.delete()
ds.close()
def test_dataset_copy_exception():
ds = Dataset("./data/test_data_cp", shape=(100,), schema=simple_schema)
ds2 = Dataset("./data/test_data_cp_2", shape=(100,), schema=simple_schema)
for i in range(100):
ds["num", i] = i
ds2["num", i] = 2 * i
ds.flush()
ds2.flush()
with pytest.raises(DirectoryNotEmptyException):
ds3 = ds.copy("./data/test_data_cp_2")
ds.delete()
ds2.delete()
def test_dataset_copy_gcs_s3():
ds = Dataset(
"s3://snark-test/cp_original_ds_s3_2", shape=(100,), schema=simple_schema
)
for i in range(100):
ds["num", i] = 2 * i
ds2 = ds.copy("gcs://snark-test/cp_copy_dataset_gcs_2")
ds3 = ds2.copy("s3://snark-test/cp_copy_ds_s3_3")
for i in range(100):
assert ds2["num", i].compute() == 2 * i
assert ds3["num", i].compute() == 2 * i
ds.delete()
ds2.delete()
ds3.delete()
def test_dataset_copy_hub_local():
password = os.getenv("ACTIVELOOP_HUB_PASSWORD")
login_fn("testingacc", password)
ds = Dataset("testingacc/cp_original_ds_hub_1", shape=(100,), schema=simple_schema)
for i in range(100):
ds["num", i] = 2 * i
ds2 = ds.copy("./data/testing/cp_copy_ds_local_5")
ds3 = ds2.copy("testingacc/cp_copy_dataset_testing_2")
for i in range(100):
assert ds2["num", i].compute() == 2 * i
assert ds3["num", i].compute() == 2 * i
ds.delete()
ds2.delete()
ds3.delete()
def test_dataset_copy_s3_local():
ds = Dataset(
"./data/testing/cp_original_data_local", shape=(100,), schema=simple_schema
)
for i in range(100):
ds["num", i] = 2 * i
ds2 = ds.copy("s3://snark-test/cp_copy_data_s3_1")
ds3 = ds2.copy("./data/testing/cp_copy_data_local_1")
for i in range(100):
assert ds2["num", i].compute() == 2 * i
assert ds3["num", i].compute() == 2 * i
ds.delete()
ds2.delete()
ds3.delete()
def test_dataset_append_and_read():
dt = {"first": "float", "second": "float"}
ds = Dataset(
schema=dt,
shape=(2,),
url="./data/test/test_dataset_append_and_read",
mode="a",
)
ds["first"][0] = 2.3
assert ds["second"][0].numpy() != 2.3
ds.close()
ds = Dataset(
url="./data/test/test_dataset_append_and_read",
mode="r",
)
ds.delete()
ds.close()
def test_dataset_copy_azure_local():
token = {"account_key": os.getenv("ACCOUNT_KEY")}
ds = Dataset(
"https://activeloop.blob.core.windows.net/activeloop-hub/cp_original_test_ds_azure_1",
token=token,
shape=(100,),
schema=simple_schema,
)
for i in range(100):
ds["num", i] = 2 * i
ds2 = ds.copy("./data/testing/cp_copy_ds_local_4")
ds3 = ds2.copy(
"https://activeloop.blob.core.windows.net/activeloop-hub/cp_copy_test_ds_azure_2",
token=token,
)
for i in range(100):
assert ds2["num", i].compute() == 2 * i
assert ds3["num", i].compute() == 2 * i
ds.delete()
ds2.delete()
ds3.delete()
def test_dataset_copy_gcs_s3():
ds = Dataset(
"s3://snark-test/cp_original_ds_s3_2_a", shape=(100,), schema=simple_schema
)
DS2_PATH = "gcs://snark-test/cp_copy_dataset_gcs_2_a"
DS3_PATH = "s3://snark-test/cp_copy_ds_s3_3_a"
for i in range(100):
ds["num", i] = 2 * i
try:
ds2 = ds.copy(DS2_PATH)
except:
dsi = Dataset(DS2_PATH)
dsi.delete()
ds2 = ds.copy(DS2_PATH)
try:
ds3 = ds2.copy(DS3_PATH)
except:
dsi = Dataset(DS3_PATH)
dsi.delete()
ds3 = ds2.copy(DS3_PATH)
for i in range(100):
assert ds2["num", i].compute() == 2 * i
assert ds3["num", i].compute() == 2 * i
ds.delete()
ds2.delete()
ds3.delete()
def test_dataset_copy_s3_local():
ds = Dataset(
"./data/testing/cp_original_data_local", shape=(100,), schema=simple_schema
)
DS2_PATH = "s3://snark-test/cp_copy_data_s3_1_a"
DS3_PATH = "./data/testing/cp_copy_data_local_1"
for i in range(100):
ds["num", i] = 2 * i
try:
ds2 = ds.copy(DS2_PATH)
except:
dsi = Dataset(DS2_PATH)
dsi.delete()
ds2 = ds.copy(DS2_PATH)
try:
ds3 = ds2.copy(DS3_PATH)
except:
dsi = Dataset(DS3_PATH)
dsi.delete()
ds3 = ds2.copy(DS3_PATH)
for i in range(100):
assert ds2["num", i].compute() == 2 * i
assert ds3["num", i].compute() == 2 * i
ds.delete()
ds2.delete()
ds3.delete()
