def test_dataset_append_and_read(): dt = {"first": "float", "second": "float"} os.makedirs("./data/test/test_dataset_append_and_read", exist_ok=True) shutil.rmtree("./data/test/test_dataset_append_and_read") ds = Dataset( schema=dt, shape=(2,), url="./data/test/test_dataset_append_and_read", mode="a", ) ds["first"][0] = 2.3 ds.meta_information["description"] = "This is my description" assert ds.meta_information["description"] == "This is my description" assert ds["second"][0].numpy() != 2.3 ds.close() ds = Dataset( url="./data/test/test_dataset_append_and_read", mode="r", ) assert ds.meta_information["description"] == "This is my description" ds.meta_information["hello"] = 5 ds.delete() ds.close()
def test_dataset_copy_exception(): ds = Dataset("./data/test_data_cp", shape=(100,), schema=simple_schema) ds2 = Dataset("./data/test_data_cp_2", shape=(100,), schema=simple_schema) for i in range(100): ds["num", i] = i ds2["num", i] = 2 * i ds.flush() ds2.flush() with pytest.raises(DirectoryNotEmptyException): ds3 = ds.copy("./data/test_data_cp_2") ds.delete() ds2.delete()
def test_dataset_copy_gcs_s3(): ds = Dataset( "s3://snark-test/cp_original_ds_s3_2", shape=(100,), schema=simple_schema ) for i in range(100): ds["num", i] = 2 * i ds2 = ds.copy("gcs://snark-test/cp_copy_dataset_gcs_2") ds3 = ds2.copy("s3://snark-test/cp_copy_ds_s3_3") for i in range(100): assert ds2["num", i].compute() == 2 * i assert ds3["num", i].compute() == 2 * i ds.delete() ds2.delete() ds3.delete()
def test_dataset_copy_hub_local(): password = os.getenv("ACTIVELOOP_HUB_PASSWORD") login_fn("testingacc", password) ds = Dataset("testingacc/cp_original_ds_hub_1", shape=(100,), schema=simple_schema) for i in range(100): ds["num", i] = 2 * i ds2 = ds.copy("./data/testing/cp_copy_ds_local_5") ds3 = ds2.copy("testingacc/cp_copy_dataset_testing_2") for i in range(100): assert ds2["num", i].compute() == 2 * i assert ds3["num", i].compute() == 2 * i ds.delete() ds2.delete() ds3.delete()
def test_dataset_copy_s3_local(): ds = Dataset( "./data/testing/cp_original_data_local", shape=(100,), schema=simple_schema ) for i in range(100): ds["num", i] = 2 * i ds2 = ds.copy("s3://snark-test/cp_copy_data_s3_1") ds3 = ds2.copy("./data/testing/cp_copy_data_local_1") for i in range(100): assert ds2["num", i].compute() == 2 * i assert ds3["num", i].compute() == 2 * i ds.delete() ds2.delete() ds3.delete()
def test_dataset_append_and_read(): dt = {"first": "float", "second": "float"} ds = Dataset( schema=dt, shape=(2,), url="./data/test/test_dataset_append_and_read", mode="a", ) ds["first"][0] = 2.3 assert ds["second"][0].numpy() != 2.3 ds.close() ds = Dataset( url="./data/test/test_dataset_append_and_read", mode="r", ) ds.delete() ds.close()
def test_dataset_copy_azure_local(): token = {"account_key": os.getenv("ACCOUNT_KEY")} ds = Dataset( "https://activeloop.blob.core.windows.net/activeloop-hub/cp_original_test_ds_azure_1", token=token, shape=(100,), schema=simple_schema, ) for i in range(100): ds["num", i] = 2 * i ds2 = ds.copy("./data/testing/cp_copy_ds_local_4") ds3 = ds2.copy( "https://activeloop.blob.core.windows.net/activeloop-hub/cp_copy_test_ds_azure_2", token=token, ) for i in range(100): assert ds2["num", i].compute() == 2 * i assert ds3["num", i].compute() == 2 * i ds.delete() ds2.delete() ds3.delete()
def test_dataset_copy_gcs_s3(): ds = Dataset( "s3://snark-test/cp_original_ds_s3_2_a", shape=(100,), schema=simple_schema ) DS2_PATH = "gcs://snark-test/cp_copy_dataset_gcs_2_a" DS3_PATH = "s3://snark-test/cp_copy_ds_s3_3_a" for i in range(100): ds["num", i] = 2 * i try: ds2 = ds.copy(DS2_PATH) except: dsi = Dataset(DS2_PATH) dsi.delete() ds2 = ds.copy(DS2_PATH) try: ds3 = ds2.copy(DS3_PATH) except: dsi = Dataset(DS3_PATH) dsi.delete() ds3 = ds2.copy(DS3_PATH) for i in range(100): assert ds2["num", i].compute() == 2 * i assert ds3["num", i].compute() == 2 * i ds.delete() ds2.delete() ds3.delete()
def test_dataset_copy_s3_local(): ds = Dataset( "./data/testing/cp_original_data_local", shape=(100,), schema=simple_schema ) DS2_PATH = "s3://snark-test/cp_copy_data_s3_1_a" DS3_PATH = "./data/testing/cp_copy_data_local_1" for i in range(100): ds["num", i] = 2 * i try: ds2 = ds.copy(DS2_PATH) except: dsi = Dataset(DS2_PATH) dsi.delete() ds2 = ds.copy(DS2_PATH) try: ds3 = ds2.copy(DS3_PATH) except: dsi = Dataset(DS3_PATH) dsi.delete() ds3 = ds2.copy(DS3_PATH) for i in range(100): assert ds2["num", i].compute() == 2 * i assert ds3["num", i].compute() == 2 * i ds.delete() ds2.delete() ds3.delete()