def test_metadata_write(storage): fs = AzureBlobFileSystem(account_name=storage.account_name, connection_string=CONN_STR) fs.mkdir("test_metadata_write") data = b"0123456789" metadata = {"meta": "data"} # standard blob type with fs.open("test_metadata_write/file.txt", "wb", metadata=metadata) as f: f.write(data) info = fs.info("test_metadata_write/file.txt") assert info["metadata"] == metadata metadata_changed_on_write = {"meta": "datum"} with fs.open("test_metadata_write/file.txt", "wb", metadata=metadata_changed_on_write) as f: f.write(data) info = fs.info("test_metadata_write/file.txt") assert info["metadata"] == metadata_changed_on_write # append blob type new_metadata = {"data": "meta"} with fs.open("test_metadata_write/append-file.txt", "ab", metadata=metadata) as f: f.write(data) # try change metadata on block appending with fs.open("test_metadata_write/append-file.txt", "ab", metadata=new_metadata) as f: f.write(data) info = fs.info("test_metadata_write/append-file.txt") # azure blob client doesn't seem to support metadata mutation when appending blocks # lets be sure this behavior doesn't change as this would imply # a potential breaking change assert info["metadata"] == metadata # getxattr / setxattr assert fs.getxattr("test_metadata_write/file.txt", "meta") == "datum" fs.setxattrs("test_metadata_write/file.txt", metadata="data2") assert fs.getxattr("test_metadata_write/file.txt", "metadata") == "data2" assert fs.info("test_metadata_write/file.txt")["metadata"] == { "metadata": "data2" } # empty file and nested directory with fs.open("test_metadata_write/a/b/c/nested-file.txt", "wb", metadata=metadata) as f: f.write(b"") assert fs.getxattr("test_metadata_write/a/b/c/nested-file.txt", "meta") == "data" fs.setxattrs("test_metadata_write/a/b/c/nested-file.txt", metadata="data2") assert fs.info( "test_metadata_write/a/b/c/nested-file.txt")["metadata"] == { "metadata": "data2" } fs.rmdir("test_metadata_write")
def test_mkdir_rmdir(storage): fs = AzureBlobFileSystem( account_name=storage.account_name, connection_string=CONN_STR, ) fs.mkdir("new-container") assert "new-container/" in fs.ls("") assert fs.ls("new-container") == [] with fs.open(path="new-container/file.txt", mode="wb") as f: f.write(b"0123456789") with fs.open("new-container/dir/file.txt", "wb") as f: f.write(b"0123456789") with fs.open("new-container/dir/file2.txt", "wb") as f: f.write(b"0123456789") # Check to verify you can skip making a directory if the container # already exists, but still create a file in that directory fs.mkdir("new-container/dir/file.txt", exist_ok=False) assert "new-container/" in fs.ls("") fs.mkdir("new-container/file2.txt", exist_ok=False) assert "new-container/file2.txt" in fs.ls("new-container") # Test to verify that the file contains expected contents with fs.open("new-container/file2.txt", "rb") as f: outfile = f.read() assert outfile == b"" # Check that trying to overwrite an existing nested file in append mode works as expected fs.mkdir("new-container/dir/file2.txt", exist_ok=False) assert "new-container/dir/file2.txt" in fs.ls("new-container/dir") # Also verify you can make a nested directory structure fs.mkdir("new-container/dir2/file.txt", exist_ok=False) with fs.open("new-container/dir2/file.txt", "wb") as f: f.write(b"0123456789") assert "new-container/dir2/file.txt" in fs.ls("new-container/dir2") fs.rm("new-container/dir2", recursive=True) fs.rm("new-container/dir", recursive=True) assert fs.ls("new-container") == [ "new-container/file.txt", "new-container/file2.txt", ] fs.rm("new-container/file.txt") fs.rm("new-container/file2.txt") fs.rmdir("new-container") assert "new-container/" not in fs.ls("")
def test_makedir_rmdir(storage, caplog): fs = AzureBlobFileSystem( account_name=storage.account_name, connection_string=CONN_STR, ) fs.makedir("new-container") assert "new-container" in fs.ls("") assert fs.ls("new-container") == [] with fs.open(path="new-container/file.txt", mode="wb") as f: f.write(b"0123456789") with fs.open("new-container/dir/file.txt", "wb") as f: f.write(b"0123456789") with fs.open("new-container/dir/file2.txt", "wb") as f: f.write(b"0123456789") # Verify that mkdir will raise an exception if the directory exists # and exist_ok is False with pytest.raises(FileExistsError): fs.makedir("new-container/dir/file.txt", exist_ok=False) # mkdir should raise an error if the container exists and # we try to create a nested directory, with exist_ok=False with pytest.raises(FileExistsError): fs.makedir("new-container/dir2", exist_ok=False) # Check that trying to overwrite an existing nested file in append mode works as expected # if exist_ok is True fs.makedir("new-container/dir/file2.txt", exist_ok=True) assert "new-container/dir/file2.txt" in fs.ls("new-container/dir") # Also verify you can make a nested directory structure with fs.open("new-container/dir2/file.txt", "wb") as f: f.write(b"0123456789") assert "new-container/dir2/file.txt" in fs.ls("new-container/dir2") fs.rm("new-container/dir2", recursive=True) fs.rm("new-container/dir", recursive=True) fs.touch("new-container/file2.txt") assert fs.ls("new-container") == [ "new-container/file.txt", "new-container/file2.txt", ] fs.rm("new-container/file.txt") fs.rm("new-container/file2.txt") fs.rmdir("new-container") assert "new-container" not in fs.ls("")
def test_mkdir_rmdir(storage): fs = AzureBlobFileSystem( account_name=storage.account_name, connection_string=CONN_STR, ) fs.mkdir("new-container") assert "new-container/" in fs.ls("") assert fs.ls("new-container") == [] with fs.open(path="new-container/file.txt", mode="wb") as f: f.write(b"0123456789") with fs.open("new-container/dir/file.txt", "wb") as f: f.write(b"0123456789") with fs.open("new-container/dir/file2.txt", "wb") as f: f.write(b"0123456789") # Check to verify you can skip making a directory if the container # already exists, but still create a file in that directory fs.mkdir("new-container/dir/file.txt", exists_ok=True) assert "new-container/" in fs.ls("") fs.mkdir("new-container/file2.txt", exists_ok=True) with fs.open("new-container/file2.txt", "wb") as f: f.write(b"0123456789") assert "new-container/file2.txt" in fs.ls("new-container") fs.mkdir("new-container/dir/file2.txt", exists_ok=True) with fs.open("new-container/dir/file2.txt", "wb") as f: f.write(b"0123456789") assert "new-container/dir/file2.txt" in fs.ls("new-container/dir") # Also verify you can make a nested directory structure fs.mkdir("new-container/dir2/file.txt", exists_ok=True) with fs.open("new-container/dir2/file.txt", "wb") as f: f.write(b"0123456789") assert "new-container/dir2/file.txt" in fs.ls("new-container/dir2") fs.rm("new-container/dir2", recursive=True) fs.rm("new-container/dir", recursive=True) assert fs.ls("new-container") == [ "new-container/file.txt", "new-container/file2.txt", ] fs.rm("new-container/file.txt") fs.rm("new-container/file2.txt") fs.rmdir("new-container") assert "new-container/" not in fs.ls("")
def test_dask_parquet(storage): fs = AzureBlobFileSystem(account_name=storage.account_name, connection_string=CONN_STR) fs.mkdir("test") STORAGE_OPTIONS = { "account_name": "devstoreaccount1", "connection_string": CONN_STR, } df = pd.DataFrame({ "col1": [1, 2, 3, 4], "col2": [2, 4, 6, 8], "index_key": [1, 1, 2, 2], "partition_key": [1, 1, 2, 2], }) dask_dataframe = dd.from_pandas(df, npartitions=1) for protocol in ["abfs", "az"]: dask_dataframe.to_parquet( "{}://test/test_group.parquet".format(protocol), storage_options=STORAGE_OPTIONS, engine="pyarrow", ) fs = AzureBlobFileSystem(**STORAGE_OPTIONS) assert fs.ls("test/test_group.parquet") == [ "test/test_group.parquet/_common_metadata", "test/test_group.parquet/_metadata", "test/test_group.parquet/part.0.parquet", ] fs.rm("test/test_group.parquet") df_test = dd.read_parquet( "abfs://test/test_group.parquet", storage_options=STORAGE_OPTIONS, engine="pyarrow", ).compute() assert_frame_equal(df, df_test) A = np.random.randint(0, 100, size=(10000, 4)) df2 = pd.DataFrame(data=A, columns=list("ABCD")) ddf2 = dd.from_pandas(df2, npartitions=4) dd.to_parquet( ddf2, "abfs://test/test_group2.parquet", storage_options=STORAGE_OPTIONS, engine="pyarrow", ) assert fs.ls("test/test_group2.parquet") == [ "test/test_group2.parquet/_common_metadata", "test/test_group2.parquet/_metadata", "test/test_group2.parquet/part.0.parquet", "test/test_group2.parquet/part.1.parquet", "test/test_group2.parquet/part.2.parquet", "test/test_group2.parquet/part.3.parquet", ] df2_test = dd.read_parquet( "abfs://test/test_group2.parquet", storage_options=STORAGE_OPTIONS, engine="pyarrow", ).compute() assert_frame_equal(df2, df2_test) a = np.full(shape=(10000, 1), fill_value=1) b = np.full(shape=(10000, 1), fill_value=2) c = np.full(shape=(10000, 1), fill_value=3) d = np.full(shape=(10000, 1), fill_value=4) B = np.concatenate((a, b, c, d), axis=1) df3 = pd.DataFrame(data=B, columns=list("ABCD")) ddf3 = dd.from_pandas(df3, npartitions=4) dd.to_parquet( ddf3, "abfs://test/test_group3.parquet", partition_on=["A", "B"], storage_options=STORAGE_OPTIONS, engine="pyarrow", ) assert fs.glob("test/test_group3.parquet/*") == [ "test/test_group3.parquet/A=1", "test/test_group3.parquet/_common_metadata", "test/test_group3.parquet/_metadata", ] df3_test = dd.read_parquet( "abfs://test/test_group3.parquet", filters=[("A", "=", 1)], storage_options=STORAGE_OPTIONS, engine="pyarrow", ).compute() df3_test = df3_test[["A", "B", "C", "D"]] df3_test = df3_test[["A", "B", "C", "D"]].astype(int) assert_frame_equal(df3, df3_test) A = np.random.randint(0, 100, size=(10000, 4)) df4 = pd.DataFrame(data=A, columns=list("ABCD")) ddf4 = dd.from_pandas(df4, npartitions=4) dd.to_parquet( ddf4, "abfs://test/test_group4.parquet", storage_options=STORAGE_OPTIONS, engine="pyarrow", flavor="spark", write_statistics=False, ) fs.rmdir("test/test_group4.parquet/_common_metadata", recursive=True) fs.rmdir("test/test_group4.parquet/_metadata", recursive=True) fs.rm("test/test_group4.parquet/_common_metadata") fs.rm("test/test_group4.parquet/_metadata") assert fs.ls("test/test_group4.parquet") == [ "test/test_group4.parquet/part.0.parquet", "test/test_group4.parquet/part.1.parquet", "test/test_group4.parquet/part.2.parquet", "test/test_group4.parquet/part.3.parquet", ] df4_test = dd.read_parquet( "abfs://test/test_group4.parquet", storage_options=STORAGE_OPTIONS, engine="pyarrow", ).compute() assert_frame_equal(df4, df4_test) A = np.random.randint(0, 100, size=(10000, 4)) df5 = pd.DataFrame(data=A, columns=list("ABCD")) ddf5 = dd.from_pandas(df5, npartitions=4) dd.to_parquet( ddf5, "abfs://test/test group5.parquet", storage_options=STORAGE_OPTIONS, engine="pyarrow", ) assert fs.ls("test/test group5.parquet") == [ "test/test group5.parquet/_common_metadata", "test/test group5.parquet/_metadata", "test/test group5.parquet/part.0.parquet", "test/test group5.parquet/part.1.parquet", "test/test group5.parquet/part.2.parquet", "test/test group5.parquet/part.3.parquet", ] df5_test = dd.read_parquet( "abfs://test/test group5.parquet", storage_options=STORAGE_OPTIONS, engine="pyarrow", ).compute() assert_frame_equal(df5, df5_test)
def test_makedir_rmdir(storage, caplog): fs = AzureBlobFileSystem( account_name=storage.account_name, connection_string=CONN_STR, ) fs.makedir("new-container") assert "new-container/" in fs.ls("") assert fs.ls("new-container") == [] with fs.open(path="new-container/file.txt", mode="wb") as f: f.write(b"0123456789") with fs.open("new-container/dir/file.txt", "wb") as f: f.write(b"0123456789") with fs.open("new-container/dir/file2.txt", "wb") as f: f.write(b"0123456789") # Verify that mkdir will raise an exception if the directory exists # and exist_ok is False with pytest.raises(FileExistsError): fs.makedir("new-container/dir/file.txt", exist_ok=False) # Verify that mkdir creates a directory if exist_ok is False and the # directory does not exist fs.makedir("new-container/file2.txt", exist_ok=False) assert "new-container/file2.txt" in fs.ls("new-container") # Verify that mkdir will silently ignore an existing directory if # the directory exists and exist_ok is True fs.makedir("new-container/dir", exist_ok=True) assert "new-container/dir/" in fs.ls("new-container") # Test to verify that the file contains expected contents with fs.open("new-container/file2.txt", "rb") as f: outfile = f.read() assert outfile == b"" # Check that trying to overwrite an existing nested file in append mode works as expected # if exist_ok is True fs.makedir("new-container/dir/file2.txt", exist_ok=True) assert "new-container/dir/file2.txt" in fs.ls("new-container/dir") # Also verify you can make a nested directory structure fs.makedir("new-container/dir2/file.txt", exist_ok=False) with fs.open("new-container/dir2/file.txt", "wb") as f: f.write(b"0123456789") assert "new-container/dir2/file.txt" in fs.ls("new-container/dir2") fs.rm("new-container/dir2", recursive=True) fs.rm("new-container/dir", recursive=True) assert fs.ls("new-container") == [ "new-container/file.txt", "new-container/file2.txt", ] fs.rm("new-container/file.txt") fs.rm("new-container/file2.txt") fs.rmdir("new-container") assert "new-container/" not in fs.ls("")