def test_metadata_write(storage):
fs = AzureBlobFileSystem(account_name=storage.account_name,
connection_string=CONN_STR)
fs.mkdir("test_metadata_write")
data = b"0123456789"
metadata = {"meta": "data"}
# standard blob type
with fs.open("test_metadata_write/file.txt", "wb", metadata=metadata) as f:
f.write(data)
info = fs.info("test_metadata_write/file.txt")
assert info["metadata"] == metadata
metadata_changed_on_write = {"meta": "datum"}
with fs.open("test_metadata_write/file.txt",
"wb",
metadata=metadata_changed_on_write) as f:
f.write(data)
info = fs.info("test_metadata_write/file.txt")
assert info["metadata"] == metadata_changed_on_write
# append blob type
new_metadata = {"data": "meta"}
with fs.open("test_metadata_write/append-file.txt",
"ab",
metadata=metadata) as f:
f.write(data)
# try change metadata on block appending
with fs.open("test_metadata_write/append-file.txt",
"ab",
metadata=new_metadata) as f:
f.write(data)
info = fs.info("test_metadata_write/append-file.txt")
# azure blob client doesn't seem to support metadata mutation when appending blocks
# lets be sure this behavior doesn't change as this would imply
# a potential breaking change
assert info["metadata"] == metadata
# getxattr / setxattr
assert fs.getxattr("test_metadata_write/file.txt", "meta") == "datum"
fs.setxattrs("test_metadata_write/file.txt", metadata="data2")
assert fs.getxattr("test_metadata_write/file.txt", "metadata") == "data2"
assert fs.info("test_metadata_write/file.txt")["metadata"] == {
"metadata": "data2"
}
# empty file and nested directory
with fs.open("test_metadata_write/a/b/c/nested-file.txt",
"wb",
metadata=metadata) as f:
f.write(b"")
assert fs.getxattr("test_metadata_write/a/b/c/nested-file.txt",
"meta") == "data"
fs.setxattrs("test_metadata_write/a/b/c/nested-file.txt", metadata="data2")
assert fs.info(
"test_metadata_write/a/b/c/nested-file.txt")["metadata"] == {
"metadata": "data2"
}
fs.rmdir("test_metadata_write")
def test_mkdir_rmdir(storage):
fs = AzureBlobFileSystem(
account_name=storage.account_name,
connection_string=CONN_STR,
)
fs.mkdir("new-container")
assert "new-container/" in fs.ls("")
assert fs.ls("new-container") == []
with fs.open(path="new-container/file.txt", mode="wb") as f:
f.write(b"0123456789")
with fs.open("new-container/dir/file.txt", "wb") as f:
f.write(b"0123456789")
with fs.open("new-container/dir/file2.txt", "wb") as f:
f.write(b"0123456789")
# Check to verify you can skip making a directory if the container
# already exists, but still create a file in that directory
fs.mkdir("new-container/dir/file.txt", exist_ok=False)
assert "new-container/" in fs.ls("")
fs.mkdir("new-container/file2.txt", exist_ok=False)
assert "new-container/file2.txt" in fs.ls("new-container")
# Test to verify that the file contains expected contents
with fs.open("new-container/file2.txt", "rb") as f:
outfile = f.read()
assert outfile == b""
# Check that trying to overwrite an existing nested file in append mode works as expected
fs.mkdir("new-container/dir/file2.txt", exist_ok=False)
assert "new-container/dir/file2.txt" in fs.ls("new-container/dir")
# Also verify you can make a nested directory structure
fs.mkdir("new-container/dir2/file.txt", exist_ok=False)
with fs.open("new-container/dir2/file.txt", "wb") as f:
f.write(b"0123456789")
assert "new-container/dir2/file.txt" in fs.ls("new-container/dir2")
fs.rm("new-container/dir2", recursive=True)
fs.rm("new-container/dir", recursive=True)
assert fs.ls("new-container") == [
"new-container/file.txt",
"new-container/file2.txt",
]
fs.rm("new-container/file.txt")
fs.rm("new-container/file2.txt")
fs.rmdir("new-container")
assert "new-container/" not in fs.ls("")
def test_makedir_rmdir(storage, caplog):
fs = AzureBlobFileSystem(
account_name=storage.account_name,
connection_string=CONN_STR,
)
fs.makedir("new-container")
assert "new-container" in fs.ls("")
assert fs.ls("new-container") == []
with fs.open(path="new-container/file.txt", mode="wb") as f:
f.write(b"0123456789")
with fs.open("new-container/dir/file.txt", "wb") as f:
f.write(b"0123456789")
with fs.open("new-container/dir/file2.txt", "wb") as f:
f.write(b"0123456789")
# Verify that mkdir will raise an exception if the directory exists
# and exist_ok is False
with pytest.raises(FileExistsError):
fs.makedir("new-container/dir/file.txt", exist_ok=False)
# mkdir should raise an error if the container exists and
# we try to create a nested directory, with exist_ok=False
with pytest.raises(FileExistsError):
fs.makedir("new-container/dir2", exist_ok=False)
# Check that trying to overwrite an existing nested file in append mode works as expected
# if exist_ok is True
fs.makedir("new-container/dir/file2.txt", exist_ok=True)
assert "new-container/dir/file2.txt" in fs.ls("new-container/dir")
# Also verify you can make a nested directory structure
with fs.open("new-container/dir2/file.txt", "wb") as f:
f.write(b"0123456789")
assert "new-container/dir2/file.txt" in fs.ls("new-container/dir2")
fs.rm("new-container/dir2", recursive=True)
fs.rm("new-container/dir", recursive=True)
fs.touch("new-container/file2.txt")
assert fs.ls("new-container") == [
"new-container/file.txt",
"new-container/file2.txt",
]
fs.rm("new-container/file.txt")
fs.rm("new-container/file2.txt")
fs.rmdir("new-container")
assert "new-container" not in fs.ls("")
def test_mkdir_rmdir(storage):
fs = AzureBlobFileSystem(
account_name=storage.account_name,
connection_string=CONN_STR,
)
fs.mkdir("new-container")
assert "new-container/" in fs.ls("")
assert fs.ls("new-container") == []
with fs.open(path="new-container/file.txt", mode="wb") as f:
f.write(b"0123456789")
with fs.open("new-container/dir/file.txt", "wb") as f:
f.write(b"0123456789")
with fs.open("new-container/dir/file2.txt", "wb") as f:
f.write(b"0123456789")
# Check to verify you can skip making a directory if the container
# already exists, but still create a file in that directory
fs.mkdir("new-container/dir/file.txt", exists_ok=True)
assert "new-container/" in fs.ls("")
fs.mkdir("new-container/file2.txt", exists_ok=True)
with fs.open("new-container/file2.txt", "wb") as f:
f.write(b"0123456789")
assert "new-container/file2.txt" in fs.ls("new-container")
fs.mkdir("new-container/dir/file2.txt", exists_ok=True)
with fs.open("new-container/dir/file2.txt", "wb") as f:
f.write(b"0123456789")
assert "new-container/dir/file2.txt" in fs.ls("new-container/dir")
# Also verify you can make a nested directory structure
fs.mkdir("new-container/dir2/file.txt", exists_ok=True)
with fs.open("new-container/dir2/file.txt", "wb") as f:
f.write(b"0123456789")
assert "new-container/dir2/file.txt" in fs.ls("new-container/dir2")
fs.rm("new-container/dir2", recursive=True)
fs.rm("new-container/dir", recursive=True)
assert fs.ls("new-container") == [
"new-container/file.txt",
"new-container/file2.txt",
]
fs.rm("new-container/file.txt")
fs.rm("new-container/file2.txt")
fs.rmdir("new-container")
assert "new-container/" not in fs.ls("")
def test_dask_parquet(storage):
fs = AzureBlobFileSystem(account_name=storage.account_name,
connection_string=CONN_STR)
fs.mkdir("test")
STORAGE_OPTIONS = {
"account_name": "devstoreaccount1",
"connection_string": CONN_STR,
}
df = pd.DataFrame({
"col1": [1, 2, 3, 4],
"col2": [2, 4, 6, 8],
"index_key": [1, 1, 2, 2],
"partition_key": [1, 1, 2, 2],
})
dask_dataframe = dd.from_pandas(df, npartitions=1)
for protocol in ["abfs", "az"]:
dask_dataframe.to_parquet(
"{}://test/test_group.parquet".format(protocol),
storage_options=STORAGE_OPTIONS,
engine="pyarrow",
)
fs = AzureBlobFileSystem(**STORAGE_OPTIONS)
assert fs.ls("test/test_group.parquet") == [
"test/test_group.parquet/_common_metadata",
"test/test_group.parquet/_metadata",
"test/test_group.parquet/part.0.parquet",
]
fs.rm("test/test_group.parquet")
df_test = dd.read_parquet(
"abfs://test/test_group.parquet",
storage_options=STORAGE_OPTIONS,
engine="pyarrow",
).compute()
assert_frame_equal(df, df_test)
A = np.random.randint(0, 100, size=(10000, 4))
df2 = pd.DataFrame(data=A, columns=list("ABCD"))
ddf2 = dd.from_pandas(df2, npartitions=4)
dd.to_parquet(
ddf2,
"abfs://test/test_group2.parquet",
storage_options=STORAGE_OPTIONS,
engine="pyarrow",
)
assert fs.ls("test/test_group2.parquet") == [
"test/test_group2.parquet/_common_metadata",
"test/test_group2.parquet/_metadata",
"test/test_group2.parquet/part.0.parquet",
"test/test_group2.parquet/part.1.parquet",
"test/test_group2.parquet/part.2.parquet",
"test/test_group2.parquet/part.3.parquet",
]
df2_test = dd.read_parquet(
"abfs://test/test_group2.parquet",
storage_options=STORAGE_OPTIONS,
engine="pyarrow",
).compute()
assert_frame_equal(df2, df2_test)
a = np.full(shape=(10000, 1), fill_value=1)
b = np.full(shape=(10000, 1), fill_value=2)
c = np.full(shape=(10000, 1), fill_value=3)
d = np.full(shape=(10000, 1), fill_value=4)
B = np.concatenate((a, b, c, d), axis=1)
df3 = pd.DataFrame(data=B, columns=list("ABCD"))
ddf3 = dd.from_pandas(df3, npartitions=4)
dd.to_parquet(
ddf3,
"abfs://test/test_group3.parquet",
partition_on=["A", "B"],
storage_options=STORAGE_OPTIONS,
engine="pyarrow",
)
assert fs.glob("test/test_group3.parquet/*") == [
"test/test_group3.parquet/A=1",
"test/test_group3.parquet/_common_metadata",
"test/test_group3.parquet/_metadata",
]
df3_test = dd.read_parquet(
"abfs://test/test_group3.parquet",
filters=[("A", "=", 1)],
storage_options=STORAGE_OPTIONS,
engine="pyarrow",
).compute()
df3_test = df3_test[["A", "B", "C", "D"]]
df3_test = df3_test[["A", "B", "C", "D"]].astype(int)
assert_frame_equal(df3, df3_test)
A = np.random.randint(0, 100, size=(10000, 4))
df4 = pd.DataFrame(data=A, columns=list("ABCD"))
ddf4 = dd.from_pandas(df4, npartitions=4)
dd.to_parquet(
ddf4,
"abfs://test/test_group4.parquet",
storage_options=STORAGE_OPTIONS,
engine="pyarrow",
flavor="spark",
write_statistics=False,
)
fs.rmdir("test/test_group4.parquet/_common_metadata", recursive=True)
fs.rmdir("test/test_group4.parquet/_metadata", recursive=True)
fs.rm("test/test_group4.parquet/_common_metadata")
fs.rm("test/test_group4.parquet/_metadata")
assert fs.ls("test/test_group4.parquet") == [
"test/test_group4.parquet/part.0.parquet",
"test/test_group4.parquet/part.1.parquet",
"test/test_group4.parquet/part.2.parquet",
"test/test_group4.parquet/part.3.parquet",
]
df4_test = dd.read_parquet(
"abfs://test/test_group4.parquet",
storage_options=STORAGE_OPTIONS,
engine="pyarrow",
).compute()
assert_frame_equal(df4, df4_test)
A = np.random.randint(0, 100, size=(10000, 4))
df5 = pd.DataFrame(data=A, columns=list("ABCD"))
ddf5 = dd.from_pandas(df5, npartitions=4)
dd.to_parquet(
ddf5,
"abfs://test/test group5.parquet",
storage_options=STORAGE_OPTIONS,
engine="pyarrow",
)
assert fs.ls("test/test group5.parquet") == [
"test/test group5.parquet/_common_metadata",
"test/test group5.parquet/_metadata",
"test/test group5.parquet/part.0.parquet",
"test/test group5.parquet/part.1.parquet",
"test/test group5.parquet/part.2.parquet",
"test/test group5.parquet/part.3.parquet",
]
df5_test = dd.read_parquet(
"abfs://test/test group5.parquet",
storage_options=STORAGE_OPTIONS,
engine="pyarrow",
).compute()
assert_frame_equal(df5, df5_test)
def test_makedir_rmdir(storage, caplog):
fs = AzureBlobFileSystem(
account_name=storage.account_name,
connection_string=CONN_STR,
)
fs.makedir("new-container")
assert "new-container/" in fs.ls("")
assert fs.ls("new-container") == []
with fs.open(path="new-container/file.txt", mode="wb") as f:
f.write(b"0123456789")
with fs.open("new-container/dir/file.txt", "wb") as f:
f.write(b"0123456789")
with fs.open("new-container/dir/file2.txt", "wb") as f:
f.write(b"0123456789")
# Verify that mkdir will raise an exception if the directory exists
# and exist_ok is False
with pytest.raises(FileExistsError):
fs.makedir("new-container/dir/file.txt", exist_ok=False)
# Verify that mkdir creates a directory if exist_ok is False and the
# directory does not exist
fs.makedir("new-container/file2.txt", exist_ok=False)
assert "new-container/file2.txt" in fs.ls("new-container")
# Verify that mkdir will silently ignore an existing directory if
# the directory exists and exist_ok is True
fs.makedir("new-container/dir", exist_ok=True)
assert "new-container/dir/" in fs.ls("new-container")
# Test to verify that the file contains expected contents
with fs.open("new-container/file2.txt", "rb") as f:
outfile = f.read()
assert outfile == b""
# Check that trying to overwrite an existing nested file in append mode works as expected
# if exist_ok is True
fs.makedir("new-container/dir/file2.txt", exist_ok=True)
assert "new-container/dir/file2.txt" in fs.ls("new-container/dir")
# Also verify you can make a nested directory structure
fs.makedir("new-container/dir2/file.txt", exist_ok=False)
with fs.open("new-container/dir2/file.txt", "wb") as f:
f.write(b"0123456789")
assert "new-container/dir2/file.txt" in fs.ls("new-container/dir2")
fs.rm("new-container/dir2", recursive=True)
fs.rm("new-container/dir", recursive=True)
assert fs.ls("new-container") == [
"new-container/file.txt",
"new-container/file2.txt",
]
fs.rm("new-container/file.txt")
fs.rm("new-container/file2.txt")
fs.rmdir("new-container")
assert "new-container/" not in fs.ls("")
