def test_delete(fixture_client, test_bucket):
source = f"s3://{test_bucket}/source.txt"
with tio.open(source, mode="w") as f:
f.write("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn.")
tio.remove(source)
with pytest.raises(S3Error, match="Unable to fetch the remote file"):
with tio.open(source) as f:
...
def test_delete():
source = "ftp://hostname/source.txt"
with tentaclio.open(source, mode="w") as f:
f.write("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn.")
tentaclio.remove(source)
with pytest.raises(tentaclio.clients.exceptions.FTPError):
with tentaclio.open(source) as f:
...
def transform(input_url: str, output_url: str):
print(f"transforming: input_url {input_url} output_url {output_url}")
# Data access layer
with tio.open(input_url) as reader, tio.open(output_url,
mode="w") as writer:
_transform(reader, writer)
print("done")
def test_authenticated_api_calls(fixture_client):
data = bytes("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn", "utf-8")
with tentaclio.open("s3://hostname/data.txt", mode="wb") as f:
f.write(data)
with tentaclio.open("s3://hostname/data.txt", mode="rb") as f:
result = f.read()
assert result == data
def test_copy(fixture_client, test_bucket):
source = f"s3://{test_bucket}/source.txt"
dest = f"s3://{test_bucket}/dest.txt"
with tio.open(source, mode="w") as f:
f.write("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn.")
tio.copy(source, dest)
with tio.open(dest) as f:
result = f.read()
assert result == "Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn."
def test_pickle(fixture_client):
expected = """
This is a highly convoluted test,
with multiple output...
encountered.
"""
with tentaclio.open("s3://hostname/data.pickle", mode="wb") as f:
pickle.dump(expected, f)
with tentaclio.open("s3://hostname/data.pickle", mode="rb") as f:
retrieved = pickle.load(f)
assert expected == retrieved
def test_scandir(fixture_client, test_bucket):
files = ["file1.txt", "file2.txt", "file3.txt", "file4.txt"]
folders = ["folder2", "folder3"]
base = f"s3://{test_bucket}/folder1/"
paths = []
for file_ in files:
paths.append(base + f"{file_}")
for folder in folders:
paths.append(base + f"{folder}/file_to_create_key.txt")
for path in paths:
with tio.open(path, mode="w") as f:
f.write("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn.")
entries_by_url = {str(entry.url): entry for entry in tio.scandir(base)}
for file_ in files:
key = base + f"{file_}"
assert key in entries_by_url
assert entries_by_url[key].is_file
for folder in folders:
key = base + f"{folder}"
assert key in entries_by_url
assert entries_by_url[key].is_dir
def test_mocked_api_calls(m_connect, m_get, m_put, m_remove, url, bucket, key):
"""Test api calls reaches the mocks."""
data = bytes("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn", "utf-8")
with tentaclio.open(url, mode="wb") as f:
f.write(data)
m_put.assert_called_once_with(mock.ANY, bucket, key)
with tentaclio.open(url, mode="rb") as f:
f.read()
m_get.assert_called_once_with(mock.ANY, bucket, key)
tentaclio.remove(url)
m_remove.assert_called_once_with(bucket, key)
m_get.reset_mock()
m_get.side_effect = google_exceptions.NotFound("Not found")
with pytest.raises(google_exceptions.NotFound):
tentaclio.open(url, mode="rb")
def test_authenticated_api_calls(fixture_client, fixture_df):
with tentaclio.open(
f"postgresql://hostname{fixture_client.url.path}::{TEST_TABLE_NAME}",
mode="w") as writer:
fixture_df.to_csv(writer, index=False)
with clients.PostgresClient(
credentials.authenticate(
f"postgresql://hostname{fixture_client.url.path}")) as client:
retrieved_df = client.get_df(f"select * from {TEST_TABLE_NAME}")
assert retrieved_df.equals(fixture_df)
def test_list_folders():
data = "Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn"
cli = tentaclio.clients.SFTPClient(authenticate("sftp://hostname"))
with cli:
cli.conn.makedirs("upload/folder1/folder2")
with tentaclio.open("sftp://hostname/upload/folder1/data.txt", mode="w") as f:
f.write(data)
ls = list(tentaclio.listdir("sftp://hostname/upload/folder1"))
assert any("upload/folder1/data.txt" in entry for entry in ls)
assert any("upload/folder1/folder2" in entry for entry in ls)
def test_authenticated_api_calls(gs_url, bucket_exists):
"""Test the authenticated API calls.
Skipped unless configured in conftest or TENTACLIO__CONN__GS_TEST is set.
🚨🚨🚨WARNING🚨🚨🚨
The functional test for GS will:
- create a bucket if non is found.
This will be created in the configured project (see gcloud docs).
- upload and remove a file as set in the URL
To use run the test use command:
```
env TENTACLIO__CONN__GS_TEST=gs://tentaclio-bucket/test.txt \
make functional-gs
```
You will need to have your environment configured, credentials and project.
This is done with the gcloud cli tool. See docs for more information:
https://googleapis.dev/python/google-api-core/latest/auth.html
"""
data = bytes("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn", "utf-8")
with tentaclio.open(gs_url, mode="wb") as f:
f.write(data)
with tentaclio.open(gs_url, mode="rb") as f:
result = f.read()
assert result == data
tentaclio.remove(gs_url)
with pytest.raises(google_exceptions.NotFound):
with tentaclio.open(gs_url, mode="rb") as f:
result = f.read()
