Beispiel #1
0
def test_delete(fixture_client, test_bucket):
    source = f"s3://{test_bucket}/source.txt"
    with tio.open(source, mode="w") as f:
        f.write("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn.")
    tio.remove(source)
    with pytest.raises(S3Error, match="Unable to fetch the remote file"):
        with tio.open(source) as f:
            ...
Beispiel #2
0
def test_delete():
    source = "ftp://hostname/source.txt"
    with tentaclio.open(source, mode="w") as f:
        f.write("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn.")
    tentaclio.remove(source)
    with pytest.raises(tentaclio.clients.exceptions.FTPError):
        with tentaclio.open(source) as f:
            ...
Beispiel #3
0
def transform(input_url: str, output_url: str):
    print(f"transforming: input_url {input_url}  output_url {output_url}")
    # Data access layer
    with tio.open(input_url) as reader, tio.open(output_url,
                                                 mode="w") as writer:
        _transform(reader, writer)

    print("done")
Beispiel #4
0
def test_authenticated_api_calls(fixture_client):
    data = bytes("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn", "utf-8")

    with tentaclio.open("s3://hostname/data.txt", mode="wb") as f:
        f.write(data)

    with tentaclio.open("s3://hostname/data.txt", mode="rb") as f:
        result = f.read()

    assert result == data
Beispiel #5
0
def test_copy(fixture_client, test_bucket):
    source = f"s3://{test_bucket}/source.txt"
    dest = f"s3://{test_bucket}/dest.txt"
    with tio.open(source, mode="w") as f:
        f.write("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn.")

    tio.copy(source, dest)

    with tio.open(dest) as f:
        result = f.read()

    assert result == "Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn."
def test_pickle(fixture_client):
    expected = """
    This is a highly convoluted test,
    with multiple output...
    encountered.
    """

    with tentaclio.open("s3://hostname/data.pickle", mode="wb") as f:
        pickle.dump(expected, f)

    with tentaclio.open("s3://hostname/data.pickle", mode="rb") as f:
        retrieved = pickle.load(f)

    assert expected == retrieved
Beispiel #7
0
def test_scandir(fixture_client, test_bucket):
    files = ["file1.txt", "file2.txt", "file3.txt", "file4.txt"]
    folders = ["folder2", "folder3"]
    base = f"s3://{test_bucket}/folder1/"
    paths = []
    for file_ in files:
        paths.append(base + f"{file_}")

    for folder in folders:
        paths.append(base + f"{folder}/file_to_create_key.txt")

    for path in paths:
        with tio.open(path, mode="w") as f:
            f.write("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn.")

    entries_by_url = {str(entry.url): entry for entry in tio.scandir(base)}

    for file_ in files:
        key = base + f"{file_}"
        assert key in entries_by_url
        assert entries_by_url[key].is_file

    for folder in folders:
        key = base + f"{folder}"
        assert key in entries_by_url
        assert entries_by_url[key].is_dir
Beispiel #8
0
def test_mocked_api_calls(m_connect, m_get, m_put, m_remove, url, bucket, key):
    """Test api calls reaches the mocks."""
    data = bytes("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn", "utf-8")

    with tentaclio.open(url, mode="wb") as f:
        f.write(data)
    m_put.assert_called_once_with(mock.ANY, bucket, key)

    with tentaclio.open(url, mode="rb") as f:
        f.read()
    m_get.assert_called_once_with(mock.ANY, bucket, key)

    tentaclio.remove(url)
    m_remove.assert_called_once_with(bucket, key)

    m_get.reset_mock()
    m_get.side_effect = google_exceptions.NotFound("Not found")
    with pytest.raises(google_exceptions.NotFound):
        tentaclio.open(url, mode="rb")
Beispiel #9
0
def test_authenticated_api_calls(fixture_client, fixture_df):
    with tentaclio.open(
            f"postgresql://hostname{fixture_client.url.path}::{TEST_TABLE_NAME}",
            mode="w") as writer:
        fixture_df.to_csv(writer, index=False)

    with clients.PostgresClient(
            credentials.authenticate(
                f"postgresql://hostname{fixture_client.url.path}")) as client:
        retrieved_df = client.get_df(f"select * from {TEST_TABLE_NAME}")

    assert retrieved_df.equals(fixture_df)
Beispiel #10
0
def test_list_folders():
    data = "Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn"
    cli = tentaclio.clients.SFTPClient(authenticate("sftp://hostname"))

    with cli:
        cli.conn.makedirs("upload/folder1/folder2")

    with tentaclio.open("sftp://hostname/upload/folder1/data.txt", mode="w") as f:
        f.write(data)

    ls = list(tentaclio.listdir("sftp://hostname/upload/folder1"))
    assert any("upload/folder1/data.txt" in entry for entry in ls)
    assert any("upload/folder1/folder2" in entry for entry in ls)
Beispiel #11
0
def test_authenticated_api_calls(gs_url, bucket_exists):
    """Test the authenticated API calls.

    Skipped unless configured in conftest or TENTACLIO__CONN__GS_TEST is set.

    🚨🚨🚨WARNING🚨🚨🚨
    The functional test for GS will:
        - create a bucket if non is found.
            This will be created in the configured project (see gcloud docs).
        - upload and remove a file as set in the URL

    To use run the test use command:
    ```
    env TENTACLIO__CONN__GS_TEST=gs://tentaclio-bucket/test.txt \
        make functional-gs
    ```

    You will need to have your environment configured, credentials and project.
    This is done with the gcloud cli tool. See docs for more information:
        https://googleapis.dev/python/google-api-core/latest/auth.html
    """
    data = bytes("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn", "utf-8")

    with tentaclio.open(gs_url, mode="wb") as f:
        f.write(data)

    with tentaclio.open(gs_url, mode="rb") as f:
        result = f.read()

    assert result == data

    tentaclio.remove(gs_url)

    with pytest.raises(google_exceptions.NotFound):
        with tentaclio.open(gs_url, mode="rb") as f:
            result = f.read()