def test_delete(fixture_client, test_bucket): source = f"s3://{test_bucket}/source.txt" with tio.open(source, mode="w") as f: f.write("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn.") tio.remove(source) with pytest.raises(S3Error, match="Unable to fetch the remote file"): with tio.open(source) as f: ...
def test_delete(): source = "ftp://hostname/source.txt" with tentaclio.open(source, mode="w") as f: f.write("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn.") tentaclio.remove(source) with pytest.raises(tentaclio.clients.exceptions.FTPError): with tentaclio.open(source) as f: ...
def transform(input_url: str, output_url: str): print(f"transforming: input_url {input_url} output_url {output_url}") # Data access layer with tio.open(input_url) as reader, tio.open(output_url, mode="w") as writer: _transform(reader, writer) print("done")
def test_authenticated_api_calls(fixture_client): data = bytes("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn", "utf-8") with tentaclio.open("s3://hostname/data.txt", mode="wb") as f: f.write(data) with tentaclio.open("s3://hostname/data.txt", mode="rb") as f: result = f.read() assert result == data
def test_copy(fixture_client, test_bucket): source = f"s3://{test_bucket}/source.txt" dest = f"s3://{test_bucket}/dest.txt" with tio.open(source, mode="w") as f: f.write("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn.") tio.copy(source, dest) with tio.open(dest) as f: result = f.read() assert result == "Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn."
def test_pickle(fixture_client): expected = """ This is a highly convoluted test, with multiple output... encountered. """ with tentaclio.open("s3://hostname/data.pickle", mode="wb") as f: pickle.dump(expected, f) with tentaclio.open("s3://hostname/data.pickle", mode="rb") as f: retrieved = pickle.load(f) assert expected == retrieved
def test_scandir(fixture_client, test_bucket): files = ["file1.txt", "file2.txt", "file3.txt", "file4.txt"] folders = ["folder2", "folder3"] base = f"s3://{test_bucket}/folder1/" paths = [] for file_ in files: paths.append(base + f"{file_}") for folder in folders: paths.append(base + f"{folder}/file_to_create_key.txt") for path in paths: with tio.open(path, mode="w") as f: f.write("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn.") entries_by_url = {str(entry.url): entry for entry in tio.scandir(base)} for file_ in files: key = base + f"{file_}" assert key in entries_by_url assert entries_by_url[key].is_file for folder in folders: key = base + f"{folder}" assert key in entries_by_url assert entries_by_url[key].is_dir
def test_mocked_api_calls(m_connect, m_get, m_put, m_remove, url, bucket, key): """Test api calls reaches the mocks.""" data = bytes("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn", "utf-8") with tentaclio.open(url, mode="wb") as f: f.write(data) m_put.assert_called_once_with(mock.ANY, bucket, key) with tentaclio.open(url, mode="rb") as f: f.read() m_get.assert_called_once_with(mock.ANY, bucket, key) tentaclio.remove(url) m_remove.assert_called_once_with(bucket, key) m_get.reset_mock() m_get.side_effect = google_exceptions.NotFound("Not found") with pytest.raises(google_exceptions.NotFound): tentaclio.open(url, mode="rb")
def test_authenticated_api_calls(fixture_client, fixture_df): with tentaclio.open( f"postgresql://hostname{fixture_client.url.path}::{TEST_TABLE_NAME}", mode="w") as writer: fixture_df.to_csv(writer, index=False) with clients.PostgresClient( credentials.authenticate( f"postgresql://hostname{fixture_client.url.path}")) as client: retrieved_df = client.get_df(f"select * from {TEST_TABLE_NAME}") assert retrieved_df.equals(fixture_df)
def test_list_folders(): data = "Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn" cli = tentaclio.clients.SFTPClient(authenticate("sftp://hostname")) with cli: cli.conn.makedirs("upload/folder1/folder2") with tentaclio.open("sftp://hostname/upload/folder1/data.txt", mode="w") as f: f.write(data) ls = list(tentaclio.listdir("sftp://hostname/upload/folder1")) assert any("upload/folder1/data.txt" in entry for entry in ls) assert any("upload/folder1/folder2" in entry for entry in ls)
def test_authenticated_api_calls(gs_url, bucket_exists): """Test the authenticated API calls. Skipped unless configured in conftest or TENTACLIO__CONN__GS_TEST is set. 🚨🚨🚨WARNING🚨🚨🚨 The functional test for GS will: - create a bucket if non is found. This will be created in the configured project (see gcloud docs). - upload and remove a file as set in the URL To use run the test use command: ``` env TENTACLIO__CONN__GS_TEST=gs://tentaclio-bucket/test.txt \ make functional-gs ``` You will need to have your environment configured, credentials and project. This is done with the gcloud cli tool. See docs for more information: https://googleapis.dev/python/google-api-core/latest/auth.html """ data = bytes("Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn", "utf-8") with tentaclio.open(gs_url, mode="wb") as f: f.write(data) with tentaclio.open(gs_url, mode="rb") as f: result = f.read() assert result == data tentaclio.remove(gs_url) with pytest.raises(google_exceptions.NotFound): with tentaclio.open(gs_url, mode="rb") as f: result = f.read()