def test_bucket_config_set_bucketfs_config():
bucketfs_config = create_test_bucketfs_config()
bucket_config = BucketConfig(bucket_name="default",
bucketfs_config=bucketfs_config)
with pytest.raises(AttributeError):
bucket_config.bucketfs_config = bucketfs_config
def test_upload_download_obj_from_different_instance():
connection_config = BucketFSConnectionConfig(host="localhost",
port=6666,
user="******",
pwd="write",
is_https=False)
bucketfs_config = BucketFSConfig("bfsdefault",
connection_config=connection_config)
bucket_config = BucketConfig(bucket_name="default",
bucketfs_config=bucketfs_config)
bucket_base_path = PurePosixPath("test_up_down_obj")
bucketfs_location_upload = BucketFSLocation(bucket_config,
bucket_base_path)
bucketfs_location_download = BucketFSLocation(bucket_config,
bucket_base_path)
bucket_file_path = "test_file.txt"
test_value = TestValue("test_string")
bucketfs_location_upload.upload_object_to_bucketfs_via_joblib(
test_value, bucket_file_path)
result = bucketfs_location_download.download_object_from_bucketfs_via_joblib(
bucket_file_path)
assert result == test_value
delete_testfile_from_bucketfs(
file_path=str(bucket_base_path) + "/" + bucket_file_path,
bucket_config=bucketfs_location_upload.bucket_config)
def test_bucket_config_with_bucketfs_config():
bucketfs_config = create_test_bucketfs_config()
bucket_name = "default"
bucket_config = BucketConfig(bucket_name=bucket_name,
bucketfs_config=bucketfs_config)
assert bucket_config.bucket_name == bucket_name and \
bucket_config.bucketfs_config == bucketfs_config
def create_bucketfs_location(self,
url: str,
user: str,
pwd: str,
base_path: Optional[PurePosixPath] = None):
"""
Create BucketFSLocation from the the url given. If the url has the schema http:// or https://,
this function creates a real BucketFSLocation for a url scheme file:/// we create a LocalFSMockBucketFSLocation.
For url with http:// or https:// schema you also need to provide the bucketfs-name via a url parameter.
A url would look like the following: http[s]://<host>:<port>/<bucket_name>/<path_in_bucket>;<bucketfs_name>
:param url:
:param user:
:param pwd:
:param base_path:
:return:
"""
parsed_url = urllib.parse.urlparse(url)
if parsed_url.scheme == "http" or parsed_url.scheme == "https":
is_https = parsed_url.scheme == "https"
connection_config = BucketFSConnectionConfig(
host=parsed_url.hostname,
port=parsed_url.port,
user=user,
pwd=pwd,
is_https=is_https)
url_path = PurePosixPath(parsed_url.path)
bucket_name = url_path.parts[1]
base_path_in_bucket = PurePosixPath(
url_path.parts[2]).joinpath(*url_path.parts[3:])
if base_path is not None:
base_path_in_bucket = PurePosixPath(base_path_in_bucket,
base_path)
bucketfs_name = parsed_url.params
bucketfs_config = BucketFSConfig(
bucketfs_name, connection_config=connection_config)
bucket_config = BucketConfig(bucket_name=bucket_name,
bucketfs_config=bucketfs_config)
bucketfs_location = BucketFSLocation(bucket_config,
base_path_in_bucket)
return bucketfs_location
elif parsed_url.scheme == "file":
if parsed_url.netloc != '':
raise ValueError(
f"URL '{url}' with file:// schema and netloc not support.")
base_path_in_bucket = PurePosixPath(parsed_url.path)
if base_path is not None:
base_path_in_bucket = PurePosixPath(base_path_in_bucket,
base_path)
bucketfs_location = LocalFSMockBucketFSLocation(
base_path_in_bucket)
return bucketfs_location
else:
raise ValueError(f"Invalid url schema for url {url}")
from exasol_bucketfs_utils_python import upload, download
from exasol_bucketfs_utils_python.bucket_config import BucketConfig
from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig
from exasol_bucketfs_utils_python.bucketfs_connection_config import BucketFSConnectionConfig
connection_config = BucketFSConnectionConfig(
host="localhost", port=6666,
user="******", pwd="write",
is_https=False)
bucketfs_config = BucketFSConfig(
connection_config=connection_config,
bucketfs_name="bfsdefault")
bucket_config = BucketConfig(
bucket_name="default",
bucketfs_config=bucketfs_config)
class TestClass:
def __init__(self, attribute: str):
self.attribute = attribute
def __eq__(self, other: "TestClass"):
return isinstance(other, TestClass) and self.attribute == other.attribute
test_python_object = TestClass("test_string")
path_in_bucket = "path/in/bucket/file.txt"
upload.upload_object_to_bucketfs_via_joblib(
bucket_config=bucket_config,
bucket_file_path=path_in_bucket,
object=test_python_object)
def test_bucket_config_with_None_as_bucket_name():
bucketfs_config = create_test_bucketfs_config()
with pytest.raises(TypeError):
bucket_config = BucketConfig(bucket_name=None,
bucketfs_config=bucketfs_config)
def test_bucket_config_with_empty_bucketfs_config():
with pytest.raises(TypeError):
bucket_config = BucketConfig(bucket_name="", bucketfs_config=None)
def test_bucket_config_with_empty_bucket_name():
bucketfs_config = create_test_bucketfs_config()
with pytest.raises(ValueError):
bucket_config = BucketConfig(bucket_name="",
bucketfs_config=bucketfs_config)
def test_read_files_to_str_from_bucketfs_inside_udf(upload_language_container,
pyexasol_connection):
connection_config = BucketFSConnectionConfig(host="localhost",
port=6666,
user="******",
pwd="write",
is_https=False)
bucketfs_config = BucketFSConfig("bfsdefault",
connection_config=connection_config)
bucket_config = BucketConfig(bucket_name="default",
bucketfs_config=bucketfs_config)
bucket_base_path = PurePosixPath("test_read_str")
bucketfs_location_upload = BucketFSLocation(bucket_config,
bucket_base_path)
bucket_file_path = "test_file.txt"
test_string = "test_string"
bucketfs_location_upload.upload_string_to_bucketfs(bucket_file_path,
test_string)
bucketfs_location_read = BucketFSLocation(bucket_config, bucket_base_path)
try:
# load file from udf
target_schema = "TARGET_SCHEMA"
pyexasol_connection.execute(
f"CREATE SCHEMA IF NOT EXISTS {target_schema};"
) # mynotes dropfirst?
pyexasol_connection.execute(f"OPEN SCHEMA {target_schema};")
udf_sql = textwrap.dedent(f"""
CREATE OR REPLACE PYTHON3_BFSUP SET SCRIPT {target_schema}."LoadFromLocalFS"(
"bucket_config" VARCHAR(2000000),
"path" VARCHAR(20000)
)
RETURNS VARCHAR(20000)
AS
from exasol_bucketfs_utils_python.bucket_config import BucketConfig
from exasol_bucketfs_utils_python.bucketfs_connection_config import BucketFSConnectionConfig
from exasol_bucketfs_utils_python.bucketfs_config import BucketFSConfig
from exasol_bucketfs_utils_python.bucketfs_location import BucketFSLocation
from pathlib import PurePosixPath
def get_bucket_config():
bucket_name = "default"
bucketfs_name = "bfsdefault"
connection_config = BucketFSConnectionConfig(host="localhost",
port=6666,
user="******", pwd="read",
is_https=False)
bucketfs_config = BucketFSConfig(bucketfs_name, connection_config=connection_config)
return BucketConfig(bucket_name, bucketfs_config)
def run(ctx):
bucket_config = get_bucket_config()
bucketfs_location_read = BucketFSLocation(bucket_config, PurePosixPath("test_read_str"))
output_test_string = bucketfs_location_read.read_file_from_bucketfs_to_string(ctx.path)
return output_test_string
""")
pyexasol_connection.execute(udf_sql)
output_test_string = pyexasol_connection.execute(
f"""select {target_schema}."LoadFromLocalFS"('{bucketfs_location_read.bucket_config}','{bucket_file_path}'
)""").fetchall()
assert output_test_string[0][0] == test_string
finally:
delete_testfile_from_bucketfs(
file_path=str(bucket_base_path) + "/" + bucket_file_path,
bucket_config=bucketfs_location_read.bucket_config)
pyexasol_connection.execute(
f"DROP SCHEMA IF EXISTS {target_schema} CASCADE;")