def azure_route_file(storage_account_name, storage_file_name, local_path, storage_file_path=None):
"""Route file to either storage account or local filesystem."""
connect_str = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
if storage_file_path and connect_str:
upload_to_azure_container(storage_file_name, local_path, storage_file_path)
else:
copy_to_local_dir(storage_account_name, local_path, storage_file_name)
def gcp_route_file(bucket_name, bucket_file_path, local_path):
"""Route file to either GCP bucket or local filesystem."""
if os.path.isdir(bucket_name):
copy_to_local_dir(bucket_name,
local_path,
bucket_file_path,)
else:
upload_to_gcp_storage(bucket_name,
bucket_file_path,
local_path)
def azure_route_file(storage_account_name, storage_file_name, local_path, storage_file_path=None):
"""Route file to either storage account or local filesystem."""
if os.path.isdir(storage_account_name):
copy_to_local_dir(storage_account_name,
local_path,
storage_file_name)
else:
upload_to_storage(storage_file_name,
local_path,
storage_file_path)
def aws_route_file(bucket_name, bucket_file_path, local_path):
"""Route file to either S3 bucket or local filesystem."""
if os.path.isdir(bucket_name):
copy_to_local_dir(bucket_name,
local_path,
bucket_file_path,)
else:
upload_to_s3(bucket_name,
bucket_file_path,
local_path)
def azure_route_file(storage_account_name, storage_file_name, local_path, storage_file_path=None):
"""Route file to either storage account or local filesystem."""
if storage_file_path is None:
copy_to_local_dir(storage_account_name,
local_path,
storage_file_name)
else:
upload_to_azure_container(storage_file_name,
local_path,
storage_file_path)
def test_copy_failure(self):
"""Test copy_to_local_dir method when local directory (bucket does not exist)."""
source_file = NamedTemporaryFile(delete=False)
source_file.seek(0)
source_file.write(b"cur report")
source_file.flush()
bucket_name = mkdtemp()
bad_bucket_name = bucket_name + "bad"
bucket_file_path = "/bucket_location"
success = copy_to_local_dir(bad_bucket_name, source_file.name,
bucket_file_path)
self.assertFalse(success)
shutil.rmtree(bucket_name)
os.remove(source_file.name)
def test_copy_success(self):
"""Test copy_to_local_dir method."""
source_file = NamedTemporaryFile(delete=False)
source_file.seek(0)
source_file.write(b"cur report")
source_file.flush()
source_file_name = os.path.split(source_file.name)[1]
bucket_name = mkdtemp()
bucket_file_path = "/{}/{}".format("report_name", source_file_name)
success = copy_to_local_dir(bucket_name, source_file.name,
bucket_file_path)
self.assertTrue(success)
expected_full_file_path = f"{bucket_name}{bucket_file_path}"
self.assertTrue(os.path.isfile(expected_full_file_path))
shutil.rmtree(bucket_name)
os.remove(source_file.name)