def upload(self, file_name: str, remote_path: str) -> str:
"""
Uploads given file to S3
:param file_name: Path to the file that will be uploaded
:param remote_path: be uploaded
:return: VersionId of the latest upload
"""
if self.prefix:
remote_path = "{0}/{1}".format(self.prefix, remote_path)
# Check if a file with same data exists
if not self.force_upload and self.file_exists(remote_path):
LOG.debug(
"File with same data is already exists at %s. "
"Skipping upload", remote_path)
return self.make_url(remote_path)
try:
# Default to regular server-side encryption unless customer has
# specified their own KMS keys
additional_args = {"ServerSideEncryption": "AES256"}
if self.kms_key_id:
additional_args["ServerSideEncryption"] = "aws:kms"
additional_args["SSEKMSKeyId"] = self.kms_key_id
if self.artifact_metadata:
additional_args["Metadata"] = self.artifact_metadata
if not self.bucket_name:
raise BucketNotSpecifiedError()
if not self.no_progressbar:
print_progress_callback = ProgressPercentage(
file_name, remote_path)
future = self.transfer_manager.upload(
file_name, self.bucket_name, remote_path, additional_args,
[print_progress_callback])
else:
future = self.transfer_manager.upload(file_name,
self.bucket_name,
remote_path,
additional_args)
future.result()
return self.make_url(remote_path)
except botocore.exceptions.ClientError as ex:
error_code = ex.response["Error"]["Code"]
if error_code == "NoSuchBucket":
raise NoSuchBucketError(bucket_name=self.bucket_name) from ex
raise ex
def test_s3_upload_no_bucket(self):
s3_uploader = S3Uploader(
s3_client=self.s3,
bucket_name=None,
prefix=self.prefix,
kms_key_id=self.kms_key_id,
force_upload=self.force_upload,
)
s3_uploader.artifact_metadata = {"a": "b"}
remote_path = Path.joinpath(Path(os.getcwd()), Path("tmp"))
with self.assertRaises(BucketNotSpecifiedError) as ex:
with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
s3_uploader.upload(f.name, remote_path)
self.assertEqual(BucketNotSpecifiedError().message, str(ex))
def file_exists(self, remote_path: str) -> bool:
"""
Check if the file we are trying to upload already exists in S3
:param remote_path:
:return: True, if file exists. False, otherwise
"""
try:
# Find the object that matches this ETag
if not self.bucket_name:
raise BucketNotSpecifiedError()
self.s3.head_object(Bucket=self.bucket_name, Key=remote_path)
return True
except botocore.exceptions.ClientError:
# Either File does not exist or we are unable to get
# this information.
return False
def make_url(self, obj_path: str) -> str:
if not self.bucket_name:
raise BucketNotSpecifiedError()
return "s3://{0}/{1}".format(self.bucket_name, obj_path)