def mtime(self):
if self.exists():
return self._s3c.key_last_modified(self.s3_bucket, self.s3_key)
else:
raise S3FileException(
"The file does not seem to exist remotely: %s" %
self.local_file())
def exists(self):
if self._matched_s3_path:
return self._s3c.exists_in_bucket(self.s3_bucket, self.s3_key)
else:
raise S3FileException(
"The file cannot be parsed as an s3 path in form 'bucket/key': %s"
% self.local_file())
def s3_create_stub(self):
if self._matched_s3_path:
if not self.exists:
self._s3c.download_from_s3(self.s3_bucket, self.s3_key, self.file, create_stub_only=True)
else:
raise S3FileException("The file to be downloaded cannot be parsed as an s3 path in form 'bucket/key': %s" %
self.local_file())
def download_from_s3(
self,
bucket_name,
key,
destination_path=None,
expandKeyIntoDirs=True,
make_dest_dirs=True,
create_stub_only=False,
):
""" Download a file from s3
This function downloads an object from a specified AWS S3 bucket.
Args:
bucket_name: the name of the S3 bucket to use (bucket name only, not ARN)
destination_path: If specified, the file will be saved to this path, otherwise cwd.
expandKeyIntoDirs: Since S3 keys can include slashes, if this is True (defult)
then S3 keys with slashes are expanded into directories on the receiving end.
If it is False, the key is passed to os.path.basename() to get the substring
following the last slash.
make_dest_dirs: If this is True (default) and the destination path includes directories
that do not exist, they will be created.
Returns:
The destination path of the downloaded file on the receiving end, or None if the destination_path
could not be downloaded
"""
assert bucket_name, "bucket_name must be specified"
assert key, "Key must be specified"
if destination_path:
destination_path = os.path.realpath(
os.path.expanduser(destination_path))
else:
if expandKeyIntoDirs:
destination_path = os.path.join(os.getcwd(), key)
else:
destination_path = os.path.join(os.getcwd(),
os.path.basename(key))
# if the destination path does not exist
if make_dest_dirs:
os.makedirs(os.path.dirname(destination_path), exist_ok=True)
k = self.s3.Object(bucket_name, key)
try:
if not create_stub_only:
k.download_file(destination_path)
else:
# just create an empty file with the right timestamps
with open(destination_path, "wb") as fp:
os.utime(
fp.name,
(k.last_modified.timestamp(),
k.last_modified.timestamp()),
)
return destination_path
except:
raise S3FileException(
"Error downloading file '%s' from bucket '%s'." %
(key, bucket_name))