def run(self):
if not os.path.exists(self.dest_dir):
os.makedirs(self.dest_dir)
client = boto3.client('s3')
transfer = boto3.s3.transfer.S3Transfer(client)
output_file = join(self.cache_dir, self.output_file)
transfer.download_file(BUCKET, self.input_file, self.output_path)
def downloadObj(bucket, s3_path, local_path):
try:
transfer.download_file('{bucket}'.format(bucket=bucket),
'{s3_path}'.format(s3_path=s3_path), local_path)
md5_obj = MD5parse(bucket, s3_path, local_path)
local_hash = md5_obj.localMD5()
aws_md5 = md5_obj.awsMD5()
result = aws_md5 == str(local_hash)
output = "Object Name: " + s3_path.split(
'/'
)[-1] + "\t" + "AWS MD5 Hash: " + aws_md5 + "\t" + "Local MD5 Hash: " + str(
local_hash) + "\t" + "File upload" + "\t" + "Match is " + str(
result)
writeMD5(output)
print("MD5sum match for: " + s3_path.split('/')[-1])
print(output)
print("Result is " + str(result))
except NameError:
print(
"Name error on downloading. Please specify correct path for object you are downloading and a proper name for saving"
)
print(
"""Reminder: objects are saved to the directory the Python script is run from. You cannot specify a directory such as 'home/user/download'. This will invoke a name error."""
)
return (result)
def test_progress_callback_on_download(self):
self.amount_seen = 0
lock = threading.Lock()
def progress_callback(amount):
with lock:
self.amount_seen += amount
transfer = self.create_s3_transfer()
filename = self.files.create_file_with_size('20mb.txt',
filesize=20 * 1024 * 1024)
with open(filename, 'rb') as f:
self.client.put_object(Bucket=self.bucket_name,
Key='20mb.txt',
Body=f)
self.addCleanup(self.delete_object, '20mb.txt')
download_path = os.path.join(self.files.rootdir, 'downloaded.txt')
transfer.download_file(
self.bucket_name,
'20mb.txt',
download_path,
callback=progress_callback,
)
self.assertEqual(self.amount_seen, 20 * 1024 * 1024)
def test_can_send_extra_params_on_download(self):
# We're picking the customer provided sse feature
# of S3 to test the extra_args functionality of
# S3.
key_bytes = os.urandom(32)
extra_args = {
'SSECustomerKey': key_bytes,
'SSECustomerAlgorithm': 'AES256',
}
self.client.put_object(
Bucket=self.bucket_name,
Key='foo.txt',
Body=b'hello world',
**extra_args,
)
self.addCleanup(self.delete_object, 'foo.txt')
transfer = self.create_s3_transfer()
download_path = os.path.join(self.files.rootdir, 'downloaded.txt')
self.wait_until_object_exists('foo.txt', extra_params=extra_args)
transfer.download_file(self.bucket_name,
'foo.txt',
download_path,
extra_args=extra_args)
with open(download_path, 'rb') as f:
self.assertEqual(f.read(), b'hello world')
def test_download_file_with_directory_not_exist(self):
transfer = self.create_s3_transfer()
self.client.put_object(Bucket=self.bucket_name,
Key='foo.txt',
Body=b'foo')
self.addCleanup(self.delete_object, 'foo.txt')
download_path = os.path.join(self.files.rootdir, 'a', 'b', 'c',
'downloaded.txt')
with self.assertRaises(IOError):
transfer.download_file(self.bucket_name, 'foo.txt', download_path)
def test_download_file_with_directory_not_exist(self):
transfer = self.create_s3_transfer()
self.client.put_object(Bucket=self.bucket_name,
Key='foo.txt',
Body=b'foo')
self.addCleanup(self.delete_object, 'foo.txt')
download_path = os.path.join(self.files.rootdir, 'a', 'b', 'c',
'downloaded.txt')
with self.assertRaises(IOError):
transfer.download_file(self.bucket_name, 'foo.txt', download_path)
def test_download_above_threshold(self):
transfer = self.create_s3_transfer()
filename = self.files.create_file_with_size('foo.txt',
filesize=20 * 1024 * 1024)
with open(filename, 'rb') as f:
self.client.put_object(Bucket=self.bucket_name,
Key='foo.txt',
Body=f)
self.addCleanup(self.delete_object, 'foo.txt')
download_path = os.path.join(self.files.rootdir, 'downloaded.txt')
transfer.download_file(self.bucket_name, 'foo.txt', download_path)
assert_files_equal(filename, download_path)
def test_download_large_file_directory_not_exist(self):
transfer = self.create_s3_transfer()
filename = self.files.create_file_with_size('foo.txt',
filesize=20 * 1024 * 1024)
with open(filename, 'rb') as f:
self.client.put_object(Bucket=self.bucket_name,
Key='foo.txt',
Body=f)
self.addCleanup(self.delete_object, 'foo.txt')
download_path = os.path.join(self.files.rootdir, 'a', 'b', 'c',
'downloaded.txt')
with self.assertRaises(IOError):
transfer.download_file(self.bucket_name, 'foo.txt', download_path)
def test_download_large_file_directory_not_exist(self):
transfer = self.create_s3_transfer()
filename = self.files.create_file_with_size(
'foo.txt', filesize=20 * 1024 * 1024)
with open(filename, 'rb') as f:
self.client.put_object(Bucket=self.bucket_name,
Key='foo.txt',
Body=f)
self.addCleanup(self.delete_object, 'foo.txt')
download_path = os.path.join(self.files.rootdir, 'a', 'b', 'c',
'downloaded.txt')
with self.assertRaises(IOError):
transfer.download_file(self.bucket_name, 'foo.txt', download_path)
def test_download_above_threshold(self):
transfer = self.create_s3_transfer()
filename = self.files.create_file_with_size(
'foo.txt', filesize=20 * 1024 * 1024)
with open(filename, 'rb') as f:
self.client.put_object(Bucket=self.bucket_name,
Key='foo.txt',
Body=f)
self.addCleanup(self.delete_object, 'foo.txt')
download_path = os.path.join(self.files.rootdir, 'downloaded.txt')
transfer.download_file(self.bucket_name, 'foo.txt',
download_path)
assert_files_equal(filename, download_path)
def test_progress_callback_on_download(self):
self.amount_seen = 0
lock = threading.Lock()
def progress_callback(amount):
with lock:
self.amount_seen += amount
transfer = self.create_s3_transfer()
filename = self.files.create_file_with_size(
'20mb.txt', filesize=20 * 1024 * 1024)
with open(filename, 'rb') as f:
self.client.put_object(Bucket=self.bucket_name,
Key='20mb.txt', Body=f)
self.addCleanup(self.delete_object, '20mb.txt')
transfer.download_file(self.bucket_name, '20mb.txt',
'foo.txt', callback=progress_callback)
self.assertEqual(self.amount_seen, 20 * 1024 * 1024)
def test_can_send_extra_params_on_download(self):
# We're picking the customer provided sse feature
# of S3 to test the extra_args functionality of
# S3.
key_bytes = os.urandom(32)
extra_args = {
'SSECustomerKey': key_bytes,
'SSECustomerAlgorithm': 'AES256',
}
self.client.put_object(Bucket=self.bucket_name,
Key='foo.txt',
Body=b'hello world',
**extra_args)
self.addCleanup(self.delete_object, 'foo.txt')
transfer = self.create_s3_transfer()
download_path = os.path.join(self.files.rootdir, 'downloaded.txt')
transfer.download_file(self.bucket_name, 'foo.txt',
download_path, extra_args=extra_args)
with open(download_path, 'rb') as f:
self.assertEqual(f.read(), b'hello world')
def save_to_filename(self, file_path):
client = self._connect()
transfer = boto3.s3.transfer.S3Transfer(client)
transfer.download_file(self._bucket, self._keyname, file_path)
def save_to_filename(self, file_path):
client = self._connect()
transfer = boto3.s3.transfer.S3Transfer(client)
transfer.download_file(self._bucket, self._keyname, file_path)
