def download_data(self):
if not self.empty_flag:
try:
download_from_s3(self.key_name, self.bucket_name,
aws_access=self.credentials)
self.message_dict['download_data'] = \
"Successfully downloaded data."
except Exception:
self.success = False
self.message_dict['download_data'] = tr.format_exc()
print("Checking for response message.")
resp_queue = sqs.connect_to_region(region).create_queue(proc_name +
"_response")
if resp_queue.count() > 0:
mess = resp_queue.read(10)
content = json.loads(mess.get_body())
print("Saving content.")
with open("tests/test_response.txt", "w") as f:
json.dump(content, f)
else:
print("No message received!")
# Now download the output file
download_from_s3("data_products/*",
proc_name,
aws_access={
"aws_access_key_id": key,
"aws_secret_access_key": secret
},
output_dir="tests/")
except Exception as e:
print("Failed with :")
print(e)
queue.delete()
resp_queue.delete()
remove_s3_bucket(proc_name, s3.connection.S3Connection())
print("Reached time limit. Terminating after 1 min.")
inst.terminate()
print("Checking for response message.")
resp_queue = sqs.connect_to_region(region).create_queue(proc_name + "_response")
if resp_queue.count() > 0:
mess = resp_queue.read(10)
content = json.loads(mess.get_body())
print("Saving content.")
with open("tests/test_response.txt", "w") as f:
json.dump(content, f)
else:
print("No message received!")
# Now download the output file
download_from_s3("data_products/*", proc_name,
aws_access={"aws_access_key_id": key,
"aws_secret_access_key": secret},
output_dir="tests/")
except Exception as e:
print("Failed with :")
print(e)
queue.delete()
resp_queue.delete()
remove_s3_bucket(proc_name, s3.connection.S3Connection())
#!/usr/bin/env python
'''Download data from AWS S3 bucket (added to bucket with upload_data_AWS.py)
Notes: key_name not full file name yet so just using wildcard for now.
output_dir adds that directory in place you call script for now.
'''
import sys
sys.path.append('/home/ubuntu/aws_controller')
from upload_download_s3 import download_from_s3
key_name = '*'
bucket_name = 'BUCKET_NAME'
output_dir = '/home/ubuntu/data'
download_from_s3(key_name,
bucket_name,
conn=None,
aws_access={
'aws_access_key_id': 'ACCESS_KEY_HERE',
'aws_secret_access_key': 'SECRET_ACCESS_KEY_HERE'
},
output_dir=output_dir)
#!/usr/bin/env python
'''Download data from AWS S3 bucket (added to bucket with upload_data_AWS.py)
Notes: key_name not full file name yet so just using wildcard for now.
output_dir adds that directory in place you call script for now.
'''
import sys
sys.path.append('/home/ubuntu/aws_controller')
from upload_download_s3 import download_from_s3
key_name='*'
bucket_name='BUCKET_NAME'
output_dir='/home/ubuntu/data'
download_from_s3(key_name, bucket_name, conn=None,
aws_access={'aws_access_key_id': 'ACCESS_KEY_HERE',
'aws_secret_access_key': 'SECRET_ACCESS_KEY_HERE'}, output_dir=output_dir)