def download_data(self): if not self.empty_flag: try: download_from_s3(self.key_name, self.bucket_name, aws_access=self.credentials) self.message_dict['download_data'] = \ "Successfully downloaded data." except Exception: self.success = False self.message_dict['download_data'] = tr.format_exc()
print("Checking for response message.") resp_queue = sqs.connect_to_region(region).create_queue(proc_name + "_response") if resp_queue.count() > 0: mess = resp_queue.read(10) content = json.loads(mess.get_body()) print("Saving content.") with open("tests/test_response.txt", "w") as f: json.dump(content, f) else: print("No message received!") # Now download the output file download_from_s3("data_products/*", proc_name, aws_access={ "aws_access_key_id": key, "aws_secret_access_key": secret }, output_dir="tests/") except Exception as e: print("Failed with :") print(e) queue.delete() resp_queue.delete() remove_s3_bucket(proc_name, s3.connection.S3Connection())
print("Reached time limit. Terminating after 1 min.") inst.terminate() print("Checking for response message.") resp_queue = sqs.connect_to_region(region).create_queue(proc_name + "_response") if resp_queue.count() > 0: mess = resp_queue.read(10) content = json.loads(mess.get_body()) print("Saving content.") with open("tests/test_response.txt", "w") as f: json.dump(content, f) else: print("No message received!") # Now download the output file download_from_s3("data_products/*", proc_name, aws_access={"aws_access_key_id": key, "aws_secret_access_key": secret}, output_dir="tests/") except Exception as e: print("Failed with :") print(e) queue.delete() resp_queue.delete() remove_s3_bucket(proc_name, s3.connection.S3Connection())
#!/usr/bin/env python '''Download data from AWS S3 bucket (added to bucket with upload_data_AWS.py) Notes: key_name not full file name yet so just using wildcard for now. output_dir adds that directory in place you call script for now. ''' import sys sys.path.append('/home/ubuntu/aws_controller') from upload_download_s3 import download_from_s3 key_name = '*' bucket_name = 'BUCKET_NAME' output_dir = '/home/ubuntu/data' download_from_s3(key_name, bucket_name, conn=None, aws_access={ 'aws_access_key_id': 'ACCESS_KEY_HERE', 'aws_secret_access_key': 'SECRET_ACCESS_KEY_HERE' }, output_dir=output_dir)
#!/usr/bin/env python '''Download data from AWS S3 bucket (added to bucket with upload_data_AWS.py) Notes: key_name not full file name yet so just using wildcard for now. output_dir adds that directory in place you call script for now. ''' import sys sys.path.append('/home/ubuntu/aws_controller') from upload_download_s3 import download_from_s3 key_name='*' bucket_name='BUCKET_NAME' output_dir='/home/ubuntu/data' download_from_s3(key_name, bucket_name, conn=None, aws_access={'aws_access_key_id': 'ACCESS_KEY_HERE', 'aws_secret_access_key': 'SECRET_ACCESS_KEY_HERE'}, output_dir=output_dir)