def test_s3_client(self): s3 = boto.s3.connect_to_region("us-east-1") s3.get_all_buckets() spans = self.memory_exporter.get_finished_spans() assert spans self.assertEqual(len(spans), 1) span = spans[0] assert_span_http_status_code(span, 200) self.assertEqual(span.attributes["http.method"], "GET") self.assertEqual(span.attributes["aws.operation"], "get_all_buckets") # Create a bucket command s3.create_bucket("cheese") spans = self.memory_exporter.get_finished_spans() assert spans self.assertEqual(len(spans), 2) span = spans[1] assert_span_http_status_code(span, 200) self.assertEqual(span.attributes["http.method"], "PUT") self.assertEqual(span.attributes["path"], "/") self.assertEqual(span.attributes["aws.operation"], "create_bucket") # Get the created bucket s3.get_bucket("cheese") spans = self.memory_exporter.get_finished_spans() assert spans self.assertEqual(len(spans), 3) span = spans[2] assert_span_http_status_code(span, 200) self.assertEqual( span.resource, Resource(attributes={ "endpoint": "s3", "http_method": "head" }), ) self.assertEqual(span.attributes["http.method"], "HEAD") self.assertEqual(span.attributes["aws.operation"], "head_bucket") self.assertEqual(span.name, "s3.command") # Checking for resource incase of error try: s3.get_bucket("big_bucket") except Exception: # pylint: disable=broad-except spans = self.memory_exporter.get_finished_spans() assert spans span = spans[2] self.assertEqual( span.resource, Resource(attributes={ "endpoint": "s3", "http_method": "head" }), )
def test_s3_client(self): s3 = boto.s3.connect_to_region('us-east-1') writer = self.tracer.writer Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(s3) s3.get_all_buckets() spans = writer.pop() assert spans self.assertEqual(len(spans), 1) span = spans[0] assert_is_measured(span) assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.METHOD), 'GET') self.assertEqual(span.get_tag('aws.operation'), 'get_all_buckets') # Create a bucket command s3.create_bucket('cheese') spans = writer.pop() assert spans self.assertEqual(len(spans), 1) span = spans[0] assert_is_measured(span) assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.METHOD), 'PUT') self.assertEqual(span.get_tag('path'), '/') self.assertEqual(span.get_tag('aws.operation'), 'create_bucket') # Get the created bucket s3.get_bucket('cheese') spans = writer.pop() assert spans self.assertEqual(len(spans), 1) span = spans[0] assert_is_measured(span) assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.METHOD), 'HEAD') self.assertEqual(span.get_tag('aws.operation'), 'head_bucket') self.assertEqual(span.service, 'test-boto-tracing.s3') self.assertEqual(span.resource, 's3.head') self.assertEqual(span.name, 's3.command') # Checking for resource incase of error try: s3.get_bucket('big_bucket') except Exception: spans = writer.pop() assert spans span = spans[0] self.assertEqual(span.resource, 's3.head')
def test_s3_client(self): s3 = boto.s3.connect_to_region("us-east-1") Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(s3) s3.get_all_buckets() spans = self.pop_spans() assert spans self.assertEqual(len(spans), 1) span = spans[0] assert_is_measured(span) assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.METHOD), "GET") self.assertEqual(span.get_tag("aws.operation"), "get_all_buckets") # Create a bucket command s3.create_bucket("cheese") spans = self.pop_spans() assert spans self.assertEqual(len(spans), 1) span = spans[0] assert_is_measured(span) assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.METHOD), "PUT") self.assertEqual(span.get_tag("path"), "/") self.assertEqual(span.get_tag("aws.operation"), "create_bucket") # Get the created bucket s3.get_bucket("cheese") spans = self.pop_spans() assert spans self.assertEqual(len(spans), 1) span = spans[0] assert_is_measured(span) assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.METHOD), "HEAD") self.assertEqual(span.get_tag("aws.operation"), "head_bucket") self.assertEqual(span.service, "test-boto-tracing.s3") self.assertEqual(span.resource, "s3.head") self.assertEqual(span.name, "s3.command") # Checking for resource incase of error try: s3.get_bucket("big_bucket") except Exception: spans = self.pop_spans() assert spans span = spans[0] self.assertEqual(span.resource, "s3.head")
def upload_file(s3, bucketname, file_path): b = s3.get_bucket(bucketname) filename = os.path.basename(file_path) k = b.new_key(filename) mp = b.initiate_multipart_upload(filename) source_size = os.stat(file_path).st_size bytes_per_chunk = 5000*1024*1024 chunks_count = int(math.ceil(source_size / float(bytes_per_chunk))) for i in range(chunks_count): offset = i * bytes_per_chunk remaining_bytes = source_size - offset bytes = min([bytes_per_chunk, remaining_bytes]) part_num = i + 1 print "uploading part " + str(part_num) + " of " + str(chunks_count) with open(file_path, 'r') as fp: fp.seek(offset) mp.upload_part_from_file(fp=fp, part_num=part_num, size=bytes) if len(mp.get_all_parts()) == chunks_count: mp.complete_upload() print "upload_file done" else: mp.cancel_upload() print "upload_file failed"
def getKeys(): bkt = s3.get_bucket(parm['s3_bucket']) keys = bkt.get_all_keys() li = sorted([(k.last_modified, k.name) for k in keys]) for x in li: print x return keys
def UploadServiceFilesToS3(s3_bucket, s3_path, files, verbose=False): """Upload all service files for build into separate s3_bucket/s3_path/. If verbose is True, print status updates while working.""" s3 = boto.connect_s3() s3 = boto.s3.connection.S3Connection( calling_format=boto.s3.connection. ProtocolIndependentOrdinaryCallingFormat()) bucket = s3.get_bucket(s3_bucket) for source_file in files: source_file = os.path.abspath(source_file) if not os.path.isfile(source_file): raise IOError("File not found: %s" % source_file) if re.search('(\w+)\.(mar|dmg|rpm|deb|exe|tar\.bz2)$', source_file): continue dest_file = os.path.basename(source_file) full_key_name = '/' + s3_path + '/' + dest_file bucket_key = boto.s3.key.Key(bucket) bucket_key.key = full_key_name if verbose: print "Uploading service file " + source_file bucket_key.set_contents_from_filename(source_file) if verbose: print "Upload service files complete"
def process(s3, s3BucketName, s3InputPrefix, s3OutputPrefix, fileName, workDir): s3Bucket = s3.get_bucket(s3BucketName) localInputPath = os.path.join(workDir, fileName) localOutputPath = os.path.join(workDir, fileName[:-5] + '.txt') remoteInputPath = s3InputPrefix + '/' + fileName remoteOutputPath = s3OutputPrefix + '/' + fileName[:-5] + '.txt' if not os.path.isdir(workDir): os.system('sudo mkdir work && sudo chmod 777 work') print("Downloading %s from s3://%s/%s ..." % (localInputPath, s3BucketName, remoteInputPath)) #print(remoteInputPath) key = s3Bucket.get_key(remoteInputPath) s3 = boto3.client('s3') s3.download_file(s3BucketName, remoteInputPath, localInputPath) key.get_contents_to_filename(workDir + "/" + fileName) #subprocess.call(['./darknet','detector','demo','cfg/coco.data','cfg/yolov3-tiny.cfg','yolov3-tiny.weights', localInputPath,'outpt.txt']) os.system( 'Xvfb :1 & export DISPLAY=:1 && ./darknet detector demo cfg/coco.data cfg/yolov3-tiny.cfg yolov3-tiny.weights ' + localInputPath + ' > ' + localOutputPath) parseOutput(localOutputPath) print("Uploading %s to s3://%s/%s ..." % (localOutputPath, s3BucketName, remoteOutputPath)) key = Key(s3Bucket) key.key = remoteOutputPath key.set_contents_from_filename(localOutputPath) return True
def _extractKeyInfoFromUrl(url, existing=None): """ Extracts bucket and key from URL. The existing parameter determines if a particular state of existence should be enforced. Note also that if existing is not True and the key does not exist. :param existing: determines what the state :return: (bucket, key) """ s3 = boto.connect_s3() bucket = s3.get_bucket(url.netloc) key = bucket.get_key(url.path[1:]) if existing is True: if key is None: raise RuntimeError('Key does not exist.') elif existing is False: if key is not None: raise RuntimeError('Key exists.') elif existing is None: pass else: assert False if key is None: key = bucket.new_key(url.path[1:]) return bucket, key
def test_bad_algorithms(self): kms = boto.kms.connect_to_region( "us-west-2", aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=self.s3_server.port, host="127.0.0.1") kms.auth_region_name = 'us-west-2' kms.auth_service_name = 'kms' s3 = boto.s3.connect_to_region( "us-west-2", aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=self.s3_server.port, host="127.0.0.1", calling_format=OrdinaryCallingFormat()) enc = S3ClientEncryptionHandler(kms, kms_key_id) bucket = s3.get_bucket("hello") key = bucket.new_key("bad_algorithms") try: enc.write(key, "Hello world!", wrapper_algorithm="foo") self.fail("Expected EncryptionError") except EncryptionError: pass try: enc.write(key, "Hello world!", encryption_algorithm="foo") self.fail("Expected EncryptionError") except EncryptionError: pass
def test_s3_put(self): s3 = boto.s3.connect_to_region('us-east-1') Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(s3) s3.create_bucket('mybucket') bucket = s3.get_bucket('mybucket') k = boto.s3.key.Key(bucket) k.key = 'foo' k.set_contents_from_string('bar') spans = self.pop_spans() assert spans # create bucket self.assertEqual(len(spans), 3) self.assertEqual(spans[0].get_tag('aws.operation'), 'create_bucket') assert_is_measured(spans[0]) assert_span_http_status_code(spans[0], 200) self.assertEqual(spans[0].service, 'test-boto-tracing.s3') self.assertEqual(spans[0].resource, 's3.put') # get bucket assert_is_measured(spans[1]) self.assertEqual(spans[1].get_tag('aws.operation'), 'head_bucket') self.assertEqual(spans[1].resource, 's3.head') # put object assert_is_measured(spans[2]) self.assertEqual(spans[2].get_tag('aws.operation'), '_send_file_internal') self.assertEqual(spans[2].resource, 's3.put')
def UploadFilesToS3(s3_bucket, s3_path, files, verbose=False): """Upload only mar file(s) from the list to s3_bucket/s3_path/. If verbose is True, print status updates while working.""" s3 = boto.connect_s3() s3 = boto.s3.connection.S3Connection(calling_format=boto.s3.connection.ProtocolIndependentOrdinaryCallingFormat()) bucket = s3.get_bucket(s3_bucket) try: for source_file in files: source_file = os.path.abspath(source_file) if not os.path.isfile(source_file): raise IOError("File not found: %s" % source_file) if not re.search('(\w+)\.(mar|dmg|exe|tar\.bz2)$', source_file): continue dest_file = os.path.basename(source_file) full_key_name = '/'+s3_path+'/'+dest_file bucket_key = boto.s3.key.Key(bucket) bucket_key.key = full_key_name if verbose: print "Uploading " + source_file bucket_key.set_contents_from_filename(source_file) finally: pass if verbose: print "Upload complete"
def prepare_custom_script(instance_id): print '------------------------------------' print 'Preparing Vulnpryer Custom Script' print '------------------------------------' f1 = open('custom_scripts/start_vulnpryer.py', 'r') f2 = open('temp/start_vulnpryer.py', 'w') for line in f1: line = re.sub(r"^.*--opsworks_region.*$", "parser.add_argument('-r', '--opsworks_region', type=str, default=\"" + config.get('general', 'opsworks_aws_region') + "\",", line) line = re.sub(r"^.*--instance_id.*$", "parser.add_argument('-i', '--instance_id', type=str,\n\t\t\tdefault=\"" + instance_id + "\",", line) line = re.sub(r"^.*--vulnpryer_pipeline_metric_nspace.*$", "parser.add_argument('-vn', '--vulnpryer_pipeline_metric_nspace', default=\'" + config.get('cloudwatch', 'vulnpryer_pipeline_metric_namespace') + "\',", line) line = re.sub(r"^.*--vulnpryer_pipeline_metric_name.*$", "parser.add_argument('-vm', '--vulnpryer_pipeline_metric_name', default=\'" + config.get('cloudwatch', 'vulnpryer_pipeline_metric_name') + "\',", line) f2.write(line) f1.close() f2.close() print 'Plugged in region ' + config.get('general', 'opsworks_aws_region') + ' and opsworks instance id ' + instance_id # Connect AWS S3 try: s3 = boto.s3.connect_to_region(region_name=config.get('general', 's3_aws_region'), aws_access_key_id=config.get('general', 'aws_access_key_id'), aws_secret_access_key=config.get('general', 'aws_secret_access_key')) bucket = s3.get_bucket(config.get('custom_script', 's3_bucket')) except: print "Check keys and configuration before proceeding." return False k = bucket.get_key(config.get('custom_script', 's3_bucket_directory') + 'start_vulnpryer.py') if k is None: k = bucket.new_key(config.get('custom_script', 's3_bucket_directory') + 'start_vulnpryer.py') k.set_contents_from_filename('temp/start_vulnpryer.py') print 'Uploaded script ' + str(k)
def store_graph(instance_id, filename): """ Transmits the specified graph file to internal object store on cloud controller. """ # TODO(devcamcar): Need to use an asynchronous method to make this # connection. If boto has some separate method that generates # the request it would like to make and another method to parse # the response we can make our own client that does the actual # request and hands it off to the response parser. s3 = boto.s3.connection.S3Connection( aws_access_key_id=FLAGS.aws_access_key_id, aws_secret_access_key=FLAGS.aws_secret_access_key, is_secure=False, calling_format=boto.s3.connection.OrdinaryCallingFormat(), port=FLAGS.s3_port, host=FLAGS.s3_host) bucket_name = '_%s.monitor' % instance_id # Object store isn't creating the bucket like it should currently # when it is first requested, so have to catch and create manually. try: bucket = s3.get_bucket(bucket_name) except Exception: bucket = s3.create_bucket(bucket_name) key = boto.s3.Key(bucket) key.key = os.path.basename(filename) key.set_contents_from_filename(filename)
def delete_folder(bucket_name, folder_name): """ Deletes an S3 folder and all of its contents, recursively """ s3 = boto.connect_s3() bucket = s3.get_bucket(bucket_name) bucketListResultSet = bucket.list(prefix=folder_name) result = bucket.delete_keys([key.name for key in bucketListResultSet]) return result
def test_s3_put(self): s3 = boto.s3.connect_to_region('us-east-1') writer = self.tracer.writer Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(s3) s3.create_bucket('mybucket') bucket = s3.get_bucket('mybucket') k = boto.s3.key.Key(bucket) k.key = 'foo' k.set_contents_from_string('bar') spans = writer.pop() assert spans # create bucket self.assertEqual(len(spans), 3) self.assertEqual(spans[0].get_tag('aws.operation'), 'create_bucket') self.assertEqual(spans[0].get_tag(http.STATUS_CODE), '200') self.assertEqual(spans[0].service, 'test-boto-tracing.s3') self.assertEqual(spans[0].resource, 's3.put') # get bucket self.assertEqual(spans[1].get_tag('aws.operation'), 'head_bucket') self.assertEqual(spans[1].resource, 's3.head') # put object self.assertEqual(spans[2].get_tag('aws.operation'), '_send_file_internal') self.assertEqual(spans[2].resource, 's3.put')
def uploadDir(localDir, s3BucketName, s3InputPrefix, s3OutputPrefix, sqsQueueName, awsRegion): files = os.listdir(localDir) s3 = boto.s3.connect_to_region(awsRegion) s3Bucket = s3.get_bucket(s3BucketName) sqs = boto.sqs.connect_to_region(awsRegion) sqsQueue = sqs.lookup(sqsQueueName) for fileName in files: localPath = os.path.join(localDir, fileName) remotePath = s3InputPrefix + fileName print "Uploading %s to s3://%s/%s ..." % (localPath, s3BucketName, remotePath) # Upload to S3 key = Key(s3Bucket) key.key = remotePath key.set_contents_from_filename(localPath) # Send message to SQS print "Sending message to SQS queue ..." messageBody = json.dumps( ['process', s3BucketName, s3InputPrefix, s3OutputPrefix, fileName]) m = Message() m.set_body(messageBody) sqsQueue.write(m) print "Done!" print "All done!"
def test_s3_put(self): s3 = boto.s3.connect_to_region("us-east-1") s3.create_bucket("mybucket") bucket = s3.get_bucket("mybucket") key = boto.s3.key.Key(bucket) key.key = "foo" key.set_contents_from_string("bar") spans = self.memory_exporter.get_finished_spans() assert spans # create bucket self.assertEqual(len(spans), 3) self.assertEqual(spans[0].attributes["aws.operation"], "create_bucket") assert_span_http_status_code(spans[0], 200) self.assertEqual(spans[0].attributes["endpoint"], "s3") self.assertEqual(spans[0].attributes["http_method"], "put") # get bucket self.assertEqual(spans[1].attributes["aws.operation"], "head_bucket") self.assertEqual(spans[1].attributes["endpoint"], "s3") self.assertEqual(spans[1].attributes["http_method"], "head") # put object self.assertEqual( spans[2].attributes["aws.operation"], "_send_file_internal" ) self.assertEqual(spans[2].attributes["endpoint"], "s3") self.assertEqual(spans[2].attributes["http_method"], "put")
def main(): parser = optparse.OptionParser('Usage: %prog <args>...') (opts, args) = parser.parse_args() if 0 == len(args): parser.print_help() return 1 try: s3 = boto.connect_s3() for a in args: if not a.startswith('s3://'): raise Error('unsupported object path!') parts = a[5:].split('/') bucket = s3.get_bucket(parts[0]) key = bucket.get_key('/'.join(parts[1:]) if 1 < len(parts) else '') _, name = os.path.split(key.name) key.get_contents_to_filename(name) src = name + '.gz' compress_file(name, src) os.remove(name) dst = key.name + '.gz' upload_file(bucket, dst, src) os.remove(src) except (Error, Exception), err: sys.stderr.write('[ERROR] {0}\n'.format(err)) return 1
def test_s3_put(self): s3 = boto.s3.connect_to_region("us-east-1") Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(s3) s3.create_bucket("mybucket") bucket = s3.get_bucket("mybucket") k = boto.s3.key.Key(bucket) k.key = "foo" k.set_contents_from_string("bar") spans = self.pop_spans() assert spans # create bucket self.assertEqual(len(spans), 3) self.assertEqual(spans[0].get_tag("aws.operation"), "create_bucket") assert_is_measured(spans[0]) assert_span_http_status_code(spans[0], 200) self.assertEqual(spans[0].service, "test-boto-tracing.s3") self.assertEqual(spans[0].resource, "s3.put") # get bucket assert_is_measured(spans[1]) self.assertEqual(spans[1].get_tag("aws.operation"), "head_bucket") self.assertEqual(spans[1].resource, "s3.head") # put object assert_is_measured(spans[2]) self.assertEqual(spans[2].get_tag("aws.operation"), "_send_file_internal") self.assertEqual(spans[2].resource, "s3.put")
def test_s3_client(self): s3 = boto.s3.connect_to_region("us-east-1") tracer = get_dummy_tracer() writer = tracer.writer Pin(service=self.TEST_SERVICE, tracer=tracer).onto(s3) s3.get_all_buckets() spans = writer.pop() assert spans eq_(len(spans), 1) span = spans[0] eq_(span.get_tag(http.STATUS_CODE), "200") eq_(span.get_tag(http.METHOD), "GET") eq_(span.get_tag('aws.operation'), "get_all_buckets") # Create a bucket command s3.create_bucket("cheese") spans = writer.pop() assert spans eq_(len(spans), 1) span = spans[0] eq_(span.get_tag(http.STATUS_CODE), "200") eq_(span.get_tag(http.METHOD), "PUT") eq_(span.get_tag('path'), '/') eq_(span.get_tag('aws.operation'), "create_bucket") # Get the created bucket s3.get_bucket("cheese") spans = writer.pop() assert spans eq_(len(spans), 1) span = spans[0] eq_(span.get_tag(http.STATUS_CODE), "200") eq_(span.get_tag(http.METHOD), "HEAD") eq_(span.get_tag('aws.operation'), "head_bucket") eq_(span.service, "test-boto-tracing.s3") eq_(span.resource, "s3.head") eq_(span.name, "s3.command") # Checking for resource incase of error try: s3.get_bucket("big_bucket") except Exception: spans = writer.pop() assert spans span = spans[0] eq_(span.resource, "s3.head")
def from_file(path, bucket_name): bucket = s3.get_bucket(bucket_name, validate=False) output = S3Key(bucket) output.key = path try: return output.get_contents_as_string().decode('utf-8') except Exception: return None
def __init__( self, bucket_name, key_name ): super( Operation, self ).__init__( ) self.bucket_name = bucket_name self.key_name = key_name work_around_dots_in_bucket_names( ) with closing( S3Connection( ) ) as s3: bucket = s3.get_bucket( self.bucket_name ) self.bucket_location = bucket.get_location( )
def upload_file_s3(bucket_name, folder, path_to_file): s3 = boto.connect_s3() bucket = s3.get_bucket(bucket_name) k = Key(bucket) filename = folder + "/" + os.path.split(path_to_file)[-1] k.key = filename k.set_contents_from_filename(path_to_file) return filename
def to_file(string, path, bucket_name, mime): if environment.local: print('[file] {}'.format(path)) else: bucket = s3.get_bucket(bucket_name, validate=False) output = S3Key(bucket) output.key = path output.content_type = mime output.set_contents_from_string(string)
def process(s3, s3BucketName, s3InputPrefix, s3OutputPrefix, fileName, workDir): print(s3BucketName + s3InputPrefix) s3BucketInput = s3.get_bucket(s3BucketName + "-" + s3InputPrefix) s3BucketOutput = s3.get_bucket(s3BucketName + "-" + s3OutputPrefix) localInputPath = os.path.join(workDir, fileName) localOutputPath = os.path.join(workDir, fileName[:-4] + '.txt') remoteInputPath = fileName remoteOutputPath = fileName[:-4] + '.txt' if not os.path.isdir(workDir): os.system('sudo mkdir work && sudo chmod 777 work') print("Downloading %s from s3://%s/%s ..." % (localInputPath, s3BucketName, remoteInputPath)) #print(remoteInputPath) key = s3BucketInput.get_key(remoteInputPath) print("here1") s3 = boto3.client('s3') print("here2") print("s3BucketInput") print(s3BucketInput) print("remoteInputPath") print(remoteInputPath) print("localInputPath") print(localInputPath) s3.download_file(s3BucketName + "-" + "inputfolder", remoteInputPath, localInputPath) print("here3") key.get_contents_to_filename(workDir + "/" + fileName) #subprocess.call(['./darknet','detector','demo','cfg/coco.data','cfg/yolov3-tiny.cfg','yolov3-tiny.weights', localInputPath,'outpt.txt']) print("hi1") #ans = os.system('python3 classifier/image_classification.py '+localInputPath) os.system('python3 classifier/image_classification.py ' + localInputPath + ' > ' + localOutputPath) print("hi2") #print(ans) print("hi2") print(localOutputPath) print("Uploading %s to s3://%s/%s ..." % (localOutputPath, s3BucketInput, remoteOutputPath)) key = Key(s3BucketOutput) key.key = remoteOutputPath key.set_contents_from_filename(localOutputPath) return True
def main(argv): bucket = s3.get_bucket('flaskbucket') isValid = True while isValid: options = {1: upload, 2: download, 3: exit} option = input("Enter your option 1. Upload 2.Download 3. exit") if option == "3": print "Exiting..\n" isValid = None else: options[option](bucket)
def create_key(region, storage_prefix, kms_key_id): kms = boto.kms.connect_to_region(region) s3 = boto.s3.connect_to_region(region) ceh = S3ClientEncryptionHandler(kms, kms_key_id) bucket_name, prefix = storage_prefix[5:].split("/", 1) s3_object = prefix + "rolemaker.key" bucket = s3.get_bucket(bucket_name) key = bucket.new_key(s3_object) data = urandom(64) ceh.write(key, data) return 0
def _getKeyForUrl(url, existing=None): """ Extracts a key from a given s3:// URL. On return, but not on exceptions, this method leaks an S3Connection object. The caller is responsible to close that by calling key.bucket.connection.close(). :param bool existing: If True, key is expected to exist. If False, key is expected not to exists and it will be created. If None, the key will be created if it doesn't exist. :rtype: Key """ # Get the bucket's region to avoid a redirect per request with closing(boto.connect_s3()) as s3: region = bucket_location_to_region(s3.get_bucket(url.netloc).get_location()) # Note that caller is responsible for closing the connection s3 = boto.s3.connect_to_region(region) try: bucket = s3.get_bucket(url.netloc) key = bucket.get_key(url.path[1:]) if existing is True: if key is None: raise RuntimeError('Key does not exist.') elif existing is False: if key is not None: raise RuntimeError('Key exists.') elif existing is None: pass else: assert False if key is None: key = bucket.new_key(url.path[1:]) except: with panic(): s3.close() else: return key
def _upload_part( self, upload_id, part_num, buf ): """ Upload a part to S3. The given buffer's tell() is assumed to point at the first byte to be uploaded. When this method returns, the tell() method points at the end of the buffer. :return: the part object representing the uploaded part :rtype: Part """ with closing( boto.s3.connect_to_region( self.bucket_location ) ) as s3: bucket = s3.get_bucket( self.bucket_name ) headers = { } self.__add_encryption_headers( headers ) upload = self._get_upload( bucket, upload_id ) key = upload.upload_part_from_file( buf, part_num + 1, headers=headers ) return self._part_for_key( bucket, part_num, key )
def _copyKey(self, srcKey, dstBucketName, dstKeyName, headers=None): headers = headers or {} s3 = boto.connect_s3() if srcKey.size > self.outer.partSize: return copyKeyMultipart(srcKey=srcKey, dstBucketName=dstBucketName, dstKeyName=dstKeyName, headers=headers) else: dstBucket = s3.get_bucket(dstBucketName) return dstBucket.copy_key(new_key_name=dstKeyName, src_bucket_name=srcKey.bucket.name, src_key_name=srcKey.name, metadata=srcKey.metadata, headers=headers)
def _copy_part( self, upload_id, part_num, src_bucket_name, src_key_name, size ): with self._propagate_worker_exception( ): try: if error_event.is_set( ): raise BailoutException( ) log.info( 'part %i: copying', part_num ) start, end = self._get_part_range( part_num ) end = min( end, size ) part_size = end - start if part_size > 0 or part_num == 0: url = urlparse( self.url ) assert url.scheme == 's3' assert url.path.startswith( '/' ) with closing( boto.s3.connect_to_region( self.bucket_location ) ) as s3: bucket = s3.get_bucket( self.bucket_name ) upload = self._get_upload( bucket, upload_id ) headers = { } if self.sse_key: self._add_encryption_headers( self.sse_key, headers ) if part_size == 0: # Since copy_part_from_key doesn't allow empty ranges, we handle that # case by uploading an empty part. assert part_num == 0 # noinspection PyTypeChecker key = upload.upload_part_from_file( StringIO( ), part_num + 1, headers=headers ) else: if self.src_sse_key: self._add_encryption_headers( self.src_sse_key, headers, for_copy=True ) key = upload.copy_part_from_key( src_bucket_name=src_bucket_name, src_key_name=src_key_name, part_num=part_num + 1, start=start, end=end - 1, headers=headers ) # somehow copy_part_from_key doesn't set the key size key.size = part_size assert key.size == part_size self._log_progress( part_num, 'copied', part_size ) return part_num, self._part_for_key( bucket, part_num, key ) else: done_event.set( ) return part_num, None finally: download_slots_semaphore.release( )
def UploadFilesToS3(s3_bucket, s3_path, files, package, verbose=False): """Upload only mar file(s) from the list to s3_bucket/s3_path/. If verbose is True, print status updates while working.""" s3 = boto.connect_s3() s3 = boto.s3.connection.S3Connection( calling_format=boto.s3.connection. ProtocolIndependentOrdinaryCallingFormat()) bucket = s3.get_bucket(s3_bucket) properties = {} property_conditions = [ ('completeMarUrl', lambda m: m.endswith('.complete.mar')), ('partialMarUrl', lambda m: m.endswith('.mar') and '.partial.' in m), ('packageUrl', lambda m: m.endswith(package)), ] for source_file in files: source_file = os.path.abspath(source_file) if not os.path.isfile(source_file): raise IOError("File not found: %s" % source_file) if not re.search('(\w+)\.(mar|dmg|rpm|deb|exe|tar\.bz2)$', source_file): continue dest_file = os.path.basename(source_file) full_key_name = '/' + s3_path + '/' + dest_file bucket_key = boto.s3.key.Key(bucket) bucket_key.key = full_key_name if verbose: print "Uploading " + source_file bucket_key.set_contents_from_filename(source_file) m = 'https://' + s3_bucket + full_key_name for prop, condition in property_conditions: if condition(m): properties.update({prop: m}) break if verbose: print "Upload complete" return properties
def read_volume_info(self): """ Read the volname.volinfo file in the S3 bucket. """ with self.s3_pool.get_connection() as s3: bucket = s3.get_bucket(self.bucket_name) key = bucket.get_key(self.volname + ".volinfo") config = json_loads(key.get_contents_as_string()) self.block_size = int(config.get("block-size", 4096)) self.encryption = config.get("encryption") self.policy = config.get("policy", "private") self.size = int(config.get("size")) self.storage_class = config.get("storage-class", "standard") self.suffix = config.get("suffix", "." + self.volname) return
def _prepare_upload( self ): """ Prepare a new multipart upload or resume a previously interrupted one. Returns the upload ID and a dictionary mapping the 0-based index of a part to its size. """ completed_parts = { } with closing( boto.s3.connect_to_region( self.bucket_location ) ) as s3: bucket = s3.get_bucket( self.bucket_name ) uploads = self._get_uploads( bucket ) if len( uploads ) == 0: if self.resume: raise UserError( "Transfer failed. There is no pending upload to be resumed." ) else: headers = { } self.__add_encryption_headers( headers ) upload_id = bucket.initiate_multipart_upload( key_name=self.key_name, headers=headers ).id elif len( uploads ) == 1: if self.resume: upload = uploads[ 0 ] upload_id = upload.id for part in upload: completed_parts[ part.part_number - 1 ] = part # If there is an upload but no parts we can use whatever part size we want, # otherwise we need to ensure that we use the same part size as before. if len( completed_parts ) > 0: previous_part_size = self._guess_part_size( completed_parts ) if self.part_size != previous_part_size: raise UserError( "Transfer failed. The part size appears to have changed from %i " "to %i. Either resume the upload with the old part size or cancel " "the upload and restart with the new part size. " % (self.part_size, previous_part_size) ) else: raise UserError( "Transfer failed. There is a pending upload. If you would like to resume " "that upload, run {me} again with --resume. If you would like to cancel " "the upload, use '{me} cancel {bucket_name} {key_name}'. Note that " "pending uploads incur storage fees.".format( me=me, **vars( self ) ) ) else: raise RuntimeError( "Transfer failed. Detected more than one pending multipart upload. Consider " "using '{me} cancel {bucket_name} {key_name}' to delete all of them before " "trying the transfer again. Note that pending uploads incur storage " "fees.".format( me=me, **vars( self ) ) ) return upload_id, completed_parts
def copyKeyMultipart(srcKey, dstBucketName, dstKeyName, headers=None): """ Copies a key from a source key to a destination key in multiple parts. Note that if the destination key exists it will be overwritten implicitly, and if it does not exist a new key will be created. :param boto.s3.key.Key srcKey: The source key to be copied from. :param str dstBucketName: The name of the destination bucket for the copy. :param str dstKeyName: The name of the destination key that will be created or overwritten. :param dict headers: Any headers that should be passed. :rtype: boto.s3.multipart.CompletedMultiPartUpload :return: An object representing the completed upload. """ partSize = defaultPartSize # We need a location-agnostic connection to S3 so we can't use the one that we # normally use for interacting with the job store bucket. with closing(boto.connect_s3()) as s3: headers = headers or {} totalSize = srcKey.size for attempt in retry_s3(): with attempt: dstBucket = s3.get_bucket(dstBucketName) upload = dstBucket.initiate_multipart_upload(dstKeyName, headers=headers) try: start = 0 partIndex = itertools.count() while start < totalSize: end = min(start + partSize, totalSize) for attempt in retry_s3(): with attempt: upload.copy_part_from_key(src_bucket_name=srcKey.bucket.name, src_key_name=srcKey.name, src_version_id=srcKey.version_id, part_num=next(partIndex) + 1, start=start, end=end - 1, headers=headers) start += partSize except: with panic(log=log): upload.cancel_upload() else: for attempt in retry_s3(): with attempt: return upload.complete_upload()
def UploadFilesToS3(s3_bucket, s3_path, files, package, verbose=False): """Upload only mar file(s) from the list to s3_bucket/s3_path/. If verbose is True, print status updates while working.""" s3 = boto.connect_s3() s3 = boto.s3.connection.S3Connection(calling_format=boto.s3.connection.ProtocolIndependentOrdinaryCallingFormat()) bucket = s3.get_bucket(s3_bucket) properties = {} property_conditions = [ ('completeMarUrl', lambda m: m.endswith('.complete.mar')), ('partialMarUrl', lambda m: m.endswith('.mar') and '.partial.' in m), ('packageUrl', lambda m: m.endswith(package)), ] for source_file in files: source_file = os.path.abspath(source_file) if not os.path.isfile(source_file): raise IOError("File not found: %s" % source_file) if not re.search('(\w+)\.(mar|dmg|rpm|deb|exe|tar\.bz2)$', source_file): continue dest_file = os.path.basename(source_file) full_key_name = '/'+s3_path+'/'+dest_file bucket_key = boto.s3.key.Key(bucket) bucket_key.key = full_key_name if verbose: print "Uploading " + source_file bucket_key.set_contents_from_filename(source_file) m = 'http://' + s3_bucket + full_key_name for prop, condition in property_conditions: if condition(m): properties.update({prop: m}) break if verbose: print "Upload complete" return properties
def read_credentials_from_s3(self, s3_url, kms, s3): """ read_credentials_from_s3(s3_url, kms, s3) -> dict Read access-key/secret-key map from the specified S3 URL (in the form s3://bucket/key). """ assert s3_url.startswith("s3://") try: bucket_name, key_name = s3_url[5:].split("/", 1) except: raise ValueError("Invalid S3 URL: %s" % s3_url) enc = S3ClientEncryptionHandler(kms) bucket = s3.get_bucket(bucket_name) key = bucket.get_key(key_name) data = enc.read(key) self.read_credentials_from_stream(cStringIO(data))
def fetch(key, logger=None): """download and extract an archive""" template = "$AICHALLENGE_PREFIX/var/lib/aichallenge/submissions/%s" path = os.path.expandvars(template % key) if os.path.isdir(path): # already decompressed -- move on return if logger is not None: logger.info('downloading ' + key) access_key = config.get('worker', 'aws_access_key') secret_key = config.get('worker', 'aws_secret_key') bucket = config.get('worker', 'submission_bucket') prefix = config.get('worker', 'submission_prefix') s3 = boto.s3.Connection(access_key, secret_key) bucket = s3.get_bucket(bucket) s3_key = bucket.get_key(prefix + key) if s3_key is None: raise KeyError io = cStringIO.StringIO() s3_key.get_contents_to_file(io) try: io.seek(0) zip = zipfile.ZipFile(io) decompress_zip(zip, path) return except zipfile.BadZipfile: pass try: io.seek(0) tar = tarfile.open(fileobj=io) decompress_tar(tar, path) return except tarfile.TarError: pass raise ValueError, "invalid archive"
def _copyKey(self, srcKey, dstBucketName, dstKeyName, headers=None): headers = headers or {} if srcKey.size > self.outer.partSize: return copyKeyMultipart(srcKey=srcKey, dstBucketName=dstBucketName, dstKeyName=dstKeyName, headers=headers) else: # We need a location-agnostic connection to S3 so we can't use the one that we # normally use for interacting with the job store bucket. with closing(boto.connect_s3()) as s3: for attempt in retry_s3(): with attempt: dstBucket = s3.get_bucket(dstBucketName) return dstBucket.copy_key(new_key_name=dstKeyName, src_bucket_name=srcKey.bucket.name, src_version_id=srcKey.version_id, src_key_name=srcKey.name, metadata=srcKey.metadata, headers=headers)
def process(s3, s3BucketName, s3InputPrefix, s3OutputPrefix, fileName, workDir, outputExtension, command): s3Bucket = s3.get_bucket(s3BucketName) localInputPath = os.path.join(workDir, fileName) localOutputPath = localInputPath + outputExtension remoteInputPath = s3InputPrefix + fileName remoteOutputPath = s3OutputPrefix + fileName + outputExtension print "Downloading %s from s3://%s/%s ..." % (localInputPath, s3BucketName, remoteInputPath) key = s3Bucket.get_key(remoteInputPath) key.get_contents_to_filename(localInputPath) full_command = [command, localInputPath, localOutputPath] print "Executing: %s" % ' '.join(full_command) returncode = subprocess.call(full_command) if returncode != 0: print "Return Code not '0'!" return False print "Uploading %s to s3://%s/%s ..." % (localOutputPath, s3BucketName, remoteOutputPath) key = Key(s3Bucket) key.key = remoteOutputPath key.set_contents_from_filename(localOutputPath) return True
def get_instance_graph(self, region, instance_id, graph_name): # TODO(devcamcar): Need better support for multiple regions. # Need a way to get object store by region. s3 = boto.s3.connection.S3Connection ( aws_access_key_id=settings.NOVA_ACCESS_KEY, aws_secret_access_key=settings.NOVA_SECRET_KEY, is_secure=False, calling_format=boto.s3.connection.OrdinaryCallingFormat(), port=3333, host=settings.NOVA_CLC_IP ) key = '_%s.monitor' % instance_id try: bucket = s3.get_bucket(key, validate=False) except boto.exception.S3ResponseError, e: if e.code == "NoSuchBucket": return None else: raise e
def copyKeyMultipart(srcKey, dstBucketName, dstKeyName, headers=None): """ Copies a key from a source key to a destination key in multiple parts. Note that if the destination key exists it will be overwritten implicitly, and if it does not exist a new key will be created. :param boto.s3.key.Key srcKey: The source key to be copied from. :param str dstBucketName: The name of the destination bucket for the copy. :param str dstKeyName: The name of the destination key that will be created or overwritten. :param dict headers: Any headers that should be passed. :rtype: boto.s3.multipart.CompletedMultiPartUpload :return: An object representing the completed upload. """ partSize = defaultPartSize s3 = boto.connect_s3() headers = headers or {} totalSize = srcKey.size # initiate copy upload = s3.get_bucket(dstBucketName).initiate_multipart_upload( dstKeyName, headers=headers) try: start = 0 partIndex = itertools.count() while start < totalSize: end = min(start + partSize, totalSize) upload.copy_part_from_key(src_bucket_name=srcKey.bucket.name, src_key_name=srcKey.name, part_num=next(partIndex) + 1, start=start, end=end - 1, headers=headers) start += partSize except: upload.cancel_upload() raise else: return upload.complete_upload()
def uploadDir(localDir, s3BucketName, s3InputPrefix, s3OutputPrefix, sqsQueueName, awsRegion): files = os.listdir(localDir) s3 = boto.s3.connect_to_region(awsRegion) s3Bucket = s3.get_bucket(s3BucketName) sqs = boto.sqs.connect_to_region(awsRegion) sqsQueue = sqs.lookup(sqsQueueName) for fileName in files: localPath = os.path.join(localDir, fileName) remotePath = s3InputPrefix + fileName print "Uploading %s to s3://%s/%s ..." % (localPath, s3BucketName, remotePath) # Upload to S3 key = Key(s3Bucket) key.key = remotePath key.set_contents_from_filename(localPath) # Send message to SQS print "Sending message to SQS queue ..." messageBody = json.dumps(['process', s3BucketName, s3InputPrefix, s3OutputPrefix, fileName]) m = Message() m.set_body(messageBody) sqsQueue.write(m) print "Done!" print "All done!"
def test_python_write_java_read(self): kms = boto.kms.connect_to_region( "us-west-2", aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=self.s3_server.port, host="127.0.0.1") kms.auth_region_name = 'us-west-2' kms.auth_service_name = 'kms' s3 = boto.s3.connect_to_region( "us-west-2", aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, port=self.s3_server.port, host="127.0.0.1", calling_format=OrdinaryCallingFormat()) enc = S3ClientEncryptionHandler(kms, kms_key_id) bucket = s3.get_bucket("hello") key = bucket.new_key("hello") enc.write(key, "Hello world!", headers={"content-type": "text/plain"}) result = self.java_encrypted_get("hello") self.assertEqual(result, "Hello world!") return
def copyKeyMultipart(srcKey, dstBucketName, dstKeyName, headers=None): """ Copies a key from a source key to a destination key in multiple parts. Note that if the destination key exists it will be overwritten implicitly, and if it does not exist a new key will be created. :param boto.s3.key.Key srcKey: The source key to be copied from. :param str dstBucketName: The name of the destination bucket for the copy. :param str dstKeyName: The name of the destination key that will be created or overwritten. :param dict headers: Any headers that should be passed. :rtype: boto.s3.multipart.CompletedMultiPartUpload :return: An object representing the completed upload. """ partSize = defaultPartSize s3 = boto.connect_s3() headers = headers or {} totalSize = srcKey.size # initiate copy upload = s3.get_bucket(dstBucketName).initiate_multipart_upload(dstKeyName, headers=headers) try: start = 0 partIndex = itertools.count() while start < totalSize: end = min(start + partSize, totalSize) upload.copy_part_from_key(src_bucket_name=srcKey.bucket.name, src_key_name=srcKey.name, part_num=next(partIndex)+1, start=start, end=end-1, headers=headers) start += partSize except: upload.cancel_upload() raise else: return upload.complete_upload()
def get_folders(bucket_name, folder_name): """ Get a list of all S3 folders with files in them under the specified folder """ s3 = boto.connect_s3() bucket = s3.get_bucket(bucket_name) # Get all files that start with the folder name bucketListResultSet = bucket.list(prefix=folder_name) keys = [key.name for key in bucketListResultSet] # Skip the first key, because this key is always just the root path (which doesn't have any files in it) keys = keys[1:] # Get the reslts results = [] for k in keys: # Trim off the actual filename and just process the path prefix = k[:k.rindex('/')] res = 's3n://' + bucket_name + '/' + prefix # Only add the path to the results if it doesn't already exist in the list if not res in results: results.append(res) return results
def create_volume(self, block_size=4096, encryption="", policy="private", size=None, storage_class="standard", suffix=None): """ Create the volume in the S3 bucket. """ with self.s3_pool.get_connection() as s3: bucket = s3.get_bucket(self.bucket_name) key = bucket.new_key(self.volname + ".volinfo") if size is None: raise ValueError("size must be specified") config = { 'block-size': block_size, 'policy': policy, 'size': size, 'storage-class': storage_class } if encryption: config['encryption'] = encryption if suffix: config['suffix'] = suffix key.set_contents_from_string( json_dumps(config), policy=self.policy) self.block_size = block_size self.encryption = encryption self.policy = policy self.size = size self.storage_class = storage_class self.suffix = suffix return
args = parser.parse_args() # process common command line arguments log = logging.getLogger('botocross') bc.configure_logging(log, args.log_level) credentials = bc.parse_credentials(args) locations = bc.filter_regions_s3(class_iterator(Location), args.region) # execute business logic log.info("Uploading to S3 buckets named '" + args.bucket + "':") file = open(args.file, 'r') filePath, fileName = os.path.split(file.name) log.debug(filePath) log.debug(fileName) s3 = boto.connect_s3(**credentials) for location in locations: region = RegionMap[location] pprint(region, indent=2) try: bucket_name = args.bucket + '-' + region bucket = s3.get_bucket(bucket_name) key = Key(bucket) key.name = fileName print 'Uploading to bucket ' + bucket_name key.set_contents_from_filename(args.file) except boto.exception.BotoServerError, e: log.error(e.error_message)
def main(): logging.getLogger('boto').setLevel( logging.CRITICAL) # disable all boto error logging logger.level = logging.INFO formatter = logging.Formatter( '%(asctime)s %(name)s %(levelname)s %(message)s') if options.output == 'syslog': logger.addHandler( SysLogHandler(address=(options.sysloghostname, options.syslogport))) else: sh = logging.StreamHandler(sys.stderr) sh.setFormatter(formatter) logger.addHandler(sh) state = State(options.state_file_name) logger.debug('started') #logger.debug(options) try: es = pyes.ES((list('{0}'.format(s) for s in options.esservers))) role_manager = RoleManager( aws_access_key_id=options.aws_access_key_id, aws_secret_access_key=options.aws_secret_access_key) for aws_account in options.aws_accounts: if aws_account not in state.data: state.data[aws_account] = {} assumed_role_arn = get_role_arn(aws_account) #in case we don't archive files..only look at today and yesterday's files. search_dates = [ datetime.utcnow() - timedelta(days=1), datetime.utcnow() ] for region in options.regions: logger.info("Checking AWS account %s in region %s" % (aws_account, region)) #capture the time we start running so next time we catch any files created while we run. lastrun = toUTC(datetime.now()).isoformat() if region not in state.data[aws_account]: state.data[aws_account][region] = {} logger.debug( 'connecting to AWS account {0} in region {1}'.format( aws_account, region)) try: ct_credentials = role_manager.get_credentials( assumed_role_arn) ct = boto.cloudtrail.connect_to_region( region, **ct_credentials) trails = ct.describe_trails()['trailList'] except Exception as e: logger.error( "Unable to connect to cloudtrail %s in order to " "enumerate CloudTrails in region %s due to %s" % (assumed_role_arn, region, e.message)) continue for trail in trails: bucket_account = get_bucket_account( trail['S3BucketName'], role_manager.get_credentials(assumed_role_arn)) bucket_account_assumed_role_arn = get_role_arn( bucket_account) if not bucket_account_assumed_role_arn: logger.error( "Unable to determine account of S3 bucket %s for account %s in region %s. Skipping bucket" % (trail['S3BucketName'], aws_account, region)) continue try: s3 = boto.connect_s3(**role_manager.get_credentials( bucket_account_assumed_role_arn)) except boto.exception.NoAuthHandlerFound as e: logger.error( "Unable to assume role %s in order to " "fetch s3 bucket contents in bucket %s due to %s" % (bucket_account_assumed_role_arn, trail['S3BucketName'], e.message)) continue try: ctbucket = s3.get_bucket(trail['S3BucketName']) except Exception as e: logger.error( "Unable to access bucket %s as %s due to %s" % (trail['S3BucketName'], bucket_account_assumed_role_arn, e.message)) continue filelist = list() for search_date in search_dates: # TODO it's possible that if the AWS account id is 11 instead of 12 digits, CloudTrail # will either 0 pad to 12 digits or remove the 0 padding when creating the s3 bucket # directory. Depending on this behavior, # we should either aws_account.lstrip('0') or aws_account.lstrip('0').zfill(12) prefix = ( 'AWSLogs/%(accountid)s/CloudTrail/%(region)s/%(year)s/%(month)s/%(day)s/' % { 'accountid': aws_account, 'region': region, 'year': date.strftime(search_date, '%Y'), 'month': date.strftime(search_date, '%m'), 'day': date.strftime(search_date, '%d') }) try: for bfile in ctbucket.list(prefix): filelist.append(bfile.key) except Exception as e: logger.error( "Unable to list keys in s3 bucket %s " "and path %s due to %s" % (trail['S3BucketName'], prefix, e.message)) continue for afile in filelist: try: s3file = ctbucket.get_key(afile) except Exception as e: logger.error( "Unable to HEAD key %s in s3 bucket %s " "due to %s" % (afile.name, trail['S3BucketName'], e.message)) continue logger.debug('{0} {1}'.format(afile, s3file.last_modified)) if 'lastrun' not in state.data[aws_account][region]: if toUTC(s3file.last_modified) > toUTC( datetime.utcnow() - timedelta(seconds=3600)): # If we've never collected from this account/region before, grab the last hour of logs process_file(s3file, es) elif toUTC(s3file.last_modified) > toUTC( state.data[aws_account][region]['lastrun']): process_file(s3file, es) state.data[aws_account][region]['lastrun'] = lastrun state.write_state_file() except boto.exception.NoAuthHandlerFound as e: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] logger.error("No auth handler found, check your credentials. %s" % [exc_type, fname, exc_tb.tb_lineno]) except Exception as e: logger.error("Unhandled exception, terminating: %r" % e) raise
def main(): if options.output == 'syslog': logger.addHandler( SysLogHandler(address=(options.sysloghostname, options.syslogport))) else: sh = logging.StreamHandler(sys.stderr) sh.setFormatter(formatter) logger.addHandler(sh) logger.debug('started') try: esserver = options.esservers[0] s3 = boto.connect_s3( aws_access_key_id=options.aws_access_key_id, aws_secret_access_key=options.aws_secret_access_key) idate = date.strftime(datetime.utcnow() - timedelta(days=1), '%Y%m%d') bucketdate = date.strftime(datetime.utcnow() - timedelta(days=1), '%Y-%m') hostname = socket.gethostname() # Create or update snapshot configuration logger.debug('Configuring snapshot repository') snapshot_config = { "type": "s3", "settings": { "bucket": "mozdefes2backups", "base_path": "elasticsearch/{0}/{1}".format(bucketdate, hostname), "region": "{0}".format(options.aws_region) } } r = requests.put('%s/_snapshot/s3backup' % esserver, data=json.dumps(snapshot_config)) if r.json().has_key('status'): logger.error("Error while registering snapshot repo: %s" % r.text) else: logger.debug('snapshot repo registered') # do the actual snapshotting for (index, dobackup, rotation, pruning) in zip(options.indices, options.dobackup, options.rotation, options.pruning): if dobackup == '1': index_to_snapshot = index if rotation == 'daily': index_to_snapshot += '-%s' % idate elif rotation == 'monthly': index_to_snapshot += '-%s' % idate[:6] logger.debug( 'Creating %s snapshot (this may take a while)...' % index_to_snapshot) snapshot_config = {'indices': index_to_snapshot} epoch = calendar.timegm(datetime.utcnow().utctimetuple()) r = requests.put( '{0}/_snapshot/s3backup/{1}-{2}?wait_for_completion=true'. format(esserver, index_to_snapshot, epoch), data=json.dumps(snapshot_config)) if r.json().has_key('status'): logger.error('Error snapshotting %s: %s' % (index_to_snapshot, r.json())) else: logger.debug('snapshot %s finished' % index_to_snapshot) # create a restore script # referencing the latest snapshot localpath = '%s/%s-restore.sh' % (expanduser("~"), index) with open(localpath, 'w') as f: logger.debug('Writing %s' % localpath) f.write(""" #!/bin/bash echo -n "Restoring the snapshot..." curl -s -XPOST "%s/_snapshot/s3backup/%s-%s/_restore?wait_for_completion=true" echo "DONE!" """ % (esserver, index_to_snapshot, epoch)) # upload the restore script bucket = s3.get_bucket('mozdefes2backups') key = bucket.new_key( 'elasticsearch/%s/%s/%s-%s-%s-restore.sh' % (bucketdate, hostname, index, idate, epoch)) key.set_contents_from_filename(localpath) # removing local file os.remove(localpath) except boto.exception.NoAuthHandlerFound: logger.error("No auth handler found, check your credentials") except Exception as e: logger.error("Unhandled exception, terminating: %r" % e)