def test_s3_client(self):
s3 = boto.s3.connect_to_region("us-east-1")
s3.get_all_buckets()
spans = self.memory_exporter.get_finished_spans()
assert spans
self.assertEqual(len(spans), 1)
span = spans[0]
assert_span_http_status_code(span, 200)
self.assertEqual(span.attributes["http.method"], "GET")
self.assertEqual(span.attributes["aws.operation"], "get_all_buckets")
# Create a bucket command
s3.create_bucket("cheese")
spans = self.memory_exporter.get_finished_spans()
assert spans
self.assertEqual(len(spans), 2)
span = spans[1]
assert_span_http_status_code(span, 200)
self.assertEqual(span.attributes["http.method"], "PUT")
self.assertEqual(span.attributes["path"], "/")
self.assertEqual(span.attributes["aws.operation"], "create_bucket")
# Get the created bucket
s3.get_bucket("cheese")
spans = self.memory_exporter.get_finished_spans()
assert spans
self.assertEqual(len(spans), 3)
span = spans[2]
assert_span_http_status_code(span, 200)
self.assertEqual(
span.resource,
Resource(attributes={
"endpoint": "s3",
"http_method": "head"
}),
)
self.assertEqual(span.attributes["http.method"], "HEAD")
self.assertEqual(span.attributes["aws.operation"], "head_bucket")
self.assertEqual(span.name, "s3.command")
# Checking for resource incase of error
try:
s3.get_bucket("big_bucket")
except Exception: # pylint: disable=broad-except
spans = self.memory_exporter.get_finished_spans()
assert spans
span = spans[2]
self.assertEqual(
span.resource,
Resource(attributes={
"endpoint": "s3",
"http_method": "head"
}),
)
def test_s3_client(self):
s3 = boto.s3.connect_to_region('us-east-1')
writer = self.tracer.writer
Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(s3)
s3.get_all_buckets()
spans = writer.pop()
assert spans
self.assertEqual(len(spans), 1)
span = spans[0]
assert_is_measured(span)
assert_span_http_status_code(span, 200)
self.assertEqual(span.get_tag(http.METHOD), 'GET')
self.assertEqual(span.get_tag('aws.operation'), 'get_all_buckets')
# Create a bucket command
s3.create_bucket('cheese')
spans = writer.pop()
assert spans
self.assertEqual(len(spans), 1)
span = spans[0]
assert_is_measured(span)
assert_span_http_status_code(span, 200)
self.assertEqual(span.get_tag(http.METHOD), 'PUT')
self.assertEqual(span.get_tag('path'), '/')
self.assertEqual(span.get_tag('aws.operation'), 'create_bucket')
# Get the created bucket
s3.get_bucket('cheese')
spans = writer.pop()
assert spans
self.assertEqual(len(spans), 1)
span = spans[0]
assert_is_measured(span)
assert_span_http_status_code(span, 200)
self.assertEqual(span.get_tag(http.METHOD), 'HEAD')
self.assertEqual(span.get_tag('aws.operation'), 'head_bucket')
self.assertEqual(span.service, 'test-boto-tracing.s3')
self.assertEqual(span.resource, 's3.head')
self.assertEqual(span.name, 's3.command')
# Checking for resource incase of error
try:
s3.get_bucket('big_bucket')
except Exception:
spans = writer.pop()
assert spans
span = spans[0]
self.assertEqual(span.resource, 's3.head')
def test_s3_client(self):
s3 = boto.s3.connect_to_region("us-east-1")
Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(s3)
s3.get_all_buckets()
spans = self.pop_spans()
assert spans
self.assertEqual(len(spans), 1)
span = spans[0]
assert_is_measured(span)
assert_span_http_status_code(span, 200)
self.assertEqual(span.get_tag(http.METHOD), "GET")
self.assertEqual(span.get_tag("aws.operation"), "get_all_buckets")
# Create a bucket command
s3.create_bucket("cheese")
spans = self.pop_spans()
assert spans
self.assertEqual(len(spans), 1)
span = spans[0]
assert_is_measured(span)
assert_span_http_status_code(span, 200)
self.assertEqual(span.get_tag(http.METHOD), "PUT")
self.assertEqual(span.get_tag("path"), "/")
self.assertEqual(span.get_tag("aws.operation"), "create_bucket")
# Get the created bucket
s3.get_bucket("cheese")
spans = self.pop_spans()
assert spans
self.assertEqual(len(spans), 1)
span = spans[0]
assert_is_measured(span)
assert_span_http_status_code(span, 200)
self.assertEqual(span.get_tag(http.METHOD), "HEAD")
self.assertEqual(span.get_tag("aws.operation"), "head_bucket")
self.assertEqual(span.service, "test-boto-tracing.s3")
self.assertEqual(span.resource, "s3.head")
self.assertEqual(span.name, "s3.command")
# Checking for resource incase of error
try:
s3.get_bucket("big_bucket")
except Exception:
spans = self.pop_spans()
assert spans
span = spans[0]
self.assertEqual(span.resource, "s3.head")
def upload_file(s3, bucketname, file_path):
b = s3.get_bucket(bucketname)
filename = os.path.basename(file_path)
k = b.new_key(filename)
mp = b.initiate_multipart_upload(filename)
source_size = os.stat(file_path).st_size
bytes_per_chunk = 5000*1024*1024
chunks_count = int(math.ceil(source_size / float(bytes_per_chunk)))
for i in range(chunks_count):
offset = i * bytes_per_chunk
remaining_bytes = source_size - offset
bytes = min([bytes_per_chunk, remaining_bytes])
part_num = i + 1
print "uploading part " + str(part_num) + " of " + str(chunks_count)
with open(file_path, 'r') as fp:
fp.seek(offset)
mp.upload_part_from_file(fp=fp, part_num=part_num, size=bytes)
if len(mp.get_all_parts()) == chunks_count:
mp.complete_upload()
print "upload_file done"
else:
mp.cancel_upload()
print "upload_file failed"
def getKeys():
bkt = s3.get_bucket(parm['s3_bucket'])
keys = bkt.get_all_keys()
li = sorted([(k.last_modified, k.name) for k in keys])
for x in li:
print x
return keys
def UploadServiceFilesToS3(s3_bucket, s3_path, files, verbose=False):
"""Upload all service files for build into separate s3_bucket/s3_path/.
If verbose is True, print status updates while working."""
s3 = boto.connect_s3()
s3 = boto.s3.connection.S3Connection(
calling_format=boto.s3.connection.
ProtocolIndependentOrdinaryCallingFormat())
bucket = s3.get_bucket(s3_bucket)
for source_file in files:
source_file = os.path.abspath(source_file)
if not os.path.isfile(source_file):
raise IOError("File not found: %s" % source_file)
if re.search('(\w+)\.(mar|dmg|rpm|deb|exe|tar\.bz2)$', source_file):
continue
dest_file = os.path.basename(source_file)
full_key_name = '/' + s3_path + '/' + dest_file
bucket_key = boto.s3.key.Key(bucket)
bucket_key.key = full_key_name
if verbose:
print "Uploading service file " + source_file
bucket_key.set_contents_from_filename(source_file)
if verbose:
print "Upload service files complete"
def process(s3, s3BucketName, s3InputPrefix, s3OutputPrefix, fileName,
workDir):
s3Bucket = s3.get_bucket(s3BucketName)
localInputPath = os.path.join(workDir, fileName)
localOutputPath = os.path.join(workDir, fileName[:-5] + '.txt')
remoteInputPath = s3InputPrefix + '/' + fileName
remoteOutputPath = s3OutputPrefix + '/' + fileName[:-5] + '.txt'
if not os.path.isdir(workDir):
os.system('sudo mkdir work && sudo chmod 777 work')
print("Downloading %s from s3://%s/%s ..." %
(localInputPath, s3BucketName, remoteInputPath))
#print(remoteInputPath)
key = s3Bucket.get_key(remoteInputPath)
s3 = boto3.client('s3')
s3.download_file(s3BucketName, remoteInputPath, localInputPath)
key.get_contents_to_filename(workDir + "/" + fileName)
#subprocess.call(['./darknet','detector','demo','cfg/coco.data','cfg/yolov3-tiny.cfg','yolov3-tiny.weights', localInputPath,'outpt.txt'])
os.system(
'Xvfb :1 & export DISPLAY=:1 && ./darknet detector demo cfg/coco.data cfg/yolov3-tiny.cfg yolov3-tiny.weights '
+ localInputPath + ' > ' + localOutputPath)
parseOutput(localOutputPath)
print("Uploading %s to s3://%s/%s ..." %
(localOutputPath, s3BucketName, remoteOutputPath))
key = Key(s3Bucket)
key.key = remoteOutputPath
key.set_contents_from_filename(localOutputPath)
return True
def _extractKeyInfoFromUrl(url, existing=None):
"""
Extracts bucket and key from URL. The existing parameter determines if a
particular state of existence should be enforced. Note also that if existing
is not True and the key does not exist.
:param existing: determines what the state
:return: (bucket, key)
"""
s3 = boto.connect_s3()
bucket = s3.get_bucket(url.netloc)
key = bucket.get_key(url.path[1:])
if existing is True:
if key is None:
raise RuntimeError('Key does not exist.')
elif existing is False:
if key is not None:
raise RuntimeError('Key exists.')
elif existing is None:
pass
else:
assert False
if key is None:
key = bucket.new_key(url.path[1:])
return bucket, key
def test_bad_algorithms(self):
kms = boto.kms.connect_to_region(
"us-west-2", aws_access_key_id=access_key,
aws_secret_access_key=secret_key, is_secure=False,
port=self.s3_server.port, host="127.0.0.1")
kms.auth_region_name = 'us-west-2'
kms.auth_service_name = 'kms'
s3 = boto.s3.connect_to_region(
"us-west-2", aws_access_key_id=access_key,
aws_secret_access_key=secret_key, is_secure=False,
port=self.s3_server.port, host="127.0.0.1",
calling_format=OrdinaryCallingFormat())
enc = S3ClientEncryptionHandler(kms, kms_key_id)
bucket = s3.get_bucket("hello")
key = bucket.new_key("bad_algorithms")
try:
enc.write(key, "Hello world!", wrapper_algorithm="foo")
self.fail("Expected EncryptionError")
except EncryptionError:
pass
try:
enc.write(key, "Hello world!", encryption_algorithm="foo")
self.fail("Expected EncryptionError")
except EncryptionError:
pass
def test_s3_put(self):
s3 = boto.s3.connect_to_region('us-east-1')
Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(s3)
s3.create_bucket('mybucket')
bucket = s3.get_bucket('mybucket')
k = boto.s3.key.Key(bucket)
k.key = 'foo'
k.set_contents_from_string('bar')
spans = self.pop_spans()
assert spans
# create bucket
self.assertEqual(len(spans), 3)
self.assertEqual(spans[0].get_tag('aws.operation'), 'create_bucket')
assert_is_measured(spans[0])
assert_span_http_status_code(spans[0], 200)
self.assertEqual(spans[0].service, 'test-boto-tracing.s3')
self.assertEqual(spans[0].resource, 's3.put')
# get bucket
assert_is_measured(spans[1])
self.assertEqual(spans[1].get_tag('aws.operation'), 'head_bucket')
self.assertEqual(spans[1].resource, 's3.head')
# put object
assert_is_measured(spans[2])
self.assertEqual(spans[2].get_tag('aws.operation'),
'_send_file_internal')
self.assertEqual(spans[2].resource, 's3.put')
def UploadFilesToS3(s3_bucket, s3_path, files, verbose=False):
"""Upload only mar file(s) from the list to s3_bucket/s3_path/.
If verbose is True, print status updates while working."""
s3 = boto.connect_s3()
s3 = boto.s3.connection.S3Connection(calling_format=boto.s3.connection.ProtocolIndependentOrdinaryCallingFormat())
bucket = s3.get_bucket(s3_bucket)
try:
for source_file in files:
source_file = os.path.abspath(source_file)
if not os.path.isfile(source_file):
raise IOError("File not found: %s" % source_file)
if not re.search('(\w+)\.(mar|dmg|exe|tar\.bz2)$', source_file):
continue
dest_file = os.path.basename(source_file)
full_key_name = '/'+s3_path+'/'+dest_file
bucket_key = boto.s3.key.Key(bucket)
bucket_key.key = full_key_name
if verbose:
print "Uploading " + source_file
bucket_key.set_contents_from_filename(source_file)
finally:
pass
if verbose:
print "Upload complete"
def prepare_custom_script(instance_id):
print '------------------------------------'
print 'Preparing Vulnpryer Custom Script'
print '------------------------------------'
f1 = open('custom_scripts/start_vulnpryer.py', 'r')
f2 = open('temp/start_vulnpryer.py', 'w')
for line in f1:
line = re.sub(r"^.*--opsworks_region.*$", "parser.add_argument('-r', '--opsworks_region', type=str, default=\"" + config.get('general', 'opsworks_aws_region') + "\",", line)
line = re.sub(r"^.*--instance_id.*$", "parser.add_argument('-i', '--instance_id', type=str,\n\t\t\tdefault=\"" + instance_id + "\",", line)
line = re.sub(r"^.*--vulnpryer_pipeline_metric_nspace.*$", "parser.add_argument('-vn', '--vulnpryer_pipeline_metric_nspace', default=\'" + config.get('cloudwatch', 'vulnpryer_pipeline_metric_namespace') + "\',", line)
line = re.sub(r"^.*--vulnpryer_pipeline_metric_name.*$", "parser.add_argument('-vm', '--vulnpryer_pipeline_metric_name', default=\'" + config.get('cloudwatch', 'vulnpryer_pipeline_metric_name') + "\',", line)
f2.write(line)
f1.close()
f2.close()
print 'Plugged in region ' + config.get('general', 'opsworks_aws_region') + ' and opsworks instance id ' + instance_id
# Connect AWS S3
try:
s3 = boto.s3.connect_to_region(region_name=config.get('general', 's3_aws_region'), aws_access_key_id=config.get('general', 'aws_access_key_id'), aws_secret_access_key=config.get('general', 'aws_secret_access_key'))
bucket = s3.get_bucket(config.get('custom_script', 's3_bucket'))
except:
print "Check keys and configuration before proceeding."
return False
k = bucket.get_key(config.get('custom_script', 's3_bucket_directory') + 'start_vulnpryer.py')
if k is None:
k = bucket.new_key(config.get('custom_script', 's3_bucket_directory') + 'start_vulnpryer.py')
k.set_contents_from_filename('temp/start_vulnpryer.py')
print 'Uploaded script ' + str(k)
def store_graph(instance_id, filename):
"""
Transmits the specified graph file to internal object store on cloud
controller.
"""
# TODO(devcamcar): Need to use an asynchronous method to make this
# connection. If boto has some separate method that generates
# the request it would like to make and another method to parse
# the response we can make our own client that does the actual
# request and hands it off to the response parser.
s3 = boto.s3.connection.S3Connection(
aws_access_key_id=FLAGS.aws_access_key_id,
aws_secret_access_key=FLAGS.aws_secret_access_key,
is_secure=False,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
port=FLAGS.s3_port,
host=FLAGS.s3_host)
bucket_name = '_%s.monitor' % instance_id
# Object store isn't creating the bucket like it should currently
# when it is first requested, so have to catch and create manually.
try:
bucket = s3.get_bucket(bucket_name)
except Exception:
bucket = s3.create_bucket(bucket_name)
key = boto.s3.Key(bucket)
key.key = os.path.basename(filename)
key.set_contents_from_filename(filename)
def delete_folder(bucket_name, folder_name): """ Deletes an S3 folder and all of its contents, recursively """ s3 = boto.connect_s3() bucket = s3.get_bucket(bucket_name) bucketListResultSet = bucket.list(prefix=folder_name) result = bucket.delete_keys([key.name for key in bucketListResultSet]) return result
def test_s3_put(self):
s3 = boto.s3.connect_to_region('us-east-1')
writer = self.tracer.writer
Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(s3)
s3.create_bucket('mybucket')
bucket = s3.get_bucket('mybucket')
k = boto.s3.key.Key(bucket)
k.key = 'foo'
k.set_contents_from_string('bar')
spans = writer.pop()
assert spans
# create bucket
self.assertEqual(len(spans), 3)
self.assertEqual(spans[0].get_tag('aws.operation'), 'create_bucket')
self.assertEqual(spans[0].get_tag(http.STATUS_CODE), '200')
self.assertEqual(spans[0].service, 'test-boto-tracing.s3')
self.assertEqual(spans[0].resource, 's3.put')
# get bucket
self.assertEqual(spans[1].get_tag('aws.operation'), 'head_bucket')
self.assertEqual(spans[1].resource, 's3.head')
# put object
self.assertEqual(spans[2].get_tag('aws.operation'),
'_send_file_internal')
self.assertEqual(spans[2].resource, 's3.put')
def uploadDir(localDir, s3BucketName, s3InputPrefix, s3OutputPrefix,
sqsQueueName, awsRegion):
files = os.listdir(localDir)
s3 = boto.s3.connect_to_region(awsRegion)
s3Bucket = s3.get_bucket(s3BucketName)
sqs = boto.sqs.connect_to_region(awsRegion)
sqsQueue = sqs.lookup(sqsQueueName)
for fileName in files:
localPath = os.path.join(localDir, fileName)
remotePath = s3InputPrefix + fileName
print "Uploading %s to s3://%s/%s ..." % (localPath, s3BucketName,
remotePath)
# Upload to S3
key = Key(s3Bucket)
key.key = remotePath
key.set_contents_from_filename(localPath)
# Send message to SQS
print "Sending message to SQS queue ..."
messageBody = json.dumps(
['process', s3BucketName, s3InputPrefix, s3OutputPrefix, fileName])
m = Message()
m.set_body(messageBody)
sqsQueue.write(m)
print "Done!"
print "All done!"
def store_graph(instance_id, filename):
"""
Transmits the specified graph file to internal object store on cloud
controller.
"""
# TODO(devcamcar): Need to use an asynchronous method to make this
# connection. If boto has some separate method that generates
# the request it would like to make and another method to parse
# the response we can make our own client that does the actual
# request and hands it off to the response parser.
s3 = boto.s3.connection.S3Connection(
aws_access_key_id=FLAGS.aws_access_key_id,
aws_secret_access_key=FLAGS.aws_secret_access_key,
is_secure=False,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
port=FLAGS.s3_port,
host=FLAGS.s3_host)
bucket_name = '_%s.monitor' % instance_id
# Object store isn't creating the bucket like it should currently
# when it is first requested, so have to catch and create manually.
try:
bucket = s3.get_bucket(bucket_name)
except Exception:
bucket = s3.create_bucket(bucket_name)
key = boto.s3.Key(bucket)
key.key = os.path.basename(filename)
key.set_contents_from_filename(filename)
def test_s3_put(self):
s3 = boto.s3.connect_to_region("us-east-1")
s3.create_bucket("mybucket")
bucket = s3.get_bucket("mybucket")
key = boto.s3.key.Key(bucket)
key.key = "foo"
key.set_contents_from_string("bar")
spans = self.memory_exporter.get_finished_spans()
assert spans
# create bucket
self.assertEqual(len(spans), 3)
self.assertEqual(spans[0].attributes["aws.operation"], "create_bucket")
assert_span_http_status_code(spans[0], 200)
self.assertEqual(spans[0].attributes["endpoint"], "s3")
self.assertEqual(spans[0].attributes["http_method"], "put")
# get bucket
self.assertEqual(spans[1].attributes["aws.operation"], "head_bucket")
self.assertEqual(spans[1].attributes["endpoint"], "s3")
self.assertEqual(spans[1].attributes["http_method"], "head")
# put object
self.assertEqual(
spans[2].attributes["aws.operation"], "_send_file_internal"
)
self.assertEqual(spans[2].attributes["endpoint"], "s3")
self.assertEqual(spans[2].attributes["http_method"], "put")
def _extractKeyInfoFromUrl(url, existing=None):
"""
Extracts bucket and key from URL. The existing parameter determines if a
particular state of existence should be enforced. Note also that if existing
is not True and the key does not exist.
:param existing: determines what the state
:return: (bucket, key)
"""
s3 = boto.connect_s3()
bucket = s3.get_bucket(url.netloc)
key = bucket.get_key(url.path[1:])
if existing is True:
if key is None:
raise RuntimeError('Key does not exist.')
elif existing is False:
if key is not None:
raise RuntimeError('Key exists.')
elif existing is None:
pass
else:
assert False
if key is None:
key = bucket.new_key(url.path[1:])
return bucket, key
def main():
parser = optparse.OptionParser('Usage: %prog <args>...')
(opts, args) = parser.parse_args()
if 0 == len(args):
parser.print_help()
return 1
try:
s3 = boto.connect_s3()
for a in args:
if not a.startswith('s3://'):
raise Error('unsupported object path!')
parts = a[5:].split('/')
bucket = s3.get_bucket(parts[0])
key = bucket.get_key('/'.join(parts[1:]) if 1 < len(parts) else '')
_, name = os.path.split(key.name)
key.get_contents_to_filename(name)
src = name + '.gz'
compress_file(name, src)
os.remove(name)
dst = key.name + '.gz'
upload_file(bucket, dst, src)
os.remove(src)
except (Error, Exception), err:
sys.stderr.write('[ERROR] {0}\n'.format(err))
return 1
def test_s3_put(self):
s3 = boto.s3.connect_to_region("us-east-1")
Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(s3)
s3.create_bucket("mybucket")
bucket = s3.get_bucket("mybucket")
k = boto.s3.key.Key(bucket)
k.key = "foo"
k.set_contents_from_string("bar")
spans = self.pop_spans()
assert spans
# create bucket
self.assertEqual(len(spans), 3)
self.assertEqual(spans[0].get_tag("aws.operation"), "create_bucket")
assert_is_measured(spans[0])
assert_span_http_status_code(spans[0], 200)
self.assertEqual(spans[0].service, "test-boto-tracing.s3")
self.assertEqual(spans[0].resource, "s3.put")
# get bucket
assert_is_measured(spans[1])
self.assertEqual(spans[1].get_tag("aws.operation"), "head_bucket")
self.assertEqual(spans[1].resource, "s3.head")
# put object
assert_is_measured(spans[2])
self.assertEqual(spans[2].get_tag("aws.operation"),
"_send_file_internal")
self.assertEqual(spans[2].resource, "s3.put")
def test_s3_client(self):
s3 = boto.s3.connect_to_region("us-east-1")
tracer = get_dummy_tracer()
writer = tracer.writer
Pin(service=self.TEST_SERVICE, tracer=tracer).onto(s3)
s3.get_all_buckets()
spans = writer.pop()
assert spans
eq_(len(spans), 1)
span = spans[0]
eq_(span.get_tag(http.STATUS_CODE), "200")
eq_(span.get_tag(http.METHOD), "GET")
eq_(span.get_tag('aws.operation'), "get_all_buckets")
# Create a bucket command
s3.create_bucket("cheese")
spans = writer.pop()
assert spans
eq_(len(spans), 1)
span = spans[0]
eq_(span.get_tag(http.STATUS_CODE), "200")
eq_(span.get_tag(http.METHOD), "PUT")
eq_(span.get_tag('path'), '/')
eq_(span.get_tag('aws.operation'), "create_bucket")
# Get the created bucket
s3.get_bucket("cheese")
spans = writer.pop()
assert spans
eq_(len(spans), 1)
span = spans[0]
eq_(span.get_tag(http.STATUS_CODE), "200")
eq_(span.get_tag(http.METHOD), "HEAD")
eq_(span.get_tag('aws.operation'), "head_bucket")
eq_(span.service, "test-boto-tracing.s3")
eq_(span.resource, "s3.head")
eq_(span.name, "s3.command")
# Checking for resource incase of error
try:
s3.get_bucket("big_bucket")
except Exception:
spans = writer.pop()
assert spans
span = spans[0]
eq_(span.resource, "s3.head")
def test_s3_client(self):
s3 = boto.s3.connect_to_region("us-east-1")
tracer = get_dummy_tracer()
writer = tracer.writer
Pin(service=self.TEST_SERVICE, tracer=tracer).onto(s3)
s3.get_all_buckets()
spans = writer.pop()
assert spans
eq_(len(spans), 1)
span = spans[0]
eq_(span.get_tag(http.STATUS_CODE), "200")
eq_(span.get_tag(http.METHOD), "GET")
eq_(span.get_tag('aws.operation'), "get_all_buckets")
# Create a bucket command
s3.create_bucket("cheese")
spans = writer.pop()
assert spans
eq_(len(spans), 1)
span = spans[0]
eq_(span.get_tag(http.STATUS_CODE), "200")
eq_(span.get_tag(http.METHOD), "PUT")
eq_(span.get_tag('path'), '/')
eq_(span.get_tag('aws.operation'), "create_bucket")
# Get the created bucket
s3.get_bucket("cheese")
spans = writer.pop()
assert spans
eq_(len(spans), 1)
span = spans[0]
eq_(span.get_tag(http.STATUS_CODE), "200")
eq_(span.get_tag(http.METHOD), "HEAD")
eq_(span.get_tag('aws.operation'), "head_bucket")
eq_(span.service, "test-boto-tracing.s3")
eq_(span.resource, "s3.head")
eq_(span.name, "s3.command")
# Checking for resource incase of error
try:
s3.get_bucket("big_bucket")
except Exception:
spans = writer.pop()
assert spans
span = spans[0]
eq_(span.resource, "s3.head")
def from_file(path, bucket_name):
bucket = s3.get_bucket(bucket_name, validate=False)
output = S3Key(bucket)
output.key = path
try:
return output.get_contents_as_string().decode('utf-8')
except Exception:
return None
def __init__( self, bucket_name, key_name ):
super( Operation, self ).__init__( )
self.bucket_name = bucket_name
self.key_name = key_name
work_around_dots_in_bucket_names( )
with closing( S3Connection( ) ) as s3:
bucket = s3.get_bucket( self.bucket_name )
self.bucket_location = bucket.get_location( )
def upload_file_s3(bucket_name, folder, path_to_file):
s3 = boto.connect_s3()
bucket = s3.get_bucket(bucket_name)
k = Key(bucket)
filename = folder + "/" + os.path.split(path_to_file)[-1]
k.key = filename
k.set_contents_from_filename(path_to_file)
return filename
def to_file(string, path, bucket_name, mime):
if environment.local:
print('[file] {}'.format(path))
else:
bucket = s3.get_bucket(bucket_name, validate=False)
output = S3Key(bucket)
output.key = path
output.content_type = mime
output.set_contents_from_string(string)
def process(s3, s3BucketName, s3InputPrefix, s3OutputPrefix, fileName,
workDir):
print(s3BucketName + s3InputPrefix)
s3BucketInput = s3.get_bucket(s3BucketName + "-" + s3InputPrefix)
s3BucketOutput = s3.get_bucket(s3BucketName + "-" + s3OutputPrefix)
localInputPath = os.path.join(workDir, fileName)
localOutputPath = os.path.join(workDir, fileName[:-4] + '.txt')
remoteInputPath = fileName
remoteOutputPath = fileName[:-4] + '.txt'
if not os.path.isdir(workDir):
os.system('sudo mkdir work && sudo chmod 777 work')
print("Downloading %s from s3://%s/%s ..." %
(localInputPath, s3BucketName, remoteInputPath))
#print(remoteInputPath)
key = s3BucketInput.get_key(remoteInputPath)
print("here1")
s3 = boto3.client('s3')
print("here2")
print("s3BucketInput")
print(s3BucketInput)
print("remoteInputPath")
print(remoteInputPath)
print("localInputPath")
print(localInputPath)
s3.download_file(s3BucketName + "-" + "inputfolder", remoteInputPath,
localInputPath)
print("here3")
key.get_contents_to_filename(workDir + "/" + fileName)
#subprocess.call(['./darknet','detector','demo','cfg/coco.data','cfg/yolov3-tiny.cfg','yolov3-tiny.weights', localInputPath,'outpt.txt'])
print("hi1")
#ans = os.system('python3 classifier/image_classification.py '+localInputPath)
os.system('python3 classifier/image_classification.py ' + localInputPath +
' > ' + localOutputPath)
print("hi2")
#print(ans)
print("hi2")
print(localOutputPath)
print("Uploading %s to s3://%s/%s ..." %
(localOutputPath, s3BucketInput, remoteOutputPath))
key = Key(s3BucketOutput)
key.key = remoteOutputPath
key.set_contents_from_filename(localOutputPath)
return True
def main(argv):
bucket = s3.get_bucket('flaskbucket')
isValid = True
while isValid:
options = {1: upload, 2: download, 3: exit}
option = input("Enter your option 1. Upload 2.Download 3. exit")
if option == "3":
print "Exiting..\n"
isValid = None
else:
options[option](bucket)
def create_key(region, storage_prefix, kms_key_id):
kms = boto.kms.connect_to_region(region)
s3 = boto.s3.connect_to_region(region)
ceh = S3ClientEncryptionHandler(kms, kms_key_id)
bucket_name, prefix = storage_prefix[5:].split("/", 1)
s3_object = prefix + "rolemaker.key"
bucket = s3.get_bucket(bucket_name)
key = bucket.new_key(s3_object)
data = urandom(64)
ceh.write(key, data)
return 0
def _getKeyForUrl(url, existing=None):
"""
Extracts a key from a given s3:// URL. On return, but not on exceptions, this method
leaks an S3Connection object. The caller is responsible to close that by calling
key.bucket.connection.close().
:param bool existing: If True, key is expected to exist. If False, key is expected not to
exists and it will be created. If None, the key will be created if it doesn't exist.
:rtype: Key
"""
# Get the bucket's region to avoid a redirect per request
with closing(boto.connect_s3()) as s3:
region = bucket_location_to_region(s3.get_bucket(url.netloc).get_location())
# Note that caller is responsible for closing the connection
s3 = boto.s3.connect_to_region(region)
try:
bucket = s3.get_bucket(url.netloc)
key = bucket.get_key(url.path[1:])
if existing is True:
if key is None:
raise RuntimeError('Key does not exist.')
elif existing is False:
if key is not None:
raise RuntimeError('Key exists.')
elif existing is None:
pass
else:
assert False
if key is None:
key = bucket.new_key(url.path[1:])
except:
with panic():
s3.close()
else:
return key
def _upload_part( self, upload_id, part_num, buf ):
"""
Upload a part to S3. The given buffer's tell() is assumed to point at the first byte to
be uploaded. When this method returns, the tell() method points at the end of the buffer.
:return: the part object representing the uploaded part
:rtype: Part
"""
with closing( boto.s3.connect_to_region( self.bucket_location ) ) as s3:
bucket = s3.get_bucket( self.bucket_name )
headers = { }
self.__add_encryption_headers( headers )
upload = self._get_upload( bucket, upload_id )
key = upload.upload_part_from_file( buf, part_num + 1, headers=headers )
return self._part_for_key( bucket, part_num, key )
def _copyKey(self, srcKey, dstBucketName, dstKeyName, headers=None):
headers = headers or {}
s3 = boto.connect_s3()
if srcKey.size > self.outer.partSize:
return copyKeyMultipart(srcKey=srcKey,
dstBucketName=dstBucketName,
dstKeyName=dstKeyName,
headers=headers)
else:
dstBucket = s3.get_bucket(dstBucketName)
return dstBucket.copy_key(new_key_name=dstKeyName,
src_bucket_name=srcKey.bucket.name,
src_key_name=srcKey.name,
metadata=srcKey.metadata,
headers=headers)
def _copyKey(self, srcKey, dstBucketName, dstKeyName, headers=None):
headers = headers or {}
s3 = boto.connect_s3()
if srcKey.size > self.outer.partSize:
return copyKeyMultipart(srcKey=srcKey,
dstBucketName=dstBucketName,
dstKeyName=dstKeyName,
headers=headers)
else:
dstBucket = s3.get_bucket(dstBucketName)
return dstBucket.copy_key(new_key_name=dstKeyName,
src_bucket_name=srcKey.bucket.name,
src_key_name=srcKey.name,
metadata=srcKey.metadata,
headers=headers)
def _copy_part( self, upload_id, part_num, src_bucket_name, src_key_name, size ):
with self._propagate_worker_exception( ):
try:
if error_event.is_set( ): raise BailoutException( )
log.info( 'part %i: copying', part_num )
start, end = self._get_part_range( part_num )
end = min( end, size )
part_size = end - start
if part_size > 0 or part_num == 0:
url = urlparse( self.url )
assert url.scheme == 's3'
assert url.path.startswith( '/' )
with closing( boto.s3.connect_to_region( self.bucket_location ) ) as s3:
bucket = s3.get_bucket( self.bucket_name )
upload = self._get_upload( bucket, upload_id )
headers = { }
if self.sse_key:
self._add_encryption_headers( self.sse_key, headers )
if part_size == 0:
# Since copy_part_from_key doesn't allow empty ranges, we handle that
# case by uploading an empty part.
assert part_num == 0
# noinspection PyTypeChecker
key = upload.upload_part_from_file(
StringIO( ), part_num + 1,
headers=headers )
else:
if self.src_sse_key:
self._add_encryption_headers( self.src_sse_key, headers,
for_copy=True )
key = upload.copy_part_from_key(
src_bucket_name=src_bucket_name,
src_key_name=src_key_name,
part_num=part_num + 1,
start=start,
end=end - 1,
headers=headers )
# somehow copy_part_from_key doesn't set the key size
key.size = part_size
assert key.size == part_size
self._log_progress( part_num, 'copied', part_size )
return part_num, self._part_for_key( bucket, part_num, key )
else:
done_event.set( )
return part_num, None
finally:
download_slots_semaphore.release( )
def UploadFilesToS3(s3_bucket, s3_path, files, package, verbose=False):
"""Upload only mar file(s) from the list to s3_bucket/s3_path/.
If verbose is True, print status updates while working."""
s3 = boto.connect_s3()
s3 = boto.s3.connection.S3Connection(
calling_format=boto.s3.connection.
ProtocolIndependentOrdinaryCallingFormat())
bucket = s3.get_bucket(s3_bucket)
properties = {}
property_conditions = [
('completeMarUrl', lambda m: m.endswith('.complete.mar')),
('partialMarUrl', lambda m: m.endswith('.mar') and '.partial.' in m),
('packageUrl', lambda m: m.endswith(package)),
]
for source_file in files:
source_file = os.path.abspath(source_file)
if not os.path.isfile(source_file):
raise IOError("File not found: %s" % source_file)
if not re.search('(\w+)\.(mar|dmg|rpm|deb|exe|tar\.bz2)$',
source_file):
continue
dest_file = os.path.basename(source_file)
full_key_name = '/' + s3_path + '/' + dest_file
bucket_key = boto.s3.key.Key(bucket)
bucket_key.key = full_key_name
if verbose:
print "Uploading " + source_file
bucket_key.set_contents_from_filename(source_file)
m = 'https://' + s3_bucket + full_key_name
for prop, condition in property_conditions:
if condition(m):
properties.update({prop: m})
break
if verbose:
print "Upload complete"
return properties
def read_volume_info(self):
"""
Read the volname.volinfo file in the S3 bucket.
"""
with self.s3_pool.get_connection() as s3:
bucket = s3.get_bucket(self.bucket_name)
key = bucket.get_key(self.volname + ".volinfo")
config = json_loads(key.get_contents_as_string())
self.block_size = int(config.get("block-size", 4096))
self.encryption = config.get("encryption")
self.policy = config.get("policy", "private")
self.size = int(config.get("size"))
self.storage_class = config.get("storage-class", "standard")
self.suffix = config.get("suffix", "." + self.volname)
return
def _prepare_upload( self ):
"""
Prepare a new multipart upload or resume a previously interrupted one. Returns the upload ID
and a dictionary mapping the 0-based index of a part to its size.
"""
completed_parts = { }
with closing( boto.s3.connect_to_region( self.bucket_location ) ) as s3:
bucket = s3.get_bucket( self.bucket_name )
uploads = self._get_uploads( bucket )
if len( uploads ) == 0:
if self.resume:
raise UserError( "Transfer failed. There is no pending upload to be resumed." )
else:
headers = { }
self.__add_encryption_headers( headers )
upload_id = bucket.initiate_multipart_upload( key_name=self.key_name,
headers=headers ).id
elif len( uploads ) == 1:
if self.resume:
upload = uploads[ 0 ]
upload_id = upload.id
for part in upload:
completed_parts[ part.part_number - 1 ] = part
# If there is an upload but no parts we can use whatever part size we want,
# otherwise we need to ensure that we use the same part size as before.
if len( completed_parts ) > 0:
previous_part_size = self._guess_part_size( completed_parts )
if self.part_size != previous_part_size:
raise UserError(
"Transfer failed. The part size appears to have changed from %i "
"to %i. Either resume the upload with the old part size or cancel "
"the upload and restart with the new part size. "
% (self.part_size, previous_part_size) )
else:
raise UserError(
"Transfer failed. There is a pending upload. If you would like to resume "
"that upload, run {me} again with --resume. If you would like to cancel "
"the upload, use '{me} cancel {bucket_name} {key_name}'. Note that "
"pending uploads incur storage fees.".format( me=me, **vars( self ) ) )
else:
raise RuntimeError(
"Transfer failed. Detected more than one pending multipart upload. Consider "
"using '{me} cancel {bucket_name} {key_name}' to delete all of them before "
"trying the transfer again. Note that pending uploads incur storage "
"fees.".format( me=me, **vars( self ) ) )
return upload_id, completed_parts
def copyKeyMultipart(srcKey, dstBucketName, dstKeyName, headers=None):
"""
Copies a key from a source key to a destination key in multiple parts. Note that if the
destination key exists it will be overwritten implicitly, and if it does not exist a new
key will be created.
:param boto.s3.key.Key srcKey: The source key to be copied from.
:param str dstBucketName: The name of the destination bucket for the copy.
:param str dstKeyName: The name of the destination key that will be created or overwritten.
:param dict headers: Any headers that should be passed.
:rtype: boto.s3.multipart.CompletedMultiPartUpload
:return: An object representing the completed upload.
"""
partSize = defaultPartSize
# We need a location-agnostic connection to S3 so we can't use the one that we
# normally use for interacting with the job store bucket.
with closing(boto.connect_s3()) as s3:
headers = headers or {}
totalSize = srcKey.size
for attempt in retry_s3():
with attempt:
dstBucket = s3.get_bucket(dstBucketName)
upload = dstBucket.initiate_multipart_upload(dstKeyName, headers=headers)
try:
start = 0
partIndex = itertools.count()
while start < totalSize:
end = min(start + partSize, totalSize)
for attempt in retry_s3():
with attempt:
upload.copy_part_from_key(src_bucket_name=srcKey.bucket.name,
src_key_name=srcKey.name,
src_version_id=srcKey.version_id,
part_num=next(partIndex) + 1,
start=start,
end=end - 1,
headers=headers)
start += partSize
except:
with panic(log=log):
upload.cancel_upload()
else:
for attempt in retry_s3():
with attempt:
return upload.complete_upload()
def UploadFilesToS3(s3_bucket, s3_path, files, package, verbose=False):
"""Upload only mar file(s) from the list to s3_bucket/s3_path/.
If verbose is True, print status updates while working."""
s3 = boto.connect_s3()
s3 = boto.s3.connection.S3Connection(calling_format=boto.s3.connection.ProtocolIndependentOrdinaryCallingFormat())
bucket = s3.get_bucket(s3_bucket)
properties = {}
property_conditions = [
('completeMarUrl', lambda m: m.endswith('.complete.mar')),
('partialMarUrl', lambda m: m.endswith('.mar') and '.partial.' in m),
('packageUrl', lambda m: m.endswith(package)),
]
for source_file in files:
source_file = os.path.abspath(source_file)
if not os.path.isfile(source_file):
raise IOError("File not found: %s" % source_file)
if not re.search('(\w+)\.(mar|dmg|rpm|deb|exe|tar\.bz2)$', source_file):
continue
dest_file = os.path.basename(source_file)
full_key_name = '/'+s3_path+'/'+dest_file
bucket_key = boto.s3.key.Key(bucket)
bucket_key.key = full_key_name
if verbose:
print "Uploading " + source_file
bucket_key.set_contents_from_filename(source_file)
m = 'http://' + s3_bucket + full_key_name
for prop, condition in property_conditions:
if condition(m):
properties.update({prop: m})
break
if verbose:
print "Upload complete"
return properties
def read_credentials_from_s3(self, s3_url, kms, s3):
"""
read_credentials_from_s3(s3_url, kms, s3) -> dict
Read access-key/secret-key map from the specified S3 URL (in the form
s3://bucket/key).
"""
assert s3_url.startswith("s3://")
try:
bucket_name, key_name = s3_url[5:].split("/", 1)
except:
raise ValueError("Invalid S3 URL: %s" % s3_url)
enc = S3ClientEncryptionHandler(kms)
bucket = s3.get_bucket(bucket_name)
key = bucket.get_key(key_name)
data = enc.read(key)
self.read_credentials_from_stream(cStringIO(data))
def fetch(key, logger=None):
"""download and extract an archive"""
template = "$AICHALLENGE_PREFIX/var/lib/aichallenge/submissions/%s"
path = os.path.expandvars(template % key)
if os.path.isdir(path):
# already decompressed -- move on
return
if logger is not None:
logger.info('downloading ' + key)
access_key = config.get('worker', 'aws_access_key')
secret_key = config.get('worker', 'aws_secret_key')
bucket = config.get('worker', 'submission_bucket')
prefix = config.get('worker', 'submission_prefix')
s3 = boto.s3.Connection(access_key, secret_key)
bucket = s3.get_bucket(bucket)
s3_key = bucket.get_key(prefix + key)
if s3_key is None:
raise KeyError
io = cStringIO.StringIO()
s3_key.get_contents_to_file(io)
try:
io.seek(0)
zip = zipfile.ZipFile(io)
decompress_zip(zip, path)
return
except zipfile.BadZipfile:
pass
try:
io.seek(0)
tar = tarfile.open(fileobj=io)
decompress_tar(tar, path)
return
except tarfile.TarError:
pass
raise ValueError, "invalid archive"
def _copyKey(self, srcKey, dstBucketName, dstKeyName, headers=None):
headers = headers or {}
if srcKey.size > self.outer.partSize:
return copyKeyMultipart(srcKey=srcKey,
dstBucketName=dstBucketName,
dstKeyName=dstKeyName,
headers=headers)
else:
# We need a location-agnostic connection to S3 so we can't use the one that we
# normally use for interacting with the job store bucket.
with closing(boto.connect_s3()) as s3:
for attempt in retry_s3():
with attempt:
dstBucket = s3.get_bucket(dstBucketName)
return dstBucket.copy_key(new_key_name=dstKeyName,
src_bucket_name=srcKey.bucket.name,
src_version_id=srcKey.version_id,
src_key_name=srcKey.name,
metadata=srcKey.metadata,
headers=headers)
def process(s3, s3BucketName, s3InputPrefix, s3OutputPrefix, fileName, workDir, outputExtension, command):
s3Bucket = s3.get_bucket(s3BucketName)
localInputPath = os.path.join(workDir, fileName)
localOutputPath = localInputPath + outputExtension
remoteInputPath = s3InputPrefix + fileName
remoteOutputPath = s3OutputPrefix + fileName + outputExtension
print "Downloading %s from s3://%s/%s ..." % (localInputPath, s3BucketName, remoteInputPath)
key = s3Bucket.get_key(remoteInputPath)
key.get_contents_to_filename(localInputPath)
full_command = [command, localInputPath, localOutputPath]
print "Executing: %s" % ' '.join(full_command)
returncode = subprocess.call(full_command)
if returncode != 0:
print "Return Code not '0'!"
return False
print "Uploading %s to s3://%s/%s ..." % (localOutputPath, s3BucketName, remoteOutputPath)
key = Key(s3Bucket)
key.key = remoteOutputPath
key.set_contents_from_filename(localOutputPath)
return True
def get_instance_graph(self, region, instance_id, graph_name):
# TODO(devcamcar): Need better support for multiple regions.
# Need a way to get object store by region.
s3 = boto.s3.connection.S3Connection (
aws_access_key_id=settings.NOVA_ACCESS_KEY,
aws_secret_access_key=settings.NOVA_SECRET_KEY,
is_secure=False,
calling_format=boto.s3.connection.OrdinaryCallingFormat(),
port=3333,
host=settings.NOVA_CLC_IP
)
key = '_%s.monitor' % instance_id
try:
bucket = s3.get_bucket(key, validate=False)
except boto.exception.S3ResponseError, e:
if e.code == "NoSuchBucket":
return None
else:
raise e
def copyKeyMultipart(srcKey, dstBucketName, dstKeyName, headers=None):
"""
Copies a key from a source key to a destination key in multiple parts. Note that if the
destination key exists it will be overwritten implicitly, and if it does not exist a new
key will be created.
:param boto.s3.key.Key srcKey: The source key to be copied from.
:param str dstBucketName: The name of the destination bucket for the copy.
:param str dstKeyName: The name of the destination key that will be created or overwritten.
:param dict headers: Any headers that should be passed.
:rtype: boto.s3.multipart.CompletedMultiPartUpload
:return: An object representing the completed upload.
"""
partSize = defaultPartSize
s3 = boto.connect_s3()
headers = headers or {}
totalSize = srcKey.size
# initiate copy
upload = s3.get_bucket(dstBucketName).initiate_multipart_upload(
dstKeyName, headers=headers)
try:
start = 0
partIndex = itertools.count()
while start < totalSize:
end = min(start + partSize, totalSize)
upload.copy_part_from_key(src_bucket_name=srcKey.bucket.name,
src_key_name=srcKey.name,
part_num=next(partIndex) + 1,
start=start,
end=end - 1,
headers=headers)
start += partSize
except:
upload.cancel_upload()
raise
else:
return upload.complete_upload()
def uploadDir(localDir, s3BucketName, s3InputPrefix, s3OutputPrefix, sqsQueueName, awsRegion):
files = os.listdir(localDir)
s3 = boto.s3.connect_to_region(awsRegion)
s3Bucket = s3.get_bucket(s3BucketName)
sqs = boto.sqs.connect_to_region(awsRegion)
sqsQueue = sqs.lookup(sqsQueueName)
for fileName in files:
localPath = os.path.join(localDir, fileName)
remotePath = s3InputPrefix + fileName
print "Uploading %s to s3://%s/%s ..." % (localPath, s3BucketName, remotePath)
# Upload to S3
key = Key(s3Bucket)
key.key = remotePath
key.set_contents_from_filename(localPath)
# Send message to SQS
print "Sending message to SQS queue ..."
messageBody = json.dumps(['process', s3BucketName, s3InputPrefix, s3OutputPrefix, fileName])
m = Message()
m.set_body(messageBody)
sqsQueue.write(m)
print "Done!"
print "All done!"
def test_python_write_java_read(self):
kms = boto.kms.connect_to_region(
"us-west-2", aws_access_key_id=access_key,
aws_secret_access_key=secret_key, is_secure=False,
port=self.s3_server.port, host="127.0.0.1")
kms.auth_region_name = 'us-west-2'
kms.auth_service_name = 'kms'
s3 = boto.s3.connect_to_region(
"us-west-2", aws_access_key_id=access_key,
aws_secret_access_key=secret_key, is_secure=False,
port=self.s3_server.port, host="127.0.0.1",
calling_format=OrdinaryCallingFormat())
enc = S3ClientEncryptionHandler(kms, kms_key_id)
bucket = s3.get_bucket("hello")
key = bucket.new_key("hello")
enc.write(key, "Hello world!", headers={"content-type": "text/plain"})
result = self.java_encrypted_get("hello")
self.assertEqual(result, "Hello world!")
return
def copyKeyMultipart(srcKey, dstBucketName, dstKeyName, headers=None):
"""
Copies a key from a source key to a destination key in multiple parts. Note that if the
destination key exists it will be overwritten implicitly, and if it does not exist a new
key will be created.
:param boto.s3.key.Key srcKey: The source key to be copied from.
:param str dstBucketName: The name of the destination bucket for the copy.
:param str dstKeyName: The name of the destination key that will be created or overwritten.
:param dict headers: Any headers that should be passed.
:rtype: boto.s3.multipart.CompletedMultiPartUpload
:return: An object representing the completed upload.
"""
partSize = defaultPartSize
s3 = boto.connect_s3()
headers = headers or {}
totalSize = srcKey.size
# initiate copy
upload = s3.get_bucket(dstBucketName).initiate_multipart_upload(dstKeyName, headers=headers)
try:
start = 0
partIndex = itertools.count()
while start < totalSize:
end = min(start + partSize, totalSize)
upload.copy_part_from_key(src_bucket_name=srcKey.bucket.name,
src_key_name=srcKey.name,
part_num=next(partIndex)+1,
start=start,
end=end-1,
headers=headers)
start += partSize
except:
upload.cancel_upload()
raise
else:
return upload.complete_upload()
def process(s3, s3BucketName, s3InputPrefix, s3OutputPrefix, fileName, workDir,
outputExtension, command):
s3Bucket = s3.get_bucket(s3BucketName)
localInputPath = os.path.join(workDir, fileName)
localOutputPath = localInputPath + outputExtension
remoteInputPath = s3InputPrefix + fileName
remoteOutputPath = s3OutputPrefix + fileName + outputExtension
print "Downloading %s from s3://%s/%s ..." % (localInputPath, s3BucketName,
remoteInputPath)
key = s3Bucket.get_key(remoteInputPath)
key.get_contents_to_filename(localInputPath)
full_command = [command, localInputPath, localOutputPath]
print "Executing: %s" % ' '.join(full_command)
returncode = subprocess.call(full_command)
if returncode != 0:
print "Return Code not '0'!"
return False
print "Uploading %s to s3://%s/%s ..." % (localOutputPath, s3BucketName,
remoteOutputPath)
key = Key(s3Bucket)
key.key = remoteOutputPath
key.set_contents_from_filename(localOutputPath)
return True
def get_folders(bucket_name, folder_name):
""" Get a list of all S3 folders with files in them under the specified folder """
s3 = boto.connect_s3()
bucket = s3.get_bucket(bucket_name)
# Get all files that start with the folder name
bucketListResultSet = bucket.list(prefix=folder_name)
keys = [key.name for key in bucketListResultSet]
# Skip the first key, because this key is always just the root path (which doesn't have any files in it)
keys = keys[1:]
# Get the reslts
results = []
for k in keys:
# Trim off the actual filename and just process the path
prefix = k[:k.rindex('/')]
res = 's3n://' + bucket_name + '/' + prefix
# Only add the path to the results if it doesn't already exist in the list
if not res in results:
results.append(res)
return results
def create_volume(self, block_size=4096, encryption="",
policy="private", size=None,
storage_class="standard", suffix=None):
"""
Create the volume in the S3 bucket.
"""
with self.s3_pool.get_connection() as s3:
bucket = s3.get_bucket(self.bucket_name)
key = bucket.new_key(self.volname + ".volinfo")
if size is None:
raise ValueError("size must be specified")
config = {
'block-size': block_size,
'policy': policy,
'size': size,
'storage-class': storage_class
}
if encryption:
config['encryption'] = encryption
if suffix:
config['suffix'] = suffix
key.set_contents_from_string(
json_dumps(config), policy=self.policy)
self.block_size = block_size
self.encryption = encryption
self.policy = policy
self.size = size
self.storage_class = storage_class
self.suffix = suffix
return
args = parser.parse_args()
# process common command line arguments
log = logging.getLogger('botocross')
bc.configure_logging(log, args.log_level)
credentials = bc.parse_credentials(args)
locations = bc.filter_regions_s3(class_iterator(Location), args.region)
# execute business logic
log.info("Uploading to S3 buckets named '" + args.bucket + "':")
file = open(args.file, 'r')
filePath, fileName = os.path.split(file.name)
log.debug(filePath)
log.debug(fileName)
s3 = boto.connect_s3(**credentials)
for location in locations:
region = RegionMap[location]
pprint(region, indent=2)
try:
bucket_name = args.bucket + '-' + region
bucket = s3.get_bucket(bucket_name)
key = Key(bucket)
key.name = fileName
print 'Uploading to bucket ' + bucket_name
key.set_contents_from_filename(args.file)
except boto.exception.BotoServerError, e:
log.error(e.error_message)
def main():
logging.getLogger('boto').setLevel(
logging.CRITICAL) # disable all boto error logging
logger.level = logging.INFO
formatter = logging.Formatter(
'%(asctime)s %(name)s %(levelname)s %(message)s')
if options.output == 'syslog':
logger.addHandler(
SysLogHandler(address=(options.sysloghostname,
options.syslogport)))
else:
sh = logging.StreamHandler(sys.stderr)
sh.setFormatter(formatter)
logger.addHandler(sh)
state = State(options.state_file_name)
logger.debug('started')
#logger.debug(options)
try:
es = pyes.ES((list('{0}'.format(s) for s in options.esservers)))
role_manager = RoleManager(
aws_access_key_id=options.aws_access_key_id,
aws_secret_access_key=options.aws_secret_access_key)
for aws_account in options.aws_accounts:
if aws_account not in state.data:
state.data[aws_account] = {}
assumed_role_arn = get_role_arn(aws_account)
#in case we don't archive files..only look at today and yesterday's files.
search_dates = [
datetime.utcnow() - timedelta(days=1),
datetime.utcnow()
]
for region in options.regions:
logger.info("Checking AWS account %s in region %s" %
(aws_account, region))
#capture the time we start running so next time we catch any files created while we run.
lastrun = toUTC(datetime.now()).isoformat()
if region not in state.data[aws_account]:
state.data[aws_account][region] = {}
logger.debug(
'connecting to AWS account {0} in region {1}'.format(
aws_account, region))
try:
ct_credentials = role_manager.get_credentials(
assumed_role_arn)
ct = boto.cloudtrail.connect_to_region(
region, **ct_credentials)
trails = ct.describe_trails()['trailList']
except Exception as e:
logger.error(
"Unable to connect to cloudtrail %s in order to "
"enumerate CloudTrails in region %s due to %s" %
(assumed_role_arn, region, e.message))
continue
for trail in trails:
bucket_account = get_bucket_account(
trail['S3BucketName'],
role_manager.get_credentials(assumed_role_arn))
bucket_account_assumed_role_arn = get_role_arn(
bucket_account)
if not bucket_account_assumed_role_arn:
logger.error(
"Unable to determine account of S3 bucket %s for account %s in region %s. Skipping bucket"
% (trail['S3BucketName'], aws_account, region))
continue
try:
s3 = boto.connect_s3(**role_manager.get_credentials(
bucket_account_assumed_role_arn))
except boto.exception.NoAuthHandlerFound as e:
logger.error(
"Unable to assume role %s in order to "
"fetch s3 bucket contents in bucket %s due to %s" %
(bucket_account_assumed_role_arn,
trail['S3BucketName'], e.message))
continue
try:
ctbucket = s3.get_bucket(trail['S3BucketName'])
except Exception as e:
logger.error(
"Unable to access bucket %s as %s due to %s" %
(trail['S3BucketName'],
bucket_account_assumed_role_arn, e.message))
continue
filelist = list()
for search_date in search_dates:
# TODO it's possible that if the AWS account id is 11 instead of 12 digits, CloudTrail
# will either 0 pad to 12 digits or remove the 0 padding when creating the s3 bucket
# directory. Depending on this behavior,
# we should either aws_account.lstrip('0') or aws_account.lstrip('0').zfill(12)
prefix = (
'AWSLogs/%(accountid)s/CloudTrail/%(region)s/%(year)s/%(month)s/%(day)s/'
% {
'accountid': aws_account,
'region': region,
'year': date.strftime(search_date, '%Y'),
'month': date.strftime(search_date, '%m'),
'day': date.strftime(search_date, '%d')
})
try:
for bfile in ctbucket.list(prefix):
filelist.append(bfile.key)
except Exception as e:
logger.error(
"Unable to list keys in s3 bucket %s "
"and path %s due to %s" %
(trail['S3BucketName'], prefix, e.message))
continue
for afile in filelist:
try:
s3file = ctbucket.get_key(afile)
except Exception as e:
logger.error(
"Unable to HEAD key %s in s3 bucket %s "
"due to %s" %
(afile.name, trail['S3BucketName'], e.message))
continue
logger.debug('{0} {1}'.format(afile,
s3file.last_modified))
if 'lastrun' not in state.data[aws_account][region]:
if toUTC(s3file.last_modified) > toUTC(
datetime.utcnow() -
timedelta(seconds=3600)):
# If we've never collected from this account/region before, grab the last hour of logs
process_file(s3file, es)
elif toUTC(s3file.last_modified) > toUTC(
state.data[aws_account][region]['lastrun']):
process_file(s3file, es)
state.data[aws_account][region]['lastrun'] = lastrun
state.write_state_file()
except boto.exception.NoAuthHandlerFound as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logger.error("No auth handler found, check your credentials. %s" %
[exc_type, fname, exc_tb.tb_lineno])
except Exception as e:
logger.error("Unhandled exception, terminating: %r" % e)
raise
def main():
if options.output == 'syslog':
logger.addHandler(
SysLogHandler(address=(options.sysloghostname,
options.syslogport)))
else:
sh = logging.StreamHandler(sys.stderr)
sh.setFormatter(formatter)
logger.addHandler(sh)
logger.debug('started')
try:
esserver = options.esservers[0]
s3 = boto.connect_s3(
aws_access_key_id=options.aws_access_key_id,
aws_secret_access_key=options.aws_secret_access_key)
idate = date.strftime(datetime.utcnow() - timedelta(days=1), '%Y%m%d')
bucketdate = date.strftime(datetime.utcnow() - timedelta(days=1),
'%Y-%m')
hostname = socket.gethostname()
# Create or update snapshot configuration
logger.debug('Configuring snapshot repository')
snapshot_config = {
"type": "s3",
"settings": {
"bucket": "mozdefes2backups",
"base_path":
"elasticsearch/{0}/{1}".format(bucketdate, hostname),
"region": "{0}".format(options.aws_region)
}
}
r = requests.put('%s/_snapshot/s3backup' % esserver,
data=json.dumps(snapshot_config))
if r.json().has_key('status'):
logger.error("Error while registering snapshot repo: %s" % r.text)
else:
logger.debug('snapshot repo registered')
# do the actual snapshotting
for (index, dobackup, rotation,
pruning) in zip(options.indices, options.dobackup,
options.rotation, options.pruning):
if dobackup == '1':
index_to_snapshot = index
if rotation == 'daily':
index_to_snapshot += '-%s' % idate
elif rotation == 'monthly':
index_to_snapshot += '-%s' % idate[:6]
logger.debug(
'Creating %s snapshot (this may take a while)...' %
index_to_snapshot)
snapshot_config = {'indices': index_to_snapshot}
epoch = calendar.timegm(datetime.utcnow().utctimetuple())
r = requests.put(
'{0}/_snapshot/s3backup/{1}-{2}?wait_for_completion=true'.
format(esserver, index_to_snapshot, epoch),
data=json.dumps(snapshot_config))
if r.json().has_key('status'):
logger.error('Error snapshotting %s: %s' %
(index_to_snapshot, r.json()))
else:
logger.debug('snapshot %s finished' % index_to_snapshot)
# create a restore script
# referencing the latest snapshot
localpath = '%s/%s-restore.sh' % (expanduser("~"), index)
with open(localpath, 'w') as f:
logger.debug('Writing %s' % localpath)
f.write("""
#!/bin/bash
echo -n "Restoring the snapshot..."
curl -s -XPOST "%s/_snapshot/s3backup/%s-%s/_restore?wait_for_completion=true"
echo "DONE!"
""" % (esserver, index_to_snapshot, epoch))
# upload the restore script
bucket = s3.get_bucket('mozdefes2backups')
key = bucket.new_key(
'elasticsearch/%s/%s/%s-%s-%s-restore.sh' %
(bucketdate, hostname, index, idate, epoch))
key.set_contents_from_filename(localpath)
# removing local file
os.remove(localpath)
except boto.exception.NoAuthHandlerFound:
logger.error("No auth handler found, check your credentials")
except Exception as e:
logger.error("Unhandled exception, terminating: %r" % e)
