def bucket_key_exists(bucket_name, key_name, local):
"""
bucket_key_exists first establishes a connection with S3; from a tron
server creds are stored in the environment, otherwise, get them from
the session_file. Once we have the connection, test if the bucket
exists, and then if the key exists, and return True if both exist
otherwise return False.
Args:
bucket_name -- the name of the bucket
key_name -- the name of a key
run_local -- whether this is run on a dev box (run_local=True) or instance
Returns:
True if the bucket key pair exists, otherwise False
"""
if local:
conn = S3Connection(**get_boto_creds())
else:
conn = S3Connection()
if conn.lookup(bucket_name) is None:
return False
s3_bucket = conn.get_bucket(bucket_name, validate=False)
exists = (s3_bucket.get_key(key_name) is not None)
conn.close()
return exists
def bucket_key_exists(bucket_name, key_name, local):
"""
bucket_key_exists first establishes a connection with S3; from a tron
server creds are stored in the environment, otherwise, get them from
the session_file. Once we have the connection, test if the bucket
exists, and then if the key exists, and return True if both exist
otherwise return False.
Args:
bucket_name -- the name of the bucket
key_name -- the name of a key
run_local -- whether this is run on a dev box (run_local=True) or instance
Returns:
True if the bucket key pair exists, otherwise False
"""
if local:
conn = S3Connection(**get_boto_creds())
else:
conn = S3Connection()
if conn.lookup(bucket_name) is None:
return False
s3_bucket = conn.get_bucket(bucket_name, validate=False)
exists = (s3_bucket.get_key(key_name) is not None)
conn.close()
return exists
def get_sqs_connection():
'''
:returns: sqs connection
'''
return boto.sqs.connect_to_region(
read_string('aws_config.region'),
**get_boto_creds()
)
def get_dynamodb_connection():
'''
:returns: dynamodb2 connection
'''
return boto.dynamodb2.connect_to_region(
read_string('aws_config.region'),
**get_boto_creds()
)
def load_from_s3_file(s3_uri):
"""Load data from S3
Useful for loading small config or schema files
:param s3_uri: path to S3 uri
:returns: file contents
"""
_, _, path = s3_uri.partition('://')
bucket_name, _, key_name = path.partition('/')
# if region is in a bucket name, put that region first
def preferred_region(item):
return item.name not in bucket_name
boto_creds = get_boto_creds()
for region in sorted(boto.s3.regions(), key=preferred_region):
try:
conn = boto.s3.connect_to_region(region.name, **boto_creds)
return _load_from_s3_region(conn, bucket_name, key_name)
except S3ResponseError as e:
# skip to next region if access is not allowed from this one
if e.status not in [403, 301]:
raise
raise ValueError("{0}: No valid region found".format(s3_uri))
def load_from_s3_file(s3_uri):
"""Load data from S3
Useful for loading small config or schema files
:param s3_uri: path to S3 uri
:returns: file contents
"""
_, _, path = s3_uri.partition('://')
bucket_name, _, key_name = path.partition('/')
# if region is in a bucket name, put that region first
def preferred_region(item):
return item.name not in bucket_name
boto_creds = get_boto_creds()
for region in sorted(boto.s3.regions(), key=preferred_region):
try:
conn = boto.s3.connect_to_region(region.name, **boto_creds)
return _load_from_s3_region(conn, bucket_name, key_name)
except S3ResponseError as e:
# skip to next region if access is not allowed from this one
if e.status not in [403, 301]:
raise
raise ValueError("{0}: No valid region found".format(s3_uri))
def get_s3_connection():
'''
:returns: s3 connection
'''
return boto.s3.connect_to_region(read_string('aws_config.region'),
**get_boto_creds())
def get_dynamodb_connection():
'''
:returns: dynamodb2 connection
'''
return boto.dynamodb2.connect_to_region(read_string('aws_config.region'),
**get_boto_creds())
