def __init__(self, bucket=STORAGE_BUCKET_NAME, access_key=None,
secret_key=None, acl=DEFAULT_ACL, headers=HEADERS,
gzip=IS_GZIPPED, gzip_content_types=GZIP_CONTENT_TYPES):
self.acl = acl
self.headers = headers
self.gzip = gzip
self.gzip_content_types = gzip_content_types
if not access_key and not secret_key:
access_key, secret_key = self._get_access_keys()
# Ignore CALLING_FORMAT for uploads - only use it for constructing download URLs. LOL IDK
upload_klass = getattr(connection, 'SubdomainCallingFormat')
self.upload_connection = connection.S3Connection(access_key, secret_key, calling_format=upload_klass(), is_secure=False)
try:
klass = getattr(connection, CALLING_FORMAT)
except AttributeError:
raise ImproperlyConfigured("Invalid CallingFormat subclass: %s\
\nValid choices: SubdomainCallingFormat, VHostCallingFormat, OrdinaryCallingFormat")
self.connection = connection.S3Connection(access_key, secret_key, calling_format=klass())
self.upload_bucket = self.upload_connection.get_bucket(bucket, validate=False)
#self.bucket = self._get_or_create_bucket(bucket)
self.bucket = self.connection.get_bucket(bucket, validate=False)
def setUp(self):
trytond.tests.test_tryton.install_module('nereid_s3')
self.static_file = POOL.get('nereid.static.file')
self.static_folder = POOL.get('nereid.static.folder')
# Mock S3Connection
self.s3_api_patcher = patch(
'boto.s3.connection.S3Connection', autospec=True
)
PatchedS3 = self.s3_api_patcher.start()
# Mock S3Key
self.s3_key_patcher = patch(
'boto.s3.key.Key', autospec=True
)
PatchedS3Key = self.s3_key_patcher.start()
PatchedS3.return_value = connection.S3Connection('ABCD', '123XYZ')
PatchedS3.return_value.get_bucket = lambda bucket_name: Bucket(
PatchedS3.return_value, 'tryton-test-s3'
)
PatchedS3Key.return_value = Key(
Bucket(PatchedS3.return_value, 'tryton-test-s3'), 'some key'
)
PatchedS3Key.return_value.key = "some key"
PatchedS3Key.return_value.get_contents_as_string = lambda *a: 'testfile'
PatchedS3Key.return_value.set_contents_from_string = \
lambda value: 'testfile'
def get_bucket(self):
'''
Return an S3 bucket for the static file
'''
s3_conn = connection.S3Connection(self.s3_access_key,
self.s3_secret_key)
return s3_conn.get_bucket(self.s3_bucket_name)
def restore_backup(days_ago):
"""
Restores the last backup that is at least `days_ago` days old.
"""
# set up django environment
management.setup_environ(scoryst_settings)
from django.conf import settings
connection = s3.S3Connection(settings.AWS_S3_ACCESS_KEY_ID,
settings.AWS_S3_SECRET_ACCESS_KEY)
bucket = connection.get_bucket(settings.AWS_PRODUCTION_BACKUPS_BUCKET_NAME)
keys = bucket.list()
key = get_last_key(keys, days_ago)
if key == None:
print 'No key found to take a backup from!'
return
# Fetch backup from S3. Macs can't handle ':' in file names, so we replace them.
temp_file_name = '/tmp/%s' % key.name
print 'Fetching file %s to %s' % (key.name, temp_file_name)
key.get_contents_to_filename(temp_file_name)
# restore backup
print 'Restoring backup...'
print subprocess.check_output(['psql', '-f', temp_file_name, 'postgres'])
def setUp(self):
"""Setup users, projects, and start a test server."""
super(S3APITestCase, self).setUp()
tempdir = self.useFixture(fixtures.TempDir())
conf = self.useFixture(config_fixture.Config())
conf.config(buckets_path=tempdir.path,
s3_listen='127.0.0.1',
s3_listen_port=0)
self.server = s3server.get_wsgi_server()
# NOTE(ft): this requires eventlet.monkey_patch, which is called in
# tests/unit/__init__.py. Remove it out from there if you get these
# tests rid of server run
self.server.start()
self.addCleanup(self.server.stop)
if not boto.config.has_section('Boto'):
boto.config.add_section('Boto')
boto.config.set('Boto', 'num_retries', '0')
conn = s3.S3Connection(aws_access_key_id='fake',
aws_secret_access_key='fake',
host=CONF.s3_listen,
port=self.server.port,
is_secure=False,
calling_format=s3.OrdinaryCallingFormat())
self.conn = conn
def get_http_connection(*args):
"""Get a new S3 connection, don't attempt to reuse connections."""
return self.conn.new_http_connection(*args)
self.conn.get_http_connection = get_http_connection
def setUp(self):
"""Setup users, projects, and start a test server."""
super(S3APITestCase, self).setUp()
self.flags(buckets_path=os.path.join(OSS_TEMPDIR, 'buckets'),
s3_host='127.0.0.1')
shutil.rmtree(CONF.buckets_path)
os.mkdir(CONF.buckets_path)
router = s3server.S3Application(CONF.buckets_path)
self.server = wsgi.Server("S3 Objectstore",
router,
host=CONF.s3_host,
port=0)
self.server.start()
if not boto.config.has_section('Boto'):
boto.config.add_section('Boto')
boto.config.set('Boto', 'num_retries', '0')
conn = s3.S3Connection(aws_access_key_id='fake',
aws_secret_access_key='fake',
host=CONF.s3_host,
port=self.server.port,
is_secure=False,
calling_format=s3.OrdinaryCallingFormat())
self.conn = conn
def get_http_connection(*args):
"""Get a new S3 connection, don't attempt to reuse connections."""
return self.conn.new_http_connection(*args)
self.conn.get_http_connection = get_http_connection
def _upload_photo(image, image_type, id=None, file_path=None):
conn = connection.S3Connection(settings.AWS_ACCESS_KEY,
settings.AWS_SECRET_KEY)
s3bucket = conn.create_bucket(settings.AWS_PHOTO_UPLOAD_BUCKET)
filename = str(uuid.uuid4())
tmp = tempfile.NamedTemporaryFile('w+b', -1, '.jpg')
image = image.convert("RGB")
image.save(tmp.name)
key = s3key.Key(s3bucket)
if file_path:
key.key = '%s.jpg' % file_path
elif id:
key.key = '%s/%s_%s.jpg' % (image_type, filename, id)
else:
key.key = '%s/%s.jpg' % (image_type, filename)
try:
key.set_contents_from_filename(tmp.name, None, True, None, 10,
'public-read', None)
except:
raise
key.close()
tmp.close()
return "http://%s.s3.amazonaws.com/%s" % (settings.AWS_PHOTO_UPLOAD_BUCKET,
key.key)
def get_connection(host_or_region):
# first case: we got a valid DNS (host)
if "." in host_or_region:
return connection.S3Connection(host=host_or_region)
# second case: we got a region
return connect_to_region(host_or_region)
def setUp(self):
"""Setup users, projects, and start a test server."""
super(S3APITestCase, self).setUp()
self.flags(auth_driver='nova.auth.ldapdriver.FakeLdapDriver',
buckets_path=os.path.join(OSS_TEMPDIR, 'buckets'),
s3_host='127.0.0.1')
self.auth_manager = manager.AuthManager()
self.admin_user = self.auth_manager.create_user('admin', admin=True)
self.admin_project = self.auth_manager.create_project('admin',
self.admin_user)
shutil.rmtree(FLAGS.buckets_path)
os.mkdir(FLAGS.buckets_path)
router = s3server.S3Application(FLAGS.buckets_path)
server = wsgi.Server()
server.start(router, FLAGS.s3_port, host=FLAGS.s3_host)
if not boto.config.has_section('Boto'):
boto.config.add_section('Boto')
boto.config.set('Boto', 'num_retries', '0')
conn = s3.S3Connection(aws_access_key_id=self.admin_user.access,
aws_secret_access_key=self.admin_user.secret,
host=FLAGS.s3_host,
port=FLAGS.s3_port,
is_secure=False,
calling_format=s3.OrdinaryCallingFormat())
self.conn = conn
def get_http_connection(host, is_secure):
"""Get a new S3 connection, don't attempt to reuse connections."""
return self.conn.new_http_connection(host, is_secure)
self.conn.get_http_connection = get_http_connection
def resource_download(self, id, resource_id, filename=None):
"""
Provides a direct download by either redirecting the user to the url stored
or downloading an uploaded file directly.
"""
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'auth_user_obj': c.userobj}
try:
rsc = get_action('resource_show')(context, {'id': resource_id})
pkg = get_action('package_show')(context, {'id': id})
except NotFound:
abort(404, _('Resource not found'))
except NotAuthorized:
abort(401, _('Unauthorized to read resource %s') % id)
if rsc.get('url_type') == 'upload':
upload = uploader.ResourceUpload(rsc)
filepath = upload.get_path(rsc['id'])
#### s3archive new code
access_key = config.get('ckanext.s3archive.access_key')
secret_key = config.get('ckanext.s3archive.secret_key')
bucket_name = config.get('ckanext.s3archive.bucket')
if not os.path.exists(filepath):
content_type, content_enc = mimetypes.guess_type(rsc.get('url',''))
key_name = filepath[len(filepath)-39:]
conn = s3connection.S3Connection(access_key, secret_key)
bucket = conn.get_bucket(bucket_name)
key = None
for key in bucket.list(prefix=key_name.lstrip('/')):
pass
if not key:
abort(404, _('Resource data not found'))
headers = {}
if content_type:
headers['response-content-type'] = content_type
url = key.generate_url(300, method='GET', response_headers=headers)
redirect(url)
#### code finish
fileapp = paste.fileapp.FileApp(filepath)
try:
status, headers, app_iter = request.call_application(fileapp)
except OSError:
abort(404, _('Resource data not found'))
response.headers.update(dict(headers))
content_type, content_enc = mimetypes.guess_type(rsc.get('url',''))
response.headers['Content-Type'] = content_type
response.status = status
return app_iter
elif not 'url' in rsc:
abort(404, _('No download is available'))
redirect(rsc['url'])
def _connect_to_s3(self, credentials): """ Connects to S3 with the given credentials. Returns a tuple of Boto (connection, bucket) objects. """ connection = s3.S3Connection(credentials['token'], credentials['secret']) bucket = connection.get_bucket(credentials['bucket']) return connection, bucket
def __init__(self, region, key=None, secret=None, host=None):
if region is None and host is not None:
self.connection = connection.S3Connection(
host=host, aws_access_key_id=key, aws_secret_access_key=secret)
else:
self.connection = connect_to_region(region,
aws_access_key_id=key,
aws_secret_access_key=secret)
def archive(self):
access_key = config.get('ckanext.s3archive.access_key')
secret_key = config.get('ckanext.s3archive.secret_key')
bucket_name = config.get('ckanext.s3archive.bucket')
if not access_key:
print 'ckanext.s3archive.access_key config argument not set'
return
if not secret_key:
print 'ckanext.s3archive.secret_key config argument not set'
return
if not bucket_name:
print 'ckanext.s3archive.bucket config argument not set'
return
storage_path = get_storage_path()
if not storage_path:
print 'ckan.storage_path not set in config'
return
resource_path = os.path.join(storage_path, 'resources')
def walk(bucket, dir, files):
for file in files:
full_path = os.path.join(resource_path, dir, file)
if not os.path.isfile(full_path) or full_path.endswith('~'):
continue
key_name = full_path[len(resource_path):]
for key in bucket.list(prefix=key_name.lstrip('/')):
key.delete()
resource_id = key_name.replace('/', '')
resource = model.Resource.get(resource_id)
if not resource:
continue
last_part = resource.url.split('/')[-1]
file_name = munge.munge_filename(last_part)
key_name = key_name + '/' + file_name
key = s3key.Key(bucket)
key.key = key_name
key.set_contents_from_filename(full_path)
print 'Archived %s' % key_name
os.remove(full_path)
conn = s3connection.S3Connection(
access_key,
secret_key,
calling_format=OrdinaryCallingFormat(),
host='object.auckland.ac.nz',
port=443)
bucket = conn.get_bucket(bucket_name)
try:
os.path.walk(resource_path, walk, bucket)
finally:
conn.close()
def __init__(self, scraper_name):
self.scraper_name = scraper_name
c = ConfigParser.ConfigParser()
c.read(ffs.Path('~/.dc.ini').abspath)
self.bucket_name = c.get('ckan', 'aws_bucket')
self.conn = s3connection.S3Connection(c.get('ckan', 'aws_access_key'),
c.get('ckan', 'aws_secret_key'))
self.bucket = self.conn.get_bucket(self.bucket_name)
def __connect(self):
'''Connect, leaving self.__conn and self.__bucket valid.'''
self.__conn = None # make sure it's bound even on failure
self.__bucket = None # ditto
log.debug('attempting to connect to s3')
self.__conn = connection.S3Connection(host=self.__opts.get(
's3_host', connection.S3Connection.DefaultHost))
self.__conn.calling_format = connection.SubdomainCallingFormat()
self.__bucket = self.__conn.lookup(self.bucket_name)
def _get_conn(self):
"""
Open an S3 connection and caches on this instance.
:return: the S3 connection.
"""
if self._connection is None:
self._connection = s3_conn.S3Connection(
aws_access_key_id=self._aws_access_key_id,
aws_secret_access_key=self._aws_secret_access_key
)
return self._connection
def get_connection(self):
"""
Creates a connection to S3
Returns:
boto.s3.connection.S3Connection: a connection to S3
"""
self._validate()
return connection.S3Connection(
settings.EXAMS_AUDIT_AWS_ACCESS_KEY_ID,
settings.EXAMS_AUDIT_AWS_SECRET_ACCESS_KEY
)
def get_modeldata(s3_data):
try:
s3_conn = connection.S3Connection(settings.AWS_ACCESS_KEY,
settings.AWS_SECRET_KEY)
key = s3_key.Key(s3_conn.get_bucket(settings.S3_BUCKET))
key.key = s3_data
data = key.get_contents_as_string()
except (S3ResponseError, S3DataError):
raise FileBackendError()
if s3_data.endswith('.gz'):
data = gzip_to_str(data)
return json.loads(data)
def sign_s3_get(key):
rval = cache.get('signed_url:%s:%s' % (settings.S3_BUCKET, key))
if rval is None:
life_time = 3600
s3_conn = connection.S3Connection(settings.AWS_ACCESS_KEY,
settings.AWS_SECRET_KEY)
bkey = s3_key.Key(s3_conn.get_bucket(settings.S3_BUCKET))
bkey.key = key
rval = bkey.generate_url(life_time, query_auth=True, force_http=True)
cache.set('signed_url:%s:%s' % (settings.S3_BUCKET, key), rval,
life_time - 60)
return rval
def __init__(self, region, key=None, secret=None, host=None):
if region is None and host is not None:
self.connection = connection.S3Connection(
host=host,
aws_access_key_id=key,
aws_secret_access_key=secret,
calling_format=ProtocolIndependentOrdinaryCallingFormat())
else:
self.connection = connect_to_region(
region,
aws_access_key_id=key,
aws_secret_access_key=secret,
calling_format=ProtocolIndependentOrdinaryCallingFormat())
def sanitize_bucket_and_host(bucket):
# type: (str) -> Tuple[str, str]
"""
if bucket is in following format : 'xxx.amazonaws.com/bucket_name',
Returns a 2-values tuple ('bucket_name', 'xxx.amazonaws.com')
"""
# first case: we got a "<host>/<bucket_name>" input
if "/" in bucket:
host, bucket = bucket.split('/', 1)
if "/" in bucket:
raise ValueError(
'{} should contains only one slash separator'.format(bucket))
if not host.endswith('amazonaws.com'):
raise ValueError('host should be a *.amazonaws.com URL')
return bucket, host
# return location from cache is possible, so we don't issue "GetBucketLocation"
# calls with each other S3 call
if bucket in BUCKET_LOCATIONS_CACHE:
return bucket, BUCKET_LOCATIONS_CACHE[bucket]
# second case: we got a bucket name, we need to figure out which region it's in
try:
conn0 = connection.S3Connection()
bucket_obj = conn0.get_bucket(bucket, validate=False)
# get_location() returns a region or an empty string for us-east-1,
# historically named "US Standard" in some places. Maybe other S3
# calls support an empty string as region, but I prefer to be
# explicit here.
location = bucket_obj.get_location() or "us-east-1"
# save location for later use
BUCKET_LOCATIONS_CACHE[bucket] = location
except S3ResponseError as e:
if e.error_code == "AccessDenied":
# probably not allowed to perform GetBucketLocation on this bucket
logger.warning(
"Access denied while trying to get location of bucket {}".
format(bucket))
location = ""
else:
raise
# fallback for backward compatibility
if not location:
location = settings.SIMPLEFLOW_S3_HOST
return bucket, location
def get_or_create_bucket(self, name):
"""
Gets an S3 bucket of the given name, creating one if it doesn't already exist.
Should be called with a role, if AWS credentials are stored in role settings. e.g.
fab local s3.get_or_create_bucket:mybucket
"""
from boto.s3 import connection
if self.dryrun:
print('boto.connect_s3().create_bucket(%s)' % repr(name))
else:
conn = connection.S3Connection(self.genv.aws_access_key_id,
self.genv.aws_secret_access_key)
bucket = conn.create_bucket(name)
return bucket
def __connect(self):
'''Connect, leaving self.__conn and self.__bucket valid.'''
self.__conn = None # make sure it's bound even on failure
self.__the_bucket = None # ditto
log.debug('attempting to connect to s3')
self.__conn = connection.S3Connection(
host=self.__opts.get('s3_host',
connection.S3Connection.DefaultHost),
proxy=self.__opts.get('proxy', None),
proxy_port=self.__opts.get('proxy_port', None),
is_secure=self.__opts.get('https', True),
aws_access_key_id=self.__opts.get('aws_access_key_id', None),
aws_secret_access_key=self.__opts.get('aws_secret_access_key',
None))
self.__conn.calling_format = connection.SubdomainCallingFormat()
self.__the_bucket = self.__conn.lookup(self.bucket_name)
def _connect_secureish(*args, **kwargs):
"""Connect using the safest available options.
This turns on encryption (works in all supported boto versions)
and certificate validation (in the subset of supported boto
versions that can handle certificate validation, namely, those
after 2.6.0).
Versions below 2.6 don't support the validate_certs option to
S3Connection, and enable it via configuration option just seems to
cause an error.
"""
if tuple(int(x) for x in boto.__version__.split('.')) >= (2, 6, 0):
kwargs['validate_certs'] = True
kwargs['is_secure'] = True
return connection.S3Connection(*args, **kwargs)
def delete_file(self, entry, path):
# Move any bucket subdirectories to the filename
bucket_name, filename = move_bucket_subdirs_to_path(entry.bucket, path)
# fixme: use temporary token if configured for it
secret = aes_decrypt(entry.secret)
conn = connection.S3Connection(entry.access_key, secret)
bucket = connection.Bucket(conn, bucket_name)
s3key = connection.Key(bucket)
s3key.key = filename
try:
bucket.delete_key(s3key)
except boto.exception.BotoServerError as ex:
raise IOError(
("Failed to delete '%s' from S3 Cloud Storage " + \
"bucket '%s'. %s: %s") % \
(filename, bucket_name, ex.reason, ex.message))
return {'status': 'OK'}
def connect(self, creds):
"""Return a boto S3Connection set up with great care.
This includes TLS settings, calling format selection, and
region detection.
The credentials are applied by the caller because in many
cases (instance-profile IAM) it is possible for those
credentials to fluctuate rapidly. By comparison, region
fluctuations of a bucket name are not nearly so likely versus
the gains of not looking up a bucket's region over and over.
"""
def _conn_help(*args, **kwargs):
return _connect_secureish(
*args,
provider=creds,
calling_format=self.calling_format(),
auth_region_name=self.region,
**kwargs)
# If WALE_S3_ENDPOINT is set, do not attempt to guess
# the right calling conventions and instead honor the explicit
# settings within WALE_S3_ENDPOINT.
impl = os.getenv('WALE_S3_ENDPOINT')
if impl:
return connection.S3Connection(**_s3connection_opts_from_uri(impl))
# Check if subdomain format compatible: if so, use the
# BUCKETNAME.s3.amazonaws.com hostname to communicate with the
# bucket.
if self.calling_format is connection.SubdomainCallingFormat:
return _conn_help(host='s3.amazonaws.com')
# Check if OrdinaryCallingFormat compatible, but also see if
# the endpoint has already been set, in which case only
# setting the host= flag is necessary.
assert self.calling_format is connection.OrdinaryCallingFormat
assert self.ordinary_endpoint is not None
return _conn_help(host=self.ordinary_endpoint)
#f_out.flush()
#f_out.close()
#soCredentials2 = __get_aws_credentials__(_contents_,method=method)
#assert soCredentials.AWSAccessKeyId == soCredentials2.AWSAccessKeyId, 'WARNING: Something is wrong with the Improved method.'
#assert soCredentials.AWSSecretKey == soCredentials2.AWSSecretKey, 'WARNING: Something is wrong with the Improved method.'
#dP = HashedLists2(fromDict=dE).asDict()
#for k,v in dP.iteritems():
#_k_ = _utils.ascii_only(blowfish.decryptData(oodb.hexToStr(k), __passphrase__))
#dP[_k_] = _utils.ascii_only(blowfish.decryptData(oodb.hexToStr(v), __passphrase__))
#del dP[k]
#assert dP[_k_] == d[_k_], 'WARNING: Expected %s to match %s.' % (dP[_k_],d[_k_])
#if (os.path.exists(fnameOut)):
#soCredentials = get_aws_credentials(filename='./credentials_secure.txt',method=method)
if (1):
soCredentials = get_aws_credentials(url=_url_(),
filename=None,
method=Methods.default)
soCredentials2 = get_aws_credentials(url=_url2_(),
filename=None,
method=Methods.improved)
assert soCredentials.AWSAccessKeyId == soCredentials2.AWSAccessKeyId, 'WARNING: Something is wrong with the Improved method.'
assert soCredentials.AWSSecretKey == soCredentials2.AWSSecretKey, 'WARNING: Something is wrong with the Improved method.'
from boto.s3 import connection
aConnection = connection.S3Connection(
aws_access_key_id=soCredentials2.AWSAccessKeyId,
aws_secret_access_key=soCredentials2.AWSSecretKey)
buckets = aConnection.get_all_buckets()
print 'buckets=%s' % (buckets)
pass
def upload(bucket,
aws_access_key,
aws_secret_key,
iterable,
key,
progress_cb=None,
threads=5,
replace=False,
secure=True,
connection=None):
''' Upload data to s3 using the s3 multipart upload API.
Args:
bucket: name of s3 bucket
aws_access_key: aws access key
aws_secret_key: aws secret key
iterable: The data to upload. Each 'part' in the list
will be uploaded in parallel. Each part must be at
least 5242880 bytes (5mb).
key: the name of the key to create in the s3 bucket
progress_cb: will be called with (part_no, uploaded, total)
each time a progress update is available.
threads: the number of threads to use while uploading. (Default is 5)
replace: will replace the key in s3 if set to true. (Default is false)
secure: use ssl when talking to s3. (Default is true)
connection: used for testing
'''
if not connection:
from boto.s3 import connection
c = connection.S3Connection(aws_access_key,
aws_secret_key,
is_secure=secure)
b = c.get_bucket(bucket)
if not replace and b.lookup(key):
raise Exception('s3 key ' + key + ' already exists')
multipart_obj = b.initiate_multipart_upload(key)
err_queue = Queue.Queue()
lock = threading.Lock()
upload.counter = 0
try:
tpool = pool.ThreadPool(processes=threads)
def check_errors():
try:
exc = err_queue.get(block=False)
except Queue.Empty:
pass
else:
raise exc
def waiter():
while upload.counter >= threads:
check_errors()
time.sleep(0.1)
def cb(err):
if err: err_queue.put(err)
with lock:
upload.counter -= 1
args = [multipart_obj.upload_part_from_file, progress_cb]
for part_no, part in enumerate(iterable):
part_no += 1
tpool.apply_async(upload_part, args + [part_no, part], callback=cb)
with lock:
upload.counter += 1
waiter()
tpool.close()
tpool.join()
# Check for thread errors before completing the upload,
# sometimes an error can be left unchecked until we
# get to this point.
check_errors()
multipart_obj.complete_upload()
except:
multipart_obj.cancel_upload()
tpool.terminate()
raise
from boto.s3 import connection
import seizure.config
class KaggleToS3(Object):
def __init__(self,
aws_access_key,
aws_secret_key,
chunk_size=1024)
self.aws_access_key = aws_access_key
self.aws_secret_key = aws_secret_key
self.chunk_size = chunk_size
self.connection = connection.S3Connection(aws_access_key,
aws_secret_key)
def transfer_from_urls(urls):
for url in urls:
transfer_from_url(url)
def transfer_from_url(url):
pass
def upload_chunk(chunk, key):
pass
def download_chunk(url):
# TODO
return chunk
def main():
pass
def get_connection(host):
return connection.S3Connection(host=host)