def s3_get(conf, s3url, dest, etag=None):
"""
High-speed download from S3 that can use multiple simultaneous
download threads to optimize the downloading of a single file.
S3 file is given in s3url (using s3://BUCKET/FILE naming
convention) and will be saved in dest. If etag from previous
download is provided, and file hasn't changed since then, don't
download the file and instead raise an exception of type
paracurl.Exception where the first element of the exception
tuple == paracurl.PC_ERR_ETAG_MATCH. Returns tuple of
(file_length, etag).
"""
import paracurl
paracurl_kw = {
'max_threads' : int(conf.get('CURL_MAX_THREADS', '16')),
'n_retries' : int(conf.get('CURL_N_RETRIES', '4')),
'debug' : int(conf.get('CURL_DEBUG', '1'))
}
if etag:
paracurl_kw['etag'] = etag
s3tup = parse_s3_url(s3url)
if not s3tup or len(s3tup) != 2:
raise ValueError("s3_get: bad s3 url: %r" % (s3url,))
conn = get_s3_conn(conf)
buck = conn.get_bucket(s3tup[0])
k = boto.s3.key.Key(buck)
k.key = s3tup[1]
url = k.generate_url(600, force_http=True)
return paracurl.download(dest, url, **paracurl_kw)
def s3_get(conf, s3url, dest, etag=None):
"""
High-speed download from S3 that can use multiple simultaneous
download threads to optimize the downloading of a single file.
S3 file is given in s3url (using s3://BUCKET/FILE naming
convention) and will be saved in dest. If etag from previous
download is provided, and file hasn't changed since then, don't
download the file and instead raise an exception of type
paracurl.Exception where the first element of the exception
tuple == paracurl.PC_ERR_ETAG_MATCH. Returns tuple of
(file_length, etag).
"""
import paracurl
paracurl_kw = {
'max_threads': int(conf.get('CURL_MAX_THREADS', '16')),
'n_retries': int(conf.get('CURL_N_RETRIES', '4')),
'debug': int(conf.get('CURL_DEBUG', '1'))
}
if etag:
paracurl_kw['etag'] = etag
s3tup = parse_s3_url(s3url)
if not s3tup or len(s3tup) != 2:
raise ValueError("s3_get: bad s3 url: %r" % (s3url, ))
conn = get_s3_conn(conf)
buck = conn.get_bucket(s3tup[0])
k = boto.s3.key.Key(buck)
k.key = s3tup[1]
url = k.generate_url(600, force_http=True)
return paracurl.download(dest, url, **paracurl_kw)
import boto
import paracurl
conn = boto.connect_s3()
buck = conn.get_bucket('marble_factory')
k = boto.s3.key.Key(buck)
k.key = 'blend.gz'
url = k.generate_url(600, force_http=True)
try:
etag = '779ccf330e7c227ebbf7b34f0a6bae79foo'
status = paracurl.download('foo.dat',
url,
max_threads=4,
n_retries=4,
etag=etag,
debug=2)
except paracurl.Exception, e:
print "Caught Exception:", e
else:
print "Return Value:", status
import boto
import paracurl
conn = boto.connect_s3()
buck = conn.get_bucket('marble_factory')
k = boto.s3.key.Key(buck)
k.key = 'marble-factory-lossless.mov'
url = k.generate_url(600, force_http=True)
try:
status = paracurl.download('foo.dat', url, max_threads=64, debug=1)
except paracurl.Exception, e:
print "Caught Exception:", e
else:
print "Return Value:", status
import boto
import paracurl
conn = boto.connect_s3()
buck = conn.get_bucket('marble_factory')
k = boto.s3.key.Key(buck)
k.key = 'blend.gz'
url = k.generate_url(600, force_http=True)
try:
etag = '779ccf330e7c227ebbf7b34f0a6bae79foo'
status = paracurl.download('foo.dat', url, max_threads=4, n_retries=4, etag=etag, debug=2)
except paracurl.Exception, e:
print "Caught Exception:", e
else:
print "Return Value:", status
