def s3_get(conf, s3url, dest, etag=None): """ High-speed download from S3 that can use multiple simultaneous download threads to optimize the downloading of a single file. S3 file is given in s3url (using s3://BUCKET/FILE naming convention) and will be saved in dest. If etag from previous download is provided, and file hasn't changed since then, don't download the file and instead raise an exception of type paracurl.Exception where the first element of the exception tuple == paracurl.PC_ERR_ETAG_MATCH. Returns tuple of (file_length, etag). """ import paracurl paracurl_kw = { 'max_threads' : int(conf.get('CURL_MAX_THREADS', '16')), 'n_retries' : int(conf.get('CURL_N_RETRIES', '4')), 'debug' : int(conf.get('CURL_DEBUG', '1')) } if etag: paracurl_kw['etag'] = etag s3tup = parse_s3_url(s3url) if not s3tup or len(s3tup) != 2: raise ValueError("s3_get: bad s3 url: %r" % (s3url,)) conn = get_s3_conn(conf) buck = conn.get_bucket(s3tup[0]) k = boto.s3.key.Key(buck) k.key = s3tup[1] url = k.generate_url(600, force_http=True) return paracurl.download(dest, url, **paracurl_kw)
def s3_get(conf, s3url, dest, etag=None): """ High-speed download from S3 that can use multiple simultaneous download threads to optimize the downloading of a single file. S3 file is given in s3url (using s3://BUCKET/FILE naming convention) and will be saved in dest. If etag from previous download is provided, and file hasn't changed since then, don't download the file and instead raise an exception of type paracurl.Exception where the first element of the exception tuple == paracurl.PC_ERR_ETAG_MATCH. Returns tuple of (file_length, etag). """ import paracurl paracurl_kw = { 'max_threads': int(conf.get('CURL_MAX_THREADS', '16')), 'n_retries': int(conf.get('CURL_N_RETRIES', '4')), 'debug': int(conf.get('CURL_DEBUG', '1')) } if etag: paracurl_kw['etag'] = etag s3tup = parse_s3_url(s3url) if not s3tup or len(s3tup) != 2: raise ValueError("s3_get: bad s3 url: %r" % (s3url, )) conn = get_s3_conn(conf) buck = conn.get_bucket(s3tup[0]) k = boto.s3.key.Key(buck) k.key = s3tup[1] url = k.generate_url(600, force_http=True) return paracurl.download(dest, url, **paracurl_kw)
import boto import paracurl conn = boto.connect_s3() buck = conn.get_bucket('marble_factory') k = boto.s3.key.Key(buck) k.key = 'blend.gz' url = k.generate_url(600, force_http=True) try: etag = '779ccf330e7c227ebbf7b34f0a6bae79foo' status = paracurl.download('foo.dat', url, max_threads=4, n_retries=4, etag=etag, debug=2) except paracurl.Exception, e: print "Caught Exception:", e else: print "Return Value:", status
import boto import paracurl conn = boto.connect_s3() buck = conn.get_bucket('marble_factory') k = boto.s3.key.Key(buck) k.key = 'marble-factory-lossless.mov' url = k.generate_url(600, force_http=True) try: status = paracurl.download('foo.dat', url, max_threads=64, debug=1) except paracurl.Exception, e: print "Caught Exception:", e else: print "Return Value:", status