def save(self, username, course_id, filepath):
"""Save certificate."""
try:
bucket = self.conn.get_bucket(self.bucket_name)
except S3ResponseError as e:
if e.status == 404:
bucket = self.conn.create_bucket(self.bucket_name,
location=self.location)
log.info("Cresate bucket(%s)", self.bucket_name)
else:
return json.dumps({"error": "{}".format(e)})
try:
s3key = Key(bucket)
s3key.key = "{cid}/{name}.pdf".format(cid=course_id, name=username)
# headers meta? encrypt_key true?
s3key.set_contents_from_filename(filepath)
url = s3key.generate_url(expires_in=0,
query_auth=False,
force_http=True)
finally:
s3key.close()
return json.dumps({
'download_url': url,
})
def save(self, username, course_id, filepath):
"""Save certificate."""
try:
bucket = self.conn.get_bucket(self.bucket_name)
except S3ResponseError as e:
if e.status == 404:
bucket = self.conn.create_bucket(
self.bucket_name,
location=self.location)
log.info("Cresate bucket(%s)", self.bucket_name)
else:
return json.dumps({"error": "{}".format(e)})
try:
s3key = Key(bucket)
s3key.key = "{cid}/{name}.pdf".format(
cid=course_id, name=username)
# headers meta? encrypt_key true?
s3key.set_contents_from_filename(filepath)
url = s3key.generate_url(
expires_in=0, query_auth=False, force_http=True)
finally:
s3key.close()
return json.dumps({'download_url': url, })
def main(stream_url: str, stream_name: str, bucket_name: str, duration: str):
temp_file = 'temp.m4a'
print('beginning rip')
code = subprocess.call(['ffmpeg',
'-i', stream_url,
'-t', duration,
'-acodec', 'copy',
'-absf', 'aac_adtstoasc',
temp_file])
assert code == 0, 'stream rip failed with code ' + str(code)
print('connecting to s3')
conn = S3Connection(is_secure=False) # AWS uses invalid certs
bucket = conn.get_bucket(bucket_name)
print('writing recorded file to s3')
m4a = Key(bucket)
m4a.name = datetime.datetime.utcnow().strftime(stream_name + '--%Y-%m-%d.m4a')
m4a.content_type = MIME_TYPE
m4a.metadata = {'Content-Type': MIME_TYPE}
m4a.storage_class = 'STANDARD_IA'
m4a.set_contents_from_filename(temp_file)
m4a.close()
print('generating new feed.xml from s3 bucket list')
feed_xml = Key(bucket)
feed_xml.name = 'feed.xml'
feed_xml.content_type = 'application/rss+xml'
feed_xml.set_contents_from_string(
rss_xml(stream_name, bucket_name, bucket.list()))
feed_xml.close()
def submit():
if len(request.files) < 1:
return 'OK'
name = request.form.get('name')
if not name:
abort(405)
root = '%s/' % name
for key in request.files:
file = request.files[key]
key = '%s%s' % (root, file.filename)
key = Key(bucket=app.bucket, name=key)
size = file.content_length
headers = None
(mimetype, encoding) = mimetypes.guess_type(file.filename)
if encoding == 'gzip':
mimetype = 'application/x-gzip'
if mimetype:
headers = {
'Content-Type': mimetype
}
key.set_contents_from_file(file.stream, size=size, headers=headers)
file.close()
key.close()
return 'OK'
def _getImageFromS3(bucket, name):
num_retries = 0
max_retries = 5
while True:
try:
key = Key(bucket, name)
data = key.get_contents_as_string()
key.close()
return data
except Exception as e:
logs.warning("S3 Exception: %s" % e)
num_retries += 1
if num_retries > max_retries:
msg = "Unable to connect to S3 after %d retries (%s)" % (max_retries, self.__class__.__name__)
logs.warning(msg)
raise Exception(msg)
logs.info("Retrying (%s)" % (num_retries))
time.sleep(0.5)
finally:
try:
if not key.closed:
key.close()
except Exception:
logs.warning("Error closing key")
def _addImageToS3(bucket, name, data):
num_retries = 0
max_retries = 5
while True:
try:
key = Key(bucket, name)
key.set_metadata("Content-Type", "image/jpeg")
key.set_contents_from_string(data.getvalue(), policy="public-read")
key.close()
return key
except Exception as e:
logs.warning("S3 Exception: %s" % e)
num_retries += 1
if num_retries > max_retries:
msg = "Unable to connect to S3 after %d retries (%s)" % (max_retries, self.__class__.__name__)
logs.warning(msg)
raise Exception(msg)
logs.info("Retrying (%s)" % (num_retries))
time.sleep(0.5)
finally:
try:
if not key.closed:
key.close()
except Exception:
logs.warning("Error closing key")
def addDataToS3(self, name, data, contentType):
num_retries = 0
max_retries = 5
while True:
try:
conn = S3Connection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY)
bucket = conn.lookup(self.bucket_name)
key = Key(bucket, name)
key.set_metadata('Content-Type', contentType)
# for some reason, if we use set_contents_from_file here, an empty file is created
key.set_contents_from_string(data.getvalue(), policy='public-read')
#key.set_contents_from_file(data, policy='public-read')
key.close()
return "%s/%s" % (self.base_url, name)
except Exception as e:
logs.warning('S3 Exception: %s' % e)
num_retries += 1
if num_retries > max_retries:
msg = "Unable to connect to S3 after %d retries (%s)" % \
(max_retries, self.__class__.__name__)
logs.warning(msg)
raise Exception(msg)
logs.info("Retrying (%s)" % (num_retries))
time.sleep(0.5)
finally:
try:
if not key.closed:
key.close()
except Exception:
logs.warning("Error closing key")
def submit():
if len(request.files) < 1:
return 'OK'
name = request.form.get('name')
if not name:
abort(405)
root = '%s/' % name
for key in request.files:
file = request.files[key]
key = '%s%s' % (root, file.filename)
key = Key(bucket=app.bucket, name=key)
size = file.content_length
headers = None
(mimetype, encoding) = mimetypes.guess_type(file.filename)
if encoding == 'gzip':
mimetype = 'application/x-gzip'
if mimetype:
headers = {'Content-Type': mimetype}
key.set_contents_from_file(file.stream, size=size, headers=headers)
file.close()
key.close()
return 'OK'
def send_to_s3(items=None, is_binary=False):
"""
For items in an iterable, send them to your s3 account
"""
conn, bucket = s3_init(AWS_ACCESS_KEY_ID, AWS_SECRET_KEY, BUCKET_NAME)
for label, data in items:
key = Key(bucket)
key.key = label
for item in bucket.list():
local_md5 = hashlib.md5(data).hexdigest()
if item.name == label:
key.open()
key.close(); #loads key.etag
# remote hash
remote_md5 = key.etag.replace('\"','') # clears quote marks
# If new backup is different than the last saved one, update it
if local_md5 != remote_md5:
if is_binary:
key.set_contents_from_filename(data)
else:
key.set_contents_from_string(data)
else:
if is_binary:
key.set_contents_from_filename(data)
else:
key.set_contents_from_string(data)
def export_job(request, job):
account = request.user.account
conn = S3Connection(aws_access_key_id=account.aws_access_key_id, aws_secret_access_key=account.aws_secret_access_key)
bucket = conn.get_bucket('lx-pilot')
key = Key(bucket)
key.key = job.cache_key
string = gzip.decompress(key.get_contents_as_string())
result = json.loads(string.decode('utf-8'))
rows = result.get('rows')
rows = [rm_dict_row(row) for row in rows]
output = StringIO()
writer = csv.writer(output)
writer.writerows(rows)
now = timezone.now()
key_string = 'exports/' + str(now.year) + '/' + str(now.month) + '/' + str(now.day) + '/' + str(uuid.uuid4())
export = JobExport(job=job, created_by=request.user, key=key_string)
key = Key(bucket)
key.key = export.key
key.set_metadata('Content-Type', 'text/csv')
key.set_metadata('Content-Encoding', 'gzip')
key.set_contents_from_string(gzip.compress(bytes(output.getvalue(), 'utf-8')))
key.close()
key = Key(bucket)
key.key = export.key
export.save()
return export
def write_string_to_key(bucket_name, key_name, string):
s3_conn = get_conn_s3()
bucket = s3_conn.get_bucket(bucket_name)
key = Key(bucket)
key.key = key_name
key.set_contents_from_string(string + "\n")
key.close()
def save_image(self, image, filename):
from boto.s3.key import Key
key = Key(self.bucket, image.hashed_filename())
key.set_contents_from_filename(filename,
{'Cache-Control': 'public, max-age=%d' % settings.S3_EXPIRES,
'Expires': time.asctime(time.gmtime(time.time() + settings.S3_EXPIRES)) })
key.close()
def save_image(self, hash, filename):
from boto.s3.key import Key
key = Key(self.bucket, hash)
key.set_contents_from_filename(filename,
{'Cache-Control': 'public, max-age=7200',
'Expires': time.asctime(time.gmtime(time.time() + 7200)) })
key.close()
def _put_S3(self, msg):
key = '/'.join([self.application, uuid.uuid4().hex])
key_object = Key(self.bucket_conn)
key_object.key = key
key_object.content_type = 'application/json'
key_object.set_contents_from_string(msg)
key_object.close()
return json.dumps({'Bucket': self.bucket_name, 'Key': key})
def upload(ext):
filename = os.path.join(settings.LOCAL_PHOTO_BUCKETS_BASE_PATH, old_photo_bucket, photo.photo_id + ext)
print "Uploading: " + filename
key = Key(s3_bucket, new_storage_id + ext)
key.metadata = {"Content-Type": "image/jpeg"}
key.set_contents_from_filename(filename)
key.close()
def backup_data_file(self, file_path, remote_key_name, remote_bucket=None):
stat = os.stat(file_path)
fp = file(file_path)
key = Key(remote_bucket, remote_key_name)
key.metadata = {'uid': stat.st_uid, 'gid': stat.st_gid}
self.set_contents_from_file(key, fp)
key.close()
fp.close()
def get_image_url(self, hash):
key = Key(self.bucket, hash)
ret = key.generate_url(settings.S3_EXPIRES, 'GET',
{'Cache-Control': 'public, max-age=7200',
'Expires': time.asctime(time.gmtime(time.time() + 7200)) })
key.close()
return ret
def s3_exists(self, key_name):
key = None
try:
logging.debug("Checking if key exists s3://%s%s" % (self.bucket_name, key_name))
key = Key(bucket=self.bucket, name=key_name)
return key.exists()
finally:
if key:
key.close()
def upload_file_to_s3(bucket, filename, filepath):
try:
s3key = Key(bucket)
s3key.key = "dump_oa_scores/{0}".format(filename)
s3key.set_contents_from_filename(filepath)
except:
raise
finally:
s3key.close()
print "Successfully uplaoded file to S3: %s/%s" % (bucket.name, s3key.key)
def upload_file_to_s3(bucket, filename, filepath):
try:
s3key = Key(bucket)
s3key.key = "dump_oa_scores/{0}".format(filename)
s3key.set_contents_from_filename(filepath)
except:
raise
finally:
s3key.close()
print "Successfully uplaoded file to S3: %s/%s" % (bucket.name, s3key.key)
def s3_exists(self, key_name):
key = None
try:
logging.debug("Checking if key exists s3://%s%s" %
(self.bucket_name, key_name))
key = Key(bucket=self.bucket, name=key_name)
return key.exists()
finally:
if key:
key.close()
def run(self):
try:
tries = 0
exception = None
while tries < self.retries:
if self.do_stop:
break
try:
if self.multipart_id and self.multipart_num and self.multipart_parts:
mp_log_info = "s3://%s%s (multipart: %d/%d, size: %.2fmb)" % (
self.bucket_name, self.short_key_name(self.key_name), self.multipart_num,
self.multipart_parts, float(self.byte_count / 1024.00 / 1024.00))
for mp in self.bucket.get_all_multipart_uploads():
if mp.id == self.multipart_id:
logging.info("Uploading AWS S3 key: %s" % mp_log_info)
callback_count = 10
if self.target_bandwidth is not None:
# request a callback every 0.5MB to allow for somewhat decent throttling
callback_count = self.byte_count / 1024 / 1024 / 0.5
with FileChunkIO(self.file_name, 'r', offset=self.multipart_offset, bytes=self.byte_count) as fp:
mp.upload_part_from_file(fp=fp, cb=self.status, num_cb=callback_count, part_num=self.multipart_num)
break
else:
raise OperationError("Missing multipart upload id %s for %s in S3 response." %
(self.multipart_id, mp_log_info))
else:
key = None
try:
logging.info("Uploading AWS S3 key: %s (multipart: None, size: %.2fmb)" % (
self.short_key_name(self.key_name),
float(self.byte_count / 1024.00 / 1024.00)
))
key = Key(bucket=self.bucket, name=self.key_name)
callback_count = 10
if self.target_bandwidth is not None:
# request a callback every 0.5MB to allow for somewhat decent throttling
callback_count = self.byte_count / 1024.00 / 1024.00 / 0.5
key.set_contents_from_filename(self.file_name, cb=self.status, num_cb=callback_count)
finally:
if key:
key.close()
break
except (httplib.HTTPException, exceptions.IOError, socket.error, socket.gaierror) as e:
logging.error("Got exception during upload: '%s', retrying upload" % e)
exception = e
finally:
sleep(self.retry_sleep_secs)
tries += 1
if tries >= self.retries and exception:
raise exception
except Exception as e:
logging.fatal("AWS S3 upload failed after %i retries! Error: %s" % (self.retries, e))
raise e
return self.file_name, self.key_name, self.multipart_num
def _save(self, name, content, public=False):
""" called by Storage.save() returns name for storage """
from boto.s3.key import Key
bucket = self._bucket()
i = Key(bucket, self._keyname(name))
i.set_contents_from_file(content, headers={"Content-Type": "image/png"})
if self.public or public:
i.set_acl("public-read")
i.close()
return name
def restore_file(self, deis_bucket_name, key_name, deis_bucket=None, remote_bucket=None):
key = Key(remote_bucket, key_name)
deis_key = Key(deis_bucket, self.get_deis_key_name(deis_bucket_name, key.key))
temp = tempfile.SpooledTemporaryFile(self._max_spool_bytes)
key.get_contents_to_file(temp)
self.set_contents_from_file(deis_key, temp)
temp.close()
key.close()
deis_key.close()
def get_image_url(self, hash):
from boto.s3.key import Key
key = Key(self.bucket, hash)
ret = key.generate_url(settings.S3_EXPIRES, 'GET',
{'Cache-Control': 'public, max-age=%d' % settings.S3_EXPIRES,
'Expires': time.asctime(time.gmtime(time.time() + settings.S3_EXPIRES)) })
key.close()
return ret
def upload_photo(username, photo):
data = base64.b64decode(photo)
s3 = boto.connect_s3()
bucket = s3.get_bucket(BUCKET_NAME)
key = Key(bucket)
key.content_type = 'image/jpg'
key.key = 'photos/%s/%s.jpg' % (username, random_hex())
key.set_contents_from_string(data)
key.close()
key.make_public()
return AWS_URL_TEMPLATE % key.key
def upload_file_to_s3_as_single(self, source_filename, destination_path):
destination_file = self.s3_location(destination_path)
source = os.path.abspath(source_filename)
source_size = os.stat(source).st_size
log.info("Uploading from %s (%s) to %s in one go", source, humanize.naturalsize(source_size), destination_file.full)
bucket = self.get_bucket(destination_file.bucket)
key = Key(bucket)
key.name = destination_file.key
key.set_contents_from_filename(source_filename, policy='authenticated-read')
key.close()
def upload_product(product, bucket_name):
s3conn = boto.connect_s3()
s3bucket = s3conn.get_bucket(bucket_name)
""" Upload the build artifacts to S3 for deployment. """
if not os.path.exists(product):
raise Exception('Build product %s does not exist, aborting.' % product)
(path, sep, filename) = product.rpartition('/')
print "Upload", filename, "to S3"
s3key = Key(s3bucket)
s3key.key = filename
s3key.set_contents_from_filename(product, replace=True)
s3key.close()
def restore_data_file(self, file_path, remote_key_name, remote_bucket=None):
fp = file(file_path, 'w')
key = Key(remote_bucket, remote_key_name)
self.get_contents_to_file(key, fp)
try:
uid = int(key.get_metadata('uid'))
gid = int(key.get_metadata('gid'))
os.chown(file_path, uid, gid)
except:
pass
key.close()
fp.close()
def upload(filename, birthday):
# Test that it's a date.
datetime.date(*map(int, birthday.replace('18', '19').split('-')))
k = Key(b)
k.key = os.path.join('data', 'bornon', birthday + '.json.gz')
k.storage_class = 'REDUCED_REDUNDANCY'
f = open(filename)
k.set_contents_from_string(f.read(), HEADERS, replace=True)
f.close()
k.close()
def backup_database_sql(self):
print('backing up database sql')
env = os.environ.copy()
env['PGHOST'] = self.get_etcd_value('/deis/database/host')
env['PGPORT'] = self.get_etcd_value('/deis/database/port')
env['PGUSER'] = self.get_etcd_value('/deis/database/adminUser')
env['PGPASSWORD'] = self.get_etcd_value('/deis/database/adminPass')
proc = subprocess.Popen('pg_dumpall', env=env, stdout=subprocess.PIPE)
(output, err) = proc.communicate()
bucket = self.get_remote_s3_bucket()
key = Key(bucket, self.get_remote_key_name('other', 'database.sql'))
self.set_contents_from_string(key, output)
key.close()
def put(self, key, content):
k = Key(self.bucket)
k.key = key
# print "Uploading some data to " + BUCKET_NAME + " with key: " + k.key
try:
k.set_contents_from_file(content)
# self.logger.write("not an error put key: " + key)
except:
return False
finally:
k.delete()
k.close()
return True
def save(self, course_id, data, filename):
try:
bucket = self.conn.get_bucket(self.bucket_name)
except:
raise
try:
s3key = Key(bucket)
s3key.key = "{dir}/{cid}/{filename}".format(
dir=settings.TRANSCRIPTS_BACKUP_DIR, cid=course_id, filename=filename)
s3key.set_contents_from_string(data)
except:
raise
finally:
s3key.close()
def update_projects():
conn = S3Connection(AWS_KEY, AWS_SECRET)
bucket = conn.get_bucket(BUCKET)
pj_list = Key(bucket)
pj_list.key = 'projects.json'
project_list = json.loads(pj_list.get_contents_as_string())
pj_list.close()
details = []
for project_url in project_list:
try:
pj_details = update_project(project_url)
except IOError:
return 'Github is throttling. Just gonna try again after limit is reset.'
if pj_details:
details.append(pj_details)
pj_details = Key(bucket)
pj_details.key = 'project_details.json'
pj_details.set_contents_from_string(json.dumps(details))
pj_details.set_metadata('Content-Type', 'application/json')
pj_details.set_acl('public-read')
pj_details.close()
people_list = Key(bucket)
people_list.key = 'people.json'
people_list.set_contents_from_string(json.dumps(get_people_totals(details)))
people_list.set_metadata('Content-Type', 'application/json')
people_list.set_acl('public-read')
people_list.close()
org_list = Key(bucket)
org_list.key = 'organizations.json'
org_list.set_contents_from_string(json.dumps(get_org_totals(details)))
org_list.set_metadata('Content-Type', 'application/json')
org_list.set_acl('public-read')
org_list.close()
return 'Updated'
def Download(self, sS3Key, sLocalFn):
#{
#try:
#{
key = Key(self.m_bucket)
key.key = sS3Key
key.get_contents_to_filename(sLocalFn)
key.close()
#}
#except:
#{
# return False
#}
return True
def delete(self, username, course_id):
"""Delete certificate."""
try:
bucket = self.conn.get_bucket(self.bucket_name)
s3key = Key(bucket)
s3key.key = "{cid}/{name}.pdf".format(cid=course_id, name=username)
if s3key.exists():
s3key.delete()
else:
return json.dumps(
{'error': "file does not exists.({})".format(s3key.key)})
finally:
s3key.close()
return json.dumps({'error': None})
def Upload(self, sLocalFn, sKey, sPolicy=None):
#{
# try:
# #{
key = Key(self.m_bucket)
key.key = sKey
key.set_contents_from_filename(sLocalFn, policy=sPolicy)
key.close()
# #}
# except:
# #{
# return False
# #}
return True
def delete(self, username, course_id):
"""Delete certificate."""
try:
bucket = self.conn.get_bucket(self.bucket_name)
s3key = Key(bucket)
s3key.key = "{cid}/{name}.pdf".format(
cid=course_id, name=username)
if s3key.exists():
s3key.delete()
else:
return json.dumps({'error': "file does not exists.({})".format(
s3key.key)})
finally:
s3key.close()
return json.dumps({'error': None})
def upload_video_s3(video_id):
filename = video_id + VIDEO_EXT
filepath = "tmp/" + filename
conn = S3Connection(os.environ.get("AWS_ACCESS_KEY_ID"), os.environ.get("AWS_SECRET_ACCESS_KEY"))
bucket_input = conn.get_bucket(BUCKET_NAME_INPUT)
# Remove all movie on input/output dir
for a in bucket_input.list():
bucket_input.delete_key(a)
f = Key(bucket_input)
f.key = filename
f.set_contents_from_filename(filepath)
#f.make_public()
url = "https://s3-us-west-1.amazonaws.com/" + BUCKET_NAME_INPUT + "/" + filename
f.close()
print url
def create_edx_course(request):
try:
data = json.loads(request.body)
title = data['title']
org = data['org']
course = data['course']
run = data['run']
key_version = data['key_version']
body = json.loads(data['body'])
edx_course, __ = EdxCourse.objects.get_or_create(
title=title,
org=org,
course=course,
run=run,
key_version=key_version)
# TODO Improve logging, especially for S3 functionality
output_filename = '%s.json' % edx_course.id
output = json.dumps(body, indent=4)
utf8_output = output.encode('utf-8')
courses_bucket_name = getattr(settings, 'COURSES_BUCKET', None)
# get the bucket
log.info("writing file to s3")
conn = S3Connection()
courses_bucket = conn.get_bucket(courses_bucket_name)
path = getattr(settings, 'COURSES_FOLDER', None)
full_key_name = os.path.join(path, output_filename)
k = Key(courses_bucket)
k.key = full_key_name
k.content_type = 'application/json'
k.content_encoding = 'UTF-8'
k.set_contents_from_string(utf8_output)
k.close()
except Exception as e:
log.info("{}".format(e))
return http.HttpResponseBadRequest()
return HttpResponse(status=201)
def s3_uploader(db_backup_bucket, gpg_file_path, update_seq, checksum):
if db_backup_bucket not in con_s3.get_all_buckets():
print 'Backup bucket is missing, creating new bucket ', db_backup_bucket
con_s3.create_bucket(db_backup_bucket)
bucket = con_s3.get_bucket(db_backup_bucket)
else:
bucket = con_s3.get_bucket(db_backup_bucket)
lifecycle = Lifecycle()
lifecycle.add_rule('14 Days CouchDB Expiration',
os.path.basename(gpg_file_path), 'Enabled', 14)
bucket.configure_lifecycle(lifecycle)
key = Key(bucket)
key.key = os.path.basename(gpg_file_path)
key.set_acl('authenticated-read')
key.set_metadata('UpdateSeq', update_seq)
key.set_metadata('Checksum', checksum)
key.set_contents_from_file(gpg_file_path)
key.close()
print 'Finished uploading backup to S3'
def get_edx_course(request):
"""
Load and parse an edX course.
Returns a JSON representation of the edX course structure. Note that this
JSON object is a direct parsing of the edX course XML structure, and may
change with little or no warning if the edX export format is modified.
"""
try:
course_id = request.GET['edx_course_id']
except KeyError:
return http.HttpResponseBadRequest()
try:
edx_course = EdxCourse.objects.get(id=course_id)
except EdxCourse.DoesNotExist:
return http.HttpResponseNotFound()
try:
# TODO Improve logging, especially for S3 functionality
input_filename = '%s.json' % course_id
courses_bucket_name = getattr(settings, 'COURSES_BUCKET', None)
# get the bucket
log.info("reading file from s3")
conn = S3Connection()
courses_bucket = conn.get_bucket(courses_bucket_name)
path = getattr(settings, 'COURSES_FOLDER', None)
full_key_name = os.path.join(path, input_filename)
k = Key(courses_bucket)
k.key = full_key_name
k.content_type = 'application/json'
k.content_encoding = 'UTF-8'
parsed = json.loads(k.get_contents_as_string())
k.close()
parsed['id'] = course_id
return http.JsonResponse(parsed, safe=False)
except IOError:
return http.HttpResponseNotFound()
def upload(self, id, max_size=10):
if not self.s3_conn or not self.bucket:
if self.failover == '1':
return super(S3Upload, self).upload(id, max_size)
elif self.failover == '2':
abort('404', 'Problem with cloud')
directory = 'resource'
try:
bucket_key = Key(self.bucket)
if self.filename:
filepath = '/'.join((directory, id, self.filename))
bucket_key.key = filepath
if self.content_type:
bucket_key.content_type = self.content_type
self.upload_file.seek(0)
bucket_key.set_contents_from_file(self.upload_file)
bucket_key.make_public()
bucket_key.close()
return self.bucket_name + '/' + filepath
except Exception, e:
log.warn(e)
def run(self):
try:
tries = 0
exception = None
while tries < self.retries:
if self.do_stop:
break
try:
if self.multipart_id and self.multipart_num and self.multipart_parts:
mp_log_info = "s3://%s%s (multipart: %d/%d, size: %.2fmb)" % (
self.bucket_name, self.short_key_name(
self.key_name), self.multipart_num,
self.multipart_parts,
float(self.byte_count / 1024.00 / 1024.00))
for mp in self.bucket.get_all_multipart_uploads():
if mp.id == self.multipart_id:
logging.info("Uploading AWS S3 key: %s" %
mp_log_info)
callback_count = 10
if self.target_bandwidth is not None:
# request a callback every 0.5MB to allow for somewhat decent throttling
callback_count = self.byte_count / 1024 / 1024 / 0.5
with FileChunkIO(self.file_name,
'r',
offset=self.multipart_offset,
bytes=self.byte_count) as fp:
mp.upload_part_from_file(
fp=fp,
cb=self.status,
num_cb=callback_count,
part_num=self.multipart_num)
break
else:
raise OperationError(
"Missing multipart upload id %s for %s in S3 response."
% (self.multipart_id, mp_log_info))
else:
key = None
try:
logging.info(
"Uploading AWS S3 key: %s (multipart: None, size: %.2fmb)"
% (self.short_key_name(self.key_name),
float(self.byte_count / 1024.00 / 1024.00)))
key = Key(bucket=self.bucket, name=self.key_name)
callback_count = 10
if self.target_bandwidth is not None:
# request a callback every 0.5MB to allow for somewhat decent throttling
callback_count = self.byte_count / 1024.00 / 1024.00 / 0.5
key.set_contents_from_filename(
self.file_name,
cb=self.status,
num_cb=callback_count)
finally:
if key:
key.close()
break
except (httplib.HTTPException, exceptions.IOError,
socket.error, socket.gaierror) as e:
logging.error(
"Got exception during upload: '%s', retrying upload" %
e)
exception = e
finally:
sleep(self.retry_sleep_secs)
tries += 1
if tries >= self.retries and exception:
raise exception
except Exception as e:
logging.fatal("AWS S3 upload failed after %i retries! Error: %s" %
(self.retries, e))
raise e
return self.file_name, self.key_name, self.multipart_num
def do_upload(self, real_path, upload_path):
print 'uploading %s to %s' % (real_path, upload_path)
key = Key(self.bucket, name=upload_path)
key.set_contents_from_filename(real_path, policy="public-read")
key.close()
query1 = dict({"$match": {"$or": None}})
query2 = dict({"$unwind": '$arr'})
#query3= dict({"$group":{"_id":"null","arr":{"$addToSet":"$arr"}}})
#"arr":{"$addToSet":"$arr"}
query3 = dict({"$group": {"_id": {}, "arr": {"$addToSet": "$arr"}}})
i = 0
l = list([])
while i < (len(sys.argv) - 1):
l.append({'word': sys.argv[i + 1]})
i += 1
query1["$match"]["$or"] = l
cur = dbmc.invertedIndex.aggregate([query1, query2, query3])
#k.key='10001'
#print k.get_contents_as_string()
strx = None
for doc in cur:
y = list(doc['arr'])
for i in y:
k.key = str(i)
try:
print(k.get_contents_as_string())
k.close()
except Exception:
print "waste"
class S3utils(object):
"""
S3 Utils
A simple user friendly interface to Amazon S3.
S3 utils methods are made similar to Linux commands
so it is easier to use/remember for simple file operations
on S3 buckets.
"""
def __init__(
self, AWS_ACCESS_KEY_ID=getattr(settings, "AWS_ACCESS_KEY_ID", ""),
AWS_SECRET_ACCESS_KEY=getattr(settings, "AWS_SECRET_ACCESS_KEY", ""),
AWS_STORAGE_BUCKET_NAME=getattr(settings, "AWS_STORAGE_BUCKET_NAME", ""),
S3UTILS_DEBUG_LEVEL=getattr(settings, "S3UTILS_DEBUG_LEVEL", 0),
):
"""
Parameters
----------
AWS_ACCESS_KEY_ID : string
AWS Access key. If it is defined in your Django settings, it will grab it from there.
Otherwise you need to specify it here.
AWS_SECRET_ACCESS_KEY : string
AWS secret. If it is defined in your Django settings, it will grab it from there.
Otherwise you need to specify it here.
AWS_STORAGE_BUCKET_NAME : string
AWS Bucket name. If it is defined in your Django settings, it will grab it from there.
Otherwise you need to specify it here.
"""
self.AWS_ACCESS_KEY_ID = AWS_ACCESS_KEY_ID
self.AWS_SECRET_ACCESS_KEY = AWS_SECRET_ACCESS_KEY
self.AWS_STORAGE_BUCKET_NAME = AWS_STORAGE_BUCKET_NAME
self.S3UTILS_DEBUG_LEVEL = S3UTILS_DEBUG_LEVEL
self.conn = None
self.conn_cloudfront = None
# setting the logging level based on S3UTILS_DEBUG_LEVEL
try:
if (S3UTILS_DEBUG_LEVEL == 0):
logger.setLevel(logging.ERROR)
else:
logger.setLevel(logging.INFO)
except AttributeError:
pass
def __del__(self):
if self.conn:
self.disconnect()
def printv(self, msg):
if self.S3UTILS_DEBUG_LEVEL:
print(msg)
logger.info(msg)
def connect(self):
"""
Establish the connection. This is done automatically for you.
If you lose the connection, you can manually run this to be re-connected.
"""
self.conn = boto.connect_s3(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, debug=self.S3UTILS_DEBUG_LEVEL)
self.bucket = self.conn.get_bucket(self.AWS_STORAGE_BUCKET_NAME)
self.k = Key(self.bucket)
def disconnect(self):
"""
Close the connection.
This is normally done automatically when the garbage collector is deleting s3utils object.
"""
self.bucket.connection.connection.close()
self.conn = None
def connect_cloudfront(self):
"Connect to Cloud Front. This is done automatically for you when needed."
self.conn_cloudfront = connect_cloudfront(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, debug=self.S3UTILS_DEBUG_LEVEL)
@connectit
def mkdir(self, target_folder):
"""
Create a folder on S3.
Examples
--------
>>> s3utils.mkdir("path/to/my_folder")
Making directory: path/to/my_folder
"""
self.printv("Making directory: %s" % target_folder)
self.k.key = re.sub(r"^/|/$", "", target_folder) + "/"
self.k.set_contents_from_string('')
self.k.close()
@connectit
def rm(self, path):
"""
Delete the path and anything under the path.
Example
-------
>>> s3utils.rm("path/to/file_or_folder")
"""
list_of_files = list(self.ls(path))
if list_of_files:
if len(list_of_files) == 1:
self.bucket.delete_key(list_of_files[0])
else:
self.bucket.delete_keys(list_of_files)
self.printv("Deleted: %s" % list_of_files)
else:
logger.error("There was nothing to remove under %s", path)
@connectit
def __put_key(self, local_file, target_file, acl='public-read', del_after_upload=False, overwrite=True, source="filename"):
"""Copy a file to s3."""
action_word = "moving" if del_after_upload else "copying"
try:
self.k.key = target_file # setting the path (key) of file in the container
if source == "filename":
# grabs the contents from local_file address. Note that it loads the whole file into memory
self.k.set_contents_from_filename(local_file)
elif source == "fileobj":
self.k.set_contents_from_file(local_file)
elif source == "string":
self.k.set_contents_from_string(local_file)
else:
raise Exception("%s is not implemented as a source." % source)
self.k.set_acl(acl) # setting the file permissions
self.k.close() # not sure if it is needed. Somewhere I read it is recommended.
self.printv("%s %s to %s" % (action_word, local_file, target_file))
# if it is supposed to delete the local file after uploading
if del_after_upload and source == "filename":
try:
os.remove(local_file)
except:
logger.error("Unable to delete the file: ", local_file, exc_info=True)
return True
except:
logger.error("Error in writing to %s", target_file, exc_info=True)
return False
def cp(self, local_path, target_path, acl='public-read',
del_after_upload=False, overwrite=True, invalidate=False):
"""
Copy a file or folder from local to s3.
Parameters
----------
local_path : string
Path to file or folder. Or if you want to copy only the contents of folder, add /* at the end of folder name
target_path : string
Target path on S3 bucket.
acl : string, optional
File permissions on S3. Default is public-read
options:
- private: Owner gets FULL_CONTROL. No one else has any access rights.
- public-read: Owners gets FULL_CONTROL and the anonymous principal is granted READ access.
- public-read-write: Owner gets FULL_CONTROL and the anonymous principal is granted READ and WRITE access.
- authenticated-read: Owner gets FULL_CONTROL and any principal authenticated as a registered Amazon S3 user is granted READ access
del_after_upload : boolean, optional
delete the local file after uploading. This is effectively like moving the file.
You can use s3utils.mv instead of s3utils.cp to move files from local to S3.
It basically sets this flag to True.
default = False
overwrite : boolean, optional
overwrites files on S3 if set to True. Default is True
invalidate : boolean, optional
invalidates the CDN (a.k.a Distribution) cache if the file already exists on S3
default = False
Note that invalidation might take up to 15 minutes to take place. It is easier and faster to use cache buster
to grab lastest version of your file on CDN than invalidation.
**Returns**
Nothing on success but it will return what went wrong if something fails.
Examples
--------
>>> s3utils.cp("path/to/folder","/test/")
copying /path/to/myfolder/test2.txt to test/myfolder/test2.txt
copying /path/to/myfolder/test.txt to test/myfolder/test.txt
copying /path/to/myfolder/hoho/photo.JPG to test/myfolder/hoho/photo.JPG
copying /path/to/myfolder/hoho/haha/ff to test/myfolder/hoho/haha/ff
>>> # When overwrite is set to False, it returns the file(s) that were already existing on s3 and were not overwritten.
>>> s3utils.cp("/tmp/test3.txt", "test3.txt", overwrite=False)
ERROR:root:test3.txt already exist. Not overwriting.
>>> {'existing_files': {'test3.txt'}}
>>> # To overwrite the files on S3 and invalidate the CDN (cloudfront) cache so the new file goes on CDN:
>>> s3utils.cp("path/to/folder","/test/", invalidate=True)
copying /path/to/myfolder/test2.txt to test/myfolder/test2.txt
copying /path/to/myfolder/test.txt to test/myfolder/test.txt
copying /path/to/myfolder/hoho/photo.JPG to test/myfolder/hoho/photo.JPG
copying /path/to/myfolder/hoho/haha/ff to test/myfolder/hoho/haha/ff
>>> # When file does not exist, it returns a dictionary of what went wrong.
>>> s3utils.cp("/tmp/does_not_exist", "somewhere")
ERROR:root:trying to upload to s3 but file doesn't exist: /tmp/does_not_exist
>>> {'file_does_not_exist': '/tmp/does_not_exist'}
"""
result = None
if overwrite:
list_of_files = []
else:
list_of_files = self.ls(folder=target_path, begin_from_file="", num=-1, get_grants=False, all_grant_data=False)
# copying the contents of the folder and not folder itself
if local_path.endswith("/*"):
local_path = local_path[:-2]
target_path = re.sub(r"^/|/$", "", target_path) # Amazon S3 doesn't let the name to begin with /
# copying folder too
else:
local_base_name = os.path.basename(local_path)
local_path = re.sub(r"/$", "", local_path)
target_path = re.sub(r"^/", "", target_path)
if not target_path.endswith(local_base_name):
target_path = os.path.join(target_path, local_base_name)
if os.path.exists(local_path):
result = self.__find_files_and_copy(local_path, target_path, acl, del_after_upload, overwrite, invalidate, list_of_files)
else:
result = {'file_does_not_exist': local_path}
logger.error("trying to upload to s3 but file doesn't exist: %s" % local_path)
return result
def __find_files_and_copy(self, local_path, target_path, acl='public-read', del_after_upload=False, overwrite=True, invalidate=False, list_of_files=[]):
files_to_be_invalidated = []
failed_to_copy_files = set([])
existing_files = set([])
def check_for_overwrite_then_write():
if overwrite or (not overwrite and target_file not in list_of_files):
success = self.__put_key(
local_file,
target_file=target_file,
acl=acl,
del_after_upload=del_after_upload,
overwrite=overwrite,
)
if not success:
failed_to_copy_files.add(target_file)
else:
existing_files.add(target_file)
logger.error("%s already exist. Not overwriting.", target_file)
if overwrite and target_file in list_of_files and invalidate:
files_to_be_invalidated.append(target_file)
first_local_root = None
# if it is a folder
if os.path.isdir(local_path):
for local_root, directories, files in os.walk(local_path):
if not first_local_root:
first_local_root = local_root
# if folder is not empty
if files:
# iterating over the files in the folder
for a_file in files:
local_file = os.path.join(local_root, a_file)
target_file = os.path.join(
target_path + local_root.replace(first_local_root, ""),
a_file
)
check_for_overwrite_then_write()
# if folder is empty
else:
target_file = target_path + local_root.replace(first_local_root, "") + "/"
if target_file not in list_of_files:
self.mkdir(target_file)
if del_after_upload:
rmtree(local_path)
# if it is a file
else:
local_file = local_path
target_file = target_path
check_for_overwrite_then_write()
if invalidate and files_to_be_invalidated:
self.invalidate(files_to_be_invalidated)
items = ('failed_to_copy_files', 'existing_files')
local_vars = locals()
result = {}
for i in items:
val = local_vars.get(i)
if val:
result[i] = val
result = None if result == {} else result
return result
def echo(self, content, target_path, acl='public-read',
overwrite=True, invalidate=False):
"""
Similar to Linux Echo command.
Puts the string into the target path on s3
Parameters
----------
content : string
The content to be put on the s3 bucket.
target_path : string
Target path on S3 bucket.
acl : string, optional
File permissions on S3. Default is public-read
options:
- private: Owner gets FULL_CONTROL. No one else has any access rights.
- public-read: (Default) Owners gets FULL_CONTROL and the anonymous principal is granted READ access.
- public-read-write: Owner gets FULL_CONTROL and the anonymous principal is granted READ and WRITE access.
- authenticated-read: Owner gets FULL_CONTROL and any principal authenticated as a registered Amazon S3 user is granted READ access
overwrite : boolean, optional
overwrites files on S3 if set to True. Default is True
invalidate : boolean, optional
invalidates the CDN (a.k.a Distribution) cache if the file already exists on S3
default = False
Note that invalidation might take up to 15 minutes to take place. It is easier and faster to use cache buster
to serve the lastest version of your file on CDN than invalidation.
**Returns:**
Nothing on success, otherwise it returns what went wrong.
Return type:
dict
Examples
--------
>>> # On success returns nothing:
>>> s3utils.echo("Hello World!","/test.txt")
>>> # On failure returns what went wrong
>>> s3utils.echo("Hello World!","/test/")
{'InvalidS3Path': "path on S3 can not end in /"}
"""
result = None
if target_path.endswith('/') or target_path.endswith('*'):
result = {'InvalidS3Path': "Path on S3 can not end in /"}
if not overwrite and not result:
file_exists = self.ls(target_path)
if file_exists:
logger.error("%s already exist. Not overwriting.", target_path)
result = {'existing_files': target_path}
if content and not result:
if isinstance(content, strings):
result = self.__put_key(content, target_path, acl=acl,
del_after_upload=False, overwrite=overwrite,
source="string")
else:
result = {"TypeError": "Content is not string"}
return result
def mv(self, local_file, target_file, acl='public-read', overwrite=True, invalidate=False):
"""
Similar to Linux mv command.
Move the file to the S3 and deletes the local copy
It is basically s3utils.cp that has del_after_upload=True
Examples
--------
>>> s3utils.mv("path/to/folder","/test/")
moving /path/to/myfolder/test2.txt to test/myfolder/test2.txt
moving /path/to/myfolder/test.txt to test/myfolder/test.txt
moving /path/to/myfolder/hoho/photo.JPG to test/myfolder/hoho/photo.JPG
moving /path/to/myfolder/hoho/haha/ff to test/myfolder/hoho/haha/ff
**Returns:**
Nothing on success, otherwise what went wrong.
Return type:
dict
"""
self.cp(local_file, target_file, acl=acl, del_after_upload=True, overwrite=overwrite, invalidate=invalidate)
@connectit
def cp_cropduster_image(self, the_image_path, del_after_upload=False, overwrite=False, invalidate=False):
"""
Deal with saving cropduster images to S3. Cropduster is a Django library for resizing editorial images.
S3utils was originally written to put cropduster images on S3 bucket.
Extra Items in your Django Settings
-----------------------------------
MEDIA_ROOT : string
Django media root.
Currently it is ONLY used in cp_cropduster_image method.
NOT any other method as this library was originally made to put Django cropduster images on s3 bucket.
S3_ROOT_BASE : string
S3 media root base. This will be the root folder in S3.
Currently it is ONLY used in cp_cropduster_image method.
NOT any other method as this library was originally made to put Django cropduster images on s3 bucket.
"""
local_file = os.path.join(settings.MEDIA_ROOT, the_image_path)
# only try to upload things if the origin cropduster file exists (so it is not already uploaded to the CDN)
if os.path.exists(local_file):
the_image_crops_path = os.path.splitext(the_image_path)[0]
the_image_crops_path_full_path = os.path.join(settings.MEDIA_ROOT, the_image_crops_path)
self.cp(local_path=local_file,
target_path=os.path.join(settings.S3_ROOT_BASE, the_image_path),
del_after_upload=del_after_upload,
overwrite=overwrite,
invalidate=invalidate,
)
self.cp(local_path=the_image_crops_path_full_path + "/*",
target_path=os.path.join(settings.S3_ROOT_BASE, the_image_crops_path),
del_after_upload=del_after_upload,
overwrite=overwrite,
invalidate=invalidate,
)
def __get_grants(self, target_file, all_grant_data):
"""
Return grant permission, grant owner, grant owner email and grant id as a list.
It needs you to set k.key to a key on amazon (file path) before running this.
note that Amazon returns a list of grants for each file.
options:
- private: Owner gets FULL_CONTROL. No one else has any access rights.
- public-read: Owners gets FULL_CONTROL and the anonymous principal is granted READ access.
- public-read-write: Owner gets FULL_CONTROL and the anonymous principal is granted READ and WRITE access.
- authenticated-read: Owner gets FULL_CONTROL and any principal authenticated as a registered Amazon S3 user is granted READ access
"""
self.k.key = target_file
the_grants = self.k.get_acl().acl.grants
grant_list = []
for grant in the_grants:
if all_grant_data:
grant_list.append(
{"permission": grant.permission, "name": grant.display_name, "email": grant.email_address, "id": grant.id})
else:
grant_list.append({"permission": grant.permission, "name": grant.display_name})
return grant_list
@connectit
def chmod(self, target_file, acl='public-read'):
"""
sets permissions for a file on S3
Parameters
----------
target_file : string
Path to file on S3
acl : string, optional
File permissions on S3. Default is public-read
options:
- private: Owner gets FULL_CONTROL. No one else has any access rights.
- public-read: Owners gets FULL_CONTROL and the anonymous principal is granted READ access.
- public-read-write: Owner gets FULL_CONTROL and the anonymous principal is granted READ and WRITE access.
- authenticated-read: Owner gets FULL_CONTROL and any principal authenticated as a registered Amazon S3 user is granted READ access
Examples
--------
>>> s3utils.chmod("path/to/file","private")
"""
self.k.key = target_file # setting the path (key) of file in the container
self.k.set_acl(acl) # setting the file permissions
self.k.close()
@connectit
def ls(self, folder="", begin_from_file="", num=-1, get_grants=False, all_grant_data=False):
"""
gets the list of file names (keys) in a s3 folder
Parameters
----------
folder : string
Path to file on S3
num: integer, optional
number of results to return, by default it returns all results.
begin_from_file: string, optional
which file to start from on S3.
This is usedful in case you are iterating over lists of files and you need to page the result by
starting listing from a certain file and fetching certain num (number) of files.
Examples
--------
>>> from s3utils import S3utils
>>> s3utils = S3utils(
... AWS_ACCESS_KEY_ID = 'your access key',
... AWS_SECRET_ACCESS_KEY = 'your secret key',
... AWS_STORAGE_BUCKET_NAME = 'your bucket name',
... S3UTILS_DEBUG_LEVEL = 1, #change it to 0 for less verbose
... )
>>> print(s3utils.ls("test/"))
{u'test/myfolder/', u'test/myfolder/em/', u'test/myfolder/hoho/', u'test/myfolder/hoho/.DS_Store', u'test/myfolder/hoho/haha/', u'test/myfolder/hoho/haha/ff', u'test/myfolder/hoho/haha/photo.JPG'}
"""
# S3 object key can't start with /
folder = re.sub(r"^/", "", folder)
bucket_files = self.bucket.list(prefix=folder, marker=begin_from_file)
# in case listing grants
if get_grants:
list_of_files = OrderedDict()
for (i, v) in enumerate(bucket_files):
file_info = {v.name: self.__get_grants(v.name, all_grant_data)}
list_of_files.update(file_info)
if i == num:
break
else:
list_of_files = set([])
for (i, v) in enumerate(bucket_files):
list_of_files.add(v.name)
if i == num:
break
return list_of_files
def ll(self, folder="", begin_from_file="", num=-1, all_grant_data=False):
"""
Get the list of files and permissions from S3.
This is similar to LL (ls -lah) in Linux: List of files with permissions.
Parameters
----------
folder : string
Path to file on S3
num: integer, optional
number of results to return, by default it returns all results.
begin_from_file : string, optional
which file to start from on S3.
This is usedful in case you are iterating over lists of files and you need to page the result by
starting listing from a certain file and fetching certain num (number) of files.
all_grant_data : Boolean, optional
More detailed file permission data will be returned.
Examples
--------
>>> from s3utils import S3utils
>>> s3utils = S3utils(
... AWS_ACCESS_KEY_ID = 'your access key',
... AWS_SECRET_ACCESS_KEY = 'your secret key',
... AWS_STORAGE_BUCKET_NAME = 'your bucket name',
... S3UTILS_DEBUG_LEVEL = 1, #change it to 0 for less verbose
... )
>>> import json
>>> # We use json.dumps to print the results more readable:
>>> my_folder_stuff = s3utils.ll("/test/")
>>> print(json.dumps(my_folder_stuff, indent=2))
{
"test/myfolder/": [
{
"name": "owner's name",
"permission": "FULL_CONTROL"
}
],
"test/myfolder/em/": [
{
"name": "owner's name",
"permission": "FULL_CONTROL"
}
],
"test/myfolder/hoho/": [
{
"name": "owner's name",
"permission": "FULL_CONTROL"
}
],
"test/myfolder/hoho/.DS_Store": [
{
"name": "owner's name",
"permission": "FULL_CONTROL"
},
{
"name": null,
"permission": "READ"
}
],
"test/myfolder/hoho/haha/": [
{
"name": "owner's name",
"permission": "FULL_CONTROL"
}
],
"test/myfolder/hoho/haha/ff": [
{
"name": "owner's name",
"permission": "FULL_CONTROL"
},
{
"name": null,
"permission": "READ"
}
],
"test/myfolder/hoho/photo.JPG": [
{
"name": "owner's name",
"permission": "FULL_CONTROL"
},
{
"name": null,
"permission": "READ"
}
],
}
"""
return self.ls(folder=folder, begin_from_file=begin_from_file, num=num, get_grants=True, all_grant_data=all_grant_data)
@connectit_cloudfront
def invalidate(self, files_to_be_invalidated):
"""
Invalidate the CDN (distribution) cache for a certain file of files. This might take up to 15 minutes to be effective.
You can check for the invalidation status using check_invalidation_request.
Examples
--------
>>> from s3utils import S3utils
>>> s3utils = S3utils(
... AWS_ACCESS_KEY_ID = 'your access key',
... AWS_SECRET_ACCESS_KEY = 'your secret key',
... AWS_STORAGE_BUCKET_NAME = 'your bucket name',
... S3UTILS_DEBUG_LEVEL = 1, #change it to 0 for less verbose
... )
>>> aa = s3utils.invalidate("test/myfolder/hoho/photo.JPG")
>>> print(aa)
('your distro id', u'your request id')
>>> invalidation_request_id = aa[1]
>>> bb = s3utils.check_invalidation_request(*aa)
>>> for inval in bb:
... print('Object: %s, ID: %s, Status: %s' % (inval, inval.id, inval.status))
"""
if not isinstance(files_to_be_invalidated, Iterable):
files_to_be_invalidated = (files_to_be_invalidated,)
# Your CDN is called distribution on Amazaon. And you can have more than one distro
all_distros = self.conn_cloudfront.get_all_distributions()
for distro in all_distros:
invalidation_request = self.conn_cloudfront.create_invalidation_request(distro.id, files_to_be_invalidated)
return (distro.id, invalidation_request.id)
@connectit_cloudfront
def check_invalidation_request(self, distro, request_id):
return self.conn_cloudfront.get_invalidation_requests(distro, request_id)
def run(self):
try:
tries = 0
exception = None
while tries < self.retries:
if self.do_stop:
break
try:
if self.multipart_id and self.multipart_num and self.multipart_parts:
for mp in self.bucket.get_all_multipart_uploads():
if mp.id == self.multipart_id:
logging.info(
"Uploading AWS S3 key: s3://%s%s (multipart: %d/%d, size: %.2fmb)"
%
(self.bucket_name,
self.short_key_name(self.key_name),
self.multipart_num, self.multipart_parts,
float(
self.byte_count / 1024.00 / 1024.00)))
with FileChunkIO(self.file_name,
'r',
offset=self.multipart_offset,
bytes=self.byte_count) as fp:
mp.upload_part_from_file(
fp=fp,
cb=self.status,
num_cb=10,
part_num=self.multipart_num)
break
else:
key = None
try:
logging.info(
"Uploading AWS S3 key: %s (multipart: None, size: %.2fmb)"
% (self.short_key_name(self.key_name),
float(self.byte_count / 1024.00 / 1024.00)))
key = Key(bucket=self.bucket, name=self.key_name)
key.set_contents_from_filename(self.file_name,
cb=self.status,
num_cb=10)
finally:
if key:
key.close()
break
except (httplib.HTTPException, exceptions.IOError,
socket.error, socket.gaierror) as e:
logging.error(
"Got exception during upload: '%s', retrying upload" %
e)
exception = e
finally:
sleep(self.retry_sleep_secs)
tries += 1
if tries >= self.retries and exception:
raise exception
except Exception as e:
logging.fatal("AWS S3 upload failed after %i retries! Error: %s" %
(self.retries, e))
raise e
return self.file_name, self.key_name, self.multipart_num
print "- [Info] This video has been already locally saved. Skipping..."
else:
videos_s3_folder_path = 'concatenated_replays'
video_s3_filename = "{0}/{1}".format(videos_s3_folder_path, video_id + '.mp4')
s3_interface = S3Interface('periscope-demo')
video_key = Key(s3_interface.bucket, video_s3_filename)
if not video_key.exists():
print "- [Error] s3 key does not exist. Skipping..."
continue
try:
video_key.get_contents_to_filename(local_video_file)
except boto_exception.S3ResponseError:
print "- [Error] Can not download s3 object. Skipping..."
continue
finally:
video_key.close()
# - get number of frames
cap = cv2.VideoCapture(local_video_file)
if not cap.isOpened():
print "- [Error] Could not open video file={}. Skipping...".format(local_video_file)
continue
length = int(cap.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT))
width = int(cap.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT))
fps = cap.get(cv2.cv.CV_CAP_PROP_FPS)
num_frames = int(cap.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT))
print "- [Info] num_frames={}".format(num_frames)
