def save_object(self, file_id, mime, content_encoding, data, replace=False):
msg = 'configuring' if replace else 'updating'
try:
k = Key(bucket=self.bucket)
k.key = file_id
k.set_metadata('Content-Type', mime)
k.content_type = mime
k.content_encoding = content_encoding
k.set_metadata('Content-Encoding', content_encoding)
k.set_contents_from_string(data, headers=self.default_headers, replace=replace)
except Exception as e:
raise exc.HTTPInternalServerError('Error while %s S3 key (%s) %s' % (msg, file_id, e))
def _upload_file(self, f):
filename = os.path.basename(f)
try:
k = Key(self.bucket, filename)
k.content_encoding = 'utf-8'
k.set_contents_from_filename(f) # Uploads the key
except:
print >> sys.stderr, 'Problem storing key %s' % filename
return
if verbose: print '%d %s' % (self.counter, filename)
self.counter += 1
def run(self):
counter = 1
while 1:
data = self.writer_queue.get()
if not data: # check for sentinel to tell us we're done
break
title, text = data # unpack the set
logger.debug('%d: %s', counter, title) # log the article title
k = Key(self.bucket, title)
k.content_encoding = 'utf-8'
k.set_contents_from_string(text) # Uploads the key
counter += 1
def _save_to_s3(self, data, mime, update=False, compress=True):
ziped_data = None
content_encoding = None
headers = {}
if compress and mime == 'application/vnd.google-earth.kml+xml':
ziped_data = self._gzip_data(data)
content_encoding = 'gzip'
headers['Content-Encoding'] = 'gzip'
if not update:
if content_encoding == 'gzip' and ziped_data is not None:
data = ziped_data
try:
k = Key(bucket=self.bucket)
k.key = self.file_id
k.set_metadata('Content-Type', mime)
k.content_type = mime
k.content_encoding = content_encoding
k.set_metadata('Content-Encoding', content_encoding)
k.set_contents_from_string(data, replace=False)
key = self.bucket.get_key(k.key)
last_updated = parse_ts(key.last_modified)
except Exception as e:
raise exc.HTTPInternalServerError('Error while configuring S3 key (%s) %s' % (self.file_id, e))
try:
_save_item(self.admin_id, file_id=self.file_id, last_updated=last_updated)
except Exception as e:
raise exc.HTTPInternalServerError('Cannot create file on Dynamodb (%s)' % e)
else:
try:
if content_encoding == 'gzip' and ziped_data is not None:
data = ziped_data
# Inconsistant behaviour with metadata, see https://github.com/boto/boto/issues/2798
self.key.content_encoding = content_encoding
self.key.set_metadata('Content-Encoding', content_encoding)
self.key.set_contents_from_string(data, replace=True)
key = self.bucket.get_key(self.key.key)
last_updated = parse_ts(key.last_modified)
except Exception as e:
raise exc.HTTPInternalServerError('Error while updating S3 key (%s) %s' % (self.key.key, e))
try:
_save_item(self.admin_id, last_updated=last_updated)
except Exception as e:
raise exc.HTTPInternalServerError('Cannot update file on Dynamodb (%s) %s' % (self.file_id, e))
def save_object(self,
file_id,
mime,
content_encoding,
data,
replace=False):
msg = 'configuring' if replace else 'updating'
try:
k = Key(bucket=self.bucket)
k.key = file_id
k.set_metadata('Content-Type', mime)
k.content_type = mime
k.content_encoding = content_encoding
k.set_metadata('Content-Encoding', content_encoding)
k.set_contents_from_string(data,
headers=self.default_headers,
replace=replace)
except Exception as e:
raise exc.HTTPInternalServerError('Error while %s S3 key (%s) %s' %
(msg, file_id, e))
def create_edx_course(request):
try:
data = json.loads(request.body)
title = data['title']
org = data['org']
course = data['course']
run = data['run']
key_version = data['key_version']
body = json.loads(data['body'])
edx_course, __ = EdxCourse.objects.get_or_create(
title=title,
org=org,
course=course,
run=run,
key_version=key_version
)
# TODO Improve logging, especially for S3 functionality
output_filename = '%s.json' % edx_course.id
output = json.dumps(body, indent=4)
utf8_output = output.encode('utf-8')
courses_bucket_name = getattr(settings, 'COURSES_BUCKET', None)
# get the bucket
log.info("writing file to s3")
conn = S3Connection()
courses_bucket = conn.get_bucket(courses_bucket_name)
path = getattr(settings, 'COURSES_FOLDER', None)
full_key_name = os.path.join(path, output_filename)
k = Key(courses_bucket)
k.key = full_key_name
k.content_type = 'application/json'
k.content_encoding = 'UTF-8'
k.set_contents_from_string(utf8_output)
k.close()
except Exception as e:
log.info("{}".format(e))
return http.HttpResponseBadRequest()
return HttpResponse(status=201)
def create_edx_course(request):
try:
data = json.loads(request.body)
title = data['title']
org = data['org']
course = data['course']
run = data['run']
key_version = data['key_version']
body = json.loads(data['body'])
edx_course, __ = EdxCourse.objects.get_or_create(
title=title,
org=org,
course=course,
run=run,
key_version=key_version)
# TODO Improve logging, especially for S3 functionality
output_filename = '%s.json' % edx_course.id
output = json.dumps(body, indent=4)
utf8_output = output.encode('utf-8')
courses_bucket_name = getattr(settings, 'COURSES_BUCKET', None)
# get the bucket
log.info("writing file to s3")
conn = S3Connection()
courses_bucket = conn.get_bucket(courses_bucket_name)
path = getattr(settings, 'COURSES_FOLDER', None)
full_key_name = os.path.join(path, output_filename)
k = Key(courses_bucket)
k.key = full_key_name
k.content_type = 'application/json'
k.content_encoding = 'UTF-8'
k.set_contents_from_string(utf8_output)
k.close()
except Exception as e:
log.info("{}".format(e))
return http.HttpResponseBadRequest()
return HttpResponse(status=201)
def get_edx_course(request):
"""
Load and parse an edX course.
Returns a JSON representation of the edX course structure. Note that this
JSON object is a direct parsing of the edX course XML structure, and may
change with little or no warning if the edX export format is modified.
"""
try:
course_id = request.GET['edx_course_id']
except KeyError:
return http.HttpResponseBadRequest()
try:
edx_course = EdxCourse.objects.get(id=course_id)
except EdxCourse.DoesNotExist:
return http.HttpResponseNotFound()
try:
# TODO Improve logging, especially for S3 functionality
input_filename = '%s.json' % course_id
courses_bucket_name = getattr(settings, 'COURSES_BUCKET', None)
# get the bucket
log.info("reading file from s3")
conn = S3Connection()
courses_bucket = conn.get_bucket(courses_bucket_name)
path = getattr(settings, 'COURSES_FOLDER', None)
full_key_name = os.path.join(path, input_filename)
k = Key(courses_bucket)
k.key = full_key_name
k.content_type = 'application/json'
k.content_encoding = 'UTF-8'
parsed = json.loads(k.get_contents_as_string())
k.close()
parsed['id'] = course_id
return http.JsonResponse(parsed, safe=False)
except IOError:
return http.HttpResponseNotFound()
def get_edx_course(request):
"""
Load and parse an edX course.
Returns a JSON representation of the edX course structure. Note that this
JSON object is a direct parsing of the edX course XML structure, and may
change with little or no warning if the edX export format is modified.
"""
try:
course_id = request.GET['edx_course_id']
except KeyError:
return http.HttpResponseBadRequest()
try:
edx_course = EdxCourse.objects.get(id=course_id)
except EdxCourse.DoesNotExist:
return http.HttpResponseNotFound()
try:
# TODO Improve logging, especially for S3 functionality
input_filename = '%s.json' % course_id
courses_bucket_name = getattr(settings, 'COURSES_BUCKET', None)
# get the bucket
log.info("reading file from s3")
conn = S3Connection()
courses_bucket = conn.get_bucket(courses_bucket_name)
path = getattr(settings, 'COURSES_FOLDER', None)
full_key_name = os.path.join(path, input_filename)
k = Key(courses_bucket)
k.key = full_key_name
k.content_type = 'application/json'
k.content_encoding = 'UTF-8'
parsed = json.loads(k.get_contents_as_string())
k.close()
parsed['id'] = course_id
return http.JsonResponse(parsed, safe=False)
except IOError:
return http.HttpResponseNotFound()
