def create_perseus_zip(ccnode, exercise_data, write_to_path): with zipfile.ZipFile(write_to_path, "w") as zf: try: exercise_context = { 'exercise': json.dumps(exercise_data, sort_keys=True, indent=4) } exercise_result = render_to_string('perseus/exercise.json', exercise_context) write_to_zipfile("exercise.json", exercise_result, zf) for question in ccnode.assessment_items.prefetch_related('files').all().order_by('order'): try: for image in question.files.filter(preset_id=format_presets.EXERCISE_IMAGE).order_by('checksum'): image_name = "images/{}.{}".format(image.checksum, image.file_format_id) if image_name not in zf.namelist(): with storage.open(ccmodels.generate_object_storage_name(image.checksum, str(image)), 'rb') as content: write_to_zipfile(image_name, content.read(), zf) for image in question.files.filter(preset_id=format_presets.EXERCISE_GRAPHIE).order_by('checksum'): svg_name = "images/{0}.svg".format(image.original_filename) json_name = "images/{0}-data.json".format(image.original_filename) if svg_name not in zf.namelist() or json_name not in zf.namelist(): with storage.open(ccmodels.generate_object_storage_name(image.checksum, str(image)), 'rb') as content: content = content.read() content = content.split(exercises.GRAPHIE_DELIMITER) write_to_zipfile(svg_name, content[0], zf) write_to_zipfile(json_name, content[1], zf) write_assessment_item(question, zf) except Exception as e: logging.error("Publishing error: {}".format(str(e))) finally: zf.close()
def create_perseus_zip(ccnode, exercise_data, write_to_path): with zipfile.ZipFile(write_to_path, "w") as zf: try: exercise_context = { 'exercise': json.dumps(exercise_data, sort_keys=True, indent=4) } exercise_result = render_to_string('perseus/exercise.json', exercise_context) write_to_zipfile("exercise.json", exercise_result, zf) for question in ccnode.assessment_items.prefetch_related('files').all().order_by('order'): try: for image in question.files.filter(preset_id=format_presets.EXERCISE_IMAGE).order_by('checksum'): image_name = "images/{}.{}".format(image.checksum, image.file_format_id) if image_name not in zf.namelist(): with storage.open(ccmodels.generate_object_storage_name(image.checksum, str(image)), 'rb') as content: write_to_zipfile(image_name, content.read(), zf) for image in question.files.filter(preset_id=format_presets.EXERCISE_GRAPHIE).order_by('checksum'): svg_name = "images/{0}.svg".format(image.original_filename) json_name = "images/{0}-data.json".format(image.original_filename) if svg_name not in zf.namelist() or json_name not in zf.namelist(): with storage.open(ccmodels.generate_object_storage_name(image.checksum, str(image)), 'rb') as content: content = content.read() content = content.split(exercises.GRAPHIE_DELIMITER) write_to_zipfile(svg_name, content[0], zf) write_to_zipfile(json_name, content[1], zf) write_assessment_item(question, zf) except Exception as e: logging.error("Publishing error: {}".format(str(e))) finally: zf.close()
def create_perseus_zip(ccnode, exercise_data, write_to_path): with zipfile.ZipFile(write_to_path, "w") as zf: try: exercise_context = { 'exercise': json.dumps(exercise_data, sort_keys=True, indent=4) } exercise_result = render_to_string('perseus/exercise.json', exercise_context) write_to_zipfile("exercise.json", exercise_result, zf) for question in ccnode.assessment_items.prefetch_related( 'files').all().order_by('order'): try: for image in question.files.filter( preset_id=format_presets.EXERCISE_IMAGE).order_by( 'checksum'): image_name = "images/{}.{}".format( image.checksum, image.file_format_id) if image_name not in zf.namelist(): with storage.open( ccmodels.generate_object_storage_name( image.checksum, str(image)), 'rb') as content: write_to_zipfile(image_name, content.read(), zf) for image in question.files.filter( preset_id=format_presets.EXERCISE_GRAPHIE ).order_by('checksum'): svg_name = "images/{0}.svg".format( image.original_filename) json_name = "images/{0}-data.json".format( image.original_filename) if svg_name not in zf.namelist( ) or json_name not in zf.namelist(): with storage.open( ccmodels.generate_object_storage_name( image.checksum, str(image)), 'rb') as content: content = content.read() # in Python 3, delimiter needs to be in bytes format content = content.split( exercises.GRAPHIE_DELIMITER.encode( 'ascii')) write_to_zipfile(svg_name, content[0], zf) write_to_zipfile(json_name, content[1], zf) write_assessment_item(question, zf) except Exception as e: logging.error("Publishing error: {}".format(str(e))) logging.error(traceback.format_exc()) # In production, these errors have historically been handled silently. # Retain that behavior for now, but raise an error locally so we can # better understand the cases in which this might happen. report_exception(e) if os.environ.get('BRANCH_ENVIRONMENT', '') != "master": raise finally: zf.close()
def test_can_upload_file_to_presigned_url(self): """ Test that we can get a 200 OK when we upload a file to the URL returned by get_presigned_upload_url. """ file_contents = b"blahfilecontents" file = BytesIO(file_contents) # S3 expects a base64-encoded MD5 checksum md5 = hashlib.md5(file_contents) md5_checksum = md5.hexdigest() md5_checksum_base64 = codecs.encode(codecs.decode(md5_checksum, "hex"), "base64").decode() filename = "blahfile.jpg" filepath = generate_object_storage_name(md5_checksum, filename) ret = get_presigned_upload_url(filepath, md5_checksum_base64, 1000, len(file_contents)) url = ret["uploadURL"] content_type = ret["mimetype"] resp = requests.put(url, data=file, headers={ "Content-Type": content_type, }) resp.raise_for_status()
def map_files_to_assessment_item(user, assessment_item, data): """ Generate files referenced in given assesment item (a.k.a. question). """ if settings.DEBUG: # assert that our parameters match expected values assert isinstance(user, User) assert isinstance(assessment_item, AssessmentItem) assert isinstance(data, list) # filter out file that are empty valid_data = filter_out_nones(data) for file_data in valid_data: filename = file_data["filename"] checksum, ext = filename.split(".") file_path = generate_object_storage_name(checksum, filename) storage = default_storage if not storage.exists(file_path): raise IOError('{} not found'.format(file_path)) resource_obj = File( checksum=checksum, assessment_item=assessment_item, file_format_id=ext, original_filename=file_data.get('original_filename') or 'file', source_url=file_data.get('source_url'), file_size=file_data['size'], preset_id=file_data["preset"], # assessment_item-files always have a preset uploaded_by=user, ) resource_obj.file_on_disk.name = file_path resource_obj.save()
def map_files_to_assessment_item(user, assessment_item, data): """ Generate files referenced in given assesment item (a.k.a. question). """ if settings.DEBUG: # assert that our parameters match expected values assert isinstance(user, User) assert isinstance(assessment_item, AssessmentItem) assert isinstance(data, list) # filter out file that are empty valid_data = filter_out_nones(data) for file_data in valid_data: filename = file_data["filename"] checksum, ext = filename.split(".") file_path = generate_object_storage_name(checksum, filename) storage = default_storage if not storage.exists(file_path): raise IOError('{} not found'.format(file_path)) resource_obj = File( checksum=checksum, assessment_item=assessment_item, file_format_id=ext, original_filename=file_data.get('original_filename') or 'file', source_url=file_data.get('source_url'), file_size=file_data['size'], preset_id=file_data[ "preset"], # assessment_item-files always have a preset uploaded_by=user, ) resource_obj.file_on_disk.name = file_path resource_obj.save()
def map_files_to_slideshow_slide_item(user, node, slides, files): """ Generate files referenced in given slideshow slide """ for file_data in files: filename = file_data["filename"] checksum, ext = filename.split(".") matching_slide = next( (slide for slide in slides if slide.metadata["checksum"] == checksum), None) if not matching_slide: # TODO(Jacob) Determine proper error type... raise it. print("NO MATCH") file_path = generate_object_storage_name(checksum, filename) storage = default_storage if not storage.exists(file_path): raise IOError('{} not found'.format(file_path)) file_obj = File(slideshow_slide=matching_slide, checksum=checksum, file_format_id=ext, original_filename=file_data.get("original_filename") or "file", source_url=file_data.get("source_url"), file_size=file_data["size"], preset_id=file_data["preset"], uploaded_by=user) file_obj.file_on_disk.name = file_path file_obj.save()
def update(self, instance, validated_data): ret = [] nodes_to_check = [] with transaction.atomic(): # Get files that have the same contentnode, preset, and language as the files that are now attached to this node for item in validated_data: file_obj = File.objects.get(pk=item['id']) file_obj.language_id = item.get('language') and item['language']['id'] file_obj.contentnode = item['contentnode'] if item['contentnode'] not in nodes_to_check: nodes_to_check.append(item['contentnode']) # Make sure file exists file_path = generate_object_storage_name(file_obj.checksum, str(file_obj)) if not default_storage.exists(file_path): raise OSError("Error: file {} was not found".format(str(file_obj))) # Replace existing files files_to_replace = item['contentnode'].files.exclude(pk=file_obj.pk)\ .filter(preset_id=file_obj.preset_id, language_id=file_obj.language_id) files_to_replace.delete() file_obj.save() ret.append(file_obj) # Remove items that are not in the validated data (file has been removed) for node in nodes_to_check: file_ids = [f['id'] for f in validated_data if f['contentnode'].pk == node.pk] node.files.exclude(pk__in=file_ids).delete() return ret
def process_image_strings(content, zf): image_list = [] content = content.replace(exercises.CONTENT_STORAGE_PLACEHOLDER, PERSEUS_IMG_DIR) for match in re.finditer(ur'!\[(?:[^\]]*)]\(([^\)]+)\)', content): img_match = re.search( ur'(.+/images/[^\s]+)(?:\s=([0-9\.]+)x([0-9\.]+))*', match.group(1)) if img_match: # Add any image files that haven't been written to the zipfile filename = img_match.group(1).split('/')[-1] checksum, ext = os.path.splitext(filename) image_name = "images/{}.{}".format(checksum, ext[1:]) if image_name not in zf.namelist(): with storage.open( ccmodels.generate_object_storage_name( checksum, filename), 'rb') as imgfile: write_to_zipfile(image_name, imgfile.read(), zf) # Add resizing data if img_match.group(2) and img_match.group(3): image_data = {'name': img_match.group(1)} image_data.update({'width': float(img_match.group(2))}) image_data.update({'height': float(img_match.group(3))}) image_list.append(image_data) content = content.replace(match.group(1), img_match.group(1))
def write_file_to_storage(fobj, check_valid=False, name=None): fobj.seek(0) # Make sure reading file from beginning # Check that hash is valid checksum = hashlib.md5() for chunk in iter(lambda: fobj.read(4096), b""): checksum.update(chunk) name = name or fobj._name or "" filename, ext = os.path.splitext(name) hashed_filename = checksum.hexdigest() full_filename = "{}{}".format(hashed_filename, ext.lower()) fobj.seek(0) if check_valid and hashed_filename != filename: raise SuspiciousOperation("Failed to upload file {0}: hash is invalid".format(name)) # Get location of file file_path = models.generate_object_storage_name(hashed_filename, full_filename) # Write file storage = default_storage if storage.exists(file_path): logging.info("{} exists in Google Cloud Storage, so it's not saved again.".format(file_path)) else: storage.save(file_path, fobj) return full_filename
def convert_channel_thumbnail(channel): """ encode_thumbnail: gets base64 encoding of thumbnail Args: thumbnail (str): file path or url to channel's thumbnail Returns: base64 encoding of thumbnail """ encoding = None if not channel.thumbnail or channel.thumbnail == '' or 'static' in channel.thumbnail: return "" if channel.thumbnail_encoding: thumbnail_data = ast.literal_eval(channel.thumbnail_encoding) if thumbnail_data.get("base64"): return thumbnail_data["base64"] checksum, ext = os.path.splitext(channel.thumbnail) with storage.open( ccmodels.generate_object_storage_name(checksum, channel.thumbnail), 'rb') as file_obj: with Image.open(file_obj) as image, tempfile.NamedTemporaryFile( suffix=ext, delete=False) as tempf: cover = resizeimage.resize_cover( image, [THUMBNAIL_DIMENSION, THUMBNAIL_DIMENSION]) cover.save(tempf.name, image.format) encoding = base64.b64encode(tempf.read()).decode('utf-8') tempname = tempf.name os.unlink(tempname) return "data:image/png;base64," + encoding
def setUp(self): super(FileThumbnailTestCase, self).setUp() self.thumbnail_fobj = create_thumbnail_from_base64(base64encoding()) filepath = generate_object_storage_name(self.thumbnail_fobj.checksum, str(self.thumbnail_fobj)) with default_storage.open(filepath, 'rb') as fobj: self.thumbnail_contents = fobj.read()
def download_file(filename, download_url=None, contentnode=None, assessment_item=None, preset=None, file_size=None, lang_id=None): checksum, extension = os.path.splitext(filename) extension = extension.lstrip('.') filepath = models.generate_object_storage_name(checksum, filename) # Download file if it hasn't already been downloaded if download_url and not default_storage.exists(filepath): buffer = BytesIO() response = requests.get('{}/content/storage/{}/{}/{}'.format( download_url, filename[0], filename[1], filename)) for chunk in response: buffer.write(chunk) checksum, _, filepath = write_raw_content_to_storage(buffer.getvalue(), ext=extension) buffer.close() # Save values to new file object file_obj = models.File( file_format_id=extension, file_size=file_size or default_storage.size(filepath), contentnode=contentnode, assessment_item=assessment_item, language_id=lang_id, preset_id=preset or "", ) file_obj.file_on_disk.name = filepath file_obj.save()
def fileobj_video(contents=None): """ Create an "mp4" video file on storage, and then create a File model pointing to it. if contents is given and is a string, then write said contents to the file. If not given, a random string is generated and set as the contents of the file. """ if contents: filecontents = contents else: filecontents = "".join(random.sample(string.printable, 20)) fileobj = StringIO(filecontents) digest = md5.new(filecontents).hexdigest() filename = "{}.mp4".format(digest) storage_file_path = cc.generate_object_storage_name(digest, filename) # Write out the file bytes on to object storage, with a filename specified with randomfilename default_storage.save(storage_file_path, fileobj) # then create a File object with that db_file_obj = mixer.blend(cc.File, file_format=fileformat_mp4(), preset=preset_video(), file_on_disk=storage_file_path) yield db_file_obj
def write_file_to_storage(fobj, check_valid=False, name=None): fobj.seek(0) # Make sure reading file from beginning # Check that hash is valid checksum = hashlib.md5() for chunk in iter(lambda: fobj.read(4096), b""): checksum.update(chunk) name = name or fobj._name or "" filename, ext = os.path.splitext(name) hashed_filename = checksum.hexdigest() full_filename = "{}{}".format(hashed_filename, ext.lower()) fobj.seek(0) if check_valid and hashed_filename != filename: raise SuspiciousOperation("Failed to upload file {0}: hash is invalid".format(name)) # Get location of file file_path = models.generate_object_storage_name(hashed_filename, full_filename) # Write file storage = default_storage if storage.exists(file_path): logging.info("{} exists in Google Cloud Storage, so it's not saved again.".format(file_path)) else: storage.save(file_path, fobj) return full_filename
def create_temp_file(filebytes, kind='text', ext='txt', mimetype='text/plain'): """ Create a file and store it in Django's object db temporarily for tests. :param filebytes: The data to be stored in the file, as a series of bytes :param kind: String identifying the kind of file :param ext: File extension, omitting the initial period :param mimetype: Mimetype of the file :return: A dict containing the keys name (filename), data (actual bytes), file (StringIO obj) and db_file (File object in db) of the temp file. """ fileobj = StringIO(filebytes) checksum = hashlib.md5(filebytes) digest = checksum.hexdigest() filename = "{}.{}".format(digest, ext) storage_file_path = cc.generate_object_storage_name(digest, filename) # Write out the file bytes on to object storage, with a filename specified with randomfilename default_storage.save(storage_file_path, fileobj) assert default_storage.exists(storage_file_path) file_kind = mixer.blend(cc.ContentKind, kind=kind) file_format = mixer.blend(cc.FileFormat, extension=ext, mimetype=mimetype) preset = mixer.blend(cc.FormatPreset, id=ext, kind=file_kind) # then create a File object with that db_file_obj = mixer.blend(cc.File, file_format=file_format, preset=preset, file_on_disk=storage_file_path) return {'name': os.path.basename(storage_file_path), 'data': filebytes, 'file': fileobj, 'db_file': db_file_obj}
def create_test_file(filebytes, ext='pdf'): """ Create a temporary file with contents of `filebytes` for use in tests. :param filebytes: The data to be stored in the file (as bytes) :param ext: File extension, omitting the initial period Returns a dict containing the following: - checksum (str): md5 hash of file contents - name (str): the filename within the content storage system (= checksum + . + ext ) - storagepath (str): the relative storage path for this file storage/c/h/checksum.ext - data (bytes): file content (echo of `filebytes`) - file (file): an instance of TemporaryFile object that you can read/write """ hash = hashlib.md5(filebytes) checksum = hash.hexdigest() filename = "{}.{}".format(checksum, ext) storage_file_path = cc.generate_object_storage_name(checksum, filename) fileobj = TemporaryFile() fileobj.write(filebytes) fileobj.seek(0) return { 'checksum': checksum, 'name': os.path.basename(storage_file_path), 'storagepath': storage_file_path, 'data': filebytes, 'file': fileobj }
def create_temp_file(filebytes, preset='document', ext='pdf', original_filename=None): """ Create a file and store it in Django's object db temporarily for tests. :param filebytes: The data to be stored in the file (as bytes) :param preset: String identifying the format preset (defaults to ``document``) :param ext: File extension, omitting the initial period :param original_filename: Original filename (needed for exercise_images) :return: A dict containing the keys name (filename), data (actual bytes), file (StringIO obj) and db_file (File object in db) of the temp file. """ fileobj = StringIO(filebytes) hash = hashlib.md5(filebytes) checksum = hash.hexdigest() filename = "{}.{}".format(checksum, ext) storage_file_path = cc.generate_object_storage_name(checksum, filename) # 1. Write out the file bytes on to object storage default_storage.save(storage_file_path, fileobj) assert default_storage.exists(storage_file_path) # 2. Get the minimum required Studio meta fields for a File object preset = cc.FormatPreset.objects.get(id=preset) file_format = cc.FileFormat.objects.get(extension=ext) if original_filename is None: original_filename = 'somefile.' + ext # 3. Create a File object db_file_obj = mixer.blend(cc.File, checksum=checksum, file_format=file_format, preset=preset, original_filename=original_filename, file_on_disk=storage_file_path) return {'name': os.path.basename(storage_file_path), 'data': filebytes, 'file': fileobj, 'db_file': db_file_obj}
def setUp(self): super(GetFileDiffTestCase, self).setUp() storage = default_storage # Upload some pieces of content, as our test data self.existing_content = "dowereallyexist.jpg" self.existing_content_path = generate_object_storage_name("dowereallyexist", self.existing_content) storage.save(self.existing_content_path, StringIO("maybe"))
def check_file_url(f): filepath = generate_object_storage_name(os.path.splitext(f)[0], f) url = "/".join([ settings.AWS_S3_ENDPOINT_URL, settings.AWS_S3_BUCKET_NAME, filepath ]) resp = session.head(url) if resp.status_code != 200: ret.append(f)
def map_files_to_node(user, node, data): """ Generate files that reference the content node. """ if settings.DEBUG: # assert that our parameters match expected values assert isinstance(user, User) assert isinstance(node, ContentNode) assert isinstance(data, list) # filter out file that are empty valid_data = filter_out_nones(data) for file_data in valid_data: filename = file_data["filename"] checksum, ext1 = os.path.splitext(filename) ext = ext1.lstrip(".") # Determine a preset if none is given kind_preset = FormatPreset.get_preset(file_data["preset"]) or FormatPreset.guess_format_preset(filename) file_path = generate_object_storage_name(checksum, filename) storage = default_storage if not storage.exists(file_path): raise IOError('{} not found'.format(file_path)) try: if file_data.get('language'): # TODO: Remove DB call per file? file_data['language'] = Language.objects.get(pk=file_data['language']) except ObjectDoesNotExist: invalid_lang = file_data.get('language') logging.warning("file_data with language {} does not exist.".format(invalid_lang)) return ValidationError("file_data given was invalid; expected string, got {}".format(invalid_lang)) resource_obj = File( checksum=checksum, contentnode=node, file_format_id=ext, original_filename=file_data.get('original_filename') or 'file', source_url=file_data.get('source_url'), file_size=file_data['size'], preset=kind_preset, language_id=file_data.get('language'), uploaded_by=user, ) resource_obj.file_on_disk.name = file_path resource_obj.save() # Handle thumbnail if resource_obj.preset and resource_obj.preset.thumbnail: node.thumbnail_encoding = json.dumps({ 'base64': get_thumbnail_encoding(str(resource_obj)), 'points': [], 'zoom': 0 }) node.save()
def map_files_to_node(user, node, data): """ Generate files that reference the content node. """ if settings.DEBUG: # assert that our parameters match expected values assert isinstance(user, User) assert isinstance(node, ContentNode) assert isinstance(data, list) # filter out file that are empty valid_data = filter_out_nones(data) for file_data in valid_data: filename = file_data["filename"] checksum, ext1 = os.path.splitext(filename) ext = ext1.lstrip(".") # Determine a preset if none is given kind_preset = FormatPreset.get_preset(file_data["preset"]) or FormatPreset.guess_format_preset(filename) file_path = generate_object_storage_name(checksum, filename) storage = default_storage if not storage.exists(file_path): raise IOError('{} not found'.format(file_path)) try: if file_data.get('language'): # TODO: Remove DB call per file? file_data['language'] = Language.objects.get(pk=file_data['language']) except ObjectDoesNotExist: invalid_lang = file_data.get('language') logging.warning("file_data with language {} does not exist.".format(invalid_lang)) return ValidationError("file_data given was invalid; expected string, got {}".format(invalid_lang)) resource_obj = File( checksum=checksum, contentnode=node, file_format_id=ext, original_filename=file_data.get('original_filename') or 'file', source_url=file_data.get('source_url'), file_size=file_data['size'], preset=kind_preset, language_id=file_data.get('language'), uploaded_by=user, ) resource_obj.file_on_disk.name = file_path resource_obj.save() # Handle thumbnail if resource_obj.preset and resource_obj.preset.thumbnail: node.thumbnail_encoding = json.dumps({ 'base64': get_thumbnail_encoding(str(resource_obj)), 'points': [], 'zoom': 0 }) node.save()
def setUp(self): super(GetFileDiffTestCase, self).setUp() storage = default_storage # Upload some pieces of content, as our test data self.existing_content = "dowereallyexist.jpg" self.existing_content_path = generate_object_storage_name( "dowereallyexist", self.existing_content) storage.save(self.existing_content_path, StringIO("maybe"))
def create_studio_file(filebytes, preset='document', ext='pdf', original_filename=None): """ Create a file with contents of `filebytes` and the associated cc.File object for it. :param filebytes: The data to be stored in the file (as bytes) :param preset: String identifying the format preset (defaults to ``document``) :param ext: File extension, omitting the initial period :param original_filename: Original filename (needed for exercise_images) Returns a dict containing the following: - name (str): the filename within the content storage system (= md5 hash of the contents + .ext ) - data (bytes): file content (echo of `filebytes`) - file (file): a basic BytesIO file-like object that you can read/write - db_file (cc.File): a Studio File object saved in DB """ try: filebytes = filebytes.encode('utf-8') except: # noqa pass fileobj = BytesIO(filebytes) # Every time the BytesIO object is read from or appended to, we need to reset the seek position, # otherwise, it will start reading from the end of the file. fileobj.seek(0) hash = hashlib.md5(filebytes) checksum = hash.hexdigest() filename = "{}.{}".format(checksum, ext) storage_file_path = cc.generate_object_storage_name(checksum, filename) # 1. Write out the file bytes on to object storage fileobj.seek(0) default_storage.save(storage_file_path, fileobj) fileobj.seek(0) assert default_storage.exists(storage_file_path) # 2. Get the minimum required Studio meta fields for a File object preset = cc.FormatPreset.objects.get(id=preset) file_format = cc.FileFormat.objects.get(extension=ext) if original_filename is None: original_filename = 'somefile.' + ext # 3. Create a File object db_file_obj = mixer.blend(cc.File, checksum=checksum, file_format=file_format, preset=preset, original_filename=original_filename, file_on_disk=storage_file_path) return { 'name': os.path.basename(storage_file_path), 'data': filebytes, 'file': fileobj, 'db_file': db_file_obj }
def test_generate_object_storage_name(object_storage_name_tests): for filename, checksum, file_format_id, expected_name in object_storage_name_tests: default_ext = '' if file_format_id: default_ext = '.{}'.format(file_format_id) actual_name = generate_object_storage_name(checksum, filename, default_ext) assert actual_name == expected_name, \ "Storage names don't match: Expected: '{}' Actual '{}'".format(expected_name, actual_name)
def debug_serve_file(request, path): # There's a problem with loading exercise images, so use this endpoint # to serve the image files to the /content/storage url filename = os.path.basename(path) checksum, _ext = os.path.splitext(filename) filepath = generate_object_storage_name(checksum, filename) if not default_storage.exists(filepath): raise Http404("The object requested does not exist.") with default_storage.open(filepath, 'rb') as fobj: response = HttpResponse(FileWrapper(fobj), content_type="application/octet-stream") return response
def debug_serve_file(request, path): # There's a problem with loading exercise images, so use this endpoint # to serve the image files to the /content/storage url filename = os.path.basename(path) checksum, _ext = os.path.splitext(filename) filepath = generate_object_storage_name(checksum, filename) if not default_storage.exists(filepath): raise Http404("The object requested does not exist.") with default_storage.open(filepath, 'rb') as fobj: response = HttpResponse(FileWrapper(fobj), content_type="application/octet-stream") return response
def map_files_to_node(user, node, data): """ Generate files that reference the content node """ # filter for file data that's not empty; valid_data = (d for d in data if d) for file_data in valid_data: file_name_parts = file_data['filename'].split(".") # Determine a preset if none is given kind_preset = None if file_data['preset'] is None: kind_preset = FormatPreset.objects.filter( kind=node.kind, allowed_formats__extension__contains=file_name_parts[1], display=True).first() else: kind_preset = FormatPreset.objects.get(id=file_data['preset']) file_path = generate_object_storage_name(file_name_parts[0], file_data['filename']) storage = default_storage if not storage.exists(file_path): return IOError('{} not found'.format(file_path)) try: if file_data.get('language'): # TODO: Remove DB call per file? file_data['language'] = Language.objects.get( pk=file_data['language']) except ObjectDoesNotExist as e: invalid_lang = file_data.get('language') logging.warning( "file_data with language {} does not exist.".format( invalid_lang)) return ValidationError( "file_data given was invalid; expected string, got {}".format( invalid_lang)) resource_obj = File( checksum=file_name_parts[0], contentnode=node, file_format_id=file_name_parts[1], original_filename=file_data.get('original_filename') or 'file', source_url=file_data.get('source_url'), file_size=file_data['size'], file_on_disk=DjFile(storage.open(file_path, 'rb')), preset=kind_preset, language_id=file_data.get('language'), uploaded_by=user, ) resource_obj.file_on_disk.name = file_path resource_obj.save()
def create_files(cursor, contentnode, indent=0, download_url=None): """ create_files: Get license Args: cursor (sqlite3.Connection): connection to export database contentnode (models.ContentNode): node file references indent (int): How far to indent print statements Returns: None """ # Parse database for files referencing content node and make file models sql_command = 'SELECT checksum, extension, file_size, contentnode_id, '\ 'lang_id, preset FROM {table} WHERE contentnode_id=\'{id}\';'\ .format(table=FILE_TABLE, id=contentnode.node_id) query = cursor.execute(sql_command).fetchall() for checksum, extension, file_size, contentnode_id, lang_id, preset in query: filename = "{}.{}".format(checksum, extension) print("{indent} * FILE {filename}...".format(indent=" |" * indent, filename=filename)) try: filepath = models.generate_object_storage_name(checksum, filename) # Download file first if download_url and not default_storage.exists(filepath): buffer = StringIO() response = requests.get('{}/content/storage/{}/{}/{}'.format( download_url, filename[0], filename[1], filename)) for chunk in response: buffer.write(chunk) create_file_from_contents(buffer.getvalue(), ext=extension, node=contentnode, preset_id=preset or "") buffer.close() else: # Save values to new or existing file object file_obj = models.File( file_format_id=extension, file_size=file_size, contentnode=contentnode, language_id=lang_id, preset_id=preset or "", ) file_obj.file_on_disk.name = filepath file_obj.save() except IOError as e: logging.warning("\b FAILED (check logs for more details)") sys.stderr.write( "Restoration Process Error: Failed to save file object {}: {}". format(filename, os.strerror(e.errno))) continue
def get_thumbnail_encoding(filename, dimension=THUMBNAIL_DIMENSION): """ Generates a base64 encoding for a thumbnail Args: filename (str): thumbnail to generate encoding from (must be in storage already) dimension (int): how big resized image should be Returns base64 encoding of resized thumbnail """ if filename.startswith("data:image"): return filename checksum, ext = os.path.splitext(filename.split("?")[0]) inbuffer = StringIO.StringIO() outbuffer = StringIO.StringIO() try: if not filename.startswith(settings.STATIC_ROOT): filename = generate_object_storage_name(checksum, filename) with default_storage.open(filename) as localtempf: inbuffer.write(localtempf.read()) else: with open(filename, 'rb') as fobj: inbuffer.write(fobj.read()) with Image.open(inbuffer) as image: image_format = image.format width, height = image.size dimension = min([dimension, width, height]) size = [dimension, dimension] ratio = Fraction(*size) # Crop image the aspect ratio is different if width > ratio * height: x, y = (width - ratio * height) // 2, 0 else: x, y = 0, (height - width / ratio) // 2 image = image.crop((x, y, width - x, height - y)) if image.size > size: image.thumbnail(size, Image.ANTIALIAS) else: image.thumbnail((dimension, dimension), Image.ANTIALIAS) image.save(outbuffer, image_format) return "data:image/{};base64,{}".format( ext[1:], base64.b64encode(outbuffer.getvalue())) finally: inbuffer.close() outbuffer.close()
def get_thumbnail_encoding(filename, dimension=THUMBNAIL_DIMENSION): """ Generates a base64 encoding for a thumbnail Args: filename (str): thumbnail to generate encoding from (must be in storage already) dimension (int): how big resized image should be Returns base64 encoding of resized thumbnail """ if filename.startswith("data:image"): return filename checksum, ext = os.path.splitext(filename.split("?")[0]) inbuffer = StringIO.StringIO() outbuffer = StringIO.StringIO() try: if not filename.startswith(settings.STATIC_ROOT): filename = generate_object_storage_name(checksum, filename) with default_storage.open(filename) as localtempf: inbuffer.write(localtempf.read()) else: with open(filename, 'rb') as fobj: inbuffer.write(fobj.read()) with Image.open(inbuffer) as image: image_format = image.format width, height = image.size dimension = min([dimension, width, height]) size = [dimension, dimension] ratio = Fraction(*size) # Crop image the aspect ratio is different if width > ratio * height: x, y = (width - ratio * height) // 2, 0 else: x, y = 0, (height - width / ratio) // 2 image = image.crop((x, y, width - x, height - y)) if image.size > size: image.thumbnail(size, Image.ANTIALIAS) else: image.thumbnail((dimension, dimension), Image.ANTIALIAS) image.save(outbuffer, image_format) return "data:image/{};base64,{}".format(ext[1:], base64.b64encode(outbuffer.getvalue())) finally: inbuffer.close() outbuffer.close()
def get_thumbnail_encoding(filename, dimension=THUMBNAIL_WIDTH): """ Generates a base64 encoding for a thumbnail Args: filename (str): thumbnail to generate encoding from (must be in storage already) dimension (int, optional): desired width of thumbnail. Defaults to 400. Returns base64 encoding of resized thumbnail """ if filename.startswith("data:image"): return filename checksum, ext = os.path.splitext(filename.split("?")[0]) outbuffer = BytesIO() # make sure the aspect ratio between width and height is 16:9 thumbnail_size = [dimension, round(dimension / 1.77)] try: if not filename.startswith(settings.STATIC_ROOT): filename = generate_object_storage_name(checksum, filename) inbuffer = default_storage.open(filename, 'rb') else: inbuffer = open(filename, 'rb') assert inbuffer with Image.open(inbuffer) as image: image_format = image.format # Note: Image.thumbnail ensures that the image will fit in the # specified thumbnail size, but it retains the original image's # aspect ratio. So a square image will remain square rather # than being distorted to a 16:9 aspect ratio. This removes # the need to make any changes like cropping the image. image.thumbnail(thumbnail_size, Image.ANTIALIAS) image.save(outbuffer, image_format) return "data:image/{};base64,{}".format( ext[1:], base64.b64encode(outbuffer.getvalue()).decode('utf-8')) finally: # Try to close the inbuffer if it has been created try: inbuffer.close() except UnboundLocalError: pass outbuffer.close()
def write_raw_content_to_storage(contents, ext=None): # Check that hash is valid checksum = hashlib.md5() checksum.update(contents) hashed_filename = checksum.hexdigest() full_filename = "{}.{}".format(hashed_filename, ext.lower()) # Get location of file file_path = models.generate_object_storage_name(hashed_filename, full_filename) # Write file storage = default_storage storage.save(file_path, StringIO(contents)) return hashed_filename, full_filename, file_path
def get_file_diff(files): """Given a list of filenames as strings, find the filenames that aren't in our storage, and return. """ storage = default_storage # Try to be storage system agnostic, in case we're using either the Object Storage, # or FileSystemStorage ret = [] for f in files: filepath = generate_object_storage_name(os.path.splitext(f)[0], f) if not storage.exists(filepath) or storage.size(filepath) == 0: ret.append(f) return ret
def upload_url(self, request): try: size = request.data["size"] checksum = request.data["checksum"] filename = request.data["name"] file_format = request.data["file_format"] preset = request.data["preset"] except KeyError: raise HttpResponseBadRequest( reason= "Must specify: size, checksum, name, file_format, and preset") try: request.user.check_space(float(size), checksum) except PermissionDenied as e: return HttpResponseBadRequest(reason=str(e), status=418) might_skip = File.objects.filter(checksum=checksum).exists() filepath = generate_object_storage_name(checksum, filename) checksum_base64 = codecs.encode(codecs.decode(checksum, "hex"), "base64").decode() retval = get_presigned_upload_url(filepath, checksum_base64, 600, content_length=size) file = File( file_size=size, checksum=checksum, original_filename=filename, file_on_disk=filepath, file_format_id=file_format, preset_id=preset, uploaded_by=request.user, ) # Avoid using our file_on_disk attribute for checks file.save(set_by_file_on_disk=False) retval.update({ "might_skip": might_skip, "file": self.serialize_object(id=file.id) }) return Response(retval)
def write_raw_content_to_storage(contents, ext=None): # Check that hash is valid checksum = hashlib.md5() checksum.update(contents) hashed_filename = checksum.hexdigest() full_filename = "{}.{}".format(hashed_filename, ext.lower()) # Get location of file file_path = models.generate_object_storage_name(hashed_filename, full_filename) # Write file storage = default_storage if storage.exists(file_path): logging.info("{} exists in Google Cloud Storage, so it's not saved again.".format(file_path)) else: storage.save(file_path, StringIO(contents)) return hashed_filename, full_filename, file_path
def write_raw_content_to_storage(contents, ext=None): # Check that hash is valid checksum = hashlib.md5() checksum.update(contents) hashed_filename = checksum.hexdigest() full_filename = "{}.{}".format(hashed_filename, ext.lower()) # Get location of file file_path = models.generate_object_storage_name(hashed_filename, full_filename) # Write file storage = default_storage if storage.exists(file_path): logging.info("{} exists in Google Cloud Storage, so it's not saved again.".format(file_path)) else: storage.save(file_path, StringIO(contents)) return hashed_filename, full_filename, file_path
def create_temp_file(filebytes, preset='document', ext='pdf', original_filename=None): """ Create a file and store it in Django's object db temporarily for tests. :param filebytes: The data to be stored in the file (as bytes) :param preset: String identifying the format preset (defaults to ``document``) :param ext: File extension, omitting the initial period :param original_filename: Original filename (needed for exercise_images) :return: A dict containing the keys name (filename), data (actual bytes), file (StringIO obj) and db_file (File object in db) of the temp file. """ fileobj = StringIO(filebytes) hash = hashlib.md5(filebytes) checksum = hash.hexdigest() filename = "{}.{}".format(checksum, ext) storage_file_path = cc.generate_object_storage_name(checksum, filename) # 1. Write out the file bytes on to object storage default_storage.save(storage_file_path, fileobj) assert default_storage.exists(storage_file_path) # 2. Get the minimum required Studio meta fields for a File object preset = cc.FormatPreset.objects.get(id=preset) file_format = cc.FileFormat.objects.get(extension=ext) if original_filename is None: original_filename = 'somefile.' + ext # 3. Create a File object db_file_obj = mixer.blend(cc.File, checksum=checksum, file_format=file_format, preset=preset, original_filename=original_filename, file_on_disk=storage_file_path) return { 'name': os.path.basename(storage_file_path), 'data': filebytes, 'file': fileobj, 'db_file': db_file_obj }
def process_image_strings(content, zf): image_list = [] content = content.replace(exercises.CONTENT_STORAGE_PLACEHOLDER, PERSEUS_IMG_DIR) for match in re.finditer(ur'!\[(?:[^\]]*)]\(([^\)]+)\)', content): img_match = re.search(ur'(.+/images/[^\s]+)(?:\s=([0-9\.]+)x([0-9\.]+))*', match.group(1)) if img_match: # Add any image files that haven't been written to the zipfile filename = img_match.group(1).split('/')[-1] checksum, ext = os.path.splitext(filename) image_name = "images/{}.{}".format(checksum, ext[1:]) if image_name not in zf.namelist(): with storage.open(ccmodels.generate_object_storage_name(checksum, filename), 'rb') as imgfile: write_to_zipfile(image_name, imgfile.read(), zf) # Add resizing data if img_match.group(2) and img_match.group(3): image_data = {'name': img_match.group(1)} image_data.update({'width': float(img_match.group(2))}) image_data.update({'height': float(img_match.group(3))}) image_list.append(image_data) content = content.replace(match.group(1), img_match.group(1))
def get_node_data_from_file(file_name): file_path = generate_object_storage_name(file_name.split('.')[0], file_name) if not default_storage.exists(file_path): raise IOError('{} not found.'.format(file_path)) with default_storage.open(file_path, 'rb') as file_obj: node_data = json.loads(file_obj.read().decode('utf-8')) if node_data is None: raise IOError('{} is empty or could not be read.'.format(file_path)) # Make sure license is valid license = None license_name = node_data['license'] if license_name is not None: try: license = License.objects.get(license_name__iexact=license_name) except ObjectDoesNotExist: raise ObjectDoesNotExist('Invalid license found') node_data['license'] = license return node_data
def map_files_to_assessment_item(user, question, data): """ Generate files that reference the content node's assessment items """ for file_data in data: file_name_parts = file_data['filename'].split(".") file_path = generate_object_storage_name(file_name_parts[0], file_data['filename']) if not os.path.isfile(file_path): return IOError('{} not found'.format(file_path)) resource_obj = File( checksum=file_name_parts[0], assessment_item=question, file_format_id=file_name_parts[1], original_filename=file_data.get('original_filename') or 'file', source_url=file_data.get('source_url'), file_size=file_data['size'], file_on_disk=DjFile(open(file_path, 'rb')), preset_id=file_data['preset'], uploaded_by=user, ) resource_obj.file_on_disk.name = file_path resource_obj.save()
def fileobj_video(contents=None): """ Create an "mp4" video file on storage, and then create a File model pointing to it. if contents is given and is a string, then write said contents to the file. If not given, a random string is generated and set as the contents of the file. """ if contents: filecontents = contents else: filecontents = "".join(random.sample(string.printable, 20)) fileobj = StringIO(filecontents) digest = md5.new(filecontents).hexdigest() filename = "{}.mp4".format(digest) storage_file_path = cc.generate_object_storage_name(digest, filename) # Write out the file bytes on to object storage, with a filename specified with randomfilename default_storage.save(storage_file_path, fileobj) # then create a File object with that db_file_obj = mixer.blend(cc.File, file_format=fileformat_mp4(), preset=preset_video(), file_on_disk=storage_file_path) return db_file_obj
def check_file_url(f): filepath = generate_object_storage_name(os.path.splitext(f)[0], f) url = "/".join([settings.AWS_S3_ENDPOINT_URL, settings.AWS_S3_BUCKET_NAME, filepath]) resp = session.head(url) if resp.status_code != 200: ret.append(f)
def get(self, request, zipped_filename, embedded_filepath): """ Handles GET requests and serves a static file from within the zip file. """ if not VALID_STORAGE_FILENAME.match(zipped_filename): return HttpResponseNotFound("'{}' is not a valid URL for this zip file".format(zipped_filename)) storage = default_storage # calculate the local file path to the zip file filename, ext = os.path.splitext(zipped_filename) zipped_path = generate_object_storage_name(filename, zipped_filename) # file size file_size = 0 # if the zipfile does not exist on disk, return a 404 if not storage.exists(zipped_path): return HttpResponseNotFound('"%(filename)s" does not exist in storage' % {'filename': zipped_path}) # if client has a cached version, use that (we can safely assume nothing has changed, due to MD5) if request.META.get('HTTP_IF_MODIFIED_SINCE'): return HttpResponseNotModified() zf_obj = storage.open(zipped_path) try: with zipfile.ZipFile(zf_obj) as zf: # if no path, or a directory, is being referenced, look for an index.html file if not embedded_filepath or embedded_filepath.endswith("/"): embedded_filepath += "index.html" # get the details about the embedded file, and ensure it exists try: info = zf.getinfo(embedded_filepath) except KeyError: return HttpResponseNotFound('"{}" does not exist inside "{}"'.format(embedded_filepath, zipped_filename)) # try to guess the MIME type of the embedded file being referenced content_type = mimetypes.guess_type(embedded_filepath)[0] or 'application/octet-stream' if not os.path.splitext(embedded_filepath)[1] == '.json': # generate a streaming response object, pulling data from within the zip file response = FileResponse(zf.open(info), content_type=content_type) file_size = info.file_size else: # load the stream from json file into memory, replace the path_place_holder. content = zf.open(info).read() str_to_be_replaced = ('$' + exercises.IMG_PLACEHOLDER).encode() zipcontent = ('/' + request.resolver_match.url_name + "/" + zipped_filename).encode() content_with_path = content.replace(str_to_be_replaced, zipcontent) response = HttpResponse(content_with_path, content_type=content_type) file_size = len(content_with_path) except zipfile.BadZipfile: just_downloaded = getattr(zf_obj, 'just_downloaded', "Unknown (Most likely local file)") client.captureMessage("Unable to open zip file. File info: name={}, size={}, mode={}, just_downloaded={}".format( zf_obj.name, zf_obj.size, zf_obj.mode, just_downloaded)) return HttpResponseServerError("Attempt to open zip file failed. Please try again, and if you continue to receive this message, please check that the zip file is valid.") # set the last-modified header to the date marked on the embedded file if info.date_time: response["Last-Modified"] = http_date(time.mktime(datetime.datetime(*info.date_time).timetuple())) # cache these resources forever; this is safe due to the MD5-naming used on content files response["Expires"] = "Sun, 17-Jan-2038 19:14:07 GMT" # set the content-length header to the size of the embedded file if file_size: response["Content-Length"] = file_size # ensure the browser knows not to try byte-range requests, as we don't support them here response["Accept-Ranges"] = "none" _add_access_control_headers(request, response) # restrict CSP to only allow resources to be loaded from the Studio host, to prevent info leakage # (e.g. via passing user info out as GET parameters to an attacker's server), or inadvertent data usage host = request.build_absolute_uri('/').strip("/") response["Content-Security-Policy"] = "default-src 'self' 'unsafe-inline' 'unsafe-eval' data: " + host return response
def setUp(self): super(FileThumbnailTestCase, self).setUp() self.thumbnail_fobj = create_thumbnail_from_base64(base64encoding()) filepath = generate_object_storage_name(self.thumbnail_fobj.checksum, str(self.thumbnail_fobj)) with default_storage.open(filepath, 'rb') as fobj: self.thumbnail_contents = fobj.read()