def image_upload(request): if request.method != 'POST': return HttpResponseBadRequest( "Only POST requests are allowed on this endpoint.") fobj = request.FILES.values()[0] name, ext = os.path.splitext( fobj._name) # gets file extension without leading period checksum = get_hash(DjFile(fobj)) request.user.check_space(fobj._size, checksum) file_object = File(contentnode_id=request.META.get('HTTP_NODE'), original_filename=name, preset_id=request.META.get('HTTP_PRESET'), file_on_disk=DjFile(request.FILES.values()[0]), file_format_id=ext[1:].lower(), uploaded_by=request.user) file_object.save() return HttpResponse( json.dumps({ "success": True, "file": JSONRenderer().render(FileSerializer(file_object).data), "path": generate_storage_url(str(file_object)), }))
def exercise_image_upload(request): if request.method != 'POST': return HttpResponseBadRequest( "Only POST requests are allowed on this endpoint.") fobj = request.FILES.values()[0] name, ext = os.path.splitext(fobj._name) checksum = get_hash(DjFile(fobj)) file_object = File( preset_id=format_presets.EXERCISE_IMAGE, file_on_disk=DjFile(request.FILES.values()[0]), file_format_id=ext[1:].lower(), ) file_object.save() return HttpResponse( json.dumps({ "success": True, "formatted_filename": exercises.CONTENT_STORAGE_FORMAT.format(str(file_object)), "file_id": file_object.pk, "path": generate_storage_url(str(file_object)), }))
def file_upload(request): if request.method == 'POST': # Implement logic for switching out files without saving it yet filename, ext = os.path.splitext(request.FILES.values()[0]._name) size = request.FILES.values()[0]._size contentfile = DjFile(request.FILES.values()[0]) checksum = get_hash(contentfile) request.user.check_space(size, checksum) file_object = File( file_size=size, file_on_disk=contentfile, checksum=checksum, file_format_id=ext[1:].lower(), original_filename=request.FILES.values()[0]._name, preset_id=request.META.get('HTTP_PRESET'), language_id=request.META.get('HTTP_LANGUAGE'), uploaded_by=request.user, ) file_object.save() return HttpResponse( json.dumps({ "success": True, "filename": str(file_object), "file": JSONRenderer().render(FileSerializer(file_object).data) }))
def create_file(display_name, preset_id, ext, user=None): with tempfile.NamedTemporaryFile(suffix=".{}".format(ext), mode="wb+", delete=False) as f: f.write(b":)") f.flush() size = f.tell() filename = write_file_to_storage(f, name=f.name) checksum, _ext = os.path.splitext(filename) f.seek(0) file_object = File( file_size=size, file_on_disk=DjFile(f), checksum=checksum, file_format_id=ext, original_filename=display_name, preset_id=preset_id, uploaded_by=user, language_id="mul" if FormatPreset.objects.filter( id=preset_id, multi_language=True).exists() else None, ) file_object.save() f.close() os.unlink(f.name) return file_object
def create_file(display_name, preset_id, ext, user=None): with tempfile.NamedTemporaryFile(suffix=".{}".format(ext), mode='w+t', delete=False) as f: f.write(":)") f.flush() size = f.tell() filename = write_file_to_storage(f, name=f.name) checksum, _ext = os.path.splitext(filename) f.seek(0) file_object = File( file_size=size, file_on_disk=DjFile(f), checksum=checksum, file_format_id=ext, original_filename=display_name, preset_id=preset_id, uploaded_by=user, ) file_object.save() f.close() os.unlink(f.name) return file_object
def create_file_from_contents(contents, ext=None, node=None, preset_id=None): checksum, filename, path = write_raw_content_to_storage(contents, ext=ext) with open(path, 'rb') as new_file: return File.objects.create(file_on_disk=DjFile(new_file), file_format_id=ext, file_size=os.path.getsize(path), checksum=checksum, preset_id=preset_id, contentnode=node)
def file_create(request): if request.method == 'POST': original_filename, ext = os.path.splitext( request.FILES.values()[0]._name) size = request.FILES.values()[0]._size presets = FormatPreset.objects.filter( allowed_formats__extension__contains=ext[1:].lower()) kind = presets.first().kind preferences = json.loads(request.user.preferences) author = preferences.get('author') or "" license = License.objects.filter( license_name=preferences.get('license')).first( ) # Use filter/first in case preference hasn't been set license_id = license.pk if license else settings.DEFAULT_LICENSE new_node = ContentNode( title=original_filename, kind=kind, license_id=license_id, author=author, copyright_holder=preferences.get('copyright_holder')) if license.license_name == licenses.SPECIAL_PERMISSIONS: new_node.license_description = preferences.get( 'license_description') new_node.save() file_object = File(file_on_disk=DjFile(request.FILES.values()[0]), file_format_id=ext[1:].lower(), original_filename=request.FILES.values()[0]._name, contentnode=new_node, file_size=size) file_object.save() if kind.pk == content_kinds.VIDEO: file_object.preset_id = guess_video_preset_by_resolution( str(file_object.file_on_disk)) elif presets.filter(supplementary=False).count() == 1: file_object.preset = presets.filter(supplementary=False).first() file_object.save() try: if preferences.get('auto_derive_video_thumbnail') and new_node.kind_id == content_kinds.VIDEO \ or preferences.get('auto_derive_audio_thumbnail') and new_node.kind_id == content_kinds.AUDIO \ or preferences.get('auto_derive_html5_thumbnail') and new_node.kind_id == content_kinds.HTML5 \ or preferences.get('auto_derive_document_thumbnail') and new_node.kind_id == content_kinds.DOCUMENT: generate_thumbnail_from_node(new_node, set_node=True) except Exception: pass return HttpResponse( json.dumps({ "success": True, "node": JSONRenderer().render( ContentNodeEditSerializer(new_node).data) }))
def exercise_image_upload(request): if request.method == 'POST': ext = os.path.splitext(request.FILES.values()[0]._name)[1][1:] # gets file extension without leading period file_object = File(preset_id=format_presets.EXERCISE_IMAGE, file_on_disk=DjFile(request.FILES.values()[0]), file_format_id=ext) file_object.save() return HttpResponse(json.dumps({ "success": True, "formatted_filename": exercises.CONTENT_STORAGE_FORMAT.format(str(file_object)), "file_id": file_object.pk, "path": generate_storage_url(str(file_object)), }))
def map_files_to_node(user, node, data): """ Generate files that reference the content node """ # filter for file data that's not empty; valid_data = (d for d in data if d) for file_data in valid_data: file_name_parts = file_data['filename'].split(".") # Determine a preset if none is given kind_preset = None if file_data['preset'] is None: kind_preset = FormatPreset.objects.filter( kind=node.kind, allowed_formats__extension__contains=file_name_parts[1], display=True).first() else: kind_preset = FormatPreset.objects.get(id=file_data['preset']) file_path = generate_object_storage_name(file_name_parts[0], file_data['filename']) storage = default_storage if not storage.exists(file_path): return IOError('{} not found'.format(file_path)) try: if file_data.get('language'): # TODO: Remove DB call per file? file_data['language'] = Language.objects.get( pk=file_data['language']) except ObjectDoesNotExist as e: invalid_lang = file_data.get('language') logging.warning( "file_data with language {} does not exist.".format( invalid_lang)) return ValidationError( "file_data given was invalid; expected string, got {}".format( invalid_lang)) resource_obj = File( checksum=file_name_parts[0], contentnode=node, file_format_id=file_name_parts[1], original_filename=file_data.get('original_filename') or 'file', source_url=file_data.get('source_url'), file_size=file_data['size'], file_on_disk=DjFile(storage.open(file_path, 'rb')), preset=kind_preset, language_id=file_data.get('language'), uploaded_by=user, ) resource_obj.file_on_disk.name = file_path resource_obj.save()
def create_file_from_contents(contents, ext=None, node=None, preset_id=None, uploaded_by=None): checksum, _, path = write_raw_content_to_storage(contents, ext=ext) with default_storage.open(path, 'rb') as new_file: return File.objects.create(file_on_disk=DjFile(new_file), file_format_id=ext, file_size=default_storage.size(path), checksum=checksum, preset_id=preset_id, contentnode=node, uploaded_by=uploaded_by)
def thumbnail_upload(request): if request.method == 'POST': fobj = request.FILES.values()[0] checksum = get_hash(DjFile(fobj)) request.user.check_space(fobj._size, checksum) formatted_filename = write_file_to_storage(fobj) return HttpResponse( json.dumps({ "success": True, "formatted_filename": formatted_filename, "file": None, "path": generate_storage_url(formatted_filename), }))
def update_content_copy(file_object=None, content_copy=None): """ Update the File object you pass in with the content copy You can pass None on content_copy to remove the associated file on disk. :param file_object: File :param content_copy: str """ if not file_object: raise TypeError("must provide a File object to update content copy") if content_copy: file_object.content_copy = DjFile(open(content_copy, 'rb')) else: file_object.content_copy = None file_object.save()
def compress_video_wrapper(file_object): with tempfile.NamedTemporaryFile(suffix=".{}".format(file_formats.MP4)) as tempf: tempf.close() compress_video(str(file_object.file_on_disk), tempf.name, overwrite=True) filename = write_file_to_storage(open(tempf.name, 'rb'), name=tempf.name) checksum, ext = os.path.splitext(filename) file_location = generate_file_on_disk_name(checksum, filename) low_res_object = File( file_on_disk=DjFile(open(file_location, 'rb')), file_format_id=file_formats.MP4, original_filename=file_object.original_filename, contentnode=file_object.contentnode, file_size=os.path.getsize(file_location), preset_id=format_presets.VIDEO_LOW_RES, ) low_res_object.save() return low_res_object
def thumbnail_upload(request): # Used for channels if request.method != 'POST': return HttpResponseBadRequest("Only POST requests are allowed on this endpoint.") fobj = request.FILES.values()[0] checksum = get_hash(DjFile(fobj)) request.user.check_space(fobj._size, checksum) formatted_filename = write_file_to_storage(fobj) return HttpResponse(json.dumps({ "success": True, "formatted_filename": formatted_filename, "file": None, "path": generate_storage_url(formatted_filename), "encoding": get_thumbnail_encoding(formatted_filename), }))
def map_files_to_node(node, data): """ Generate files that reference the content node """ # filter for file data that's not empty; valid_data = (d for d in data if d) for file_data in valid_data: file_hash = file_data['filename'].split(".") # Determine a preset if none is given kind_preset = None if file_data['preset'] is None: kind_preset = FormatPreset.objects.filter(kind=node.kind, allowed_formats__extension__contains=file_hash[1], display=True).first() else: kind_preset = FormatPreset.objects.get(id=file_data['preset']) file_path=generate_file_on_disk_name(file_hash[0], file_data['filename']) if not os.path.isfile(file_path): raise IOError('{} not found'.format(file_path)) language = None try: if file_data.get('language'): language = Language.objects.get(pk=file_data['language']) except ObjectDoesNotExist as e: invalid_lang = file_data.get('language') logging.warning("file_data with language {} does not exist.".format(invalid_lang)) raise ValidationError("file_data given was invalid; expected string, got {}".format(invalid_lang)) file_obj = File( checksum=file_hash[0], contentnode=node, file_format_id=file_hash[1], original_filename=file_data.get('original_filename') or 'file', source_url=file_data.get('source_url'), file_size = file_data['size'], file_on_disk=DjFile(open(file_path, 'rb')), preset=kind_preset, language_id=file_data.get('language'), ) file_obj.save()
def map_files_to_assessment_item(question, data): """ Generate files that reference the content node's assessment items """ for file_data in data: file_hash = file_data['filename'].split(".") file_path = generate_file_on_disk_name(file_hash[0], file_data['filename']) if not os.path.isfile(file_path): raise IOError('{} not found'.format(file_path)) file_obj = File( checksum=file_hash[0], assessment_item=question, file_format_id=file_hash[1], original_filename=file_data.get('original_filename') or 'file', source_url=file_data.get('source_url'), file_size = file_data['size'], file_on_disk=DjFile(open(file_path, 'rb')), preset_id=file_data['preset'], ) file_obj.save()
def image_upload(request): if request.method == 'POST': name, ext = os.path.splitext( request.FILES.values() [0]._name) # gets file extension without leading period file_object = File(contentnode_id=request.META.get('HTTP_NODE'), original_filename=name, preset_id=request.META.get('HTTP_PRESET'), file_on_disk=DjFile(request.FILES.values()[0]), file_format_id=ext[1:]) file_object.save() return HttpResponse( json.dumps({ "success": True, "file": JSONRenderer().render(FileSerializer(file_object).data), "path": generate_storage_url(str(file_object)), }))
def map_files_to_assessment_item(user, question, data): """ Generate files that reference the content node's assessment items """ for file_data in data: file_name_parts = file_data['filename'].split(".") file_path = generate_object_storage_name(file_name_parts[0], file_data['filename']) if not os.path.isfile(file_path): return IOError('{} not found'.format(file_path)) resource_obj = File( checksum=file_name_parts[0], assessment_item=question, file_format_id=file_name_parts[1], original_filename=file_data.get('original_filename') or 'file', source_url=file_data.get('source_url'), file_size=file_data['size'], file_on_disk=DjFile(open(file_path, 'rb')), preset_id=file_data['preset'], uploaded_by=user, ) resource_obj.file_on_disk.name = file_path resource_obj.save()
def update(self, instance, validated_data): ret = [] update_files = {} with transaction.atomic(): for item in validated_data: item.update({ 'preset_id': item['preset']['id'], 'language_id': item.get('language')['id'] if item.get('language') else None }) # User should not be able to change files without a display if item['preset']['display']: if 'id' in item: update_files[item['id']] = item else: # create new nodes ret.append(File.objects.create(**item)) item.pop('preset', None) item.pop('language', None) files_to_delete = [] nodes_to_parse = [] current_files = [f['id'] for f in validated_data] # Get files that have the same contentnode, preset, and language as the files that are now attached to this node for file_obj in validated_data: delete_queryset = File.objects.filter( Q(contentnode=file_obj['contentnode']) & # Get files that are associated with this node (Q(preset_id=file_obj['preset_id']) | Q(preset=None)) & # Look at files that have the same preset as this file Q(language_id=file_obj.get('language_id')) & # Look at files with the same language as this file ~Q(id=file_obj['id']) # Remove the file if it's not this file ) files_to_delete += [f for f in delete_queryset.all()] if file_obj['contentnode'] not in nodes_to_parse: nodes_to_parse.append(file_obj['contentnode']) # Delete removed files for node in nodes_to_parse: previous_files = node.files.all() for f in previous_files: if f.id not in current_files: files_to_delete.append(f) for to_delete in files_to_delete: to_delete.delete() if update_files: with transaction.atomic(): for file_id, data in update_files.items(): file_obj, is_new = File.objects.get_or_create(pk=file_id) # potential optimization opportunity for attr, value in data.items(): if attr != "preset" and attr != "language": setattr(file_obj, attr, value) file_path = generate_file_on_disk_name( file_obj.checksum, str(file_obj)) if os.path.isfile(file_path): file_obj.file_on_disk = DjFile(open(file_path, 'rb')) else: raise OSError("Error: file {} was not found".format( str(file_obj))) file_obj.save() ret.append(file_obj) return ret
def subtitle_upload(request): # File will be converted to VTT format ext = file_formats.VTT language_id = request.META.get('HTTP_LANGUAGE') content_file = request.FILES.values()[0] with NamedTemporaryFile() as temp_file: try: converter = build_subtitle_converter( unicode(content_file.read(), 'utf-8')) convert_language_code = language_id # We're making the assumption here that language the user selected is truly the caption # file's language if it's unknown if len(converter.get_language_codes()) == 1 \ and converter.has_language(LANGUAGE_CODE_UNKNOWN): converter.replace_unknown_language(language_id) # determine if the request language exists by another code, otherwise we can't continue if not converter.has_language(convert_language_code): for language_code in converter.get_language_codes(): language = getlang_by_alpha2(language_code) if language and language.code == language_id: convert_language_code = language_code break else: return HttpResponseBadRequest( "Language '{}' not present in subtitle file".format( language_id)) converter.write(temp_file.name, convert_language_code) except InvalidSubtitleFormatError as ex: return HttpResponseBadRequest( "Subtitle conversion failed: {}".format(ex)) temp_file.seek(0) converted_file = DjFile(temp_file) checksum = get_hash(converted_file) size = converted_file.size request.user.check_space(size, checksum) file_object = File( file_size=size, file_on_disk=converted_file, checksum=checksum, file_format_id=ext, original_filename=request.FILES.values()[0]._name, preset_id=request.META.get('HTTP_PRESET'), language_id=language_id, uploaded_by=request.user, ) file_object.save() return HttpResponse( json.dumps({ "success": True, "filename": str(file_object), "file": JSONRenderer().render(FileSerializer(file_object).data) }))
def file_create(request): if request.method == 'POST': original_filename, ext = os.path.splitext( request.FILES.values()[0]._name) size = request.FILES.values()[0]._size contentfile = DjFile(request.FILES.values()[0]) checksum = get_hash(contentfile) request.user.check_space(size, checksum) presets = FormatPreset.objects.filter( allowed_formats__extension__contains=ext[1:].lower()) kind = presets.first().kind preferences = json.loads(request.META.get('HTTP_PREFERENCES')) author = preferences.get('author') or "" license = License.objects.filter( license_name=preferences.get('license')).first( ) # Use filter/first in case preference hasn't been set license_id = license.pk if license else None new_node = ContentNode( title=original_filename, kind=kind, license_id=license_id, author=author, copyright_holder=preferences.get('copyright_holder'), ) if license and license.is_custom: new_node.license_description = preferences.get( 'license_description') new_node.save() file_object = File( file_on_disk=contentfile, checksum=checksum, file_format_id=ext[1:].lower(), original_filename=request.FILES.values()[0]._name, contentnode=new_node, file_size=size, uploaded_by=request.user, ) file_object.save() if kind.pk == content_kinds.VIDEO: file_object.preset_id = guess_video_preset_by_resolution( str(file_object.file_on_disk)) elif presets.filter(supplementary=False).count() == 1: file_object.preset = presets.filter(supplementary=False).first() file_object.save() thumbnail = None try: if preferences.get('auto_derive_video_thumbnail') and new_node.kind_id == content_kinds.VIDEO \ or preferences.get('auto_derive_audio_thumbnail') and new_node.kind_id == content_kinds.AUDIO \ or preferences.get('auto_derive_html5_thumbnail') and new_node.kind_id == content_kinds.HTML5 \ or preferences.get('auto_derive_document_thumbnail') and new_node.kind_id == content_kinds.DOCUMENT: thumbnail = generate_thumbnail_from_node(new_node, set_node=True) request.user.check_space(thumbnail.file_size, thumbnail.checksum) except Exception: if thumbnail: thumbnail.delete() return HttpResponse( json.dumps({ "success": True, "node": JSONRenderer().render( ContentNodeEditSerializer(new_node).data) }))
def file_create(request): if request.method != 'POST': return HttpResponseBadRequest( "Only POST requests are allowed on this endpoint.") original_filename, ext = os.path.splitext(request.FILES.values()[0]._name) size = request.FILES.values()[0]._size contentfile = DjFile(request.FILES.values()[0]) checksum = get_hash(contentfile) request.user.check_space(size, checksum) presets = FormatPreset.objects.filter( allowed_formats__extension__contains=ext[1:].lower()) kind = presets.first().kind preferences = json.loads( request.POST.get('content_defaults', None) or "{}") license = License.objects.filter( license_name=preferences.get('license')).first( ) # Use filter/first in case preference hasn't been set license_id = license.pk if license else None new_node = ContentNode( title=original_filename, kind=kind, license_id=license_id, author=preferences.get('author') or "", aggregator=preferences.get('aggregator') or "", provider=preferences.get('provider') or "", copyright_holder=preferences.get('copyright_holder'), parent_id=settings.ORPHANAGE_ROOT_ID, ) if license and license.is_custom: new_node.license_description = preferences.get('license_description') # The orphanage is not an actual tree but just a long list of items. with ContentNode.objects.disable_mptt_updates(): new_node.save() file_object = File( file_on_disk=contentfile, checksum=checksum, file_format_id=ext[1:].lower(), original_filename=request.FILES.values()[0]._name, contentnode=new_node, file_size=size, uploaded_by=request.user, ) file_object.save() if kind.pk == content_kinds.VIDEO: file_object.preset_id = guess_video_preset_by_resolution( str(file_object.file_on_disk)) elif presets.filter(supplementary=False).count() == 1: file_object.preset = presets.filter(supplementary=False).first() file_object.save() thumbnail = None try: if preferences.get('auto_derive_video_thumbnail') and new_node.kind_id == content_kinds.VIDEO \ or preferences.get('auto_derive_audio_thumbnail') and new_node.kind_id == content_kinds.AUDIO \ or preferences.get('auto_derive_html5_thumbnail') and new_node.kind_id == content_kinds.HTML5 \ or preferences.get('auto_derive_document_thumbnail') and new_node.kind_id == content_kinds.DOCUMENT: thumbnail = generate_thumbnail_from_node(new_node, set_node=True) request.user.check_space(thumbnail.file_size, thumbnail.checksum) except Exception: if thumbnail: thumbnail.delete() return HttpResponse( json.dumps({ "success": True, "node": JSONRenderer().render(ContentNodeEditSerializer(new_node).data) }))