class ChunkedS3VideoUploader: def __init__(self, djfile, key, name): self.s3 = boto3.client( "s3", aws_access_key_id=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY ) self.file = UploadedFile( djfile, djfile.name, djfile.content_type, djfile.size, djfile.charset, djfile.content_type_extra ) self.key = key + "/" + name self.video_name = djfile.video_name self.user_email = djfile.user_email self.bucket_and_key_mixin = { "Bucket": settings.AWS_STORAGE_BUCKET_NAME, "Key": self.key } def upload(self): self.file.seek(0, os.SEEK_SET) m_upload = self.s3.create_multipart_upload(**self.bucket_and_key_mixin) uid = m_upload["UploadId"] chunks = self.file.chunks(settings.CHUNK_SIZE) chunks_list = list() for chunk in chunks: chunks_list.append(chunk.hex()) try: finish_file_upload.delay( bucket_and_key=self.bucket_and_key_mixin, upload_id=uid, file_chunks=chunks_list, video_name=self.video_name, user_email=self.user_email ) except Exception as exc: print(exc) self.s3.abort_multipart_upload( Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=m_upload["Key"], UploadId=uid ) return self.s3.generate_presigned_url( "get_object", Params=self.bucket_and_key_mixin, HttpMethod="GET" )
def document_validator(filepath, ex_first_row, ex_first_col): try: with open(os.path.join(settings.MEDIA_ROOT, filepath), 'r') as f: file = UploadedFile(f) dialect = csv.Sniffer().sniff(file.readline(), delimiters=[';', ',', '\t']) mimetype = magic.from_buffer(file.readline(), mime=True) file.seek(0) reader = csv.reader(file, dialect) temp_list = [] for line in iter(reader): if reader.line_num == 1: # save first row temp_list.append(line) # save last row temp_list.append(line) print ex_first_row if not ex_first_row: print "Ciao" # check char in first row and first col if not ex_first_row and not float(temp_list[0][-1]): print 'Hola' raise ValueError if not ex_first_col and not float(temp_list[-1][0]): print 'Hello' raise ValueError ncol = (len(temp_list[0]) - 1) if ex_first_col else len(temp_list[0]) nrow = (reader.line_num - 1) if ex_first_col else reader.line_num if nrow <= 2: print 'Hey' raise ValueError is_cubic = True if (ncol == nrow) else False return_value = {'is_valid': True, 'nrow': nrow, 'ncol': ncol, 'separator': dialect.delimiter, 'mimetype': mimetype, 'is_cubic': is_cubic} except csv.Error: print "Csv" return_value = {'is_valid': False} file = None except Exception: return_value = {'is_valid': False} print "Exc" file = None except ValueError: return_value = {'is_valid': False} print "Value" file = file return return_value, file
def smart_load_from_upload(classname: str, f: UploadedFile) -> BasePriceList: ''' Attempt to intelligently load the given Django UploadedFile, interpreting it as a price list for the given schedule class name. If interpreting it under the preferred schedule results in either a ValidationError or no valid rows, attempts will be made to re-interpret the file under different schedules. The first found that yields better interpretations of the data will be returned. If no better matches are found, the original result or exception (from interpreting the data under the preferred price list) will be returned. ''' original_error = None pricelist: Optional[BasePriceList] = None try: pricelist = load_from_upload(classname, f) except ValidationError as e: original_error = e if original_error or (pricelist and not pricelist.valid_rows): # See if any of our other registered schedules can make better sense # of it. next_best_pricelist = None for fallback, _ in CHOICES: if fallback == classname: continue try: f.seek(0) next_best_pricelist = load_from_upload(fallback, f) if next_best_pricelist.valid_rows: pricelist = next_best_pricelist break except ValidationError as e: pass if pricelist is None: default_error = ValidationError('Unrecognized price list!') raise original_error or default_error return pricelist
def make_thumbnail_with_uploaded(cls, post: Post, file_content: UploadedFile, size: tuple, order: int = 1): _, ext = os.path.splitext(file_content.name) mimetype = file_content.content_type filename = f'{uuid4().hex}{ext}' if not mimetype: raise drf_exc.ValidationError('invalid uploaded file format', 'invalid-file-format') im = Image.open(file_content) base_size = min(im.size) min_size = settings.MINIMUM_IMAGE_SIZE if base_size < min_size: raise drf_exc.ValidationError( f'image size requires to be bigger than {min_size}', 'minimum-image-size') if base_size < size[0]: return file_content.seek(0) fp = BytesIO() resized = im.crop((0, 0, base_size, base_size)) resized.thumbnail(size, Image.ANTIALIAS) resized.save(fp, format=mimetype.split('/')[-1].upper()) uploaded = InMemoryUploadedFile(fp, None, filename, mimetype, fp.tell, None) thumb = cls(post=post, content=uploaded, order=order) thumb.save() return thumb
def _uploaded_file_obj_to_buffer( uploaded_file: UploadedFile, sample_size: int = 8096 ) -> (bytes, bytes): """ Optimize memory usage by only returning the first and last 8096 bytes (default). """ head = uploaded_file.read(sample_size) try: # Try to position to the almost end of the file seek_pos = uploaded_file.seek(-sample_size, os.SEEK_END) if seek_pos == 0: # The file was smaller that the sample size return head, b"" except IOError: return head, b"" # Continue reading from the end position found foot = uploaded_file.read() return head, foot
def get_minecraft_avatar(minecraft_user, geometry_string, force=True): """ This method uses the sorl-thumbnail cache backend to prevent images from being downloaded every time. It requires an username. """ avatar_file = UploadedFile( file=StringIO(), name='%s/%s.png' % (settings.MEDIA_ROOT, tokey(minecraft_user))) try: # Check if the avatar is cached thumbnail = get_thumbnail( avatar_file, '100x100', quality=100, format='PNG') except IOError: download_thumbnail = True else: is_dummy = not hasattr(thumbnail, 'storage') if not is_dummy and not thumbnail.storage.exists(thumbnail.name): # It seems we have the avatar on cache (kvstore) # but it's not present on the storage download_thumbnail = True # Force remove thumbnail from kvstore sorl_default.kvstore.delete(thumbnail) # Log logger.warning('Avatar cache mismatch: %s (resetting)' % ( minecraft_user,)) else: logger.debug('Avatar fetched from cache: %s' % minecraft_user) download_thumbnail = False if download_thumbnail: logger.debug('Downloading avatar: %s' % minecraft_user) # Otherwise download avatar thumbnail = None try: skin_bin = requests.get( 'http://s3.amazonaws.com/MinecraftSkins/%s.png' % ( minecraft_user ) ).content except ConnectionError: return None try: skin = Image.open(StringIO(skin_bin)).convert('RGBA') except IOError: # Image not found or some permission error with S3 if minecraft_user != 'char': if not force: return None # Return default user avatar return settings.STATIC_URL + settings.DEFAULT_USER_AVATAR else: face = skin.crop((8, 8, 16, 16)) accessories = skin.crop((40, 8, 48, 16)) r, g, b, a = accessories.split() accessories = Image.merge('RGB', (r, g, b)) mask = Image.merge('L', (a,)) face.paste(accessories, (0, 0), mask) avatar = face.resize((135, 135)) avatar.save(avatar_file, 'PNG') avatar_file.seek(0) # Save through sorl backend thumbnail = get_thumbnail(avatar_file, '100x100', quality=100, format='PNG') # Use the cached file return get_thumbnail(thumbnail, geometry_string, quality=100, format='PNG').url
def get_content_type(file: UploadedFile): content_type = magic.from_buffer(file.read(), mime=True) file.seek(0) return content_type