class ChunkedS3VideoUploader:
def __init__(self, djfile, key, name):
self.s3 = boto3.client(
"s3",
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY
)
self.file = UploadedFile(
djfile,
djfile.name,
djfile.content_type,
djfile.size,
djfile.charset,
djfile.content_type_extra
)
self.key = key + "/" + name
self.video_name = djfile.video_name
self.user_email = djfile.user_email
self.bucket_and_key_mixin = {
"Bucket": settings.AWS_STORAGE_BUCKET_NAME,
"Key": self.key
}
def upload(self):
self.file.seek(0, os.SEEK_SET)
m_upload = self.s3.create_multipart_upload(**self.bucket_and_key_mixin)
uid = m_upload["UploadId"]
chunks = self.file.chunks(settings.CHUNK_SIZE)
chunks_list = list()
for chunk in chunks:
chunks_list.append(chunk.hex())
try:
finish_file_upload.delay(
bucket_and_key=self.bucket_and_key_mixin,
upload_id=uid,
file_chunks=chunks_list,
video_name=self.video_name,
user_email=self.user_email
)
except Exception as exc:
print(exc)
self.s3.abort_multipart_upload(
Bucket=settings.AWS_STORAGE_BUCKET_NAME,
Key=m_upload["Key"],
UploadId=uid
)
return self.s3.generate_presigned_url(
"get_object",
Params=self.bucket_and_key_mixin,
HttpMethod="GET"
)
def document_validator(filepath, ex_first_row, ex_first_col):
try:
with open(os.path.join(settings.MEDIA_ROOT, filepath), 'r') as f:
file = UploadedFile(f)
dialect = csv.Sniffer().sniff(file.readline(), delimiters=[';', ',', '\t'])
mimetype = magic.from_buffer(file.readline(), mime=True)
file.seek(0)
reader = csv.reader(file, dialect)
temp_list = []
for line in iter(reader):
if reader.line_num == 1:
# save first row
temp_list.append(line)
# save last row
temp_list.append(line)
print ex_first_row
if not ex_first_row:
print "Ciao"
# check char in first row and first col
if not ex_first_row and not float(temp_list[0][-1]):
print 'Hola'
raise ValueError
if not ex_first_col and not float(temp_list[-1][0]):
print 'Hello'
raise ValueError
ncol = (len(temp_list[0]) - 1) if ex_first_col else len(temp_list[0])
nrow = (reader.line_num - 1) if ex_first_col else reader.line_num
if nrow <= 2:
print 'Hey'
raise ValueError
is_cubic = True if (ncol == nrow) else False
return_value = {'is_valid': True, 'nrow': nrow, 'ncol': ncol, 'separator': dialect.delimiter,
'mimetype': mimetype, 'is_cubic': is_cubic}
except csv.Error:
print "Csv"
return_value = {'is_valid': False}
file = None
except Exception:
return_value = {'is_valid': False}
print "Exc"
file = None
except ValueError:
return_value = {'is_valid': False}
print "Value"
file = file
return return_value, file
def smart_load_from_upload(classname: str, f: UploadedFile) -> BasePriceList:
'''
Attempt to intelligently load the given Django UploadedFile,
interpreting it as a price list for the given schedule class name.
If interpreting it under the preferred schedule results in either
a ValidationError or no valid rows, attempts will be made to
re-interpret the file under different schedules. The first found
that yields better interpretations of the data will be returned.
If no better matches are found, the original result or exception
(from interpreting the data under the preferred price list) will
be returned.
'''
original_error = None
pricelist: Optional[BasePriceList] = None
try:
pricelist = load_from_upload(classname, f)
except ValidationError as e:
original_error = e
if original_error or (pricelist and not pricelist.valid_rows):
# See if any of our other registered schedules can make better sense
# of it.
next_best_pricelist = None
for fallback, _ in CHOICES:
if fallback == classname:
continue
try:
f.seek(0)
next_best_pricelist = load_from_upload(fallback, f)
if next_best_pricelist.valid_rows:
pricelist = next_best_pricelist
break
except ValidationError as e:
pass
if pricelist is None:
default_error = ValidationError('Unrecognized price list!')
raise original_error or default_error
return pricelist
def make_thumbnail_with_uploaded(cls,
post: Post,
file_content: UploadedFile,
size: tuple,
order: int = 1):
_, ext = os.path.splitext(file_content.name)
mimetype = file_content.content_type
filename = f'{uuid4().hex}{ext}'
if not mimetype:
raise drf_exc.ValidationError('invalid uploaded file format',
'invalid-file-format')
im = Image.open(file_content)
base_size = min(im.size)
min_size = settings.MINIMUM_IMAGE_SIZE
if base_size < min_size:
raise drf_exc.ValidationError(
f'image size requires to be bigger than {min_size}',
'minimum-image-size')
if base_size < size[0]:
return
file_content.seek(0)
fp = BytesIO()
resized = im.crop((0, 0, base_size, base_size))
resized.thumbnail(size, Image.ANTIALIAS)
resized.save(fp, format=mimetype.split('/')[-1].upper())
uploaded = InMemoryUploadedFile(fp, None, filename, mimetype, fp.tell,
None)
thumb = cls(post=post, content=uploaded, order=order)
thumb.save()
return thumb
def _uploaded_file_obj_to_buffer(
uploaded_file: UploadedFile, sample_size: int = 8096
) -> (bytes, bytes):
"""
Optimize memory usage by only returning the first and last 8096 bytes (default).
"""
head = uploaded_file.read(sample_size)
try:
# Try to position to the almost end of the file
seek_pos = uploaded_file.seek(-sample_size, os.SEEK_END)
if seek_pos == 0: # The file was smaller that the sample size
return head, b""
except IOError:
return head, b""
# Continue reading from the end position found
foot = uploaded_file.read()
return head, foot
def get_minecraft_avatar(minecraft_user, geometry_string, force=True):
"""
This method uses the sorl-thumbnail cache backend to
prevent images from being downloaded every time.
It requires an username.
"""
avatar_file = UploadedFile(
file=StringIO(),
name='%s/%s.png' % (settings.MEDIA_ROOT, tokey(minecraft_user)))
try:
# Check if the avatar is cached
thumbnail = get_thumbnail(
avatar_file, '100x100', quality=100, format='PNG')
except IOError:
download_thumbnail = True
else:
is_dummy = not hasattr(thumbnail, 'storage')
if not is_dummy and not thumbnail.storage.exists(thumbnail.name):
# It seems we have the avatar on cache (kvstore)
# but it's not present on the storage
download_thumbnail = True
# Force remove thumbnail from kvstore
sorl_default.kvstore.delete(thumbnail)
# Log
logger.warning('Avatar cache mismatch: %s (resetting)' % (
minecraft_user,))
else:
logger.debug('Avatar fetched from cache: %s' % minecraft_user)
download_thumbnail = False
if download_thumbnail:
logger.debug('Downloading avatar: %s' % minecraft_user)
# Otherwise download avatar
thumbnail = None
try:
skin_bin = requests.get(
'http://s3.amazonaws.com/MinecraftSkins/%s.png' % (
minecraft_user
)
).content
except ConnectionError:
return None
try:
skin = Image.open(StringIO(skin_bin)).convert('RGBA')
except IOError:
# Image not found or some permission error with S3
if minecraft_user != 'char':
if not force:
return None
# Return default user avatar
return settings.STATIC_URL + settings.DEFAULT_USER_AVATAR
else:
face = skin.crop((8, 8, 16, 16))
accessories = skin.crop((40, 8, 48, 16))
r, g, b, a = accessories.split()
accessories = Image.merge('RGB', (r, g, b))
mask = Image.merge('L', (a,))
face.paste(accessories, (0, 0), mask)
avatar = face.resize((135, 135))
avatar.save(avatar_file, 'PNG')
avatar_file.seek(0)
# Save through sorl backend
thumbnail = get_thumbnail(avatar_file, '100x100',
quality=100, format='PNG')
# Use the cached file
return get_thumbnail(thumbnail, geometry_string,
quality=100, format='PNG').url
def get_content_type(file: UploadedFile):
content_type = magic.from_buffer(file.read(), mime=True)
file.seek(0)
return content_type
