def upload(file: UploadedFile) -> str:
"""
Upload an image, if it does not exist.
"""
buffer = io.BytesIO()
for chunk in file.chunks():
buffer.write(chunk)
result_id = hashlib.blake2s(buffer.getbuffer()).hexdigest()
result_path = os.path.join(MEDIA_ROOT, result_id + '.png')
if os.path.isfile(result_path):
return result_id
try:
image = Image.open(buffer)
except OSError: # This is not an image
raise
with open(result_path, 'wb') as result_file:
image.save(result_file, 'PNG')
send_to_ml(result_path)
return result_id
def save_uploaded_file(file: UploadedFile) -> str:
with tempfile.NamedTemporaryFile(mode="wb",
delete=False,
suffix=f"_{file.name}") as dest:
for chunk in file.chunks():
dest.write(chunk)
return os.path.join(tempfile.gettempdir(), dest.name)
class ChunkedS3VideoUploader:
def __init__(self, djfile, key, name):
self.s3 = boto3.client(
"s3",
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY
)
self.file = UploadedFile(
djfile,
djfile.name,
djfile.content_type,
djfile.size,
djfile.charset,
djfile.content_type_extra
)
self.key = key + "/" + name
self.video_name = djfile.video_name
self.user_email = djfile.user_email
self.bucket_and_key_mixin = {
"Bucket": settings.AWS_STORAGE_BUCKET_NAME,
"Key": self.key
}
def upload(self):
self.file.seek(0, os.SEEK_SET)
m_upload = self.s3.create_multipart_upload(**self.bucket_and_key_mixin)
uid = m_upload["UploadId"]
chunks = self.file.chunks(settings.CHUNK_SIZE)
chunks_list = list()
for chunk in chunks:
chunks_list.append(chunk.hex())
try:
finish_file_upload.delay(
bucket_and_key=self.bucket_and_key_mixin,
upload_id=uid,
file_chunks=chunks_list,
video_name=self.video_name,
user_email=self.user_email
)
except Exception as exc:
print(exc)
self.s3.abort_multipart_upload(
Bucket=settings.AWS_STORAGE_BUCKET_NAME,
Key=m_upload["Key"],
UploadId=uid
)
return self.s3.generate_presigned_url(
"get_object",
Params=self.bucket_and_key_mixin,
HttpMethod="GET"
)
def create_file(cls, path: str, file: UploadedFile) -> bool:
try:
with open(os.path.join(path, file.name), 'wb') as f:
for chunk in file.chunks():
f.write(chunk)
return True
except OSError:
return False
def handle_uploaded_file(f: UploadedFile, user_id):
# Экземпляр FileSystemStorage
fs = FileSystemStorage('fileStore/' + user_id + "/files")
# Получаем доступное имя файла
name = fs.get_available_name(f.name)
# Записываем файл
with fs.open(name, 'wb') as destination:
for chunk in f.chunks():
destination.write(chunk)
return name
def save_uploaded_file(self, uploaded_file: UploadedFile):
save_path = pathlib.Path(settings.MEDIA_ROOT, 'sources',
str(self.event_id), uploaded_file.name)
if save_path.exists():
save_path = save_path.with_name("{}_{}{}".format(
save_path.name, time.time(), save_path.suffix))
save_path.parent.mkdir(parents=True, exist_ok=True, mode=0o775)
with open(str(save_path), mode='wb') as save_file:
for chunk in uploaded_file.chunks():
save_file.write(chunk)
self.file = str(save_path.relative_to(settings.MEDIA_ROOT))
return
def write_file_to_disk(workspace: str, local_file: UploadedFile) -> None:
"""
Write file uploaded to server to unique workspace.
Parameters
----------
workspace: str
Unique workspace for current task.
local_file: UploadedFile
Uploaded file to local server.
"""
filepath = os.path.join(workspace, local_file.name)
with open(filepath, "wb+") as destination:
for chunk in local_file.chunks():
destination.write(chunk)
def get_hashsums(uploaded_file: UploadedFile):
hash_sums = {}
hash_sums['md5sum'] = hashlib.md5()
hash_sums['sha1sum'] = hashlib.sha1()
hash_sums['sha224sum'] = hashlib.sha224()
hash_sums['sha256sum'] = hashlib.sha256()
hash_sums['sha384sum'] = hashlib.sha384()
hash_sums['sha512sum'] = hashlib.sha512()
for data_chunk in uploaded_file.chunks(chunk_size=4096):
for hashsum in hash_sums.keys():
hash_sums[hashsum].update(data_chunk)
results = {}
for key,value in hash_sums.items():
results[key] = value.hexdigest()
return dict(results)
def show(request):
content = {}
if request.method == 'POST':
uploaded_file = request.FILES['file']
uploaded_file_chunks = UploadedFile.chunks(uploaded_file,
chunk_size=100)
print(uploaded_file.name)
print(uploaded_file.size)
fs = FileSystemStorage()
fs.save(uploaded_file.name, uploaded_file)
global upload
upload = Upload()
upload.title = uploaded_file.name
upload.file = uploaded_file
upload.save()
global filecontent
filecontent = file_content()
path = os.path.join(BASE_DIR, 'media/') + uploaded_file.name
with open(path) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
next(csv_reader)
for row in csv_reader:
if not is_paused:
filecontent.seq = row[0]
filecontent.First_name = row[1]
filecontent.Last_name = row[2]
filecontent.age = row[3]
filecontent.street = row[4]
filecontent.city = row[5]
filecontent.state = row[6]
filecontent.zipcode = row[7]
filecontent.dollar = row[8]
filecontent.color = row[9]
filecontent.date = row[10]
filecontent.CSV_ID = upload.pk
filecontent.save()
time.sleep(1)
if stopped:
break
while is_paused:
time.sleep(1)
print("paused")
return render(request, 'upload/upload.html', content)
def save_file_to_drive(file: UploadedFile, raw_dir: Path) -> Optional[str]:
"""
:param file: A Django UploadedFile file
:param raw_dir: A path to user uploaded unprocessed corpora files.
"""
filename = Path(file.name)
stem, suffix = slugify(filename.stem), filename.suffix
filename = f'{stem}{suffix}'
# logger.debug(f'Before: {file.name}, after: {filename}, type: {type(filename)}')
# if raw_dir.joinpath(filename).exists():
# return None
logging.info(f'Received: {filename}, size: {file.size}')
path = raw_dir / filename
with open(path, 'wb+') as fp:
for chunk in file.chunks():
fp.write(chunk)
logging.info(f'Writing to disk complete.')
return filename
def set_config_file(key: str, file: UploadedFile):
from palanaeum.models import ConfigEntry
try:
entry = ConfigEntry.objects.get(key=key)
if entry.value:
os.unlink(os.path.join(settings.MEDIA_ROOT, entry.value))
except ConfigEntry.DoesNotExist:
entry = ConfigEntry(key=key)
except FileNotFoundError:
pass
file_path = os.path.join(settings.CONFIG_UPLOADS, file.name)
os.makedirs(settings.CONFIG_UPLOADS, exist_ok=True)
entry.value = os.path.relpath(file_path, settings.MEDIA_ROOT)
with open(file_path, mode='wb') as write_file:
for chunk in file.chunks():
write_file.write(chunk)
entry.save()
cache.set(key, entry.value)
return
def accept_uploaded_photo(request, album_id):
"""
Main Multiuploader module.
Parses data from jQuery plugin and makes database changes.
"""
if request.method == 'POST':
logid = random.randint(0,1000)
log.info('[%s] received POST to main multiuploader view' % logid)
if request.FILES == None:
return HttpResponseBadRequest('Must have files attached!')
#getting file data for farther manipulations
file = request.FILES[u'files[]']
wrapped_file = UploadedFile(file)
filename = wrapped_file.name
file_size = wrapped_file.file.size
log.info ('[%s] Got file: "%s"' % (logid, str(filename)))
# Write out file to disk as a temp file
randnumber = logid # use the random number here too
temp_filename = '%stmp%s_%s' % (settings.TEMP_DIRECTORY,randnumber, filename)
log.info('[%s] Writing out to: %s' % (logid, temp_filename))
destination = open(temp_filename, 'wb+')
if wrapped_file.multiple_chunks():
for chunk in wrapped_file.chunks():
destination.write(chunk)
else:
destination.write(wrapped_file.read())
destination.close()
# Dump out EXIF Tags
# im = Image.open(temp_filename)
# if hasattr( im, '_getexif' ):
# exifinfo = im._getexif()
# if exifinfo:
# for tag, value in exifinfo.items():
# decoded = ExifTags.TAGS.get(tag, tag)
# log.info('Found tag: %s, value: %s' % (decoded,value))
orientation = None
date_taken = None
# Make full size and thumbsize
try:
im = Image.open(temp_filename)
except IOError:
log.info('[%s] Error opening file %s: %s %s' % (logid, temp_filename, e.errno, e))
return HttpResponseBadRequest('Could not read file')
if hasattr( im, '_getexif' ):
exifinfo = im._getexif()
if exifinfo:
for tag, value in exifinfo.items():
decoded = ExifTags.TAGS.get(tag, tag)
# if decoded != 'MakerNote':
# if decoded != 'UserComment':
# log.info('Found tag: %s, value: %s' % (decoded,value))
if decoded == 'Orientation':
orientation = value
log.info('[%s] Found tag: %s, value: %s' % (logid,decoded,value))
elif decoded == 'DateTime':
date_taken = datetime.strptime(value, "%Y:%m:%d %H:%M:%S")
log.info('[%s] Found tag: %s, value: %s, date_taken=%s' % (logid,decoded,value,date_taken))
# We rotate regarding to the EXIF orientation information
if orientation:
if orientation == 1:
# Nothing
log.info('[%s] Orientation: No rotation necessary' % logid)
pass
elif orientation == 2:
# Vertical Mirror
log.info('[%s] Orientation: Vertical flip' % logid)
im = im.transpose(Image.FLIP_LEFT_RIGHT)
elif orientation == 3:
# Rotation 180
log.info('[%s] Orientation: Rotation 180' % logid)
im = im.transpose(Image.ROTATE_180)
elif orientation == 4:
# Horizontal Mirror
log.info('[%s] Orientation: Horizontal Mirror' % logid)
im = im.transpose(Image.FLIP_TOP_BOTTOM)
elif orientation == 5:
# Horizontal Mirror + Rotation 270
log.info('[%s] Orientation: Flip top bottom, rot 270' % logid)
im = im.transpose(Image.FLIP_TOP_BOTTOM).transpose(Image.ROTATE_270)
elif orientation == 6:
# Rotation 270
log.info('[%s] Orientation: Rotate 270' % logid)
im = im.transpose(Image.ROTATE_270)
elif orientation == 7:
# Vertical Mirror + Rotation 270
log.info('[%s] Orientation: Flip left right, rotate 270' % logid)
im = im.transpose(Image.FLIP_LEFT_RIGHT).transpose(Image.ROTATE_270)
elif orientation == 8:
# Rotation 90
log.info('[%s] Orientation: Rotate 90' % logid)
im = im.transpose(Image.ROTATE_90)
#------------------
# Save the transposed image to disk
orig_path = '%stmp%s_mod%s' % (settings.TEMP_DIRECTORY,randnumber, filename)
# keep 100% fidelity on the image
try:
log.info('[%s] Writing corrected photo to path %s' % (logid,orig_path))
im.save(orig_path, "JPEG", quality=100)
except IOError:
log.info('[%s] Error saving file %s: %s %s' % (logid, orig_path, e.errno, e))
return HttpResponseBadRequest('Could not save file')
#------------------
# Save the photo object into the database
album = Album.objects.get(id=album_id)
photo = Photo()
photo.album = album
log.info('[%s] Determining photo order' % logid)
#------------------
# Determine where in the photo order this picture needs to be
photo.order = 0
if date_taken:
photo.photodate = date_taken
log.info('[%s] Date Taken is %s' % (logid,date_taken))
# Now try to insert the photo by date taken in the order list
prev_photo = photo.prev_photo_by_photodate()
if prev_photo:
log.info('got prev photo. id=%s, photodate=%s, order=%s' % (prev_photo.id,prev_photo.photodate,prev_photo.order))
photo.order = prev_photo.order
else:
# First in album
photo.order = 0
else:
# Last in album
photo.order = album.photo_set.count() + 1
log.info('[%s] Writing photo entry to database' % logid)
#------------------
# Now finally write the entry to the db
photo.save()
log.info('[%s] Photo object saved. id = %s, order = %s' % (logid, photo.id,photo.order))
#album.reorder_photos()
log.info('[%s] Attempting to save file %s to django model id %s' % (logid, orig_path, photo.id))
f = open(orig_path, 'r')
photo.filename.save('%s.jpg' % photo.id, File(f))
f.close()
log.info('[%s] Cleaning up files' % logid)
#clean up temp file
unlink(temp_filename)
unlink(orig_path)
#settings imports
file_delete_url = 'multi_delete/'
thumbnail_options = dict(size=(200, 200), crop=True)
thumb_url = get_thumbnailer(photo.filename).get_thumbnail(thumbnail_options).url
#generating json response array
result = []
result.append({"name":filename,
"size":file_size,
"url": thumb_url,
"thumbnail_url":thumb_url,
"delete_url":'/',
"delete_type":"POST",})
response_data = simplejson.dumps(result)
#checking for json data type
#big thanks to Guy Shapiro
if "application/json" in request.META['HTTP_ACCEPT_ENCODING']:
mimetype = 'application/json'
else:
mimetype = 'text/plain'
return HttpResponse(response_data, mimetype=mimetype)
else: #GET
return HttpResponse('Only POST accepted')
def calculate_checksum(uploaded_file: UploadedFile) -> str:
md5 = hashlib.md5()
for chunk in uploaded_file.chunks():
md5.update(chunk)
return md5.hexdigest()
def detect_mime_type(uploaded_file: UploadedFile) -> str:
"""Detect mime type of an uploaded file"""
magic = Magic(mime=True)
chunk = next(uploaded_file.chunks(chunk_size=2048))
return magic.from_buffer(chunk)
