def _upload_file(self, request):
"""
Upload file to the server.
"""
from django.core.files.move import file_move_safe
if request.method == 'POST':
folder = request.POST.get('folder')
fb_uploadurl_re = re.compile(r'^.*(%s)' % reverse("filebrowser:fb_upload", current_app=self.name))
folder = fb_uploadurl_re.sub('', folder)
abs_path = os.path.join(self.media_root, folder)
if request.FILES:
filedata = request.FILES['Filedata']
filedata.name = convert_filename(filedata.name)
self.filebrowser_pre_upload.send(sender=request, path=request.POST.get('folder'), file=filedata)
uploadedfile = handle_file_upload(abs_path, filedata, media_root=self.media_root)
# if file already exists
if os.path.isfile(smart_unicode(os.path.join(self.media_root, folder, filedata.name))):
old_file = smart_unicode(os.path.join(abs_path, filedata.name))
new_file = smart_unicode(os.path.join(abs_path, uploadedfile))
file_move_safe(new_file, old_file, allow_overwrite=True)
# POST UPLOAD SIGNAL
self.filebrowser_post_upload.send(sender=request, path=request.POST.get('folder'), file=FileObject(smart_unicode(os.path.join(folder, filedata.name)), media_root=self.media_root, media_url=self.media_url))
return HttpResponse('True')
def save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
print "Error dir is not error"
while True:
try:
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
else:
fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
except OSError, e:
if e.errno == errno.EEXIST:
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
break
def _upload_file(request):
"""
Upload file to the server.
"""
from django.core.files.move import file_move_safe
from django.http import QueryDict
if request.method == 'POST':
query = QueryDict(request.POST.get('get_query'))
folder = request.POST.get('folder')
fb_uploadurl_re = re.compile(r'^(%s)' % reverse("fb_upload"))
folder = fb_uploadurl_re.sub('', folder)
abs_path = os.path.join(MEDIA_ROOT, DIRECTORY, folder)
if request.FILES:
filedata = request.FILES['Filedata']
filedata.name = convert_filename(filedata.name)
# PRE UPLOAD SIGNAL
filebrowser_pre_upload.send(sender=request, path=request.POST.get('folder'), file=filedata)
# HANDLE UPLOAD
uploadedfile = handle_file_upload(abs_path, filedata)
# MOVE UPLOADED FILE
# if file already exists
if os.path.isfile(os.path.join(MEDIA_ROOT, DIRECTORY, folder, filedata.name)):
old_file = os.path.join(abs_path, filedata.name)
new_file = os.path.join(abs_path, uploadedfile)
file_move_safe(new_file, old_file)
# POST UPLOAD SIGNAL
filebrowser_post_upload.send(sender=request, path=request.POST.get('folder'), file=FileObject(os.path.join(DIRECTORY, folder, filedata.name)))
return HttpResponseRedirect(reverse("fb_browse") + query_helper(query, "", "filename,filetype"))
else:
return HttpResponse("Error: No files were posted")
else:
return HttpResponse("Error: No POST data")
def _save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_TRUNC \
| os.O_CREAT | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def _save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# Open file for writting in binary mode
fd = open(full_path.encode('utf-8'), "wb")
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
fd.write(chunk)
finally:
locks.unlock(fd)
fd.close()
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def update_item(item, newname):
file_root = os.path.join(settings.MEDIA_ROOT, get_storage_path(item, ''))
path = item.file.path
if not os.path.exists(path):
print newname, ": files doesn't exist: ", path
return
root, name = os.path.split(path)
base, ext = os.path.splitext(name)
renamed = os.path.join(root, u'%s%s' % (newname, ext))
cache = {}
for size in MediaSizeCache().sizes.values():
func = getattr(item, "get_%s_filename" % size.name, None)
if func:
sizename = func()
if os.path.exists(sizename) and sizename.startswith(file_root):
cache[func] = sizename
# Do the move and save
move_file(item, path, renamed)
# No need to regenerate caches
item.prevent_cache_clear = True
item.remove_deleted = False
item.save()
for key, value in cache.items():
target = key()
if not target.endswith('unconverted') and os.path.exists(value):
file_move_safe(value, target)
taken = item.date_taken
taken = taken.astimezone(get_current_timezone())
mtime = int(time.mktime(taken.timetuple()))
os.utime(item.file.path, (mtime, mtime))
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
if self.directory_permissions_mode is not None:
# os.makedirs applies the global umask, so we reset it,
# for consistency with file_permissions_mode behavior.
old_umask = os.umask(0)
try:
os.makedirs(directory, self.directory_permissions_mode)
finally:
os.umask(old_umask)
else:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
tmp_file = tempfile.mktemp()
filey = open(tmp_file, 'wb')
filey.write(content.read())
# make sure that all data is on disk
filey.flush()
os.fsync(filey.fileno())
filey.close()
file_move_safe(tmp_file, full_path, allow_overwrite=True)
return name
def _upload_file(request):
"""
Upload file to the server.
"""
from django.core.files.move import file_move_safe
if request.method == 'POST':
folder = request.POST.get('folder')
fb_uploadurl_re = re.compile(r'^.*(%s)' % reverse("fb_upload"))
folder = fb_uploadurl_re.sub('', folder)
abs_path = _check_access(request, folder)
if request.FILES:
filedata = request.FILES['Filedata']
filedata.name = convert_filename(filedata.name)
_check_access(request, abs_path, filedata.name)
# PRE UPLOAD SIGNAL
filebrowser_pre_upload.send(sender=request, path=request.POST.get('folder'), file=filedata)
# HANDLE UPLOAD
uploadedfile = handle_file_upload(abs_path, filedata)
# MOVE UPLOADED FILE
# if file already exists
if os.path.isfile(smart_str(os.path.join(fb_settings.MEDIA_ROOT, fb_settings.DIRECTORY, folder, filedata.name))):
old_file = smart_str(os.path.join(abs_path, filedata.name))
new_file = smart_str(os.path.join(abs_path, uploadedfile))
file_move_safe(new_file, old_file)
# POST UPLOAD SIGNAL
filebrowser_post_upload.send(sender=request, path=request.POST.get('folder'), file=FileObject(smart_str(os.path.join(fb_settings.DIRECTORY, folder, filedata.name))))
return HttpResponse('True')
def _save(self, name, content):
"""
Lifted partially from django/core/files/storage.py
"""
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# Ensure that content is open
content.open()
if hasattr(content, 'temporary_file_path'):
# Content has a file that we can move.
temp_data_location = content.temporary_file_path()
file_move_safe(temp_data_location, full_path, allow_overwrite=True)
else:
# Write the content stream to a temporary file and move it.
fd, tmp_path = mkstemp()
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
locks.unlock(fd)
os.close(fd)
file_move_safe(tmp_path, full_path, allow_overwrite=True)
content.close()
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def save_video(self, filename):
self.type = 2
(self.width, self.height) = get_video_size(filename)
self.length = get_media_duration(filename)
super(LibraryFile, self).save() # Intermediate save to get new ID
self.filename = "adex%s_%s" % (self.id, self.name)
genVideoThumb(filename, settings.MEDIA_ROOT + "v/thumb/%s.jpg" % self.filename)
file_move_safe(filename, settings.MEDIA_ROOT + "v/%s" % self.filename)
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
except OSError as e:
if e.errno == errno.EEXIST:
# Ooops, the file exists. We need a new file name.
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
# OK, the file save worked. Break out of the loop.
break
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def test_file_move_overwrite(self):
handle_a, self.file_a = tempfile.mkstemp(dir=os.environ['DJANGO_TEST_TEMP_DIR'])
handle_b, self.file_b = tempfile.mkstemp(dir=os.environ['DJANGO_TEST_TEMP_DIR'])
# file_move_safe should raise an IOError exception if destination file exists and allow_overwrite is False
self.assertRaises(IOError, lambda: file_move_safe(self.file_a, self.file_b, allow_overwrite=False))
# should allow it and continue on if allow_overwrite is True
self.assertIsNone(file_move_safe(self.file_a, self.file_b, allow_overwrite=True))
def save_image(self, filename):
self.type = 1
im = pil.open(filename)
(self.width, self.height) = im.size
super(LibraryFile, self).save() # Intermediate save to get new ID
self.filename = "adex%s_%s" % (self.id, self.name)
im.thumbnail((LIBRARYFILE_THUMB_WIDTH,LIBRARYFILE_THUMB_HEIGHT), pil.ANTIALIAS)
im.save(settings.MEDIA_ROOT + "i/thumb/%s" % self.filename)
file_move_safe(filename, settings.MEDIA_ROOT + "i/%s" % self.filename)
def _save(self, name, content):
"""
Lifted partially from django/core/files/storage.py
"""
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
temp_data_location = content.temporary_file_path()
else:
tmp_prefix = "tmp_%s" % (get_valid_filename(name), )
temp_data_location = tempfile.mktemp(prefix=tmp_prefix,
dir=self.location)
try:
# This is a normal uploadedfile that we can stream.
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(temp_data_location,
os.O_WRONLY | os.O_CREAT |
os.O_EXCL | getattr(os, 'O_BINARY', 0))
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
locks.unlock(fd)
os.close(fd)
except Exception:
if os.path.exists(temp_data_location):
os.remove(temp_data_location)
raise
file_move_safe(temp_data_location, full_path, allow_overwrite=True)
content.close()
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def _save(self, name, content):
# Prepare file name from the file content hash
dir_name, file_full_name = os.path.split(name)
file_name, file_ext = os.path.splitext(file_full_name)
content_hash = hashlib.sha1()
for chunk in content.chunks():
content_hash.update(chunk)
hash_name = self._split_name(content_hash.hexdigest())
# + str(content.size) Can add size if anytime stuck into collisions.
name = os.path.join(dir_name, hash_name + file_ext.lower())
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition: it's possible that two threads
# might return the same name. In this case we just continue normally.
# NOTE! This differs considerably from the default file storage behaviour.
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def write_hosts_file(vm,password):
towrite = '[vm]' + '\n'
towrite += str(vm) + " ansible_ssh_pass='******'"
with open('hosts', 'wb') as f:
locks.lock(f, locks.LOCK_EX)
f.write(towrite)
file_move_safe("hosts", "../Ansible/hosts", allow_overwrite = True)
def write_variable_file_2(user):
towrite = '---' +'\n'
towrite += '\n'
towrite += ('user: "******"\n')
with open('test2.yml', 'wb') as f:
locks.lock(f, locks.LOCK_EX)
f.write(towrite)
file_move_safe("test2.yml", "../Ansible/variables2.yml", allow_overwrite = True)
def test_file_move_overwrite(self):
handle_a, self.file_a = tempfile.mkstemp()
handle_b, self.file_b = tempfile.mkstemp()
# file_move_safe should raise an IOError exception if destination file exists and allow_overwrite is False
self.assertRaises(IOError, lambda: file_move_safe(self.file_a, self.file_b, allow_overwrite=False))
# should allow it and continue on if allow_overwrite is True
self.assertIsNone(file_move_safe(self.file_a, self.file_b, allow_overwrite=True))
os.close(handle_a)
os.close(handle_b)
def safely_rename(self, url_request_file, new_name, clobber=False):
""" Pass a URLRequestFile, with a new filename, to move or rename. """
new_path = safe_join(
dirname(self.path(url_request_file.name)),
new_name)
file_move_safe(
self.path(url_request_file.name),
new_path,
allow_overwrite=clobber)
url_request_file.name = new_name
def save_flash(self, filename):
self.type = 4
super(LibraryFile, self).save() # Intermediate save to get new ID
self.filename = "adex%s_%s" % (self.id, self.name)
webthumb(
'http://anondex.com/flashview/?'+settings.MEDIA_URL+'tmp/'+self.name,
settings.MEDIA_ROOT+"f/thumb/%s.jpg"%self.filename,
is_flash=True,
)
file_move_safe(filename, settings.MEDIA_ROOT + "f/%s" % self.filename)
if settings.DEBUG:
return
def _save(self, name, content):
'''
Copied from super and lightly modified - Unfortunately, the default
race condition handling will lead to an infinite loop here, since
get_available_name() doesn't return unique names for identical hashes.
'''
full_path = self.path(name)
if os.path.exists(full_path):
return name
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
# handle concurrency issue where the directory is created while
# by another thread after the call to exists()
try:
os.makedirs(directory)
except OSError as e:
# ignore EEXIST, since it means that our goal here was already
# accomplished
if e.errno != errno.EEXIST:
raise
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition when saving the file; it's
# possible that two threads might try to write the file simultaneously.
# We need to try to create the file, but abort if we detect that it
# already exists (in which case we assume that another thread is
# writing it).
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
except OSError, e:
# abort and continue normally if we detect the existence of the
# file we're trying to write, otherwise raise normally
if e.errno != errno.EEXIST:
raise
def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
self._createdir() # Cache dir can be deleted at any time.
fname = self._key_to_file(key, version)
self._cull() # make some room if necessary
fd, tmp_path = tempfile.mkstemp(dir=self._dir)
renamed = False
try:
with open(fd, 'wb') as f:
self._write_content(f, timeout, value)
file_move_safe(tmp_path, fname, allow_overwrite=True)
renamed = True
finally:
if not renamed:
os.remove(tmp_path)
def test_file_move_overwrite(self):
handle_a, self.file_a = tempfile.mkstemp()
handle_b, self.file_b = tempfile.mkstemp()
# file_move_safe() raises OSError if the destination file exists and
# allow_overwrite is False.
with self.assertRaises(FileExistsError):
file_move_safe(self.file_a, self.file_b, allow_overwrite=False)
# should allow it and continue on if allow_overwrite is True
self.assertIsNone(file_move_safe(self.file_a, self.file_b, allow_overwrite=True))
os.close(handle_a)
os.close(handle_b)
def add_revision(self, content, instance, commit_msg, username):
fname = self.get_filename(instance)
full_path = os.path.join(self.location, fname)
if hasattr(content, 'temporary_file_path'):
# This file has a file path that we can move.
file_move_safe(content.temporary_file_path(), full_path)
content.close()
else:
# This is a normal uploadedfile that we can stream.
with open(full_path, 'wb') as f:
content.seek(0)
f.write(content.read())
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
self._commit(fname, commit_msg, username, 'add')
def save(self, *args, **kwargs):
# temporary values for version, date
self.version = 0
self.date = datetime.date.today()
super(CacheDatabase, self).save(*args, **kwargs)
conn = sqlite3.connect(self.file.path)
c = conn.cursor()
c.execute("SELECT name, value FROM meta")
date = None
version = None
for i in c.fetchall():
if i[0] == 'version':
version = i[1]
if i[0] == 'date':
date = i[1]
try:
if not version:
raise ValueError
self.version = int(version)
except ValueError:
raise models.IntegrityError('database does not list valid version')
try:
if not date:
raise ValueError
# MM/DD/YYYY (unfortunately... grr...)
date = map(int, date.split('/', 2))
if len(date) != 3:
raise ValueError
self.date = datetime.date(date[2], date[0], date[1])
except ValueError:
raise models.IntegrityError('database does not list valid date')
c.close()
conn.close()
# move the file, if needed
if self.file.name.startswith('databases/tmp.'):
newpath = 'databases/' + str(self)
newpathreal = self.file.storage.path(newpath)
file_move_safe(self.file.path, newpathreal)
self.file = newpath
# resave
super(CacheDatabase, self).save(*args, **kwargs)
def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
self._createdir() # Cache dir can be deleted at any time.
fname = self._key_to_file(key, version)
self._cull() # make some room if necessary
fd, tmp_path = tempfile.mkstemp(dir=self._dir)
renamed = False
try:
with io.open(fd, 'wb') as f:
expiry = self.get_backend_timeout(timeout)
f.write(pickle.dumps(expiry, pickle.HIGHEST_PROTOCOL))
f.write(zlib.compress(pickle.dumps(value, pickle.HIGHEST_PROTOCOL)))
file_move_safe(tmp_path, fname, allow_overwrite=True)
renamed = True
finally:
if not renamed:
os.remove(tmp_path)
def handle_noargs(self, **options):
processed = 0
delete_orphans = options.get("delete_orphans", False)
blank_orphans = options.get("blank_orphans", False)
if blank_orphans and delete_orphans:
raise Exception("Can't set both --blank-orphans and --delete-orphans")
sys.exit()
delete_orphans = not blank_orphans
for item in PrivateFile.objects.all():
if item.file is None:
print("ignoring", item)
continue
itempath = item.file.path
if not os.path.exists(itempath):
if delete_orphans:
print("Deleting {}".format(item.file))
item.delete()
if blank_orphans:
print("Blanking {}".format(item.file))
item.file = ""
item.save()
continue
if " " in itempath:
ext = None
dirname = os.path.dirname(itempath)
filename = os.path.basename(itempath)
if "." in filename:
filename, ext = filename.split(".")
filename = ".".join([slugify(filename), ext])
else:
filename = slugify(filename)
new_path = os.sep.join([dirname, filename])
file_move_safe(item.file.path, new_path)
item.file = new_path.replace(settings.MEDIA_ROOT + "/", "")
item.save()
processed += 1
if options["verbosity"] > 0:
print("Cleaned up {} objects".format(processed))
def _save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
# CHANGED: This un-hangs us long enough to keep things rolling.
eventlet.sleep(0)
finally:
locks.unlock(fd)
os.close(fd)
except OSError, e:
if e.errno == errno.EEXIST:
# Ooops, the file exists. We need a new file name.
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
# OK, the file save worked. Break out of the loop.
break
def _save(self, name, content):
"""
Lifted partially from django/core/files/storage.py
"""
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
temp_data_location = content.temporary_file_path()
else:
tmp_prefix = "tmp_%s" % (get_valid_filename(name), )
temp_data_location = tempfile.mktemp(prefix=tmp_prefix,
dir=self.location)
try:
# This is a normal uploadedfile that we can stream.
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(temp_data_location,
os.O_WRONLY | os.O_CREAT |
os.O_EXCL | getattr(os, 'O_BINARY', 0))
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
locks.unlock(fd)
os.close(fd)
except Exception:
if os.path.exists(temp_data_location):
os.remove(temp_data_location)
raise
file_move_safe(temp_data_location, full_path, allow_overwrite=True)
content.close()
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def _save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
try:
content.seek(0)
image = Image.open(content)
except IOError:
raise exceptions.InvalidImageUploadException()
image_size = (self.PHOTO_MAX_X, self.PHOTO_MAX_Y)
image.thumbnail(image_size, Image.ANTIALIAS)
image.save(full_path)
except OSError, e:
if e.errno == errno.EEXIST:
# Ooops, the file exists. We need a new file name.
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
# OK, the file save worked. Break out of the loop.
break
def _upload_file(request):
"""
Upload file to the server.
"""
from django.core.files.move import file_move_safe
if request.method == 'POST':
folder = request.POST.get('folder')
fb_uploadurl_re = re.compile(r'^(%s)' % reverse("fb_upload"))
folder = fb_uploadurl_re.sub('', folder)
abs_path = os.path.join(MEDIA_ROOT, DIRECTORY, folder)
if request.FILES:
filedata = request.FILES['Filedata']
filedata.name = convert_filename(filedata.name)
# PRE UPLOAD SIGNAL
filebrowser_pre_upload.send(sender=request,
path=request.POST.get('folder'),
file=filedata)
# HANDLE UPLOAD
uploadedfile = handle_file_upload(abs_path, filedata)
# MOVE UPLOADED FILE
# if file already exists
if os.path.isfile(
os.path.join(MEDIA_ROOT, DIRECTORY, folder,
filedata.name)):
old_file = os.path.join(abs_path, filedata.name)
new_file = os.path.join(abs_path, uploadedfile)
file_move_safe(new_file, old_file)
# POST UPLOAD SIGNAL
filebrowser_post_upload.send(
sender=request,
path=request.POST.get('folder'),
file=FileObject(os.path.join(DIRECTORY, folder,
filedata.name)))
return HttpResponse('True')
def _upload_file(request):
"""
Upload file to the server.
"""
if request.method == 'POST':
folder = request.GET.get('folder')
fb_uploadurl_re = re.compile(r'^.*({0})'.format(reverse("fb_upload")))
folder = fb_uploadurl_re.sub('', folder)
abs_path = _check_access(request, folder)
if request.FILES:
filedata = request.FILES['file']
filedata.name = convert_filename(filedata.name)
_check_access(request, abs_path, filedata.name)
# PRE UPLOAD SIGNAL
filebrowser_pre_upload.send(sender=request,
path=request.GET.get('folder'),
file=filedata)
# HANDLE UPLOAD
uploadedfile = handle_file_upload(abs_path, filedata)
# MOVE UPLOADED FILE
# if file already exists
if os.path.isfile(smart_str(
os.path.join(fb_settings.MEDIA_ROOT, fb_settings.DIRECTORY,
folder, filedata.name))):
old_file = smart_str(os.path.join(abs_path, filedata.name))
new_file = smart_str(os.path.join(abs_path, uploadedfile))
file_move_safe(new_file, old_file)
# POST UPLOAD SIGNAL
filebrowser_post_upload.send(sender=request,
path=request.GET.get('folder'),
file=FileObject(smart_str(
os.path.join(
fb_settings.DIRECTORY, folder,
filedata.name))))
return JsonResponse({'success': True})
class OverwriteStorage(FileSystemStorage):
"""
File storage that allows overwriting of stored files.
"""
def get_available_name(self, name):
return name
def _save(self, name, content):
"""
Lifted partially from django/core/files/storage.py
"""
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
temp_data_location = content.temporary_file_path()
else:
tmp_prefix = "tmp_%s" % (get_valid_filename(name), )
temp_data_location = tempfile.mktemp(prefix=tmp_prefix,
dir=self.location)
try:
# This is a normal uploadedfile that we can stream.
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(
temp_data_location, os.O_WRONLY | os.O_CREAT | os.O_EXCL
| getattr(os, 'O_BINARY', 0))
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
locks.unlock(fd)
os.close(fd)
except Exception, e:
if os.path.exists(temp_data_location):
os.remove(temp_data_location)
raise
file_move_safe(temp_data_location, full_path)
content.close()
return name
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
if self.directory_permissions_mode is not None:
# os.makedirs applies the global umask, so we reset it,
# for consistency with file_permissions_mode behavior.
old_umask = os.umask(0)
try:
os.makedirs(directory, self.directory_permissions_mode)
finally:
os.umask(old_umask)
else:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL
| getattr(os, 'O_BINARY', 0))
# The current umask value is masked out by os.open!
fd = os.open(full_path, flags, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk,
bytes) else 'wt'
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
except OSError as e:
if e.errno == errno.EEXIST:
# Ooops, the file exists. We need a new file name.
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
# OK, the file save worked. Break out of the loop.
break
if self.file_permissions_mode is not None:
os.chmod(full_path, self.file_permissions_mode)
return name
def _save(self, name, content):
# print(f'about to save {name} to {self.path(name)}')
if self.exists(name):
self.delete(name)
# print("deleted!!!")
full_path = self.path(name)
# Create any intermediate directories that do not exist.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
if self.directory_permissions_mode is not None:
# os.makedirs applies the global umask, so we reset it,
# for consistency with file_permissions_mode behavior.
old_umask = os.umask(0)
try:
os.makedirs(directory, self.directory_permissions_mode)
finally:
os.umask(old_umask)
else:
os.makedirs(directory)
except FileExistsError:
# There's a race between os.path.exists() and os.makedirs().
# If os.makedirs() fails with FileExistsError, the directory
# was created concurrently.
pass
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
# This is a normal uploadedfile that we can stream.
else:
# The current umask value is masked out by os.open!
fd = os.open(full_path, self.OS_OPEN_FLAGS, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk, bytes) else 'wt'
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
except FileExistsError:
# A new name is needed if the file exists.
pass
name = self.get_available_name(name)
full_path = self.path(name)
else:
# OK, the file save worked. Break out of the loop.
break
if self.file_permissions_mode is not None:
os.chmod(full_path, self.file_permissions_mode)
# Store filenames with forward slashes, even on Windows.
return name.replace('\\', '/')
class FileSystemStorage(Storage):
"""
Standard filesystem storage
"""
def __init__(self, location=None, base_url=None):
if location is None:
location = settings.MEDIA_ROOT
self.base_location = location
self.location = abspathu(self.base_location)
if base_url is None:
base_url = settings.MEDIA_URL
self.base_url = base_url
def _open(self, name, mode='rb'):
return File(open(self.path(name), mode))
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError, e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
except OSError, e:
if e.errno == errno.EEXIST:
# Ooops, the file exists. We need a new file name.
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
# OK, the file save worked. Break out of the loop.
break
def move(self, src_name, to_name):
file_move_safe(self.path(src_name), self.path(to_name))
def move(self, old_file_name, new_file_name, allow_overwrite=False):
file_move_safe(self.path(old_file_name),
self.path(new_file_name),
allow_overwrite=True)
def post(self, request, *args, **kwargs):
form = self.form_class(request.POST, request.FILES)
if form.is_valid():
try:
file_data = form.cleaned_data['zipfile']
filepath = get_uploaded_filepath(file_data)
solutions_zip = ZipFile(filepath)
# Check for any corrupted files in the zip
# testzip returns the list of corrupted ones
if solutions_zip.testzip():
messages.error(
request, '"%s" in the .zip archive is corrupt.' %
solutions_zip.testzip())
raise Exception('Corrpted archive.')
# We loop over all PDF files in the zip
for filename in [
name for name in solutions_zip.namelist()
if name.endswith('.pdf')
]:
# Check that the name is of the form
# <score>-<username>-<problem_pk>.pdf
try:
parts = filename.rstrip('.pdf').split('-')
score = int(parts[0])
username = '******'.join(parts[1:-1])
problem_pk = int(parts[-1])
except (IndexError, ValueError, AssertionError):
messages.error(
request, '"%s" is not of the correct form '
'<score>-<username>-<problem_pk>.pdf' %
remove_accents(filename))
continue
# Find the UserSolution and modify it
try:
user = User.objects.get(username=username)
solution = UserSolution.objects.get(user=user,
problem=problem_pk)
except User.DoesNotExist:
messages.error('User %s does not exist' % username)
continue
except UserSolution.DoesNotExist:
messages.error(
request, 'Solution for user %s and problem '
'%d does not exist.' % (username, problem_pk))
continue
extracted_path = solutions_zip.extract(filename,
path='/tmp')
new_path = os.path.join(
settings.SENDFILE_ROOT,
solution.get_corrected_solution_path())
file_move_safe(extracted_path,
new_path,
allow_overwrite=True)
solution.score = score
solution.corrected_solution = solution.get_corrected_solution_path(
)
solution.save()
messages.success(
request,
_("%s assigned %d points") % (solution, score))
except Exception, e:
# If any exceptions happened, errors should be in messages
messages.error(request, 'exception happened: %s' % e)
finally:
def move(source_username, source_path, target_username, target_path):
source_full_path = path_(source_username, source_path)
target_full_path = path_(target_username, target_path)
file_move_safe(source_full_path, target_full_path)
return target_full_path
def _save(self, name, content):
#if isinstance(content, ContentFile) or isinstance(content, InMemoryUploadedFile):
print("IN MOVER")
print(type(content))
print("INNER")
print(type(content.file))
print("START FILE CHECK")
if not isinstance(content.file, File):
return super()._save(name, content)
full_path = self.path(name)
# Create any intermediate directories that do not exist.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
if self.directory_permissions_mode is not None:
# os.makedirs applies the global umask, so we reset it,
# for consistency with file_permissions_mode behavior.
old_umask = os.umask(0)
try:
os.makedirs(directory, self.directory_permissions_mode)
finally:
os.umask(old_umask)
else:
os.makedirs(directory)
except FileNotFoundError:
# There's a race between os.path.exists() and os.makedirs().
# If os.makedirs() fails with FileNotFoundError, the directory
# was created concurrently.
pass
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
print("")
print("MADE IT HERE")
print("")
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
else:
input_file_path = content.file.name
# Close input file before moving
content.close()
os.rename(input_file_path, full_path)
except FileExistsError:
# A new name is needed if the file exists.
name = self.get_available_name(name)
full_path = self.path(name)
else:
break
if self.file_permissions_mode is not None:
os.chmod(full_path, self.file_permissions_mode)
# Store filenames with forward slashes, even on Windows.
return name.replace('\\', '/')
def _save_FIELD_file(self, field, filename, raw_field, save=True):
# Create the upload directory if it doesn't already exist
directory = os.path.join(settings.MEDIA_ROOT,
field.get_directory_name())
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError('%s exists and is not a directory' % directory)
# Check for old-style usage (files-as-dictionaries). Warn here first
# since there are multiple locations where we need to support both new
# and old usage.
if isinstance(raw_field, dict):
import warnings
warnings.warn(
message=
"Representing uploaded files as dictionaries is deprecated. Use django.core.files.uploadedfile.SimpleUploadedFile instead.",
category=DeprecationWarning,
stacklevel=2)
from django.core.files.uploadedfile import SimpleUploadedFile
raw_field = SimpleUploadedFile.from_dict(raw_field)
elif isinstance(raw_field, str):
import warnings
warnings.warn(
message=
"Representing uploaded files as strings is deprecated. Use django.core.files.uploadedfile.SimpleUploadedFile instead.",
category=DeprecationWarning,
stacklevel=2)
from django.core.files.uploadedfile import SimpleUploadedFile
raw_field = SimpleUploadedFile(filename, raw_field)
if filename is None:
filename = raw_field.file_name
filename = field.get_filename(filename)
# If the filename already exists, keep adding an underscore to the name
# of the file until the filename doesn't exist.
while os.path.exists(os.path.join(settings.MEDIA_ROOT, filename)):
try:
dot_index = filename.rindex('.')
except ValueError: # filename has no dot.
filename += '_'
else:
filename = filename[:dot_index] + '_' + filename[dot_index:]
# Save the file name on the object and write the file to disk.
setattr(self, field.attname, filename)
full_filename = self._get_FIELD_filename(field)
if hasattr(raw_field, 'temporary_file_path'):
# This file has a file path that we can move.
raw_field.close()
file_move_safe(raw_field.temporary_file_path(), full_filename)
else:
# This is a normal uploadedfile that we can stream.
fp = open(full_filename, 'wb')
locks.lock(fp, locks.LOCK_EX)
for chunk in raw_field.chunks():
fp.write(chunk)
locks.unlock(fp)
fp.close()
# Save the width and/or height, if applicable.
if isinstance(field, ImageField) and \
(field.width_field or field.height_field):
from django.utils.images import get_image_dimensions
width, height = get_image_dimensions(full_filename)
if field.width_field:
setattr(self, field.width_field, width)
if field.height_field:
setattr(self, field.height_field, height)
# Save the object because it has changed, unless save is False.
if save:
self.save()
def _save(self, name, content, max_length=None):
"""
Create dirs to the destination, move the file if already in MEDIA_ROOT, or copy otherwise.
Args:
name (str): Target path to which the file is copied.
content (File): Source file object.
max_length (int): Maximum supported length of file name.
Returns:
str: Final storage path.
"""
full_path = self.path(name)
# Create any intermediate directories that do not exist.
directory = os.path.dirname(full_path)
try:
if self.directory_permissions_mode is not None:
# os.makedirs applies the global umask, so we reset it,
# for consistency with file_permissions_mode behavior.
old_umask = os.umask(0)
try:
os.makedirs(directory, self.directory_permissions_mode, exist_ok=True)
finally:
os.umask(old_umask)
else:
os.makedirs(directory, exist_ok=True)
except FileExistsError:
raise FileExistsError('%s exists and is not a directory.' % directory)
try:
if hasattr(content, 'temporary_file_path') and \
content.temporary_file_path().startswith(settings.MEDIA_ROOT):
file_move_safe(content.temporary_file_path(), full_path)
else:
# This is a normal uploaded file that we can stream.
# The current umask value is masked out by os.open!
fd = os.open(full_path, self.OS_OPEN_FLAGS, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk, bytes) else 'wt'
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
except FileExistsError:
# It's a content addressable store so if the file is already in place we can do nothing
pass
if self.file_permissions_mode is not None:
os.chmod(full_path, self.file_permissions_mode)
# Store filenames with forward slashes, even on Windows.
return str(name).replace('\\', '/')
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
if self.directory_permissions_mode is not None:
# os.makedirs applies the global umask, so we reset it,
# for consistency with file_permissions_mode behavior.
old_umask = os.umask(0)
try:
os.makedirs(directory, self.directory_permissions_mode)
finally:
os.umask(old_umask)
else:
os.makedirs(directory)
except FileNotFoundError:
# There's a race between os.path.exists() and os.makedirs().
# If os.makedirs() fails with FileNotFoundError, the directory
# was created concurrently.
pass
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
# This is a normal uploadedfile that we can stream.
else:
# The current umask value is masked out by os.open!
fd = os.open(full_path, self.OS_OPEN_FLAGS, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk,
bytes) else 'wt'
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
except FileExistsError:
# A new name is needed if the file exists.
name = self.get_available_name(name)
full_path = self.path(name)
else:
# OK, the file save worked. Break out of the loop.
break
if self.file_permissions_mode is not None:
os.chmod(full_path, self.file_permissions_mode)
# Store filenames with forward slashes, even on Windows.
return name.replace('\\', '/')
def _save(self, name, content):
# e.g. name = 'images/takeaway.jpg'
#print('_save name: ' + str(name))
collection_str = os.path.dirname(name)
if (not collection_str):
raise ImproperlyConfigured(
"No collection string can be extracted from the file name '{}'. Has 'upload_to' not been declared in the Model field?"
.format(name))
#print('collection_str: ' + str(collection_str))
#print('uoload_to: ' + str(self.upload_to))
coll = self.db(collection_str)
path = None
with coll.auto_create_cb() as full_path:
#print('full_path: ' + str(full_path))
#path = full_path
# path is storing the 'name' (relpath) for return.
# One way to get this is by splitting the now-known
# fullpath (or getting the pk then calling
# coll.relpath(pk)?)
# NB: BucketDB does not guarentee the separator (unlike Django)
path_segs = full_path.split(os.path.sep)
path = os.path.join(path_segs[-3], path_segs[-2], path_segs[-1])
#print('relpath: ' + str(path))
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
#? unecessary
flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL
| getattr(os, 'O_BINARY', 0))
# The current umask value is masked out by os.open!
fd = os.open(full_path, flags, 0o666)
_file = None
try:
#locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk, bytes) else 'wt'
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
#locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
# The original code returns a name (which is 'upload_to' +
# name). We return the relpath (which is 'upload_to'
# (collection) + bucket + pk).
# In the field, the return is dumped on FieldFile.name.
# self.name is used to construct filepaths, but not in
# any other way (it is an id locator for the file).
# Replacing with the relpath means attributes like 'size'
# work without overriding.
return path
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
directory = os.path.dirname(full_path)
try:
if self.directory_permissions_mode is not None:
# Set the umask because os.makedirs() doesn't apply the "mode"
# argument to intermediate-level directories.
old_umask = os.umask(0o777 & ~self.directory_permissions_mode)
try:
os.makedirs(
directory, self.directory_permissions_mode, exist_ok=True
)
finally:
os.umask(old_umask)
else:
os.makedirs(directory, exist_ok=True)
except FileExistsError:
raise FileExistsError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, "temporary_file_path"):
file_move_safe(content.temporary_file_path(), full_path)
# This is a normal uploadedfile that we can stream.
else:
# The current umask value is masked out by os.open!
fd = os.open(full_path, self.OS_OPEN_FLAGS, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = "wb" if isinstance(chunk, bytes) else "wt"
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
except FileExistsError:
# A new name is needed if the file exists.
name = self.get_available_name(name)
full_path = self.path(name)
else:
# OK, the file save worked. Break out of the loop.
break
if self.file_permissions_mode is not None:
os.chmod(full_path, self.file_permissions_mode)
# Ensure the saved path is always relative to the storage root.
name = os.path.relpath(full_path, self.location)
# Store filenames with forward slashes, even on Windows.
return str(name).replace("\\", "/")
def upload_solutions_with_points(self, request, pk=None):
if 'file' not in request.data:
raise exceptions.ParseError(detail='No file attached')
zfile = request.data['file']
if not zipfile.is_zipfile(zfile):
raise exceptions.ParseError(
detail='Attached file is not a zip file')
with zipfile.ZipFile(zfile) as zfile:
if zfile.testzip():
raise exceptions.ParseError(detail='Zip file is corrupted')
pdf_files = [
name for name in zfile.namelist() if name.endswith('.pdf')
]
errors = []
for filename in pdf_files:
try:
parts = filename.rstrip('.pdf').split('-')
score = int(parts[0])
problem_pk = int(parts[-2])
registration_pk = int(parts[-1])
event_reg = EventRegistration.objects.get(
pk=registration_pk)
solution = Solution.objects.get(
semester_registration=event_reg, problem=problem_pk)
except (IndexError, ValueError, AssertionError):
errors.append({
'filename': filename,
'status': 'Cannot parse file'
})
continue
except EventRegistration.DoesNotExist:
errors.append({
'filename':
filename,
'status':
f'User registration with id {registration_pk} does not exist'
})
continue
except Solution.DoesNotExist:
errors.append({
'filename':
filename,
'status':
f'Solution with registration id {registration_pk}'
f'and problem id {problem_pk} does not exist'
})
continue
extracted_path = zfile.extract(filename, path='/tmp')
new_path = os.path.join(
settings.MEDIA_ROOT, 'solutions',
solution.get_corrected_solution_file_name())
file_move_safe(extracted_path, new_path, allow_overwrite=True)
solution.score = score
solution.corrected_solution = solution.get_corrected_solution_file_name(
)
solution.save()
errors.append({
'filename': filename,
'status': f'OK - points: {score}'
})
return Response(json.dumps(errors))
def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
self._createdir() # Cache dir can be deleted at any time.
fname = self._key_to_file(key, version)
self._cull() # make some room if necessary
fd, tmp_path = tempfile.mkstemp(dir=self._dir)
renamed = False
try:
<<<<<<< HEAD
with io.open(fd, 'wb') as f:
=======
with open(fd, 'wb') as f:
>>>>>>> 37c99181c9a6b95433d60f8c8ef9af5731096435
expiry = self.get_backend_timeout(timeout)
f.write(pickle.dumps(expiry, pickle.HIGHEST_PROTOCOL))
f.write(zlib.compress(pickle.dumps(value, pickle.HIGHEST_PROTOCOL)))
file_move_safe(tmp_path, fname, allow_overwrite=True)
renamed = True
finally:
if not renamed:
os.remove(tmp_path)
def delete(self, key, version=None):
self._delete(self._key_to_file(key, version))
def _delete(self, fname):
if not fname.startswith(self._dir) or not os.path.exists(fname):
return
try:
os.remove(fname)
<<<<<<< HEAD
except OSError as e:
