def save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
print "Error dir is not error"
while True:
try:
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
else:
fd = os.open(
full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL
| getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
except OSError, e:
if e.errno == errno.EEXIST:
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
break
def __get__(self, obj, type=None):
if obj is None:
return
if self.name not in obj.__dict__:
# Load data from disk
path = self.path(obj)
if not self.storage.exists(path):
value = self.default
else:
# Open the file for reading and acquire a lock on it if
# possible
fp = self.storage.open(path, "r+b")
if isinstance(self.storage, FileSystemStorage):
locks.lock(fp.file, locks.LOCK_EX)
try:
# Read file content
if self.load is not None:
value = self.load(fp)
else:
value = fp.read().decode("utf8")
finally:
# Release lock
if isinstance(self.storage, FileSystemStorage):
locks.unlock(fp.file)
# Close file
fp.close()
obj.__dict__[self.name] = value
return obj.__dict__[self.name]
def test_shared_lock(self):
file_path = Path(__file__).parent / 'test.png'
with open(file_path) as f1, open(file_path) as f2:
self.assertIs(locks.lock(f1, locks.LOCK_SH), True)
self.assertIs(locks.lock(f2, locks.LOCK_SH | locks.LOCK_NB), True)
self.assertIs(locks.unlock(f1), True)
self.assertIs(locks.unlock(f2), True)
def save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
print "Error dir is not error"
while True:
try:
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
else:
fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
except OSError, e:
if e.errno == errno.EEXIST:
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
break
def _save(self, name, content):
"""
Lifted partially from django/core/files/storage.py
"""
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
temp_data_location = content.temporary_file_path()
else:
tmp_prefix = "tmp_%s" % (get_valid_filename(name), )
temp_data_location = tempfile.mktemp(prefix=tmp_prefix,
dir=self.location)
try:
# This is a normal uploadedfile that we can stream.
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(
temp_data_location, os.O_WRONLY | os.O_CREAT | os.O_EXCL
| getattr(os, 'O_BINARY', 0))
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
locks.unlock(fd)
os.close(fd)
except Exception, e:
if os.path.exists(temp_data_location):
os.remove(temp_data_location)
raise
def file_move_safe(old_file_name, new_file_name, chunk_size=1024 * 64, allow_overwrite=False):
"""
Moves a file from one location to another in the safest way possible.
First, tries ``os.rename``, which is simple but will break across filesystems.
If that fails, streams manually from one file to another in pure Python.
If the destination file exists and ``allow_overwrite`` is ``False``, this
function will throw an ``IOError``.
"""
# There's no reason to move if we don't have to.
if _samefile(old_file_name, new_file_name):
return
try:
# If the destination file exists and allow_overwrite is False then raise an IOError
if not allow_overwrite and os.access(new_file_name, os.F_OK):
raise IOError("Destination file %s exists and allow_overwrite is False" % new_file_name)
os.rename(old_file_name, new_file_name)
return
except OSError:
# This will happen with os.rename if moving to another filesystem
# or when moving opened files on certain operating systems
pass
# first open the old file, so that it won't go away
with open(old_file_name, 'rb') as old_file:
# now open the new file, not forgetting allow_overwrite
fd = os.open(new_file_name, (os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0) |
(os.O_EXCL if not allow_overwrite else 0)))
try:
locks.lock(fd, locks.LOCK_EX)
current_chunk = None
while current_chunk != b'':
current_chunk = old_file.read(chunk_size)
os.write(fd, current_chunk)
finally:
locks.unlock(fd)
os.close(fd)
try:
copystat(old_file_name, new_file_name)
except OSError as e:
# Certain filesystems (e.g. CIFS) fail to copy the file's metadata if
# the type of the destination filesystem isn't the same as the source
# filesystem; ignore that.
if getattr(e, 'errno', 0) != errno.EPERM:
raise
try:
os.remove(old_file_name)
except OSError as e:
# Certain operating systems (Cygwin and Windows)
# fail when deleting opened files, ignore it. (For the
# systems where this happens, temporary files will be auto-deleted
# on close anyway.)
if getattr(e, 'winerror', 0) != 32 and getattr(e, 'errno', 0) != 13:
raise
def _save(self, name, content):
"""
Lifted partially from django/core/files/storage.py
"""
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
temp_data_location = content.temporary_file_path()
else:
tmp_prefix = "tmp_%s" %(get_valid_filename(name), )
temp_data_location = tempfile.mktemp(prefix=tmp_prefix,
dir=self.location)
try:
# This is a normal uploadedfile that we can stream.
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(temp_data_location,
os.O_WRONLY | os.O_CREAT |
os.O_EXCL | getattr(os, 'O_BINARY', 0))
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
locks.unlock(fd)
os.close(fd)
except Exception, e:
if os.path.exists(temp_data_location):
os.remove(temp_data_location)
raise
def _save(self, name, content):
"""
Lifted partially from django/core/files/storage.py
"""
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# Ensure that content is open
content.open()
if hasattr(content, 'temporary_file_path'):
# Content has a file that we can move.
temp_data_location = content.temporary_file_path()
file_move_safe(temp_data_location, full_path, allow_overwrite=True)
else:
# Write the content stream to a temporary file and move it.
fd, tmp_path = mkstemp()
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
locks.unlock(fd)
os.close(fd)
file_move_safe(tmp_path, full_path, allow_overwrite=True)
content.close()
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def _save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_TRUNC \
| os.O_CREAT | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def file_move_safe(old_file_name, new_file_name, chunk_size=1024 * 64, allow_overwrite=False):
"""
Moves a file from one location to another in the safest way possible.
First, tries ``os.rename``, which is simple but will break across filesystems.
If that fails, streams manually from one file to another in pure Python.
If the destination file exists and ``allow_overwrite`` is ``False``, this
function will throw an ``IOError``.
"""
# There's no reason to move if we don't have to.
if _samefile(old_file_name, new_file_name):
return
try:
# If the destination file exists and allow_overwrite is False then raise an IOError
if not allow_overwrite and os.access(new_file_name, os.F_OK):
raise IOError("Destination file %s exists and allow_overwrite is False" % new_file_name)
os.rename(old_file_name, new_file_name)
return
except OSError:
# This will happen with os.rename if moving to another filesystem
# or when moving opened files on certain operating systems
pass
# first open the old file, so that it won't go away
with open(old_file_name, 'rb') as old_file:
# now open the new file, not forgetting allow_overwrite
fd = os.open(new_file_name, (os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0) |
(os.O_EXCL if not allow_overwrite else 0)))
try:
locks.lock(fd, locks.LOCK_EX)
current_chunk = None
while current_chunk != b'':
current_chunk = old_file.read(chunk_size)
os.write(fd, current_chunk)
finally:
locks.unlock(fd)
os.close(fd)
try:
copystat(old_file_name, new_file_name)
except OSError as e:
# Certain filesystems (e.g. CIFS) fail to copy the file's metadata if
# the type of the destination filesystem isn't the same as the source
# filesystem; ignore that.
if getattr(e, 'errno', 0) != errno.EPERM:
raise
try:
os.remove(old_file_name)
except OSError as e:
# Certain operating systems (Cygwin and Windows)
# fail when deleting opened files, ignore it. (For the
# systems where this happens, temporary files will be auto-deleted
# on close anyway.)
if getattr(e, 'winerror', 0) != 32 and getattr(e, 'errno', 0) != 13:
raise
def _save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_TRUNC \
| os.O_CREAT | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def _install_extension_media(self, ext_class):
"""Installs extension static media.
This method is a wrapper around _install_extension_media_internal to
check whether we actually need to install extension media, and avoid
contention among multiple threads/processes when doing so.
We need to install extension media if it hasn't been installed yet,
or if the version of the extension media that we installed is different
from the current version of the extension.
"""
lockfile = os.path.join(tempfile.gettempdir(), ext_class.id + '.lock')
extension = ext_class.instance
cur_version = ext_class.info.version
# We only want to fetch the existing version information if the
# extension is already installed. We remove this key when
# disabling an extension, so if it were there, it was either
# copy/pasted, or something went wrong. Either way, we wouldn't
# be able to trust it.
if ext_class.registration.installed:
old_version = extension.settings.get(self.VERSION_SETTINGS_KEY)
else:
old_version = None
if old_version == cur_version:
# Nothing to do
return
if not old_version:
logging.debug('Installing extension media for %s', ext_class.info)
else:
logging.debug('Reinstalling extension media for %s because '
'version changed from %s',
ext_class.info, old_version)
while old_version != cur_version:
with open(lockfile, 'w') as f:
try:
locks.lock(f, locks.LOCK_EX)
except IOError as e:
if e.errno == errno.EINTR:
# Sleep for one second, then try again
time.sleep(1)
extension.settings.load()
old_version = extension.settings.get(
self.VERSION_SETTINGS_KEY)
continue
else:
raise e
self._install_extension_media_internal(ext_class)
extension.settings.set(self.VERSION_SETTINGS_KEY, cur_version)
extension.settings.save()
old_version = cur_version
locks.unlock(f)
os.unlink(lockfile)
def _save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# Open file for writting in binary mode
fd = open(full_path.encode('utf-8'), "wb")
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
fd.write(chunk)
finally:
locks.unlock(fd)
fd.close()
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def _save(self, name, content):
"""
Lifted partially from django/core/files/storage.py
"""
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# Ensure that content is open
content.open()
if hasattr(content, 'temporary_file_path'):
# Content has a file that we can move.
temp_data_location = content.temporary_file_path()
file_move_safe(temp_data_location, full_path, allow_overwrite=True)
else:
# Write the content stream to a temporary file and move it.
fd, tmp_path = mkstemp()
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
locks.unlock(fd)
os.close(fd)
file_move_safe(tmp_path, full_path, allow_overwrite=True)
content.close()
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
old_umask = os.umask(0)
try:
os.makedirs(directory)
finally:
os.umask(old_umask)
except FileExistsError:
pass
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL |
getattr(os, 'O_BINARY', 0))
fd = os.open(full_path, flags, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk, bytes) else 'wt'
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
return name.replace('\\', '/')
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(
full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL
| getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
except OSError as e:
if e.errno == errno.EEXIST:
# Ooops, the file exists. We need a new file name.
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
# OK, the file save worked. Break out of the loop.
break
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def _save(self, name, content):
"""
Lifted partially from django/core/files/storage.py
"""
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
temp_data_location = content.temporary_file_path()
else:
tmp_prefix = "tmp_%s" % (get_valid_filename(name), )
temp_data_location = tempfile.mktemp(prefix=tmp_prefix,
dir=self.location)
try:
# This is a normal uploadedfile that we can stream.
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(temp_data_location,
os.O_WRONLY | os.O_CREAT |
os.O_EXCL | getattr(os, 'O_BINARY', 0))
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
locks.unlock(fd)
os.close(fd)
except Exception:
if os.path.exists(temp_data_location):
os.remove(temp_data_location)
raise
file_move_safe(temp_data_location, full_path, allow_overwrite=True)
content.close()
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def file_move_safe(old_file_name,
new_file_name,
chunk_size=1024 * 64,
allow_overwrite=False):
"""
Moves a file from one location to another in the safest way possible.
First, try using ``shutils.move``, which is OS-dependent but doesn't break
if moving across filesystems. Then, try ``os.rename``, which will break
across filesystems. Finally, streams manually from one file to another in
pure Python.
If the destination file exists and ``allow_overwrite`` is ``False``, this
function will throw an ``IOError``.
"""
# There's no reason to move if we don't have to.
if _samefile(old_file_name, new_file_name):
return
try:
os.rename(old_file_name, new_file_name)
return
except OSError:
# This will happen with os.rename if moving to another filesystem
# or when moving opened files on certain operating systems
pass
# first open the old file, so that it won't go away
old_file = open(old_file_name, 'rb')
try:
# now open the new file, not forgetting allow_overwrite
fd = os.open(
new_file_name,
os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0) |
(not allow_overwrite and os.O_EXCL or 0))
try:
locks.lock(fd, locks.LOCK_EX)
current_chunk = None
while current_chunk != '':
current_chunk = old_file.read(chunk_size)
os.write(fd, current_chunk)
finally:
locks.unlock(fd)
os.close(fd)
finally:
old_file.close()
copystat(old_file_name, new_file_name)
try:
os.remove(old_file_name)
except OSError, e:
# Certain operating systems (Cygwin and Windows)
# fail when deleting opened files, ignore it
if getattr(e, 'winerror', 0) != 32:
# FIXME: should we also ignore errno 13?
raise
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
except OSError as e:
if e.errno == errno.EEXIST:
# Ooops, the file exists. We need a new file name.
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
# OK, the file save worked. Break out of the loop.
break
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def _save(self, name, content):
# Prepare file name from the file content hash
dir_name, file_full_name = os.path.split(name)
file_name, file_ext = os.path.splitext(file_full_name)
content_hash = hashlib.sha1()
for chunk in content.chunks():
content_hash.update(chunk)
hash_name = self._split_name(content_hash.hexdigest())
# + str(content.size) Can add size if anytime stuck into collisions.
name = os.path.join(dir_name, hash_name + file_ext.lower())
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition: it's possible that two threads
# might return the same name. In this case we just continue normally.
# NOTE! This differs considerably from the default file storage behaviour.
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def _save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
print(directory, full_path, '----------save_method')
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
print(content.temporary_file_path(), full_path,
'-----temp_method')
file_move_safe(content.temporary_file_path(), full_path)
content.close()
else:
flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL
| getattr(os, 'O_BINARY', 0))
fd = os.open(full_path, flags, 0o666)
try:
locks.lock(fd, locks.LOCK_EX)
_file = None
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk,
bytes) else 'wt'
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
except OSError as e:
if e.errno == errno.EEXIST:
# Ooops, the file exists. We need a new file name.
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
break
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def file_move_safe(old_file_name, new_file_name, chunk_size=1024 * 64, allow_overwrite=False):
"""
Moves a file from one location to another in the safest way possible.
First, tries ``os.rename``, which is simple but will break across filesystems.
If that fails, streams manually from one file to another in pure Python.
If the destination file exists and ``allow_overwrite`` is ``False``, this
function will throw an ``IOError``.
"""
# There's no reason to move if we don't have to.
if _samefile(old_file_name, new_file_name):
return
try:
os.rename(old_file_name, new_file_name)
return
except OSError:
# This will happen with os.rename if moving to another filesystem
# or when moving opened files on certain operating systems
pass
# first open the old file, so that it won't go away
old_file = open(old_file_name, "rb")
try:
# now open the new file, not forgetting allow_overwrite
fd = os.open(
new_file_name,
os.O_WRONLY | os.O_CREAT | getattr(os, "O_BINARY", 0) | (not allow_overwrite and os.O_EXCL or 0),
)
try:
locks.lock(fd, locks.LOCK_EX)
current_chunk = None
while current_chunk != "":
current_chunk = old_file.read(chunk_size)
os.write(fd, current_chunk)
finally:
locks.unlock(fd)
os.close(fd)
finally:
old_file.close()
copystat(old_file_name, new_file_name)
try:
os.remove(old_file_name)
except OSError as e:
# Certain operating systems (Cygwin and Windows)
# fail when deleting opened files, ignore it. (For the
# systems where this happens, temporary files will be auto-deleted
# on close anyway.)
if getattr(e, "winerror", 0) != 32 and getattr(e, "errno", 0) != 13:
raise
def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False):
"""
Moves a file from one location to another in the safest way possible.
First, try using ``shutils.move``, which is OS-dependent but doesn't break
if moving across filesystems. Then, try ``os.rename``, which will break
across filesystems. Finally, streams manually from one file to another in
pure Python.
If the destination file exists and ``allow_overwrite`` is ``False``, this
function will throw an ``IOError``.
"""
# There's no reason to move if we don't have to.
if _samefile(old_file_name, new_file_name):
return
try:
os.rename(old_file_name, new_file_name)
return
except OSError:
# This will happen with os.rename if moving to another filesystem
# or when moving opened files on certain operating systems
pass
# first open the old file, so that it won't go away
old_file = open(old_file_name, 'rb')
try:
# now open the new file, not forgetting allow_overwrite
fd = os.open(new_file_name, os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0) |
(not allow_overwrite and os.O_EXCL or 0))
try:
locks.lock(fd, locks.LOCK_EX)
current_chunk = None
while current_chunk != '':
current_chunk = old_file.read(chunk_size)
os.write(fd, current_chunk)
finally:
locks.unlock(fd)
os.close(fd)
finally:
old_file.close()
copystat(old_file_name, new_file_name)
try:
os.remove(old_file_name)
except OSError, e:
# Certain operating systems (Cygwin and Windows)
# fail when deleting opened files, ignore it
if getattr(e, 'winerror', 0) != 32:
# FIXME: should we also ignore errno 13?
raise
def _save(self, name, content):
'''
Copied from super and lightly modified - Unfortunately, the default
race condition handling will lead to an infinite loop here, since
get_available_name() doesn't return unique names for identical hashes.
'''
full_path = self.path(name)
if os.path.exists(full_path):
return name
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
# handle concurrency issue where the directory is created while
# by another thread after the call to exists()
try:
os.makedirs(directory)
except OSError as e:
# ignore EEXIST, since it means that our goal here was already
# accomplished
if e.errno != errno.EEXIST:
raise
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition when saving the file; it's
# possible that two threads might try to write the file simultaneously.
# We need to try to create the file, but abort if we detect that it
# already exists (in which case we assume that another thread is
# writing it).
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
except OSError, e:
# abort and continue normally if we detect the existence of the
# file we're trying to write, otherwise raise normally
if e.errno != errno.EEXIST:
raise
def _save(self, name, content):
""" This is similar to the parent calss but removes any safe-guards against existing files """
if not self.file_overwrite:
return super(OverwriteFileSystemStorage, self)._save(name, content)
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
flags = (os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0))
# The current umask value is masked out by os.open!
fd = os.open(full_path, flags, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk, bytes) else 'wt'
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def _save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
try:
if self.directory_permissions_mode is not None:
old_umask = os.umask(0)
try:
os.makedirs(directory,
self.directory_permissions_mode,
exist_ok=True)
finally:
os.umask(old_umask)
else:
os.makedirs(directory, exist_ok=True)
except FileExistsError:
raise FileExistsError("%s exists and is not a directory." %
directory)
while True:
try:
if hasattr(content, "temporary_file_path"):
file_move_safe(content.temporary_file_path(), full_path)
else:
fd = os.open(full_path, self.OS_OPEN_FLAGS, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = "wb" if isinstance(chunk,
bytes) else "wt"
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
except FileExistsError:
name = self.get_available_name(name)
full_path = self.path(name)
else:
break
if self.file_permissions_mode is not None:
os.chmod(full_path, self.file_permissions_mode)
return name.replace("\\", "/")
def test_exclusive_lock(self):
file_path = Path(__file__).parent / 'test.png'
with open(file_path) as f1, open(file_path) as f2:
self.assertIs(locks.lock(f1, locks.LOCK_EX), True)
self.assertIs(locks.lock(f2, locks.LOCK_EX | locks.LOCK_NB), False)
self.assertIs(locks.lock(f2, locks.LOCK_SH | locks.LOCK_NB), False)
self.assertIs(locks.unlock(f1), True)
def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None):
try:
with open(self._key_to_file(key, version), 'r+b') as f:
try:
locks.lock(f, locks.LOCK_EX)
if self._is_expired(f):
return False
else:
previous_value = pickle.loads(zlib.decompress(f.read()))
f.seek(0)
self._write_content(f, timeout, previous_value)
return True
finally:
locks.unlock(f)
except FileNotFoundError:
return False
def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None):
try:
with open(self._key_to_file(key, version), 'r+b') as f:
try:
locks.lock(f, locks.LOCK_EX)
if self._is_expired(f):
return False
else:
previous_value = pickle.loads(zlib.decompress(f.read()))
f.seek(0)
_write_content(f, self.get_backend_timeout(timeout), previous_value)
return True
finally:
locks.unlock(f)
except FileNotFoundError:
return False
def _save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
# CHANGED: This un-hangs us long enough to keep things rolling.
eventlet.sleep(0)
finally:
locks.unlock(fd)
os.close(fd)
except OSError, e:
if e.errno == errno.EEXIST:
# Ooops, the file exists. We need a new file name.
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
# OK, the file save worked. Break out of the loop.
break
def contribute_to_class(self, cls, name):
if self.filename is not None:
filename = self.filename
else:
filename = name
# Create a post_save signal handler to write data to disk
@receiver(post_save, sender=cls, weak=False,
dispatch_uid="fsfield_write_data_%s_%s_%s" %
(cls.__module__, cls.__name__, name))
def write_data(sender, instance, created, raw, using, **kwargs):
if (name not in instance.__dict__ or
instance.__dict__[name] is None):
# Nothing to save
return
# Open the file for writing and acquire a lock on it if
# possible
path = model_instance_field_path(instance, filename)
fs_storage = isinstance(self.storage, FileSystemStorage)
if fs_storage:
full_path = self.storage.path(path)
directory = op.dirname(full_path)
if not op.exists(directory):
try:
os.makedirs(directory)
except OSError, err:
# Another thread may have created the directory since
# the check
if err.errno == 17:
pass
fp = self.storage.open(path, "wb")
if fs_storage:
locks.lock(fp.file, locks.LOCK_EX)
# Write data
try:
value = instance.__dict__[name]
if self.dump is None:
fp.write(value.encode("utf8"))
else:
self.dump(value, fp)
finally:
# Release lock
if fs_storage:
locks.unlock(fp.file)
# Close file
fp.close()
class ImageStorage(FileSystemStorage):
"""
Create image file and empty thumb file.
"""
# almost all copyed from FileSystemStorage._save
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError, e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
thumb_created = False
thumb_name = get_thumb_name(full_path)
try:
fd = os.open(thumb_name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
os.close(fd)
except OSError, e:
if e.errno == errno.EEXIST:
# Ooops, the file exists. We need a new file name.
name = self.get_available_name(name)
full_path = self.path(name)
continue
else:
raise
else:
thumb_created = True
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False):
"""
Moves a file from one location to another in the safest way possible.
First, try using ``shutils.move``, which is OS-dependent but doesn't break
if moving across filesystems. Then, try ``os.rename``, which will break
across filesystems. Finally, streams manually from one file to another in
pure Python.
If the destination file exists and ``allow_overwrite`` is ``False``, this
function will throw an ``IOError``.
"""
# There's no reason to move if we don't have to.
if old_file_name == new_file_name:
return
if not allow_overwrite and os.path.exists(new_file_name):
raise IOError("Cannot overwrite existing file '%s'." % new_file_name)
try:
file_move(old_file_name, new_file_name)
return
except OSError:
# This will happen with os.rename if moving to another filesystem
pass
# If the built-in didn't work, do it the hard way.
fd = os.open(new_file_name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
old_file = open(old_file_name, 'rb')
current_chunk = None
while current_chunk != '':
current_chunk = old_file.read(chunk_size)
os.write(fd, current_chunk)
finally:
locks.unlock(fd)
os.close(fd)
old_file.close()
os.remove(old_file_name)
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL
| getattr(os, 'O_BINARY', 0))
# The current umask value is masked out by os.open!
fd = os.open(full_path, flags, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk, bytes) else 'wt'
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
file_name = self.client.put(self.path(name))
os.unlink(full_path)
return file_name
def append(self, name, content, progress_callback=None):
full_path = self.path(name)
while True:
# This file has a file path that we can move.
#if hasattr(content, 'temporary_file_path'):
# file_move_safe(content.temporary_file_path(), full_path)
#
## This is a normal uploadedfile that we can stream.
#else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
flags = (os.O_WRONLY | getattr(os, 'O_BINARY', 0))
# The current umask value is masked out by os.open!
fd = os.open(full_path, flags, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
# if isinstance(chunk, bytes) else 'wt'
mode = 'ab'
_file = os.fdopen(fd, mode)
_file.write(chunk)
break
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
break
if self.file_permissions_mode is not None:
os.chmod(full_path, self.file_permissions_mode)
# Store filenames with forward slashes, even on Windows.
return name.replace('\\', '/')
class HashFileSystemStorage(FileSystemStorage):
def __init__(self, location=None, base_url=None):
if location is None:
location = os.path.join(settings.MEDIA_ROOT, 'hash')
if base_url is None:
base_url = settings.MEDIA_URL + 'hash/'
super(HashFileSystemStorage, self).__init__(location, base_url)
def path(self, name):
name = os.path.join(name[:2], name[2:4], name)
return super(HashFileSystemStorage, self).path(name)
def save(self, name, content):
# 计算内容 HASH
md5 = hashlib.md5()
for chunk in content.chunks():
md5.update(chunk)
name = md5.hexdigest()
# 保存文件
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError, e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
try:
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
else:
fd = os.open(
full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL
| getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
except OSError, e:
if e.errno != errno.EEXIST:
raise
def snippet(request, snippet_id):
def _deleteUncompressedReport(report):
sibling_snippets = Snippet.objects.filter(report=report, status=Snippet.GENERATING)
if sibling_snippets.count() == 0:
current_dir = os.getcwd()
os.chdir(os.path.join(settings.DATA_REPORTS_ROOT, snippet.report.parentfolder()))
decompressedfolder = snippet.report.decompressedfolder()
if os.path.exists(decompressedfolder):
shutil.rmtree(decompressedfolder)
os.chdir(current_dir)
# Only by Ajax
if not(request.is_ajax()):
raise Http404
# Setup the locking mechanism
filename = os.path.join(settings.SNIPPETS_ROOT, 'snippets.lock')
lock_file = open(filename, 'wb')
try:
os.chmod(filename, 0766)
except:
pass
locks.lock(lock_file, locks.LOCK_EX)
# Retrieve the snippet
snippet = get_object_or_404(Snippet, pk=snippet_id)
# Ensure that it wasn't already generated
if snippet.status == Snippet.AVAILABLE:
locks.unlock(lock_file)
lock_file.close()
response = HttpResponse(mimetype="text/xml")
response["Cache-Control"] = "max-age=0,no-cache,no-store"
response.write(snippet.content())
return response
# Ensure that the generation didn't failed
if snippet.status == Snippet.ERROR:
locks.unlock(lock_file)
lock_file.close()
response = HttpResponse(mimetype="text/xml")
response["Cache-Control"] = "max-age=0,no-cache,no-store"
response.write('ERROR')
return response
# Ensure that the generation isn't in progress
if snippet.status == Snippet.GENERATING:
locks.unlock(lock_file)
lock_file.close()
response = HttpResponse(mimetype="text/xml")
response["Cache-Control"] = "max-age=0,no-cache,no-store"
response.write('UNAVAILABLE')
return response
# Uncompress the report files if necessary
sibling_snippets = Snippet.objects.filter(report=snippet.report, status=Snippet.GENERATING)
if sibling_snippets.count() == 0:
current_dir = os.getcwd()
os.chdir(os.path.join(settings.DATA_REPORTS_ROOT, snippet.report.parentfolder()))
decompressedfolder = snippet.report.decompressedfolder()
if not(os.path.exists(decompressedfolder)):
os.makedirs(decompressedfolder)
tar = tarfile.open(snippet.report.archivename(), 'r:gz')
tar.extractall(path=decompressedfolder)
tar.close()
os.chdir(current_dir)
# Update the status
snippet.status = Snippet.GENERATING
snippet.save()
locks.unlock(lock_file)
try:
# Execute the script of the view
if not(os.path.exists(os.path.join(settings.SNIPPETS_ROOT, snippet.folder))):
os.makedirs(os.path.join(settings.SNIPPETS_ROOT, snippet.folder))
current_dir = os.getcwd()
os.chdir(os.path.join(settings.INSTRUMENTS_ROOT, snippet.view.instrument.fullname()))
p = subprocess.Popen('./%s %s %s /snippets/%s' % (snippet.view.script, os.path.join(settings.DATA_REPORTS_ROOT, snippet.report.parentfolder(), snippet.report.decompressedfolder()),
os.path.join(settings.SNIPPETS_ROOT, snippet.folder), snippet.folder),
shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if p.wait() == 0:
locks.lock(lock_file, locks.LOCK_EX)
snippet.status = Snippet.AVAILABLE
snippet.save()
_deleteUncompressedReport(snippet.report)
locks.unlock(lock_file)
else:
print p.stdout.read()
raise IOError
os.chdir(current_dir)
except:
locks.lock(lock_file, locks.LOCK_EX)
snippet.status = Snippet.ERROR
snippet.save()
_deleteUncompressedReport(snippet.report)
locks.unlock(lock_file)
lock_file.close()
response = HttpResponse(mimetype="text/xml")
response["Cache-Control"] = "max-age=0,no-cache,no-store"
response.write('ERROR')
return response
lock_file.close()
# Return the content of the snippet
response = HttpResponse(mimetype="text/xml")
response["Cache-Control"] = "max-age=0,no-cache,no-store"
response.write(snippet.content())
return response
def _save(self, name, content):
# print(f'about to save {name} to {self.path(name)}')
if self.exists(name):
self.delete(name)
# print("deleted!!!")
full_path = self.path(name)
# Create any intermediate directories that do not exist.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
if self.directory_permissions_mode is not None:
# os.makedirs applies the global umask, so we reset it,
# for consistency with file_permissions_mode behavior.
old_umask = os.umask(0)
try:
os.makedirs(directory, self.directory_permissions_mode)
finally:
os.umask(old_umask)
else:
os.makedirs(directory)
except FileExistsError:
# There's a race between os.path.exists() and os.makedirs().
# If os.makedirs() fails with FileExistsError, the directory
# was created concurrently.
pass
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
# This is a normal uploadedfile that we can stream.
else:
# The current umask value is masked out by os.open!
fd = os.open(full_path, self.OS_OPEN_FLAGS, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk, bytes) else 'wt'
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
except FileExistsError:
# A new name is needed if the file exists.
pass
name = self.get_available_name(name)
full_path = self.path(name)
else:
# OK, the file save worked. Break out of the loop.
break
if self.file_permissions_mode is not None:
os.chmod(full_path, self.file_permissions_mode)
# Store filenames with forward slashes, even on Windows.
return name.replace('\\', '/')
def processClusteringResults(self, job):
response = job.client.waitResponse()
if response.name != 'RESULTS':
alert = Alert()
alert.message = 'Failed to perform the clustering'
alert.details = "Response: %s" % response.toString()
job.outStream.write("ERROR - %s\n" % alert.message)
job.outStream.write(" %s\n" % alert.details)
job.server_job.logs = getServerLogs(job.client)
job.server_job.save()
job.markAsFailed(alert=alert)
return
job.outStream.write("Clustering done, retrieving the results...\n")
filesize = int(response.parameters[0])
content = job.client.waitData(filesize)
if content is None:
alert = Alert()
alert.message = 'Failed to retrieve the results of the clustering'
alert.details = "Response: %s" % response.toString()
job.outStream.write("ERROR - %s\n" % alert.message)
job.outStream.write(" %s\n" % alert.details)
job.server_job.logs = getServerLogs(job.client)
job.server_job.save()
job.markAsFailed(alert=alert)
return
job.client.sendCommand('DONE')
job.client.close()
del job.client
job.client = None
# Mark the signatures as processed
for signature in job.sent_signatures:
signature.processed = True
signature.save()
# Save the results of the clustering
if not(os.path.exists(os.path.join(settings.SNIPPETS_ROOT, 'clustering'))):
os.makedirs(os.path.join(settings.SNIPPETS_ROOT, 'clustering'))
filename = '%s.dat.tmp' % job.command.parameters[0]
f = open(os.path.join(settings.SNIPPETS_ROOT, 'clustering', filename), 'w')
f.write(content)
f.close()
# Lock the snippets folder
filename = os.path.join(settings.SNIPPETS_ROOT, 'snippets.lock')
lock_file = open(filename, 'wb')
try:
os.chmod(filename, 0766)
except:
pass
locks.lock(lock_file, locks.LOCK_EX)
# Update the clustering results file used by the website
os.rename(os.path.join(settings.SNIPPETS_ROOT, 'clustering', filename),
os.path.join(settings.SNIPPETS_ROOT, 'clustering', filename[:-4]))
# Unlock the snippets folder
locks.unlock(lock_file)
# Mark the job as done
job.markAsDone()
# Test if another clustering must start
signatures_status = SignatureStatus.objects.filter(signature__experiment__status=Experiment.STATUS_DONE,
algorithm__name=job.command.parameters[0], processed=False)
if signatures_status.count() > 0:
self.channel.sendMessage(Message('CLUSTER_HEURISTICS', job.command.parameters))
# Send an event about the end of the clustering
self.channel.sendMessage(Message('EVT_HEURISTICS_CLUSTERING_DONE', job.command.parameters))
def _save_FIELD_file(self, field, filename, raw_field, save=True):
# Create the upload directory if it doesn't already exist
directory = os.path.join(settings.MEDIA_ROOT, field.get_directory_name())
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError('%s exists and is not a directory' % directory)
# Check for old-style usage (files-as-dictionaries). Warn here first
# since there are multiple locations where we need to support both new
# and old usage.
if isinstance(raw_field, dict):
import warnings
warnings.warn(
message = "Representing uploaded files as dictionaries is deprecated. Use django.core.files.uploadedfile.SimpleUploadedFile instead.",
category = DeprecationWarning,
stacklevel = 2
)
from django.core.files.uploadedfile import SimpleUploadedFile
raw_field = SimpleUploadedFile.from_dict(raw_field)
elif isinstance(raw_field, str):
import warnings
warnings.warn(
message = "Representing uploaded files as strings is deprecated. Use django.core.files.uploadedfile.SimpleUploadedFile instead.",
category = DeprecationWarning,
stacklevel = 2
)
from django.core.files.uploadedfile import SimpleUploadedFile
raw_field = SimpleUploadedFile(filename, raw_field)
if filename is None:
filename = raw_field.file_name
filename = field.get_filename(filename)
# If the filename already exists, keep adding an underscore to the name
# of the file until the filename doesn't exist.
while os.path.exists(os.path.join(settings.MEDIA_ROOT, filename)):
try:
dot_index = filename.rindex('.')
except ValueError: # filename has no dot.
filename += '_'
else:
filename = filename[:dot_index] + '_' + filename[dot_index:]
# Save the file name on the object and write the file to disk.
setattr(self, field.attname, filename)
full_filename = self._get_FIELD_filename(field)
if hasattr(raw_field, 'temporary_file_path'):
# This file has a file path that we can move.
raw_field.close()
file_move_safe(raw_field.temporary_file_path(), full_filename)
else:
# This is a normal uploadedfile that we can stream.
fp = open(full_filename, 'wb')
locks.lock(fp, locks.LOCK_EX)
for chunk in raw_field.chunks():
fp.write(chunk)
locks.unlock(fp)
fp.close()
# Save the width and/or height, if applicable.
if isinstance(field, ImageField) and \
(field.width_field or field.height_field):
from django.utils.images import get_image_dimensions
width, height = get_image_dimensions(full_filename)
if field.width_field:
setattr(self, field.width_field, width)
if field.height_field:
setattr(self, field.height_field, height)
# Save the object because it has changed, unless save is False.
if save:
self.save()
def _save(self, name, content, max_length=None):
"""
Create dirs to the destination, move the file if already in MEDIA_ROOT, or copy otherwise.
Args:
name (str): Target path to which the file is copied.
content (File): Source file object.
max_length (int): Maximum supported length of file name.
Returns:
str: Final storage path.
"""
full_path = self.path(name)
# Create any intermediate directories that do not exist.
directory = os.path.dirname(full_path)
try:
if self.directory_permissions_mode is not None:
# os.makedirs applies the global umask, so we reset it,
# for consistency with file_permissions_mode behavior.
old_umask = os.umask(0)
try:
os.makedirs(directory, self.directory_permissions_mode, exist_ok=True)
finally:
os.umask(old_umask)
else:
os.makedirs(directory, exist_ok=True)
except FileExistsError:
raise FileExistsError('%s exists and is not a directory.' % directory)
try:
if hasattr(content, 'temporary_file_path') and \
content.temporary_file_path().startswith(settings.MEDIA_ROOT):
file_move_safe(content.temporary_file_path(), full_path)
else:
# This is a normal uploaded file that we can stream.
# The current umask value is masked out by os.open!
fd = os.open(full_path, self.OS_OPEN_FLAGS, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk, bytes) else 'wt'
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
except FileExistsError:
# It's a content addressable store so if the file is already in place we can do nothing
pass
if self.file_permissions_mode is not None:
os.chmod(full_path, self.file_permissions_mode)
# Store filenames with forward slashes, even on Windows.
return str(name).replace('\\', '/')
def assign_retainer_task(request, crowd_name):
try:
# get the interface implementation from the crowd name.
interface, model_spec = CrowdRegistry.get_registry_entry(crowd_name)
context = interface.get_response_context(request)
interface.require_context(
context, ['task_id', 'worker_id'],
ValueError("retainer assignment context missing required keys."))
try:
task = (model_spec.task_model.objects.select_related(
'group__retainer_pool').get(task_id=context['task_id']))
group = task.group
pool = group.retainer_pool
worker = model_spec.worker_model.objects.get(
worker_id=context['worker_id'])
logger.info('Retainer task %s requested work.' % task)
except Exception: # Issue loading models from IDs, finish this assignment
return HttpResponse(json.dumps({
'start': False,
'pool_status': 'finished'
}),
content_type='application/json')
exp_config = json.loads(group.global_config).get('experimental')
if exp_config:
straggler_mitigation = exp_config.get('mitigate_stragglers', False)
straggler_routing_policy = exp_config.get(
'straggler_routing_policy', 'random')
churn_threshold = exp_config.get('churn_threshold')
else:
straggler_mitigation = False
churn_threshold = None
# Acquire an exclusive lock to avoid duplicate assignments
lockf = open('/tmp/ASSIGNMENT_LOCK', 'wb')
logger.debug("Locking assignment lock...")
locks.lock(lockf, locks.LOCK_EX)
# Don't assign a task if the worker is on reserve or the pool is inactive.
on_reserve = (task.assignments.filter(worker=worker,
on_reserve=True).exists()
if churn_threshold is not None else False)
pool_inactive = pool.status not in (RetainerPoolStatus.ACTIVE,
RetainerPoolStatus.REFILLING,
RetainerPoolStatus.IDLE)
no_work_response = HttpResponse(json.dumps({
'start':
False,
'pool_status':
pool.get_status_display()
}),
content_type='application/json')
if on_reserve:
logger.info("Worker on reserve: not assigning work.")
return no_work_response
if pool_inactive:
logger.info(
"Pool still recruiting or otherwise inactive: not assigning work."
)
return no_work_response
# Look for a task the worker is already assigned to
assignment_task = None
existing_assignments = (worker.assignments.filter(
finished_at__isnull=True).filter(
task__group__retainer_pool=pool).exclude(
task__task_type='retainer'))
logger.info('Looking for assignments for retainer worker...')
if existing_assignments.exists():
assignment_task = existing_assignments[0].task
logger.info('Found an existing assignment for this worker')
else: # Look for open tasks
incomplete_tasks = (
# incomplete tasks
model_spec.task_model.objects.filter(is_complete=False)
# in this pool's tasks
.filter(group__retainer_pool=pool)
# that aren't dummy retainer tasks
.exclude(task_type='retainer')
# that the worker hasn't worked on already
.exclude(assignments__worker=worker))
# First check if the open tasks haven't been assigned to enough workers.
# TODO: avoid gross SQL
non_terminated_assignments = """
SELECT COUNT(*) FROM %(crowdname)s_%(assignment_model)s
WHERE %(crowdname)s_%(assignment_model)s.terminated = False
AND %(crowdname)s_%(assignment_model)s.task_id = %(crowdname)s_%(task_model)s.task_id
""" % {
'crowdname': crowd_name,
'assignment_model':
model_spec.assignment_model.__name__.lower(),
'task_model': model_spec.task_model.__name__.lower(),
}
open_tasks = incomplete_tasks.extra(
where=["num_assignments > (%s)" % non_terminated_assignments])
if open_tasks.exists():
logger.info('Found an unassigned but open task')
assignment_task = open_tasks.order_by('?')[0]
# Then, check if there in-progress tasks with enough assignments.
elif incomplete_tasks.exists():
if not straggler_mitigation: # only assign tasks that have been abandoned
# Bad performance characteristics! consider rewriting.
active_workers = set(pool.active_workers.all())
abandoned_tasks = [
t for t in incomplete_tasks if len([
a for a in t.assignments.select_related(
'worker').all() if a.worker in active_workers
]) < t.num_assignments
]
if abandoned_tasks:
logger.info('Found an assigned but abandoned task.')
assignment_task = random.choice(abandoned_tasks)
else:
logger.info('All tasks are assigned.')
# Straggler mitigation
else:
logger.info(
'Assigning to an active task for straggler mitigation with policy %s.'
% straggler_routing_policy)
if straggler_routing_policy == 'random':
assignment_task = incomplete_tasks.order_by('?')[0]
elif straggler_routing_policy == 'oldest':
now = timezone.now()
annotated = incomplete_tasks.annotate(
start=Min('assignments__assigned_at'))
weights = [(now - t.start).total_seconds()
for t in annotated]
weights = np.array(weights) / sum(weights)
assignment_task = np.random.choice(list(annotated),
size=1,
p=weights)[0]
elif straggler_routing_policy == 'young-workers':
now = timezone.now()
weights = [
1 / (now - min([
a.worker.assignments.filter(
task__task_type='retainer',
task__group__retainer_pool=pool).order_by(
'assigned_at')[0].assigned_at
for a in task.assignments.all()
])).total_seconds() for task in incomplete_tasks
]
weights = np.array(weights) / sum(weights)
assignment_task = np.random.choice(
list(incomplete_tasks), size=1, p=weights)[0]
elif straggler_routing_policy == 'fair':
# assign to the task with the fewest assignments
assignment_task = (incomplete_tasks.extra(
select={
'n_assignments': non_terminated_assignments
},
order_by=['n_assignments']))[0]
else:
logger.info(
'Unkown straggler routing policy: %s. Using random instead...'
% straggler_routing_policy)
assignment_task = incomplete_tasks.order_by('?')[0]
# return a url to the assignment
if assignment_task:
# create the assignment if necessary
try:
logger.info('Looking up assignment...')
assignment = worker.assignments.get(task=assignment_task,
worker=worker)
if not assignment.retainer_session_task:
assignment.retainer_session_task = task
assignment.save()
except model_spec.assignment_model.DoesNotExist:
logger.info('No assignment found: creating new one.')
assignment_id = str(uuid.uuid4())
assignment = model_spec.assignment_model.objects.create(
assignment_id=assignment_id,
worker=worker,
task=assignment_task,
retainer_session_task=task)
if not assignment_task.group.work_start_time:
assignment_task.group.work_start_time = timezone.now()
assignment_task.group.save()
url_args = {
'crowd_name': crowd_name,
'worker_id': worker.worker_id,
'task_id': assignment_task.task_id,
}
response_data = json.dumps({
'start':
True,
'task_url':
reverse('basecrowd:get_retainer_assignment', kwargs=url_args),
'task_id':
assignment_task.task_id,
'pool_status':
pool.get_status_display()
})
logger.info('Linking task to assignment.')
return HttpResponse(response_data, content_type='application/json')
else:
logger.info('No tasks found!')
return no_work_response
except Exception as e:
logger.exception(e)
raise e
finally:
# Release the assignment lock--either an assignment has been created in the DB, or an error occurred.
logger.debug("Unlocking assignment lock...")
locks.unlock(lockf)
lockf.close()
def createTopic(owner_object, forum_name, owner_object_name, owner_object_url, owner_object_kind):
# Lock the forum
filename = os.path.join(settings.FORUM_ROOT, 'forum.lock')
lock_file = open(filename, 'wb')
try:
os.chmod(filename, 0766)
except:
pass
locks.lock(lock_file, locks.LOCK_EX)
# Create the owner object topic if it don't already exists (it could have been
# created while we were locked)
if owner_object.post is None:
forum = PhpbbForum.objects.get(forum_name=forum_name)
# We can't directly use the model here, since Django refuses to save a post without a topic ID,
# and a topic without at least one post
cursor = connection.cursor()
post_subject = owner_object_name
forum_id = int(forum.forum_id)
poster = User.objects.get(username=settings.SYSTEM_ACCOUNT).get_profile().forum_user
poster_id = int(poster.user_id)
post_time = int(time.time())
bbcode_uid = 'deadf00d'
post_text = "This is the official topic of the [b:%s][url=%s/%s:%s]%s[/url:%s][/b:%s] %s." % \
(bbcode_uid, settings.WEBSITE_URL_DOMAIN.replace(':', ':'), owner_object_url, bbcode_uid, owner_object_name,
bbcode_uid, bbcode_uid, owner_object_kind)
cursor.execute("INSERT INTO `%s` (`post_subject`, `forum_id`, `poster_id`, `post_time`, `post_text`, `bbcode_uid`, `bbcode_bitfield`) VALUES ('%s', %d, %d, %d, '%s', '%s', 'UA==')" % \
(settings.DATABASE_PHPBB_PREFIX + 'posts', post_subject, forum_id, poster_id, post_time, post_text, bbcode_uid))
transaction.commit_unless_managed()
cursor.execute("SELECT `post_id` FROM `%s` WHERE `post_subject`='%s' AND `forum_id`=%d" % \
(settings.DATABASE_PHPBB_PREFIX + 'posts', post_subject, forum_id))
row = cursor.fetchone()
post_id = row[0]
cursor.execute("INSERT INTO `%s` (`topic_title`, `forum_id`, `topic_poster`, `topic_time`, `topic_first_post_id`, `topic_last_post_id`, `topic_last_post_time`, `topic_first_poster_name`, `topic_last_poster_id`, `topic_last_poster_name`) VALUES ('%s', %d, %d, %d, %d, %d, %d, '%s', %d, '%s')" % \
(settings.DATABASE_PHPBB_PREFIX + 'topics', post_subject, forum_id, poster_id, post_time, post_id, post_id, post_time, poster.username, poster_id, poster.username))
transaction.commit_unless_managed()
cursor.execute("SELECT `topic_id` FROM `%s` WHERE `topic_title`='%s' AND `forum_id`=%d" % \
(settings.DATABASE_PHPBB_PREFIX + 'topics', post_subject, forum_id))
row = cursor.fetchone()
topic_id = row[0]
cursor.execute("UPDATE `%s` SET `topic_id`=%d WHERE `post_id`=%d" % \
(settings.DATABASE_PHPBB_PREFIX + 'posts', topic_id, post_id))
transaction.commit_unless_managed()
cursor.execute("UPDATE `%s` SET `forum_last_post_id`=%d, `forum_last_poster_id`=%d, `forum_last_post_subject`='%s', `forum_last_post_time`=%d, `forum_last_poster_name`='%s' WHERE `forum_id`=%d" % \
(settings.DATABASE_PHPBB_PREFIX + 'forums', post_id, poster_id, post_subject, post_time, poster.username, forum_id))
transaction.commit_unless_managed()
post = PhpbbPost.objects.get(pk=post_id)
post.forum.forum_topics += 1
post.forum.forum_topics_real += 1
post.forum.forum_posts += 1
post.forum.save()
owner_object.post = post
owner_object.save()
poster.user_posts = poster.user_posts + 1
poster.user_lastpost_time = post_time
poster.save()
# Unlock the forum
locks.unlock(lock_file)
lock_file.close()
"""
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
if self.directory_permissions_mode is not None:
# os.makedirs applies the global umask, so we reset it,
# for consistency with file_permissions_mode behavior.
old_umask = os.umask(0)
try:
os.makedirs(directory, self.directory_permissions_mode)
finally:
os.umask(old_umask)
else:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL |
getattr(os, 'O_BINARY', 0))
# The current umask value is masked out by os.open!
fd = os.open(full_path, flags, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk, bytes) else 'wt'
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
except OSError as e:
if e.errno == errno.EEXIST:
# Ooops, the file exists. We need a new file name.
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
# OK, the file save worked. Break out of the loop.
break
if self.file_permissions_mode is not None:
os.chmod(full_path, self.file_permissions_mode)
return name
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
if self.directory_permissions_mode is not None:
# os.makedirs applies the global umask, so we reset it,
# for consistency with file_permissions_mode behavior.
old_umask = os.umask(0)
try:
os.makedirs(directory, self.directory_permissions_mode)
finally:
os.umask(old_umask)
else:
os.makedirs(directory)
except FileExistsError:
# There's a race between os.path.exists() and os.makedirs().
# If os.makedirs() fails with FileExistsError, the directory
# was created concurrently.
pass
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
# This is a normal uploadedfile that we can stream.
else:
# The current umask value is masked out by os.open!
fd = os.open(full_path, self.OS_OPEN_FLAGS, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk, bytes) else 'wt'
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
except FileExistsError:
# A new name is needed if the file exists.
name = self.get_available_name(name)
full_path = self.path(name)
else:
# OK, the file save worked. Break out of the loop.
break
if self.file_permissions_mode is not None:
os.chmod(full_path, self.file_permissions_mode)
# Store filenames with forward slashes, even on Windows.
return name.replace('\\', '/')
def release(self):
locks.unlock(self.__lock_fd)
self.__lock_fd.close()
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
if self.directory_permissions_mode is not None:
# os.makedirs applies the global umask, so we reset it,
# for consistency with file_permissions_mode behavior.
old_umask = os.umask(0)
try:
os.makedirs(directory, self.directory_permissions_mode)
finally:
os.umask(old_umask)
else:
os.makedirs(directory)
except FileNotFoundError:
# There's a race between os.path.exists() and os.makedirs().
# If os.makedirs() fails with FileNotFoundError, the directory
# was created concurrently.
pass
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
# This is a normal uploadedfile that we can stream.
else:
# The current umask value is masked out by os.open!
fd = os.open(full_path, self.OS_OPEN_FLAGS, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk,
bytes) else 'wt'
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
except FileExistsError:
# A new name is needed if the file exists.
name = self.get_available_name(name)
full_path = self.path(name)
else:
# OK, the file save worked. Break out of the loop.
break
if self.file_permissions_mode is not None:
os.chmod(full_path, self.file_permissions_mode)
# Store filenames with forward slashes, even on Windows.
return name.replace('\\', '/')
def _save_FIELD_file(self, field, filename, raw_field, save=True):
# Create the upload directory if it doesn't already exist
directory = os.path.join(settings.MEDIA_ROOT,
field.get_directory_name())
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError('%s exists and is not a directory' % directory)
# Check for old-style usage (files-as-dictionaries). Warn here first
# since there are multiple locations where we need to support both new
# and old usage.
if isinstance(raw_field, dict):
import warnings
warnings.warn(
message=
"Representing uploaded files as dictionaries is deprecated. Use django.core.files.uploadedfile.SimpleUploadedFile instead.",
category=DeprecationWarning,
stacklevel=2)
from django.core.files.uploadedfile import SimpleUploadedFile
raw_field = SimpleUploadedFile.from_dict(raw_field)
elif isinstance(raw_field, str):
import warnings
warnings.warn(
message=
"Representing uploaded files as strings is deprecated. Use django.core.files.uploadedfile.SimpleUploadedFile instead.",
category=DeprecationWarning,
stacklevel=2)
from django.core.files.uploadedfile import SimpleUploadedFile
raw_field = SimpleUploadedFile(filename, raw_field)
if filename is None:
filename = raw_field.file_name
filename = field.get_filename(filename)
# If the filename already exists, keep adding an underscore to the name
# of the file until the filename doesn't exist.
while os.path.exists(os.path.join(settings.MEDIA_ROOT, filename)):
try:
dot_index = filename.rindex('.')
except ValueError: # filename has no dot.
filename += '_'
else:
filename = filename[:dot_index] + '_' + filename[dot_index:]
# Save the file name on the object and write the file to disk.
setattr(self, field.attname, filename)
full_filename = self._get_FIELD_filename(field)
if hasattr(raw_field, 'temporary_file_path'):
# This file has a file path that we can move.
raw_field.close()
file_move_safe(raw_field.temporary_file_path(), full_filename)
else:
# This is a normal uploadedfile that we can stream.
fp = open(full_filename, 'wb')
locks.lock(fp, locks.LOCK_EX)
for chunk in raw_field.chunks():
fp.write(chunk)
locks.unlock(fp)
fp.close()
# Save the width and/or height, if applicable.
if isinstance(field, ImageField) and \
(field.width_field or field.height_field):
from django.utils.images import get_image_dimensions
width, height = get_image_dimensions(full_filename)
if field.width_field:
setattr(self, field.width_field, width)
if field.height_field:
setattr(self, field.height_field, height)
# Save the object because it has changed, unless save is False.
if save:
self.save()
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
directory = os.path.dirname(full_path)
try:
if self.directory_permissions_mode is not None:
# Set the umask because os.makedirs() doesn't apply the "mode"
# argument to intermediate-level directories.
old_umask = os.umask(0o777 & ~self.directory_permissions_mode)
try:
os.makedirs(
directory, self.directory_permissions_mode, exist_ok=True
)
finally:
os.umask(old_umask)
else:
os.makedirs(directory, exist_ok=True)
except FileExistsError:
raise FileExistsError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, "temporary_file_path"):
file_move_safe(content.temporary_file_path(), full_path)
# This is a normal uploadedfile that we can stream.
else:
# The current umask value is masked out by os.open!
fd = os.open(full_path, self.OS_OPEN_FLAGS, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = "wb" if isinstance(chunk, bytes) else "wt"
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
except FileExistsError:
# A new name is needed if the file exists.
name = self.get_available_name(name)
full_path = self.path(name)
else:
# OK, the file save worked. Break out of the loop.
break
if self.file_permissions_mode is not None:
os.chmod(full_path, self.file_permissions_mode)
# Ensure the saved path is always relative to the storage root.
name = os.path.relpath(full_path, self.location)
# Store filenames with forward slashes, even on Windows.
return str(name).replace("\\", "/")
def unlock(lock):
""" release lock (unlock locked file) """
locks.unlock(lock)
import errno
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
if self.directory_permissions_mode is not None:
# os.makedirs applies the global umask, so we reset it,
# for consistency with file_permissions_mode behavior.
old_umask = os.umask(0)
try:
os.makedirs(directory, self.directory_permissions_mode)
finally:
os.umask(old_umask)
else:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL
| getattr(os, 'O_BINARY', 0))
# The current umask value is masked out by os.open!
fd = os.open(full_path, flags, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk,
bytes) else 'wt'
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
except OSError as e:
if e.errno == errno.EEXIST:
# Ooops, the file exists. We need a new file name.
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
# OK, the file save worked. Break out of the loop.
break
if self.file_permissions_mode is not None:
os.chmod(full_path, self.file_permissions_mode)
return name
>>>>>>> 37c99181c9a6b95433d60f8c8ef9af5731096435
pass
# first open the old file, so that it won't go away
with open(old_file_name, 'rb') as old_file:
# now open the new file, not forgetting allow_overwrite
fd = os.open(new_file_name, (os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0) |
(os.O_EXCL if not allow_overwrite else 0)))
try:
locks.lock(fd, locks.LOCK_EX)
current_chunk = None
while current_chunk != b'':
current_chunk = old_file.read(chunk_size)
os.write(fd, current_chunk)
finally:
locks.unlock(fd)
os.close(fd)
try:
copystat(old_file_name, new_file_name)
<<<<<<< HEAD
except OSError as e:
# Certain filesystems (e.g. CIFS) fail to copy the file's metadata if
# the type of the destination filesystem isn't the same as the source
# filesystem; ignore that.
if getattr(e, 'errno', 0) != errno.EPERM:
=======
except PermissionError as e:
# Certain filesystems (e.g. CIFS) fail to copy the file's metadata if
# the type of the destination filesystem isn't the same as the source
# filesystem; ignore that.
