def save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
print "Error dir is not error"
while True:
try:
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
else:
fd = os.open(
full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL
| getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
except OSError, e:
if e.errno == errno.EEXIST:
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
break
def release(self):
super(FileLock, self).release()
lock.acquire()
with open(self.__class__.lock_file, 'r+') as file_object:
locks.lock(f=file_object, flags=locks.LOCK_EX)
try:
file_locks = json.loads(file_object.read())
except EOFError:
file_locks = {}
if self.name in file_locks:
if file_locks[self.name]['uuid'] == self.uuid:
file_locks.pop(self.name)
else:
# Lock expired and someone else acquired it
pass
else:
# Lock expired and someone else released it
pass
file_object.seek(0)
file_object.truncate()
file_object.write(json.dumps(file_locks))
lock.release()
def __get__(self, obj, type=None):
if obj is None:
return
if self.name not in obj.__dict__:
# Load data from disk
path = self.path(obj)
if not self.storage.exists(path):
value = self.default
else:
# Open the file for reading and acquire a lock on it if
# possible
fp = self.storage.open(path, "r+b")
if isinstance(self.storage, FileSystemStorage):
locks.lock(fp.file, locks.LOCK_EX)
try:
# Read file content
if self.load is not None:
value = self.load(fp)
else:
value = fp.read().decode("utf8")
finally:
# Release lock
if isinstance(self.storage, FileSystemStorage):
locks.unlock(fp.file)
# Close file
fp.close()
obj.__dict__[self.name] = value
return obj.__dict__[self.name]
def _save(self, name, content):
"""
Lifted partially from django/core/files/storage.py
"""
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# Ensure that content is open
content.open()
if hasattr(content, 'temporary_file_path'):
# Content has a file that we can move.
temp_data_location = content.temporary_file_path()
file_move_safe(temp_data_location, full_path, allow_overwrite=True)
else:
# Write the content stream to a temporary file and move it.
fd, tmp_path = mkstemp()
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
locks.unlock(fd)
os.close(fd)
file_move_safe(tmp_path, full_path, allow_overwrite=True)
content.close()
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def lock(self):
try:
lock_name = self.get_lock_name()
# need loop to avoid races on file unlinking
while True:
f = open(lock_name, 'w+', 0)
locks.lock(f, locks.LOCK_EX | locks.LOCK_NB)
# Here is the Race:
# Previous process "A" is still running. Process "B" opens
# the file and then the process "A" finishes and deletes it.
# "B" locks the deleted file (by fd it already have) and runs,
# then the next process "C" creates _new_ file and locks it
# successfully while "B" is still running.
# We just need to check that "B" didn't lock a deleted file
# to avoid any problems. If process "C" have locked
# a new file wile "B" stats it then ok, let "B" quit and "C" run.
# We can still meet an attacker that permanently creates and deletes
# our file but we can't avoid problems in that case.
if os.path.isfile(lock_name):
st1 = os.fstat(f.fileno())
st2 = os.stat(lock_name)
if st1.st_ino == st2.st_ino:
f.write(str(os.getpid()))
self.lockfile = f
return True
# else:
# retry. Don't unlink, next process might already use it.
f.close()
except IOError as e:
if e.errno in (errno.EACCES, errno.EAGAIN):
return False
else:
e = sys.exc_info()[1]
raise e
def file_move_safe(old_file_name, new_file_name, chunk_size=1024 * 64, allow_overwrite=False):
"""
Moves a file from one location to another in the safest way possible.
First, tries ``os.rename``, which is simple but will break across filesystems.
If that fails, streams manually from one file to another in pure Python.
If the destination file exists and ``allow_overwrite`` is ``False``, this
function will throw an ``IOError``.
"""
# There's no reason to move if we don't have to.
if _samefile(old_file_name, new_file_name):
return
try:
# If the destination file exists and allow_overwrite is False then raise an IOError
if not allow_overwrite and os.access(new_file_name, os.F_OK):
raise IOError("Destination file %s exists and allow_overwrite is False" % new_file_name)
os.rename(old_file_name, new_file_name)
return
except OSError:
# This will happen with os.rename if moving to another filesystem
# or when moving opened files on certain operating systems
pass
# first open the old file, so that it won't go away
with open(old_file_name, 'rb') as old_file:
# now open the new file, not forgetting allow_overwrite
fd = os.open(new_file_name, (os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0) |
(os.O_EXCL if not allow_overwrite else 0)))
try:
locks.lock(fd, locks.LOCK_EX)
current_chunk = None
while current_chunk != b'':
current_chunk = old_file.read(chunk_size)
os.write(fd, current_chunk)
finally:
locks.unlock(fd)
os.close(fd)
try:
copystat(old_file_name, new_file_name)
except OSError as e:
# Certain filesystems (e.g. CIFS) fail to copy the file's metadata if
# the type of the destination filesystem isn't the same as the source
# filesystem; ignore that.
if getattr(e, 'errno', 0) != errno.EPERM:
raise
try:
os.remove(old_file_name)
except OSError as e:
# Certain operating systems (Cygwin and Windows)
# fail when deleting opened files, ignore it. (For the
# systems where this happens, temporary files will be auto-deleted
# on close anyway.)
if getattr(e, 'winerror', 0) != 32 and getattr(e, 'errno', 0) != 13:
raise
def _save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# Open file for writting in binary mode
fd = open(full_path.encode('utf-8'), "wb")
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
fd.write(chunk)
finally:
locks.unlock(fd)
fd.close()
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def release(self):
super(FileLock, self).release()
lock.acquire()
with open(self.__class__.lock_file, 'r+') as file_object:
locks.lock(f=file_object, flags=locks.LOCK_EX)
try:
file_locks = json.loads(file_object.read())
except EOFError:
file_locks = {}
if self.name in file_locks:
if file_locks[self.name]['uuid'] == self.uuid:
file_locks.pop(self.name)
else:
# Lock expired and someone else acquired it
pass
else:
# Lock expired and someone else released it
pass
file_object.seek(0)
file_object.truncate()
file_object.write(json.dumps(file_locks))
lock.release()
def _purge_locks(cls):
lock.acquire()
with open(file=cls.lock_file, mode='r+') as file_object:
locks.lock(f=file_object, flags=locks.LOCK_EX)
file_object.seek(0)
file_object.truncate()
lock.release()
def __init__(self, name, timeout):
self.name = name
self.timeout = timeout
self.uuid = force_text(s=uuid.uuid4())
lock.acquire()
with open(file=self.__class__.lock_file, mode='r+') as file_object:
locks.lock(f=file_object, flags=locks.LOCK_EX)
data = file_object.read()
if data:
file_locks = json.loads(s=data)
else:
file_locks = {}
if name in file_locks:
# Someone already got this lock, check to see if it is expired
if file_locks[name]['expiration'] and time.time() > file_locks[name]['expiration']:
# It expires and has expired, we re-acquired it
file_locks[name] = self._get_lock_dictionary()
else:
lock.release()
raise LockError
else:
file_locks[name] = self._get_lock_dictionary()
file_object.seek(0)
file_object.truncate()
file_object.write(json.dumps(obj=file_locks))
lock.release()
def _save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_TRUNC \
| os.O_CREAT | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def _save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_TRUNC \
| os.O_CREAT | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def test_exclusive_lock(self):
file_path = Path(__file__).parent / 'test.png'
with open(file_path) as f1, open(file_path) as f2:
self.assertIs(locks.lock(f1, locks.LOCK_EX), True)
self.assertIs(locks.lock(f2, locks.LOCK_EX | locks.LOCK_NB), False)
self.assertIs(locks.lock(f2, locks.LOCK_SH | locks.LOCK_NB), False)
self.assertIs(locks.unlock(f1), True)
def checkStatus(request, user = ''): val = "" with open(user, 'rb') as f: locks.lock(f, locks.LOCK_EX) val = f.read() return HttpResponse(val)
def test_shared_lock(self):
file_path = Path(__file__).parent / 'test.png'
with open(file_path) as f1, open(file_path) as f2:
self.assertIs(locks.lock(f1, locks.LOCK_SH), True)
self.assertIs(locks.lock(f2, locks.LOCK_SH | locks.LOCK_NB), True)
self.assertIs(locks.unlock(f1), True)
self.assertIs(locks.unlock(f2), True)
def _install_extension_media(self, ext_class):
"""Installs extension static media.
This method is a wrapper around _install_extension_media_internal to
check whether we actually need to install extension media, and avoid
contention among multiple threads/processes when doing so.
We need to install extension media if it hasn't been installed yet,
or if the version of the extension media that we installed is different
from the current version of the extension.
"""
lockfile = os.path.join(tempfile.gettempdir(), ext_class.id + '.lock')
extension = ext_class.instance
cur_version = ext_class.info.version
# We only want to fetch the existing version information if the
# extension is already installed. We remove this key when
# disabling an extension, so if it were there, it was either
# copy/pasted, or something went wrong. Either way, we wouldn't
# be able to trust it.
if ext_class.registration.installed:
old_version = extension.settings.get(self.VERSION_SETTINGS_KEY)
else:
old_version = None
if old_version == cur_version:
# Nothing to do
return
if not old_version:
logging.debug('Installing extension media for %s', ext_class.info)
else:
logging.debug('Reinstalling extension media for %s because '
'version changed from %s',
ext_class.info, old_version)
while old_version != cur_version:
with open(lockfile, 'w') as f:
try:
locks.lock(f, locks.LOCK_EX)
except IOError as e:
if e.errno == errno.EINTR:
# Sleep for one second, then try again
time.sleep(1)
extension.settings.load()
old_version = extension.settings.get(
self.VERSION_SETTINGS_KEY)
continue
else:
raise e
self._install_extension_media_internal(ext_class)
extension.settings.set(self.VERSION_SETTINGS_KEY, cur_version)
extension.settings.save()
old_version = cur_version
locks.unlock(f)
os.unlink(lockfile)
def file_move_safe(old_file_name, new_file_name, chunk_size=1024 * 64, allow_overwrite=False):
"""
Moves a file from one location to another in the safest way possible.
First, tries ``os.rename``, which is simple but will break across filesystems.
If that fails, streams manually from one file to another in pure Python.
If the destination file exists and ``allow_overwrite`` is ``False``, this
function will throw an ``IOError``.
"""
# There's no reason to move if we don't have to.
if _samefile(old_file_name, new_file_name):
return
try:
# If the destination file exists and allow_overwrite is False then raise an IOError
if not allow_overwrite and os.access(new_file_name, os.F_OK):
raise IOError("Destination file %s exists and allow_overwrite is False" % new_file_name)
os.rename(old_file_name, new_file_name)
return
except OSError:
# This will happen with os.rename if moving to another filesystem
# or when moving opened files on certain operating systems
pass
# first open the old file, so that it won't go away
with open(old_file_name, 'rb') as old_file:
# now open the new file, not forgetting allow_overwrite
fd = os.open(new_file_name, (os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0) |
(os.O_EXCL if not allow_overwrite else 0)))
try:
locks.lock(fd, locks.LOCK_EX)
current_chunk = None
while current_chunk != b'':
current_chunk = old_file.read(chunk_size)
os.write(fd, current_chunk)
finally:
locks.unlock(fd)
os.close(fd)
try:
copystat(old_file_name, new_file_name)
except OSError as e:
# Certain filesystems (e.g. CIFS) fail to copy the file's metadata if
# the type of the destination filesystem isn't the same as the source
# filesystem; ignore that.
if getattr(e, 'errno', 0) != errno.EPERM:
raise
try:
os.remove(old_file_name)
except OSError as e:
# Certain operating systems (Cygwin and Windows)
# fail when deleting opened files, ignore it. (For the
# systems where this happens, temporary files will be auto-deleted
# on close anyway.)
if getattr(e, 'winerror', 0) != 32 and getattr(e, 'errno', 0) != 13:
raise
def _save(self, name, content):
"""
Lifted partially from django/core/files/storage.py
"""
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# Ensure that content is open
content.open()
if hasattr(content, 'temporary_file_path'):
# Content has a file that we can move.
temp_data_location = content.temporary_file_path()
file_move_safe(temp_data_location, full_path, allow_overwrite=True)
else:
# Write the content stream to a temporary file and move it.
fd, tmp_path = mkstemp()
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
locks.unlock(fd)
os.close(fd)
file_move_safe(tmp_path, full_path, allow_overwrite=True)
content.close()
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
old_umask = os.umask(0)
try:
os.makedirs(directory)
finally:
os.umask(old_umask)
except FileExistsError:
pass
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL |
getattr(os, 'O_BINARY', 0))
fd = os.open(full_path, flags, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk, bytes) else 'wt'
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
return name.replace('\\', '/')
def _save(self, name, content):
"""
Lifted partially from django/core/files/storage.py
"""
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
temp_data_location = content.temporary_file_path()
else:
tmp_prefix = "tmp_%s" %(get_valid_filename(name), )
temp_data_location = tempfile.mktemp(prefix=tmp_prefix,
dir=self.location)
try:
# This is a normal uploadedfile that we can stream.
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(temp_data_location,
os.O_WRONLY | os.O_CREAT |
os.O_EXCL | getattr(os, 'O_BINARY', 0))
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
locks.unlock(fd)
os.close(fd)
except Exception, e:
if os.path.exists(temp_data_location):
os.remove(temp_data_location)
raise
def _save(self, name, content):
"""
Lifted partially from django/core/files/storage.py
"""
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
temp_data_location = content.temporary_file_path()
else:
tmp_prefix = "tmp_%s" % (get_valid_filename(name), )
temp_data_location = tempfile.mktemp(prefix=tmp_prefix,
dir=self.location)
try:
# This is a normal uploadedfile that we can stream.
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(
temp_data_location, os.O_WRONLY | os.O_CREAT | os.O_EXCL
| getattr(os, 'O_BINARY', 0))
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
locks.unlock(fd)
os.close(fd)
except Exception, e:
if os.path.exists(temp_data_location):
os.remove(temp_data_location)
raise
def __init__(self, name, timeout=None):
self.name = name
self.timeout = timeout or setting_default_lock_timeout.value
self.uuid = force_text(uuid.uuid4())
lock.acquire()
with open(self.__class__.lock_file, 'r+') as file_object:
locks.lock(f=file_object, flags=locks.LOCK_EX)
data = file_object.read()
if data:
file_locks = json.loads(data)
else:
file_locks = {}
if name in file_locks:
# Someone already got this lock, check to see if it is expired
if file_locks[name]['expiration'] and time.time() > file_locks[name]['expiration']:
# It expires and has expired, we re-acquired it
file_locks[name] = self._get_lock_dictionary()
else:
lock.release()
raise LockError
else:
file_locks[name] = self._get_lock_dictionary()
file_object.seek(0)
file_object.truncate()
file_object.write(json.dumps(file_locks))
lock.release()
def save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
print "Error dir is not error"
while True:
try:
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
else:
fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
except OSError, e:
if e.errno == errno.EEXIST:
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
break
def inner(*args, **kwargs):
from django.core.files import locks
with open('/var/tmp/ocemr_matplotlab_lock', 'wb') as f:
locks.lock(f, locks.LOCK_EX)
ret = None
ret = func(*args, **kwargs)
return ret
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(
full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL
| getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
except OSError as e:
if e.errno == errno.EEXIST:
# Ooops, the file exists. We need a new file name.
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
# OK, the file save worked. Break out of the loop.
break
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def lock(self):
lock_name = self.get_lock_name()
try:
self.__lock_fd = open(lock_name, 'w+b', 1)
locks.lock(self.__lock_fd, locks.LOCK_EX | locks.LOCK_NB)
except IOError:
return False
return True
def get_json_from_file(path):
f = open(path,'r')
locks.lock(f, locks.LOCK_EX)
try:
j = json.load(f)
finally:
f.close()
return j
def purge_locks(cls):
super(FileLock, cls).purge_locks()
lock.acquire()
with open(cls.lock_file, 'r+') as file_object:
locks.lock(f=file_object, flags=locks.LOCK_EX)
file_object.seek(0)
file_object.truncate()
lock.release()
def purge_locks(cls):
super(FileLock, cls).purge_locks()
lock.acquire()
with open(cls.lock_file, 'r+') as file_object:
locks.lock(f=file_object, flags=locks.LOCK_EX)
file_object.seek(0)
file_object.truncate()
lock.release()
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
except OSError as e:
if e.errno == errno.EEXIST:
# Ooops, the file exists. We need a new file name.
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
# OK, the file save worked. Break out of the loop.
break
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def _save(self, name, content):
"""
Lifted partially from django/core/files/storage.py
"""
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
temp_data_location = content.temporary_file_path()
else:
tmp_prefix = "tmp_%s" % (get_valid_filename(name), )
temp_data_location = tempfile.mktemp(prefix=tmp_prefix,
dir=self.location)
try:
# This is a normal uploadedfile that we can stream.
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(temp_data_location,
os.O_WRONLY | os.O_CREAT |
os.O_EXCL | getattr(os, 'O_BINARY', 0))
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
locks.unlock(fd)
os.close(fd)
except Exception:
if os.path.exists(temp_data_location):
os.remove(temp_data_location)
raise
file_move_safe(temp_data_location, full_path, allow_overwrite=True)
content.close()
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def file_move_safe(old_file_name,
new_file_name,
chunk_size=1024 * 64,
allow_overwrite=False):
"""
Moves a file from one location to another in the safest way possible.
First, try using ``shutils.move``, which is OS-dependent but doesn't break
if moving across filesystems. Then, try ``os.rename``, which will break
across filesystems. Finally, streams manually from one file to another in
pure Python.
If the destination file exists and ``allow_overwrite`` is ``False``, this
function will throw an ``IOError``.
"""
# There's no reason to move if we don't have to.
if _samefile(old_file_name, new_file_name):
return
try:
os.rename(old_file_name, new_file_name)
return
except OSError:
# This will happen with os.rename if moving to another filesystem
# or when moving opened files on certain operating systems
pass
# first open the old file, so that it won't go away
old_file = open(old_file_name, 'rb')
try:
# now open the new file, not forgetting allow_overwrite
fd = os.open(
new_file_name,
os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0) |
(not allow_overwrite and os.O_EXCL or 0))
try:
locks.lock(fd, locks.LOCK_EX)
current_chunk = None
while current_chunk != '':
current_chunk = old_file.read(chunk_size)
os.write(fd, current_chunk)
finally:
locks.unlock(fd)
os.close(fd)
finally:
old_file.close()
copystat(old_file_name, new_file_name)
try:
os.remove(old_file_name)
except OSError, e:
# Certain operating systems (Cygwin and Windows)
# fail when deleting opened files, ignore it
if getattr(e, 'winerror', 0) != 32:
# FIXME: should we also ignore errno 13?
raise
def _save(self, name, content):
# Prepare file name from the file content hash
dir_name, file_full_name = os.path.split(name)
file_name, file_ext = os.path.splitext(file_full_name)
content_hash = hashlib.sha1()
for chunk in content.chunks():
content_hash.update(chunk)
hash_name = self._split_name(content_hash.hexdigest())
# + str(content.size) Can add size if anytime stuck into collisions.
name = os.path.join(dir_name, hash_name + file_ext.lower())
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition: it's possible that two threads
# might return the same name. In this case we just continue normally.
# NOTE! This differs considerably from the default file storage behaviour.
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def write_hosts_file(vm,password):
towrite = '[vm]' + '\n'
towrite += str(vm) + " ansible_ssh_pass='******'"
with open('hosts', 'wb') as f:
locks.lock(f, locks.LOCK_EX)
f.write(towrite)
file_move_safe("hosts", "../Ansible/hosts", allow_overwrite = True)
def _save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
print(directory, full_path, '----------save_method')
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
print(content.temporary_file_path(), full_path,
'-----temp_method')
file_move_safe(content.temporary_file_path(), full_path)
content.close()
else:
flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL
| getattr(os, 'O_BINARY', 0))
fd = os.open(full_path, flags, 0o666)
try:
locks.lock(fd, locks.LOCK_EX)
_file = None
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk,
bytes) else 'wt'
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
except OSError as e:
if e.errno == errno.EEXIST:
# Ooops, the file exists. We need a new file name.
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
break
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def write_variable_file_2(user):
towrite = '---' +'\n'
towrite += '\n'
towrite += ('user: "******"\n')
with open('test2.yml', 'wb') as f:
locks.lock(f, locks.LOCK_EX)
f.write(towrite)
file_move_safe("test2.yml", "../Ansible/variables2.yml", allow_overwrite = True)
def lock(self):
if self.nb_locks == 0:
filename = os.path.join(self.root_path, self.lock_file_name)
self.lock_file = open(filename, 'wb')
try:
os.chmod(filename, 0766)
except:
pass
locks.lock(self.lock_file, locks.LOCK_EX)
self.nb_locks += 1
def file_move_safe(old_file_name, new_file_name, chunk_size=1024 * 64, allow_overwrite=False):
"""
Moves a file from one location to another in the safest way possible.
First, tries ``os.rename``, which is simple but will break across filesystems.
If that fails, streams manually from one file to another in pure Python.
If the destination file exists and ``allow_overwrite`` is ``False``, this
function will throw an ``IOError``.
"""
# There's no reason to move if we don't have to.
if _samefile(old_file_name, new_file_name):
return
try:
os.rename(old_file_name, new_file_name)
return
except OSError:
# This will happen with os.rename if moving to another filesystem
# or when moving opened files on certain operating systems
pass
# first open the old file, so that it won't go away
old_file = open(old_file_name, "rb")
try:
# now open the new file, not forgetting allow_overwrite
fd = os.open(
new_file_name,
os.O_WRONLY | os.O_CREAT | getattr(os, "O_BINARY", 0) | (not allow_overwrite and os.O_EXCL or 0),
)
try:
locks.lock(fd, locks.LOCK_EX)
current_chunk = None
while current_chunk != "":
current_chunk = old_file.read(chunk_size)
os.write(fd, current_chunk)
finally:
locks.unlock(fd)
os.close(fd)
finally:
old_file.close()
copystat(old_file_name, new_file_name)
try:
os.remove(old_file_name)
except OSError as e:
# Certain operating systems (Cygwin and Windows)
# fail when deleting opened files, ignore it. (For the
# systems where this happens, temporary files will be auto-deleted
# on close anyway.)
if getattr(e, "winerror", 0) != 32 and getattr(e, "errno", 0) != 13:
raise
def _save(self, name, content):
'''
Copied from super and lightly modified - Unfortunately, the default
race condition handling will lead to an infinite loop here, since
get_available_name() doesn't return unique names for identical hashes.
'''
full_path = self.path(name)
if os.path.exists(full_path):
return name
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
# handle concurrency issue where the directory is created while
# by another thread after the call to exists()
try:
os.makedirs(directory)
except OSError as e:
# ignore EEXIST, since it means that our goal here was already
# accomplished
if e.errno != errno.EEXIST:
raise
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition when saving the file; it's
# possible that two threads might try to write the file simultaneously.
# We need to try to create the file, but abort if we detect that it
# already exists (in which case we assume that another thread is
# writing it).
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
except OSError, e:
# abort and continue normally if we detect the existence of the
# file we're trying to write, otherwise raise normally
if e.errno != errno.EEXIST:
raise
def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False):
"""
Moves a file from one location to another in the safest way possible.
First, try using ``shutils.move``, which is OS-dependent but doesn't break
if moving across filesystems. Then, try ``os.rename``, which will break
across filesystems. Finally, streams manually from one file to another in
pure Python.
If the destination file exists and ``allow_overwrite`` is ``False``, this
function will throw an ``IOError``.
"""
# There's no reason to move if we don't have to.
if _samefile(old_file_name, new_file_name):
return
try:
os.rename(old_file_name, new_file_name)
return
except OSError:
# This will happen with os.rename if moving to another filesystem
# or when moving opened files on certain operating systems
pass
# first open the old file, so that it won't go away
old_file = open(old_file_name, 'rb')
try:
# now open the new file, not forgetting allow_overwrite
fd = os.open(new_file_name, os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0) |
(not allow_overwrite and os.O_EXCL or 0))
try:
locks.lock(fd, locks.LOCK_EX)
current_chunk = None
while current_chunk != '':
current_chunk = old_file.read(chunk_size)
os.write(fd, current_chunk)
finally:
locks.unlock(fd)
os.close(fd)
finally:
old_file.close()
copystat(old_file_name, new_file_name)
try:
os.remove(old_file_name)
except OSError, e:
# Certain operating systems (Cygwin and Windows)
# fail when deleting opened files, ignore it
if getattr(e, 'winerror', 0) != 32:
# FIXME: should we also ignore errno 13?
raise
def _save(self, name, content):
""" This is similar to the parent calss but removes any safe-guards against existing files """
if not self.file_overwrite:
return super(OverwriteFileSystemStorage, self)._save(name, content)
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
flags = (os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0))
# The current umask value is masked out by os.open!
fd = os.open(full_path, flags, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk, bytes) else 'wt'
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
if settings.FILE_UPLOAD_PERMISSIONS is not None:
os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
return name
def _save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
try:
if self.directory_permissions_mode is not None:
old_umask = os.umask(0)
try:
os.makedirs(directory,
self.directory_permissions_mode,
exist_ok=True)
finally:
os.umask(old_umask)
else:
os.makedirs(directory, exist_ok=True)
except FileExistsError:
raise FileExistsError("%s exists and is not a directory." %
directory)
while True:
try:
if hasattr(content, "temporary_file_path"):
file_move_safe(content.temporary_file_path(), full_path)
else:
fd = os.open(full_path, self.OS_OPEN_FLAGS, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = "wb" if isinstance(chunk,
bytes) else "wt"
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
except FileExistsError:
name = self.get_available_name(name)
full_path = self.path(name)
else:
break
if self.file_permissions_mode is not None:
os.chmod(full_path, self.file_permissions_mode)
return name.replace("\\", "/")
def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None):
try:
with open(self._key_to_file(key, version), 'r+b') as f:
try:
locks.lock(f, locks.LOCK_EX)
if self._is_expired(f):
return False
else:
previous_value = pickle.loads(zlib.decompress(f.read()))
f.seek(0)
self._write_content(f, timeout, previous_value)
return True
finally:
locks.unlock(f)
except FileNotFoundError:
return False
def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None):
try:
with open(self._key_to_file(key, version), 'r+b') as f:
try:
locks.lock(f, locks.LOCK_EX)
if self._is_expired(f):
return False
else:
previous_value = pickle.loads(zlib.decompress(f.read()))
f.seek(0)
_write_content(f, self.get_backend_timeout(timeout), previous_value)
return True
finally:
locks.unlock(f)
except FileNotFoundError:
return False
def contribute_to_class(self, cls, name):
if self.filename is not None:
filename = self.filename
else:
filename = name
# Create a post_save signal handler to write data to disk
@receiver(post_save, sender=cls, weak=False,
dispatch_uid="fsfield_write_data_%s_%s_%s" %
(cls.__module__, cls.__name__, name))
def write_data(sender, instance, created, raw, using, **kwargs):
if (name not in instance.__dict__ or
instance.__dict__[name] is None):
# Nothing to save
return
# Open the file for writing and acquire a lock on it if
# possible
path = model_instance_field_path(instance, filename)
fs_storage = isinstance(self.storage, FileSystemStorage)
if fs_storage:
full_path = self.storage.path(path)
directory = op.dirname(full_path)
if not op.exists(directory):
try:
os.makedirs(directory)
except OSError, err:
# Another thread may have created the directory since
# the check
if err.errno == 17:
pass
fp = self.storage.open(path, "wb")
if fs_storage:
locks.lock(fp.file, locks.LOCK_EX)
# Write data
try:
value = instance.__dict__[name]
if self.dump is None:
fp.write(value.encode("utf8"))
else:
self.dump(value, fp)
finally:
# Release lock
if fs_storage:
locks.unlock(fp.file)
# Close file
fp.close()
def load_file(cls, file_name):
try:
with open(file_name, 'r') as fp:
locks.lock(fp, locks.LOCK_EX)
content = fp.read()
except IOError as exc:
if exc.errno == errno.ENOENT:
content = '{}'
else:
raise
data = yaml.safe_load(content)
if not isinstance(data, dict):
raise TypeError(
"YAML content of {} is not a dictionary".format(file_name))
return data
def _save(self, name, content):
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
os.makedirs(directory)
elif not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
# CHANGED: This un-hangs us long enough to keep things rolling.
eventlet.sleep(0)
finally:
locks.unlock(fd)
os.close(fd)
except OSError, e:
if e.errno == errno.EEXIST:
# Ooops, the file exists. We need a new file name.
name = self.get_available_name(name)
full_path = self.path(name)
else:
raise
else:
# OK, the file save worked. Break out of the loop.
break
def file_move_safe(old_file_name,
new_file_name,
chunk_size=1024 * 64,
allow_overwrite=False):
"""
Moves a file from one location to another in the safest way possible.
First, try using ``shutils.move``, which is OS-dependent but doesn't break
if moving across filesystems. Then, try ``os.rename``, which will break
across filesystems. Finally, streams manually from one file to another in
pure Python.
If the destination file exists and ``allow_overwrite`` is ``False``, this
function will throw an ``IOError``.
"""
# There's no reason to move if we don't have to.
if old_file_name == new_file_name:
return
if not allow_overwrite and os.path.exists(new_file_name):
raise IOError("Cannot overwrite existing file '%s'." % new_file_name)
try:
file_move(old_file_name, new_file_name)
return
except OSError:
# This will happen with os.rename if moving to another filesystem
pass
# If the built-in didn't work, do it the hard way.
new_file = open(new_file_name, 'wb')
locks.lock(new_file, locks.LOCK_EX)
old_file = open(old_file_name, 'rb')
current_chunk = None
while current_chunk != '':
current_chunk = old_file.read(chunk_size)
new_file.write(current_chunk)
new_file.close()
old_file.close()
os.remove(old_file_name)
class ImageStorage(FileSystemStorage):
"""
Create image file and empty thumb file.
"""
# almost all copyed from FileSystemStorage._save
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError, e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
thumb_created = False
thumb_name = get_thumb_name(full_path)
try:
fd = os.open(thumb_name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
os.close(fd)
except OSError, e:
if e.errno == errno.EEXIST:
# Ooops, the file exists. We need a new file name.
name = self.get_available_name(name)
full_path = self.path(name)
continue
else:
raise
else:
thumb_created = True
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
def write_variable_file_1(user, password):
hashed_password = sha512_crypt.encrypt(password)
towrite = '---' +'\n'
towrite += '\n'
towrite += ('user: "******"\n')
towrite += ('password: "******"\n')
towrite += '\n'
towrite += ('dockercmd: "docker run -d --name ' + user + ' -it ubuntu"' +'\n')
towrite += '\n'
towrite += ('linetoadd1: "me=`whoami`"' + '\n')
towrite += ('linetoadd2: "if [ $me == \'' + user + '\' ]; then"' + '\n')
towrite += ('linetoadd3: " docker exec -it ' + user + ' /bin/bash\\nfi"' + '\n')
with open('test.yml', 'wb') as f:
locks.lock(f, locks.LOCK_EX)
f.write(towrite)
file_move_safe("test.yml", "../Ansible/variables.yml", allow_overwrite = True)
def set_and_save(self, key, value):
if self.data.get(key, object()) == value:
return
fd = os.open(self.file_name, os.O_RDWR | os.O_CREAT)
with os.fdopen(fd, "r+") as fp:
locks.lock(fd, locks.LOCK_EX)
data = yaml.safe_load(fp)
if data is None:
data = {}
self.data[key] = value
data[key] = value
fp.seek(0)
yaml.safe_dump(
data, fp, default_flow_style=False, allow_unicode=True, width=10000
)
def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False):
"""
Moves a file from one location to another in the safest way possible.
First, try using ``shutils.move``, which is OS-dependent but doesn't break
if moving across filesystems. Then, try ``os.rename``, which will break
across filesystems. Finally, streams manually from one file to another in
pure Python.
If the destination file exists and ``allow_overwrite`` is ``False``, this
function will throw an ``IOError``.
"""
# There's no reason to move if we don't have to.
if old_file_name == new_file_name:
return
if not allow_overwrite and os.path.exists(new_file_name):
raise IOError("Cannot overwrite existing file '%s'." % new_file_name)
try:
file_move(old_file_name, new_file_name)
return
except OSError:
# This will happen with os.rename if moving to another filesystem
pass
# If the built-in didn't work, do it the hard way.
fd = os.open(new_file_name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
old_file = open(old_file_name, 'rb')
current_chunk = None
while current_chunk != '':
current_chunk = old_file.read(chunk_size)
os.write(fd, current_chunk)
finally:
locks.unlock(fd)
os.close(fd)
old_file.close()
os.remove(old_file_name)
def createdocker(user, instance, vm, email):
try:
with open(user, 'wb') as f:
locks.lock(f, locks.LOCK_EX)
f.write('0')
os.system('ansible-playbook ../Ansible/newdock.yml -i ../Ansible/hosts')
with open(user, 'wb') as f:
locks.lock(f, locks.LOCK_EX)
f.write('1')
body = 'New Docker Instance Created\n'
body += 'IP : ' + str(instance.vm.ip) + '\n'
body += 'User : '******'\n'
body += 'Password : '******'\n'
msg = EmailMultiAlternatives(subject="DockerPaaS | New Instance Created", body=body,from_email="*****@*****.**",to=[email])
msg.send()
except:
print 'no'
instance.save()
vm.save()
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
# Note that there is a race between os.path.exists and os.makedirs:
# if os.makedirs fails with EEXIST, the directory was created
# concurrently, and we can continue normally. Refs #16082.
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL
| getattr(os, 'O_BINARY', 0))
# The current umask value is masked out by os.open!
fd = os.open(full_path, flags, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk, bytes) else 'wt'
_file = os.fdopen(fd, mode)
_file.write(chunk)
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
file_name = self.client.put(self.path(name))
os.unlink(full_path)
return file_name
def append(self, name, content, progress_callback=None):
full_path = self.path(name)
while True:
# This file has a file path that we can move.
#if hasattr(content, 'temporary_file_path'):
# file_move_safe(content.temporary_file_path(), full_path)
#
## This is a normal uploadedfile that we can stream.
#else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
flags = (os.O_WRONLY | getattr(os, 'O_BINARY', 0))
# The current umask value is masked out by os.open!
fd = os.open(full_path, flags, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
# if isinstance(chunk, bytes) else 'wt'
mode = 'ab'
_file = os.fdopen(fd, mode)
_file.write(chunk)
break
finally:
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
break
if self.file_permissions_mode is not None:
os.chmod(full_path, self.file_permissions_mode)
# Store filenames with forward slashes, even on Windows.
return name.replace('\\', '/')
def get_lock(name, timeout='forewer'):
""" Acquire and return unique lock (locked file)
If lock has not been created before timeout then IOError raised """
f = open(FILE_NAME_PREFIX + name, 'wb')
if timeout == 'forewer':
locks.lock(f, locks.LOCK_EX)
else:
#print ' waiting for lock...'
for waited in range(timeout + 1):
try:
if waited > 0: # sleep between iterations
sleep(1)
locks.lock(f, locks.LOCK_EX + locks.LOCK_NB)
break
except IOError:
pass
else:
f.close()
raise LockException('Get lock timeout')
return f
class HashFileSystemStorage(FileSystemStorage):
def __init__(self, location=None, base_url=None):
if location is None:
location = os.path.join(settings.MEDIA_ROOT, 'hash')
if base_url is None:
base_url = settings.MEDIA_URL + 'hash/'
super(HashFileSystemStorage, self).__init__(location, base_url)
def path(self, name):
name = os.path.join(name[:2], name[2:4], name)
return super(HashFileSystemStorage, self).path(name)
def save(self, name, content):
# 计算内容 HASH
md5 = hashlib.md5()
for chunk in content.chunks():
md5.update(chunk)
name = md5.hexdigest()
# 保存文件
full_path = self.path(name)
directory = os.path.dirname(full_path)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError, e:
if e.errno != errno.EEXIST:
raise
if not os.path.isdir(directory):
raise IOError("%s exists and is not a directory." % directory)
try:
if hasattr(content, 'temporary_file_path'):
file_move_safe(content.temporary_file_path(), full_path)
content.close()
else:
fd = os.open(
full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL
| getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
os.write(fd, chunk)
finally:
locks.unlock(fd)
os.close(fd)
except OSError, e:
if e.errno != errno.EEXIST:
raise
