Exemple #1
0
    def save(self, name, content):
        full_path = self.path(name)

        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            os.makedirs(directory)
        elif not os.path.isdir(directory):
            print "Error dir is not error"

        while True:
            try:
                if hasattr(content, 'temporary_file_path'):
                    file_move_safe(content.temporary_file_path(), full_path)
                    content.close()
                else:
                    fd = os.open(
                        full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL
                        | getattr(os, 'O_BINARY', 0))
                    try:
                        locks.lock(fd, locks.LOCK_EX)
                        for chunk in content.chunks():
                            os.write(fd, chunk)
                    finally:
                        locks.unlock(fd)
                        os.close(fd)
            except OSError, e:
                if e.errno == errno.EEXIST:
                    name = self.get_available_name(name)
                    full_path = self.path(name)
                else:
                    raise
            else:
                break
Exemple #2
0
    def release(self):
        super(FileLock, self).release()

        lock.acquire()
        with open(self.__class__.lock_file, 'r+') as file_object:
            locks.lock(f=file_object, flags=locks.LOCK_EX)
            try:
                file_locks = json.loads(file_object.read())
            except EOFError:
                file_locks = {}

            if self.name in file_locks:
                if file_locks[self.name]['uuid'] == self.uuid:
                    file_locks.pop(self.name)
                else:
                    # Lock expired and someone else acquired it
                    pass
            else:
                # Lock expired and someone else released it
                pass

            file_object.seek(0)
            file_object.truncate()
            file_object.write(json.dumps(file_locks))
            lock.release()
Exemple #3
0
 def __get__(self, obj, type=None):
     if obj is None:
         return
     if self.name not in obj.__dict__:
         # Load data from disk
         path = self.path(obj)
         if not self.storage.exists(path):
             value = self.default
         else:
             # Open the file for reading and acquire a lock on it if
             # possible
             fp = self.storage.open(path, "r+b")
             if isinstance(self.storage, FileSystemStorage):
                 locks.lock(fp.file, locks.LOCK_EX)
             try:
                 # Read file content
                 if self.load is not None:
                     value = self.load(fp)
                 else:
                     value = fp.read().decode("utf8")
             finally:
                 # Release lock
                 if isinstance(self.storage, FileSystemStorage):
                     locks.unlock(fp.file)
                 # Close file
                 fp.close()
         obj.__dict__[self.name] = value
     return obj.__dict__[self.name]
Exemple #4
0
    def _save(self, name, content):
        """
        Lifted partially from django/core/files/storage.py
        """
        full_path = self.path(name)

        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            os.makedirs(directory)
        elif not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # Ensure that content is open
        content.open()

        if hasattr(content, 'temporary_file_path'):
            # Content has a file that we can move.
            temp_data_location = content.temporary_file_path()
            file_move_safe(temp_data_location, full_path, allow_overwrite=True)
        else:
            # Write the content stream to a temporary file and move it.
            fd, tmp_path = mkstemp()
            locks.lock(fd, locks.LOCK_EX)
            for chunk in content.chunks():
                os.write(fd, chunk)
            locks.unlock(fd)
            os.close(fd)
            file_move_safe(tmp_path, full_path, allow_overwrite=True)

        content.close()
        if settings.FILE_UPLOAD_PERMISSIONS is not None:
            os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)

        return name
Exemple #5
0
    def lock(self):
        try:
            lock_name = self.get_lock_name()
            # need loop to avoid races on file unlinking
            while True:
                f = open(lock_name, 'w+', 0)
                locks.lock(f, locks.LOCK_EX | locks.LOCK_NB)
            # Here is the Race:
            # Previous process "A" is still running. Process "B" opens
            # the file and then the process "A" finishes and deletes it.
            # "B" locks the deleted file (by fd it already have) and runs,
            # then the next process "C" creates _new_ file and locks it
            # successfully while "B" is still running.
            # We just need to check that "B" didn't lock a deleted file
            # to avoid any problems. If process "C" have locked
            # a new file wile "B" stats it then ok, let "B" quit and "C" run.
            # We can still meet an attacker that permanently creates and deletes
            # our file but we can't avoid problems in that case.
                if os.path.isfile(lock_name):
                    st1 = os.fstat(f.fileno())
                    st2 = os.stat(lock_name)
                    if st1.st_ino == st2.st_ino:
                        f.write(str(os.getpid()))
                        self.lockfile = f
                        return True
                # else:
                # retry. Don't unlink, next process might already use it.
                f.close()

        except IOError as e:
            if e.errno in (errno.EACCES, errno.EAGAIN):
                return False
            else:
                e = sys.exc_info()[1]
                raise e
Exemple #6
0
def file_move_safe(old_file_name, new_file_name, chunk_size=1024 * 64, allow_overwrite=False):
    """
    Moves a file from one location to another in the safest way possible.

    First, tries ``os.rename``, which is simple but will break across filesystems.
    If that fails, streams manually from one file to another in pure Python.

    If the destination file exists and ``allow_overwrite`` is ``False``, this
    function will throw an ``IOError``.
    """

    # There's no reason to move if we don't have to.
    if _samefile(old_file_name, new_file_name):
        return

    try:
        # If the destination file exists and allow_overwrite is False then raise an IOError
        if not allow_overwrite and os.access(new_file_name, os.F_OK):
            raise IOError("Destination file %s exists and allow_overwrite is False" % new_file_name)

        os.rename(old_file_name, new_file_name)
        return
    except OSError:
        # This will happen with os.rename if moving to another filesystem
        # or when moving opened files on certain operating systems
        pass

    # first open the old file, so that it won't go away
    with open(old_file_name, 'rb') as old_file:
        # now open the new file, not forgetting allow_overwrite
        fd = os.open(new_file_name, (os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0) |
                                     (os.O_EXCL if not allow_overwrite else 0)))
        try:
            locks.lock(fd, locks.LOCK_EX)
            current_chunk = None
            while current_chunk != b'':
                current_chunk = old_file.read(chunk_size)
                os.write(fd, current_chunk)
        finally:
            locks.unlock(fd)
            os.close(fd)

    try:
        copystat(old_file_name, new_file_name)
    except OSError as e:
        # Certain filesystems (e.g. CIFS) fail to copy the file's metadata if
        # the type of the destination filesystem isn't the same as the source
        # filesystem; ignore that.
        if getattr(e, 'errno', 0) != errno.EPERM:
            raise

    try:
        os.remove(old_file_name)
    except OSError as e:
        # Certain operating systems (Cygwin and Windows)
        # fail when deleting opened files, ignore it.  (For the
        # systems where this happens, temporary files will be auto-deleted
        # on close anyway.)
        if getattr(e, 'winerror', 0) != 32 and getattr(e, 'errno', 0) != 13:
            raise
Exemple #7
0
    def _save(self, name, content):
        full_path = self.path(name)
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            os.makedirs(directory)
        elif not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # This file has a file path that we can move.
        if hasattr(content, 'temporary_file_path'):
            file_move_safe(content.temporary_file_path(), full_path)
            content.close()

        # This is a normal uploadedfile that we can stream.
        else:
            # Open file for writting in binary mode
            fd = open(full_path.encode('utf-8'), "wb")
            try:
                locks.lock(fd, locks.LOCK_EX)
                for chunk in content.chunks():
                    fd.write(chunk)
            finally:
                locks.unlock(fd)
                fd.close()

        if settings.FILE_UPLOAD_PERMISSIONS is not None:
            os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)

        return name
Exemple #8
0
    def release(self):
        super(FileLock, self).release()

        lock.acquire()
        with open(self.__class__.lock_file, 'r+') as file_object:
            locks.lock(f=file_object, flags=locks.LOCK_EX)
            try:
                file_locks = json.loads(file_object.read())
            except EOFError:
                file_locks = {}

            if self.name in file_locks:
                if file_locks[self.name]['uuid'] == self.uuid:
                    file_locks.pop(self.name)
                else:
                    # Lock expired and someone else acquired it
                    pass
            else:
                # Lock expired and someone else released it
                pass

            file_object.seek(0)
            file_object.truncate()
            file_object.write(json.dumps(file_locks))
            lock.release()
Exemple #9
0
 def _purge_locks(cls):
     lock.acquire()
     with open(file=cls.lock_file, mode='r+') as file_object:
         locks.lock(f=file_object, flags=locks.LOCK_EX)
         file_object.seek(0)
         file_object.truncate()
         lock.release()
Exemple #10
0
    def __init__(self, name, timeout):
        self.name = name
        self.timeout = timeout
        self.uuid = force_text(s=uuid.uuid4())

        lock.acquire()
        with open(file=self.__class__.lock_file, mode='r+') as file_object:
            locks.lock(f=file_object, flags=locks.LOCK_EX)

            data = file_object.read()

            if data:
                file_locks = json.loads(s=data)
            else:
                file_locks = {}

            if name in file_locks:
                # Someone already got this lock, check to see if it is expired
                if file_locks[name]['expiration'] and time.time() > file_locks[name]['expiration']:
                    # It expires and has expired, we re-acquired it
                    file_locks[name] = self._get_lock_dictionary()
                else:
                    lock.release()
                    raise LockError
            else:
                file_locks[name] = self._get_lock_dictionary()

            file_object.seek(0)
            file_object.truncate()
            file_object.write(json.dumps(obj=file_locks))
            lock.release()
Exemple #11
0
 def _save(self, name, content):
     full_path = self.path(name)
     directory = os.path.dirname(full_path)
     if not os.path.exists(directory):
         os.makedirs(directory)
     elif not os.path.isdir(directory):
         raise IOError("%s exists and is not a directory." % directory)
     # This file has a file path that we can move.
     if hasattr(content, 'temporary_file_path'):
         file_move_safe(content.temporary_file_path(), full_path)
         content.close()
     # This is a normal uploadedfile that we can stream.
     else:
         # This fun binary flag incantation makes os.open throw an
         # OSError if the file already exists before we open it.
         fd = os.open(full_path, os.O_WRONLY | os.O_TRUNC \
                 | os.O_CREAT | getattr(os, 'O_BINARY', 0))
         try:
             locks.lock(fd, locks.LOCK_EX)
             for chunk in content.chunks():
                 os.write(fd, chunk)
         finally:
             locks.unlock(fd)
             os.close(fd)
     if settings.FILE_UPLOAD_PERMISSIONS is not None:
         os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
     return name
Exemple #12
0
 def _save(self, name, content):
     full_path = self.path(name)
     directory = os.path.dirname(full_path)
     if not os.path.exists(directory):
         os.makedirs(directory)
     elif not os.path.isdir(directory):
         raise IOError("%s exists and is not a directory." % directory)
     # This file has a file path that we can move.
     if hasattr(content, 'temporary_file_path'):
         file_move_safe(content.temporary_file_path(), full_path)
         content.close()
     # This is a normal uploadedfile that we can stream.
     else:
         # This fun binary flag incantation makes os.open throw an
         # OSError if the file already exists before we open it.
         fd = os.open(full_path, os.O_WRONLY | os.O_TRUNC \
                 | os.O_CREAT | getattr(os, 'O_BINARY', 0))
         try:
             locks.lock(fd, locks.LOCK_EX)
             for chunk in content.chunks():
                 os.write(fd, chunk)
         finally:
             locks.unlock(fd)
             os.close(fd)
     if settings.FILE_UPLOAD_PERMISSIONS is not None:
         os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
     return name
Exemple #13
0
 def test_exclusive_lock(self):
     file_path = Path(__file__).parent / 'test.png'
     with open(file_path) as f1, open(file_path) as f2:
         self.assertIs(locks.lock(f1, locks.LOCK_EX), True)
         self.assertIs(locks.lock(f2, locks.LOCK_EX | locks.LOCK_NB), False)
         self.assertIs(locks.lock(f2, locks.LOCK_SH | locks.LOCK_NB), False)
         self.assertIs(locks.unlock(f1), True)
Exemple #14
0
def checkStatus(request, user = ''):
	val = ""
	with open(user, 'rb') as f:
		locks.lock(f, locks.LOCK_EX)
		val = f.read()

	return HttpResponse(val)
Exemple #15
0
 def test_shared_lock(self):
     file_path = Path(__file__).parent / 'test.png'
     with open(file_path) as f1, open(file_path) as f2:
         self.assertIs(locks.lock(f1, locks.LOCK_SH), True)
         self.assertIs(locks.lock(f2, locks.LOCK_SH | locks.LOCK_NB), True)
         self.assertIs(locks.unlock(f1), True)
         self.assertIs(locks.unlock(f2), True)
Exemple #16
0
    def _install_extension_media(self, ext_class):
        """Installs extension static media.

        This method is a wrapper around _install_extension_media_internal to
        check whether we actually need to install extension media, and avoid
        contention among multiple threads/processes when doing so.

        We need to install extension media if it hasn't been installed yet,
        or if the version of the extension media that we installed is different
        from the current version of the extension.
        """
        lockfile = os.path.join(tempfile.gettempdir(), ext_class.id + '.lock')
        extension = ext_class.instance

        cur_version = ext_class.info.version

        # We only want to fetch the existing version information if the
        # extension is already installed. We remove this key when
        # disabling an extension, so if it were there, it was either
        # copy/pasted, or something went wrong. Either way, we wouldn't
        # be able to trust it.
        if ext_class.registration.installed:
            old_version = extension.settings.get(self.VERSION_SETTINGS_KEY)
        else:
            old_version = None

        if old_version == cur_version:
            # Nothing to do
            return

        if not old_version:
            logging.debug('Installing extension media for %s', ext_class.info)
        else:
            logging.debug('Reinstalling extension media for %s because '
                          'version changed from %s',
                          ext_class.info, old_version)

        while old_version != cur_version:
            with open(lockfile, 'w') as f:
                try:
                    locks.lock(f, locks.LOCK_EX)
                except IOError as e:
                    if e.errno == errno.EINTR:
                        # Sleep for one second, then try again
                        time.sleep(1)
                        extension.settings.load()
                        old_version = extension.settings.get(
                            self.VERSION_SETTINGS_KEY)
                        continue
                    else:
                        raise e

                self._install_extension_media_internal(ext_class)
                extension.settings.set(self.VERSION_SETTINGS_KEY, cur_version)
                extension.settings.save()
                old_version = cur_version

                locks.unlock(f)

        os.unlink(lockfile)
Exemple #17
0
def file_move_safe(old_file_name, new_file_name, chunk_size=1024 * 64, allow_overwrite=False):
    """
    Moves a file from one location to another in the safest way possible.

    First, tries ``os.rename``, which is simple but will break across filesystems.
    If that fails, streams manually from one file to another in pure Python.

    If the destination file exists and ``allow_overwrite`` is ``False``, this
    function will throw an ``IOError``.
    """

    # There's no reason to move if we don't have to.
    if _samefile(old_file_name, new_file_name):
        return

    try:
        # If the destination file exists and allow_overwrite is False then raise an IOError
        if not allow_overwrite and os.access(new_file_name, os.F_OK):
            raise IOError("Destination file %s exists and allow_overwrite is False" % new_file_name)

        os.rename(old_file_name, new_file_name)
        return
    except OSError:
        # This will happen with os.rename if moving to another filesystem
        # or when moving opened files on certain operating systems
        pass

    # first open the old file, so that it won't go away
    with open(old_file_name, 'rb') as old_file:
        # now open the new file, not forgetting allow_overwrite
        fd = os.open(new_file_name, (os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0) |
                                     (os.O_EXCL if not allow_overwrite else 0)))
        try:
            locks.lock(fd, locks.LOCK_EX)
            current_chunk = None
            while current_chunk != b'':
                current_chunk = old_file.read(chunk_size)
                os.write(fd, current_chunk)
        finally:
            locks.unlock(fd)
            os.close(fd)

    try:
        copystat(old_file_name, new_file_name)
    except OSError as e:
        # Certain filesystems (e.g. CIFS) fail to copy the file's metadata if
        # the type of the destination filesystem isn't the same as the source
        # filesystem; ignore that.
        if getattr(e, 'errno', 0) != errno.EPERM:
            raise

    try:
        os.remove(old_file_name)
    except OSError as e:
        # Certain operating systems (Cygwin and Windows)
        # fail when deleting opened files, ignore it.  (For the
        # systems where this happens, temporary files will be auto-deleted
        # on close anyway.)
        if getattr(e, 'winerror', 0) != 32 and getattr(e, 'errno', 0) != 13:
            raise
Exemple #18
0
    def _save(self, name, content):
        """
        Lifted partially from django/core/files/storage.py
        """
        full_path = self.path(name)

        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            os.makedirs(directory)
        elif not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # Ensure that content is open
        content.open()

        if hasattr(content, 'temporary_file_path'):
            # Content has a file that we can move.
            temp_data_location = content.temporary_file_path()
            file_move_safe(temp_data_location, full_path, allow_overwrite=True)
        else:
            # Write the content stream to a temporary file and move it.
            fd, tmp_path = mkstemp()
            locks.lock(fd, locks.LOCK_EX)
            for chunk in content.chunks():
                os.write(fd, chunk)
            locks.unlock(fd)
            os.close(fd)
            file_move_safe(tmp_path, full_path, allow_overwrite=True)

        content.close()
        if settings.FILE_UPLOAD_PERMISSIONS is not None:
            os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)

        return name
Exemple #19
0
    def _save(self, name, content):
        full_path = self.path(name)

        # Create any intermediate directories that do not exist.
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                old_umask = os.umask(0)
                try:
                    os.makedirs(directory)
                finally:
                    os.umask(old_umask)
            except FileExistsError:
                pass
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)
        flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL |
                 getattr(os, 'O_BINARY', 0))
        fd = os.open(full_path, flags, 0o666)
        _file = None
        try:
            locks.lock(fd, locks.LOCK_EX)
            for chunk in content.chunks():
                if _file is None:
                    mode = 'wb' if isinstance(chunk, bytes) else 'wt'
                    _file = os.fdopen(fd, mode)
                _file.write(chunk)
        finally:
            locks.unlock(fd)
            if _file is not None:
                _file.close()
            else:
                os.close(fd)
        return name.replace('\\', '/')
 def _save(self, name, content):
     """
     Lifted partially from django/core/files/storage.py
     """ 
     full_path = self.path(name)
         
     directory = os.path.dirname(full_path)
     if not os.path.exists(directory):        
         os.makedirs(directory)
     elif not os.path.isdir(directory):
         raise IOError("%s exists and is not a directory." % directory)
             
     # This file has a file path that we can move.
     if hasattr(content, 'temporary_file_path'):
         temp_data_location = content.temporary_file_path()
     else:   
         tmp_prefix = "tmp_%s" %(get_valid_filename(name), )
         temp_data_location = tempfile.mktemp(prefix=tmp_prefix,
                                              dir=self.location)
         try:
             # This is a normal uploadedfile that we can stream.
             # This fun binary flag incantation makes os.open throw an
             # OSError if the file already exists before we open it.
             fd = os.open(temp_data_location,
                          os.O_WRONLY | os.O_CREAT |
                          os.O_EXCL | getattr(os, 'O_BINARY', 0))
             locks.lock(fd, locks.LOCK_EX)
             for chunk in content.chunks():
                 os.write(fd, chunk)
             locks.unlock(fd)
             os.close(fd)
         except Exception, e:
             if os.path.exists(temp_data_location):
                 os.remove(temp_data_location)
             raise
Exemple #21
0
    def _save(self, name, content):
        """
        Lifted partially from django/core/files/storage.py
        """
        full_path = self.path(name)

        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            os.makedirs(directory)
        elif not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # This file has a file path that we can move.
        if hasattr(content, 'temporary_file_path'):
            temp_data_location = content.temporary_file_path()
        else:
            tmp_prefix = "tmp_%s" % (get_valid_filename(name), )
            temp_data_location = tempfile.mktemp(prefix=tmp_prefix,
                                                 dir=self.location)
            try:
                # This is a normal uploadedfile that we can stream.
                # This fun binary flag incantation makes os.open throw an
                # OSError if the file already exists before we open it.
                fd = os.open(
                    temp_data_location, os.O_WRONLY | os.O_CREAT | os.O_EXCL
                    | getattr(os, 'O_BINARY', 0))
                locks.lock(fd, locks.LOCK_EX)
                for chunk in content.chunks():
                    os.write(fd, chunk)
                locks.unlock(fd)
                os.close(fd)
            except Exception, e:
                if os.path.exists(temp_data_location):
                    os.remove(temp_data_location)
                raise
Exemple #22
0
    def __init__(self, name, timeout=None):
        self.name = name
        self.timeout = timeout or setting_default_lock_timeout.value
        self.uuid = force_text(uuid.uuid4())

        lock.acquire()
        with open(self.__class__.lock_file, 'r+') as file_object:
            locks.lock(f=file_object, flags=locks.LOCK_EX)

            data = file_object.read()

            if data:
                file_locks = json.loads(data)
            else:
                file_locks = {}

            if name in file_locks:
                # Someone already got this lock, check to see if it is expired
                if file_locks[name]['expiration'] and time.time() > file_locks[name]['expiration']:
                    # It expires and has expired, we re-acquired it
                    file_locks[name] = self._get_lock_dictionary()
                else:
                    lock.release()
                    raise LockError
            else:
                file_locks[name] = self._get_lock_dictionary()

            file_object.seek(0)
            file_object.truncate()
            file_object.write(json.dumps(file_locks))
            lock.release()
Exemple #23
0
  def save(self, name, content):
    full_path = self.path(name)
 
    directory = os.path.dirname(full_path)
    if not os.path.exists(directory):
      os.makedirs(directory)
    elif not os.path.isdir(directory):
      print "Error dir is not error"
    
    while True:
      try:
        if hasattr(content, 'temporary_file_path'):
          file_move_safe(content.temporary_file_path(), full_path)
          content.close()
        else:
          fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
          try:
            locks.lock(fd, locks.LOCK_EX)
            for chunk in content.chunks():
              os.write(fd, chunk)
          finally:
            locks.unlock(fd)
            os.close(fd)
      except OSError, e:
        if e.errno == errno.EEXIST:
          name = self.get_available_name(name)
          full_path = self.path(name)
        else:
          raise
      else:
        break
Exemple #24
0
 def inner(*args, **kwargs):
     from django.core.files import locks
     with open('/var/tmp/ocemr_matplotlab_lock', 'wb') as f:
         locks.lock(f, locks.LOCK_EX)
         ret = None
         ret = func(*args, **kwargs)
         return ret
Exemple #25
0
    def _save(self, name, content):
        full_path = self.path(name)

        # Create any intermediate directories that do not exist.
        # Note that there is a race between os.path.exists and os.makedirs:
        # if os.makedirs fails with EEXIST, the directory was created
        # concurrently, and we can continue normally. Refs #16082.
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                os.makedirs(directory)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # There's a potential race condition between get_available_name and
        # saving the file; it's possible that two threads might return the
        # same name, at which point all sorts of fun happens. So we need to
        # try to create the file, but if it already exists we have to go back
        # to get_available_name() and try again.

        while True:
            try:
                # This file has a file path that we can move.
                if hasattr(content, 'temporary_file_path'):
                    file_move_safe(content.temporary_file_path(), full_path)
                    content.close()

                # This is a normal uploadedfile that we can stream.
                else:
                    # This fun binary flag incantation makes os.open throw an
                    # OSError if the file already exists before we open it.
                    fd = os.open(
                        full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL
                        | getattr(os, 'O_BINARY', 0))
                    try:
                        locks.lock(fd, locks.LOCK_EX)
                        for chunk in content.chunks():
                            os.write(fd, chunk)
                    finally:
                        locks.unlock(fd)
                        os.close(fd)
            except OSError as e:
                if e.errno == errno.EEXIST:
                    # Ooops, the file exists. We need a new file name.
                    name = self.get_available_name(name)
                    full_path = self.path(name)
                else:
                    raise
            else:
                # OK, the file save worked. Break out of the loop.
                break

        if settings.FILE_UPLOAD_PERMISSIONS is not None:
            os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)

        return name
Exemple #26
0
 def lock(self):
     lock_name = self.get_lock_name()
     try:
         self.__lock_fd = open(lock_name, 'w+b', 1)
         locks.lock(self.__lock_fd, locks.LOCK_EX | locks.LOCK_NB)
     except IOError:
         return False
     return True
Exemple #27
0
def get_json_from_file(path):
    f = open(path,'r')
    locks.lock(f, locks.LOCK_EX)
    try:
        j = json.load(f)
    finally:
        f.close()
        return j
Exemple #28
0
 def purge_locks(cls):
     super(FileLock, cls).purge_locks()
     lock.acquire()
     with open(cls.lock_file, 'r+') as file_object:
         locks.lock(f=file_object, flags=locks.LOCK_EX)
         file_object.seek(0)
         file_object.truncate()
         lock.release()
Exemple #29
0
 def purge_locks(cls):
     super(FileLock, cls).purge_locks()
     lock.acquire()
     with open(cls.lock_file, 'r+') as file_object:
         locks.lock(f=file_object, flags=locks.LOCK_EX)
         file_object.seek(0)
         file_object.truncate()
         lock.release()
Exemple #30
0
    def _save(self, name, content):
        full_path = self.path(name)

        # Create any intermediate directories that do not exist.
        # Note that there is a race between os.path.exists and os.makedirs:
        # if os.makedirs fails with EEXIST, the directory was created
        # concurrently, and we can continue normally. Refs #16082.
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                os.makedirs(directory)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # There's a potential race condition between get_available_name and
        # saving the file; it's possible that two threads might return the
        # same name, at which point all sorts of fun happens. So we need to
        # try to create the file, but if it already exists we have to go back
        # to get_available_name() and try again.

        while True:
            try:
                # This file has a file path that we can move.
                if hasattr(content, 'temporary_file_path'):
                    file_move_safe(content.temporary_file_path(), full_path)
                    content.close()

                # This is a normal uploadedfile that we can stream.
                else:
                    # This fun binary flag incantation makes os.open throw an
                    # OSError if the file already exists before we open it.
                    fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
                    try:
                        locks.lock(fd, locks.LOCK_EX)
                        for chunk in content.chunks():
                            os.write(fd, chunk)
                    finally:
                        locks.unlock(fd)
                        os.close(fd)
            except OSError as e:
                if e.errno == errno.EEXIST:
                    # Ooops, the file exists. We need a new file name.
                    name = self.get_available_name(name)
                    full_path = self.path(name)
                else:
                    raise
            else:
                # OK, the file save worked. Break out of the loop.
                break

        if settings.FILE_UPLOAD_PERMISSIONS is not None:
            os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)

        return name
Exemple #31
0
    def _save(self, name, content):
        """
        Lifted partially from django/core/files/storage.py
        """
        full_path = self.path(name)

        # Create any intermediate directories that do not exist.
        # Note that there is a race between os.path.exists and os.makedirs:
        # if os.makedirs fails with EEXIST, the directory was created
        # concurrently, and we can continue normally. Refs #16082.
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                os.makedirs(directory)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

            # There's a potential race condition between get_available_name and
            # saving the file; it's possible that two threads might return the
            # same name, at which point all sorts of fun happens. So we need to
            # try to create the file, but if it already exists we have to go back
            # to get_available_name() and try again.

            # This file has a file path that we can move.
        if hasattr(content, 'temporary_file_path'):
            temp_data_location = content.temporary_file_path()
        else:
            tmp_prefix = "tmp_%s" % (get_valid_filename(name), )
            temp_data_location = tempfile.mktemp(prefix=tmp_prefix,
                                                 dir=self.location)
            try:
                # This is a normal uploadedfile that we can stream.
                # This fun binary flag incantation makes os.open throw an
                # OSError if the file already exists before we open it.
                fd = os.open(temp_data_location,
                             os.O_WRONLY | os.O_CREAT |
                             os.O_EXCL | getattr(os, 'O_BINARY', 0))
                locks.lock(fd, locks.LOCK_EX)
                for chunk in content.chunks():
                    os.write(fd, chunk)
                locks.unlock(fd)
                os.close(fd)
            except Exception:
                if os.path.exists(temp_data_location):
                    os.remove(temp_data_location)
                raise

        file_move_safe(temp_data_location, full_path, allow_overwrite=True)
        content.close()

        if settings.FILE_UPLOAD_PERMISSIONS is not None:
            os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)

        return name
def file_move_safe(old_file_name,
                   new_file_name,
                   chunk_size=1024 * 64,
                   allow_overwrite=False):
    """
    Moves a file from one location to another in the safest way possible.

    First, try using ``shutils.move``, which is OS-dependent but doesn't break
    if moving across filesystems. Then, try ``os.rename``, which will break
    across filesystems. Finally, streams manually from one file to another in
    pure Python.

    If the destination file exists and ``allow_overwrite`` is ``False``, this
    function will throw an ``IOError``.
    """

    # There's no reason to move if we don't have to.
    if _samefile(old_file_name, new_file_name):
        return

    try:
        os.rename(old_file_name, new_file_name)
        return
    except OSError:
        # This will happen with os.rename if moving to another filesystem
        # or when moving opened files on certain operating systems
        pass

    # first open the old file, so that it won't go away
    old_file = open(old_file_name, 'rb')
    try:
        # now open the new file, not forgetting allow_overwrite
        fd = os.open(
            new_file_name,
            os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0) |
            (not allow_overwrite and os.O_EXCL or 0))
        try:
            locks.lock(fd, locks.LOCK_EX)
            current_chunk = None
            while current_chunk != '':
                current_chunk = old_file.read(chunk_size)
                os.write(fd, current_chunk)
        finally:
            locks.unlock(fd)
            os.close(fd)
    finally:
        old_file.close()
    copystat(old_file_name, new_file_name)

    try:
        os.remove(old_file_name)
    except OSError, e:
        # Certain operating systems (Cygwin and Windows)
        # fail when deleting opened files, ignore it
        if getattr(e, 'winerror', 0) != 32:
            # FIXME: should we also ignore errno 13?
            raise
Exemple #33
0
    def _save(self, name, content):
        # Prepare file name from the file content hash
        dir_name, file_full_name = os.path.split(name)
        file_name, file_ext = os.path.splitext(file_full_name)
        content_hash = hashlib.sha1()
        for chunk in content.chunks():
            content_hash.update(chunk)
        hash_name = self._split_name(content_hash.hexdigest())
            # + str(content.size) Can add size if anytime stuck into collisions.
        name = os.path.join(dir_name, hash_name + file_ext.lower())

        full_path = self.path(name)

        # Create any intermediate directories that do not exist.
        # Note that there is a race between os.path.exists and os.makedirs:
        # if os.makedirs fails with EEXIST, the directory was created
        # concurrently, and we can continue normally. Refs #16082.
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                os.makedirs(directory)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # There's a potential race condition: it's possible that two threads
        # might return the same name. In this case we just continue normally.
        # NOTE! This differs considerably from the default file storage behaviour.
        try:
            # This file has a file path that we can move.
            if hasattr(content, 'temporary_file_path'):
                file_move_safe(content.temporary_file_path(), full_path)
                content.close()

            # This is a normal uploadedfile that we can stream.
            else:
                # This fun binary flag incantation makes os.open throw an
                # OSError if the file already exists before we open it.
                fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
                try:
                    locks.lock(fd, locks.LOCK_EX)
                    for chunk in content.chunks():
                        os.write(fd, chunk)
                finally:
                    locks.unlock(fd)
                    os.close(fd)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

        if settings.FILE_UPLOAD_PERMISSIONS is not None:
            os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)

        return name
Exemple #34
0
def write_hosts_file(vm,password):

	towrite = '[vm]' + '\n'
	towrite += str(vm) + " ansible_ssh_pass='******'"

	with open('hosts', 'wb') as f:
		locks.lock(f, locks.LOCK_EX)
		f.write(towrite)

	file_move_safe("hosts", "../Ansible/hosts", allow_overwrite = True)
Exemple #35
0
    def _save(self, name, content):
        full_path = self.path(name)
        directory = os.path.dirname(full_path)
        print(directory, full_path, '----------save_method')
        if not os.path.exists(directory):
            try:
                os.makedirs(directory)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        while True:
            try:
                # This file has a file path that we can move.
                if hasattr(content, 'temporary_file_path'):
                    print(content.temporary_file_path(), full_path,
                          '-----temp_method')
                    file_move_safe(content.temporary_file_path(), full_path)
                    content.close()

                else:
                    flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL
                             | getattr(os, 'O_BINARY', 0))
                    fd = os.open(full_path, flags, 0o666)
                    try:
                        locks.lock(fd, locks.LOCK_EX)
                        _file = None
                        for chunk in content.chunks():
                            if _file is None:
                                mode = 'wb' if isinstance(chunk,
                                                          bytes) else 'wt'
                                _file = os.fdopen(fd, mode)
                            _file.write(chunk)
                    finally:
                        locks.unlock(fd)
                        if _file is not None:
                            _file.close()
                        else:
                            os.close(fd)
            except OSError as e:
                if e.errno == errno.EEXIST:
                    # Ooops, the file exists. We need a new file name.
                    name = self.get_available_name(name)
                    full_path = self.path(name)
                else:
                    raise
            else:
                break

        if settings.FILE_UPLOAD_PERMISSIONS is not None:
            os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)

        return name
Exemple #36
0
def write_variable_file_2(user):

	towrite = '---' +'\n'
	towrite += '\n'
	towrite += ('user: "******"\n')

	with open('test2.yml', 'wb') as f:
		locks.lock(f, locks.LOCK_EX)
		f.write(towrite)

	file_move_safe("test2.yml", "../Ansible/variables2.yml", allow_overwrite = True)
Exemple #37
0
    def lock(self):
        if self.nb_locks == 0:
            filename = os.path.join(self.root_path, self.lock_file_name)
            self.lock_file = open(filename, 'wb')
            try:
                os.chmod(filename, 0766)
            except:
                pass
            locks.lock(self.lock_file, locks.LOCK_EX)

        self.nb_locks += 1
Exemple #38
0
def file_move_safe(old_file_name, new_file_name, chunk_size=1024 * 64, allow_overwrite=False):
    """
    Moves a file from one location to another in the safest way possible.

    First, tries ``os.rename``, which is simple but will break across filesystems.
    If that fails, streams manually from one file to another in pure Python.

    If the destination file exists and ``allow_overwrite`` is ``False``, this
    function will throw an ``IOError``.
    """

    # There's no reason to move if we don't have to.
    if _samefile(old_file_name, new_file_name):
        return

    try:
        os.rename(old_file_name, new_file_name)
        return
    except OSError:
        # This will happen with os.rename if moving to another filesystem
        # or when moving opened files on certain operating systems
        pass

    # first open the old file, so that it won't go away
    old_file = open(old_file_name, "rb")
    try:
        # now open the new file, not forgetting allow_overwrite
        fd = os.open(
            new_file_name,
            os.O_WRONLY | os.O_CREAT | getattr(os, "O_BINARY", 0) | (not allow_overwrite and os.O_EXCL or 0),
        )
        try:
            locks.lock(fd, locks.LOCK_EX)
            current_chunk = None
            while current_chunk != "":
                current_chunk = old_file.read(chunk_size)
                os.write(fd, current_chunk)
        finally:
            locks.unlock(fd)
            os.close(fd)
    finally:
        old_file.close()
    copystat(old_file_name, new_file_name)

    try:
        os.remove(old_file_name)
    except OSError as e:
        # Certain operating systems (Cygwin and Windows)
        # fail when deleting opened files, ignore it.  (For the
        # systems where this happens, temporary files will be auto-deleted
        # on close anyway.)
        if getattr(e, "winerror", 0) != 32 and getattr(e, "errno", 0) != 13:
            raise
    def _save(self, name, content):
        '''
        Copied from super and lightly modified - Unfortunately, the default 
        race condition handling will lead to an infinite loop here, since 
        get_available_name() doesn't return unique names for identical hashes.
        '''
        full_path = self.path(name)
        if os.path.exists(full_path):
            return name

        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            # handle concurrency issue where the directory is created while
            # by another thread after the call to exists()
            try:
                os.makedirs(directory)
            except OSError as e:
                # ignore EEXIST, since it means that our goal here was already
                # accomplished
                if e.errno != errno.EEXIST:
                    raise
        elif not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)
        
        # There's a potential race condition when saving the file; it's 
        # possible that two threads might try to write the file simultaneously.
        # We need to try to create the file, but abort if we detect that it
        # already exists (in which case we assume that another thread is
        # writing it).
        try:
            # This file has a file path that we can move. 
            if hasattr(content, 'temporary_file_path'):
                file_move_safe(content.temporary_file_path(), full_path)
                content.close()

            # This is a normal uploadedfile that we can stream.
            else:
                # This fun binary flag incantation makes os.open throw an 
                # OSError if the file already exists before we open it.
                fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
                try:
                    locks.lock(fd, locks.LOCK_EX)
                    for chunk in content.chunks():
                        os.write(fd, chunk)
                finally:
                    locks.unlock(fd)
                    os.close(fd)
        except OSError, e:
            # abort and continue normally if we detect the existence of the 
            # file we're trying to write, otherwise raise normally
            if e.errno != errno.EEXIST:
                raise
Exemple #40
0
def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False):
    """
    Moves a file from one location to another in the safest way possible.

    First, try using ``shutils.move``, which is OS-dependent but doesn't break
    if moving across filesystems. Then, try ``os.rename``, which will break
    across filesystems. Finally, streams manually from one file to another in
    pure Python.

    If the destination file exists and ``allow_overwrite`` is ``False``, this
    function will throw an ``IOError``.
    """

    # There's no reason to move if we don't have to.
    if _samefile(old_file_name, new_file_name):
        return

    try:
        os.rename(old_file_name, new_file_name)
        return
    except OSError:
        # This will happen with os.rename if moving to another filesystem
        # or when moving opened files on certain operating systems
        pass

    # first open the old file, so that it won't go away
    old_file = open(old_file_name, 'rb')
    try:
        # now open the new file, not forgetting allow_overwrite
        fd = os.open(new_file_name, os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0) |
                                    (not allow_overwrite and os.O_EXCL or 0))
        try:
            locks.lock(fd, locks.LOCK_EX)
            current_chunk = None
            while current_chunk != '':
                current_chunk = old_file.read(chunk_size)
                os.write(fd, current_chunk)
        finally:
            locks.unlock(fd)
            os.close(fd)
    finally:
        old_file.close()
    copystat(old_file_name, new_file_name)

    try:
        os.remove(old_file_name)
    except OSError, e:
        # Certain operating systems (Cygwin and Windows)
        # fail when deleting opened files, ignore it
        if getattr(e, 'winerror', 0) != 32:
            # FIXME: should we also ignore errno 13?
            raise
Exemple #41
0
    def _save(self, name, content):
        """ This is similar to the parent calss but removes any safe-guards against existing files """
        if not self.file_overwrite:
            return super(OverwriteFileSystemStorage, self)._save(name, content)

        full_path = self.path(name)

        # Create any intermediate directories that do not exist.
        # Note that there is a race between os.path.exists and os.makedirs:
        # if os.makedirs fails with EEXIST, the directory was created
        # concurrently, and we can continue normally. Refs #16082.
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                os.makedirs(directory)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # This file has a file path that we can move.
        if hasattr(content, 'temporary_file_path'):
            file_move_safe(content.temporary_file_path(), full_path)
            content.close()

        # This is a normal uploadedfile that we can stream.
        else:
            flags = (os.O_WRONLY | os.O_CREAT | getattr(os, 'O_BINARY', 0))
            # The current umask value is masked out by os.open!
            fd = os.open(full_path, flags, 0o666)
            _file = None
            try:
                locks.lock(fd, locks.LOCK_EX)
                for chunk in content.chunks():
                    if _file is None:
                        mode = 'wb' if isinstance(chunk, bytes) else 'wt'
                        _file = os.fdopen(fd, mode)
                    _file.write(chunk)
            finally:
                locks.unlock(fd)
                if _file is not None:
                    _file.close()
                else:
                    os.close(fd)

        if settings.FILE_UPLOAD_PERMISSIONS is not None:
            os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)

        return name
Exemple #42
0
    def _save(self, name, content):
        full_path = self.path(name)
        directory = os.path.dirname(full_path)
        try:
            if self.directory_permissions_mode is not None:
                old_umask = os.umask(0)
                try:
                    os.makedirs(directory,
                                self.directory_permissions_mode,
                                exist_ok=True)
                finally:
                    os.umask(old_umask)
            else:
                os.makedirs(directory, exist_ok=True)
        except FileExistsError:
            raise FileExistsError("%s exists and is not a directory." %
                                  directory)
        while True:
            try:
                if hasattr(content, "temporary_file_path"):
                    file_move_safe(content.temporary_file_path(), full_path)
                else:
                    fd = os.open(full_path, self.OS_OPEN_FLAGS, 0o666)
                    _file = None
                    try:
                        locks.lock(fd, locks.LOCK_EX)
                        for chunk in content.chunks():
                            if _file is None:
                                mode = "wb" if isinstance(chunk,
                                                          bytes) else "wt"
                                _file = os.fdopen(fd, mode)
                            _file.write(chunk)
                    finally:
                        locks.unlock(fd)
                        if _file is not None:
                            _file.close()
                        else:
                            os.close(fd)
            except FileExistsError:
                name = self.get_available_name(name)
                full_path = self.path(name)
            else:
                break

        if self.file_permissions_mode is not None:
            os.chmod(full_path, self.file_permissions_mode)

        return name.replace("\\", "/")
Exemple #43
0
 def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None):
     try:
         with open(self._key_to_file(key, version), 'r+b') as f:
             try:
                 locks.lock(f, locks.LOCK_EX)
                 if self._is_expired(f):
                     return False
                 else:
                     previous_value = pickle.loads(zlib.decompress(f.read()))
                     f.seek(0)
                     self._write_content(f, timeout, previous_value)
                     return True
             finally:
                 locks.unlock(f)
     except FileNotFoundError:
         return False
 def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None):
     try:
         with open(self._key_to_file(key, version), 'r+b') as f:
             try:
                 locks.lock(f, locks.LOCK_EX)
                 if self._is_expired(f):
                     return False
                 else:
                     previous_value = pickle.loads(zlib.decompress(f.read()))
                     f.seek(0)
                     _write_content(f, self.get_backend_timeout(timeout), previous_value)
                     return True
             finally:
                 locks.unlock(f)
     except FileNotFoundError:
         return False
Exemple #45
0
    def contribute_to_class(self, cls, name):
        if self.filename is not None:
            filename = self.filename
        else:
            filename = name

        # Create a post_save signal handler to write data to disk 
        @receiver(post_save, sender=cls, weak=False,
                dispatch_uid="fsfield_write_data_%s_%s_%s" % 
                (cls.__module__, cls.__name__, name))
        def write_data(sender, instance, created, raw, using, **kwargs):
            if (name not in instance.__dict__ or
                    instance.__dict__[name] is None):
                # Nothing to save
                return
            # Open the file for writing and acquire a lock on it if
            # possible
            path = model_instance_field_path(instance, filename)
            fs_storage = isinstance(self.storage, FileSystemStorage)
            if fs_storage:
                full_path = self.storage.path(path)
                directory = op.dirname(full_path)
                if not op.exists(directory):
                    try:
                        os.makedirs(directory)
                    except OSError, err:
                        # Another thread may have created the directory since
                        # the check
                        if err.errno == 17:
                            pass
            fp = self.storage.open(path, "wb")
            if fs_storage:
                locks.lock(fp.file, locks.LOCK_EX)
            # Write data
            try:
                value = instance.__dict__[name]
                if self.dump is None:
                    fp.write(value.encode("utf8"))
                else:
                    self.dump(value, fp)
            finally:
                # Release lock
                if fs_storage:
                    locks.unlock(fp.file)
                # Close file
                fp.close()
Exemple #46
0
    def load_file(cls, file_name):
        try:
            with open(file_name, 'r') as fp:
                locks.lock(fp, locks.LOCK_EX)
                content = fp.read()
        except IOError as exc:
            if exc.errno == errno.ENOENT:
                content = '{}'
            else:
                raise

        data = yaml.safe_load(content)
        if not isinstance(data, dict):
            raise TypeError(
                "YAML content of {} is not a dictionary".format(file_name))

        return data
    def _save(self, name, content):
        full_path = self.path(name)

        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            os.makedirs(directory)
        elif not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # There's a potential race condition between get_available_name and
        # saving the file; it's possible that two threads might return the
        # same name, at which point all sorts of fun happens. So we need to
        # try to create the file, but if it already exists we have to go back
        # to get_available_name() and try again.

        while True:
            try:
                # This file has a file path that we can move.
                if hasattr(content, 'temporary_file_path'):
                    file_move_safe(content.temporary_file_path(), full_path)
                    content.close()

                # This is a normal uploadedfile that we can stream.
                else:
                    # This fun binary flag incantation makes os.open throw an
                    # OSError if the file already exists before we open it.
                    fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
                    try:
                        locks.lock(fd, locks.LOCK_EX)
                        for chunk in content.chunks():
                            os.write(fd, chunk)
                            # CHANGED: This un-hangs us long enough to keep things rolling.
                            eventlet.sleep(0)
                    finally:
                        locks.unlock(fd)
                        os.close(fd)
            except OSError, e:
                if e.errno == errno.EEXIST:
                    # Ooops, the file exists. We need a new file name.
                    name = self.get_available_name(name)
                    full_path = self.path(name)
                else:
                    raise
            else:
                # OK, the file save worked. Break out of the loop.
                break
Exemple #48
0
def file_move_safe(old_file_name,
                   new_file_name,
                   chunk_size=1024 * 64,
                   allow_overwrite=False):
    """
    Moves a file from one location to another in the safest way possible.

    First, try using ``shutils.move``, which is OS-dependent but doesn't break
    if moving across filesystems. Then, try ``os.rename``, which will break
    across filesystems. Finally, streams manually from one file to another in
    pure Python.

    If the destination file exists and ``allow_overwrite`` is ``False``, this
    function will throw an ``IOError``.
    """

    # There's no reason to move if we don't have to.
    if old_file_name == new_file_name:
        return

    if not allow_overwrite and os.path.exists(new_file_name):
        raise IOError("Cannot overwrite existing file '%s'." % new_file_name)

    try:
        file_move(old_file_name, new_file_name)
        return
    except OSError:
        # This will happen with os.rename if moving to another filesystem
        pass

    # If the built-in didn't work, do it the hard way.
    new_file = open(new_file_name, 'wb')
    locks.lock(new_file, locks.LOCK_EX)
    old_file = open(old_file_name, 'rb')
    current_chunk = None

    while current_chunk != '':
        current_chunk = old_file.read(chunk_size)
        new_file.write(current_chunk)

    new_file.close()
    old_file.close()

    os.remove(old_file_name)
Exemple #49
0
class ImageStorage(FileSystemStorage):
    """
    Create image file and empty thumb file.
    """
    # almost all copyed from FileSystemStorage._save
    def _save(self, name, content):
        full_path = self.path(name)

        # Create any intermediate directories that do not exist.
        # Note that there is a race between os.path.exists and os.makedirs:
        # if os.makedirs fails with EEXIST, the directory was created
        # concurrently, and we can continue normally. Refs #16082.
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                os.makedirs(directory)
            except OSError, e:
                if e.errno != errno.EEXIST:
                    raise
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # There's a potential race condition between get_available_name and
        # saving the file; it's possible that two threads might return the
        # same name, at which point all sorts of fun happens. So we need to
        # try to create the file, but if it already exists we have to go back
        # to get_available_name() and try again.

        while True:
            try:
                thumb_created = False
                thumb_name = get_thumb_name(full_path)
                try:
                    fd = os.open(thumb_name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
                    os.close(fd)
                except OSError, e:
                    if e.errno == errno.EEXIST:
                        # Ooops, the file exists. We need a new file name.
                        name = self.get_available_name(name)
                        full_path = self.path(name)
                        continue
                    else:
                        raise
                else:
                    thumb_created = True

                # This fun binary flag incantation makes os.open throw an
                # OSError if the file already exists before we open it.
                fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
                try:
                    locks.lock(fd, locks.LOCK_EX)
                    for chunk in content.chunks():
                        os.write(fd, chunk)
                finally:
                    locks.unlock(fd)
                    os.close(fd)
Exemple #50
0
def write_variable_file_1(user, password):
	hashed_password = sha512_crypt.encrypt(password)

	towrite = '---' +'\n'
	towrite += '\n'
	towrite += ('user: "******"\n')
	towrite += ('password: "******"\n')
	towrite += '\n'
	towrite += ('dockercmd: "docker run -d --name ' + user + ' -it ubuntu"' +'\n')
	towrite += '\n'
	towrite += ('linetoadd1: "me=`whoami`"' + '\n')
	towrite += ('linetoadd2: "if [ $me == \'' + user + '\' ]; then"' + '\n')
	towrite += ('linetoadd3: "  docker exec -it ' + user + ' /bin/bash\\nfi"' + '\n')

	with open('test.yml', 'wb') as f:
		locks.lock(f, locks.LOCK_EX)
		f.write(towrite)

	file_move_safe("test.yml", "../Ansible/variables.yml", allow_overwrite = True)
Exemple #51
0
    def set_and_save(self, key, value):
        if self.data.get(key, object()) == value:
            return

        fd = os.open(self.file_name, os.O_RDWR | os.O_CREAT)
        with os.fdopen(fd, "r+") as fp:
            locks.lock(fd, locks.LOCK_EX)

            data = yaml.safe_load(fp)
            if data is None:
                data = {}

            self.data[key] = value
            data[key] = value

            fp.seek(0)
            yaml.safe_dump(
                data, fp, default_flow_style=False, allow_unicode=True, width=10000
            )
Exemple #52
0
def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False):
    """
    Moves a file from one location to another in the safest way possible.

    First, try using ``shutils.move``, which is OS-dependent but doesn't break
    if moving across filesystems. Then, try ``os.rename``, which will break
    across filesystems. Finally, streams manually from one file to another in
    pure Python.

    If the destination file exists and ``allow_overwrite`` is ``False``, this
    function will throw an ``IOError``.
    """

    # There's no reason to move if we don't have to.
    if old_file_name == new_file_name:
        return

    if not allow_overwrite and os.path.exists(new_file_name):
        raise IOError("Cannot overwrite existing file '%s'." % new_file_name)

    try:
        file_move(old_file_name, new_file_name)
        return
    except OSError:
        # This will happen with os.rename if moving to another filesystem
        pass

    # If the built-in didn't work, do it the hard way.
    fd = os.open(new_file_name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
    try:
        locks.lock(fd, locks.LOCK_EX)
        old_file = open(old_file_name, 'rb')
        current_chunk = None
        while current_chunk != '':
            current_chunk = old_file.read(chunk_size)
            os.write(fd, current_chunk)
    finally:
        locks.unlock(fd)
        os.close(fd)
        old_file.close()

    os.remove(old_file_name)
Exemple #53
0
def createdocker(user, instance, vm, email):
	try:
		with open(user, 'wb') as f:
			locks.lock(f, locks.LOCK_EX)
			f.write('0')
		os.system('ansible-playbook ../Ansible/newdock.yml -i ../Ansible/hosts')
		with open(user, 'wb') as f:
			locks.lock(f, locks.LOCK_EX)
			f.write('1')
		body = 'New Docker Instance Created\n'
		body += 'IP : ' + str(instance.vm.ip) + '\n'
		body += 'User : '******'\n'
		body += 'Password : '******'\n'
		msg = EmailMultiAlternatives(subject="DockerPaaS | New Instance Created", body=body,from_email="*****@*****.**",to=[email])
		msg.send()
	except:
		print 'no'

	instance.save()
	vm.save()
Exemple #54
0
    def _save(self, name, content):

        full_path = self.path(name)
        # Create any intermediate directories that do not exist.
        # Note that there is a race between os.path.exists and os.makedirs:
        # if os.makedirs fails with EEXIST, the directory was created
        # concurrently, and we can continue normally. Refs #16082.
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                os.makedirs(directory)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # This fun binary flag incantation makes os.open throw an
        # OSError if the file already exists before we open it.
        flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL
                 | getattr(os, 'O_BINARY', 0))
        # The current umask value is masked out by os.open!
        fd = os.open(full_path, flags, 0o666)
        _file = None
        try:
            locks.lock(fd, locks.LOCK_EX)
            for chunk in content.chunks():
                if _file is None:
                    mode = 'wb' if isinstance(chunk, bytes) else 'wt'
                    _file = os.fdopen(fd, mode)
                _file.write(chunk)
        finally:
            locks.unlock(fd)
            if _file is not None:
                _file.close()
            else:
                os.close(fd)
        file_name = self.client.put(self.path(name))
        os.unlink(full_path)
        return file_name
Exemple #55
0
    def append(self, name, content, progress_callback=None):
        full_path = self.path(name)
        while True:

            # This file has a file path that we can move.
            #if hasattr(content, 'temporary_file_path'):
            #    file_move_safe(content.temporary_file_path(), full_path)
            #
            ## This is a normal uploadedfile that we can stream.
            #else:
            # This fun binary flag incantation makes os.open throw an
            # OSError if the file already exists before we open it.
            flags = (os.O_WRONLY | getattr(os, 'O_BINARY', 0))
            # The current umask value is masked out by os.open!
            fd = os.open(full_path, flags, 0o666)
            _file = None
            try:
                locks.lock(fd, locks.LOCK_EX)
                for chunk in content.chunks():
                    if _file is None:
                        # if isinstance(chunk, bytes) else 'wt'
                        mode = 'ab'
                        _file = os.fdopen(fd, mode)

                    _file.write(chunk)
                break
            finally:
                locks.unlock(fd)
                if _file is not None:
                    _file.close()
                else:
                    os.close(fd)
                break

        if self.file_permissions_mode is not None:
            os.chmod(full_path, self.file_permissions_mode)

        # Store filenames with forward slashes, even on Windows.
        return name.replace('\\', '/')
Exemple #56
0
def get_lock(name, timeout='forewer'):
    """ Acquire and return unique lock (locked file)
    If lock has not been created before timeout then IOError raised """

    f = open(FILE_NAME_PREFIX + name, 'wb')

    if timeout == 'forewer':
        locks.lock(f, locks.LOCK_EX)
    else:
        #print '   waiting for lock...'
        for waited in range(timeout + 1):
            try:
                if waited > 0:  # sleep between iterations
                    sleep(1)
                locks.lock(f, locks.LOCK_EX + locks.LOCK_NB)
                break
            except IOError:
                pass
        else:
            f.close()
            raise LockException('Get lock timeout')

    return f
Exemple #57
0
class HashFileSystemStorage(FileSystemStorage):
    def __init__(self, location=None, base_url=None):
        if location is None:
            location = os.path.join(settings.MEDIA_ROOT, 'hash')
        if base_url is None:
            base_url = settings.MEDIA_URL + 'hash/'
        super(HashFileSystemStorage, self).__init__(location, base_url)

    def path(self, name):
        name = os.path.join(name[:2], name[2:4], name)
        return super(HashFileSystemStorage, self).path(name)

    def save(self, name, content):
        # 计算内容 HASH
        md5 = hashlib.md5()
        for chunk in content.chunks():
            md5.update(chunk)
        name = md5.hexdigest()

        # 保存文件
        full_path = self.path(name)
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                os.makedirs(directory)
            except OSError, e:
                if e.errno != errno.EEXIST:
                    raise
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        try:
            if hasattr(content, 'temporary_file_path'):
                file_move_safe(content.temporary_file_path(), full_path)
                content.close()
            else:
                fd = os.open(
                    full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL
                    | getattr(os, 'O_BINARY', 0))
                try:
                    locks.lock(fd, locks.LOCK_EX)
                    for chunk in content.chunks():
                        os.write(fd, chunk)
                finally:
                    locks.unlock(fd)
                    os.close(fd)
        except OSError, e:
            if e.errno != errno.EEXIST:
                raise