Beispiel #1
0
 def _upload_file(self, request):
     """
     Upload file to the server.
     """
     from django.core.files.move import file_move_safe
     
     if request.method == 'POST':
         folder = request.POST.get('folder')
         fb_uploadurl_re = re.compile(r'^.*(%s)' % reverse("filebrowser:fb_upload", current_app=self.name))
         folder = fb_uploadurl_re.sub('', folder)
         abs_path = os.path.join(self.media_root, folder)
         if request.FILES:
             filedata = request.FILES['Filedata']
             filedata.name = convert_filename(filedata.name)
             self.filebrowser_pre_upload.send(sender=request, path=request.POST.get('folder'), file=filedata)
             uploadedfile = handle_file_upload(abs_path, filedata, media_root=self.media_root)
             # if file already exists
             if os.path.isfile(smart_unicode(os.path.join(self.media_root, folder, filedata.name))):
                 old_file = smart_unicode(os.path.join(abs_path, filedata.name))
                 new_file = smart_unicode(os.path.join(abs_path, uploadedfile))
                 file_move_safe(new_file, old_file, allow_overwrite=True)
             # POST UPLOAD SIGNAL
             self.filebrowser_post_upload.send(sender=request, path=request.POST.get('folder'), file=FileObject(smart_unicode(os.path.join(folder, filedata.name)), media_root=self.media_root, media_url=self.media_url))
     
     return HttpResponse('True')
Beispiel #2
0
  def save(self, name, content):
    full_path = self.path(name)
 
    directory = os.path.dirname(full_path)
    if not os.path.exists(directory):
      os.makedirs(directory)
    elif not os.path.isdir(directory):
      print "Error dir is not error"
    
    while True:
      try:
        if hasattr(content, 'temporary_file_path'):
          file_move_safe(content.temporary_file_path(), full_path)
          content.close()
        else:
          fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
          try:
            locks.lock(fd, locks.LOCK_EX)
            for chunk in content.chunks():
              os.write(fd, chunk)
          finally:
            locks.unlock(fd)
            os.close(fd)
      except OSError, e:
        if e.errno == errno.EEXIST:
          name = self.get_available_name(name)
          full_path = self.path(name)
        else:
          raise
      else:
        break
def _upload_file(request):
    """
    Upload file to the server.
    """
    
    from django.core.files.move import file_move_safe 
    from django.http import QueryDict

    if request.method == 'POST':
        query = QueryDict(request.POST.get('get_query'))
        folder = request.POST.get('folder')
        fb_uploadurl_re = re.compile(r'^(%s)' % reverse("fb_upload"))
        folder = fb_uploadurl_re.sub('', folder)
        abs_path = os.path.join(MEDIA_ROOT, DIRECTORY, folder)
        if request.FILES:
            filedata = request.FILES['Filedata']
            filedata.name = convert_filename(filedata.name)
            # PRE UPLOAD SIGNAL
            filebrowser_pre_upload.send(sender=request, path=request.POST.get('folder'), file=filedata)
            # HANDLE UPLOAD
            uploadedfile = handle_file_upload(abs_path, filedata)
            # MOVE UPLOADED FILE
            # if file already exists
            if os.path.isfile(os.path.join(MEDIA_ROOT, DIRECTORY, folder, filedata.name)):
                old_file = os.path.join(abs_path, filedata.name)
                new_file = os.path.join(abs_path, uploadedfile)
                file_move_safe(new_file, old_file)
            # POST UPLOAD SIGNAL
            filebrowser_post_upload.send(sender=request, path=request.POST.get('folder'), file=FileObject(os.path.join(DIRECTORY, folder, filedata.name)))
            return HttpResponseRedirect(reverse("fb_browse") + query_helper(query, "", "filename,filetype"))
        else:
            return HttpResponse("Error: No files were posted")
    else:
        return HttpResponse("Error: No POST data")
Beispiel #4
0
 def _save(self, name, content):
     full_path = self.path(name)
     directory = os.path.dirname(full_path)
     if not os.path.exists(directory):
         os.makedirs(directory)
     elif not os.path.isdir(directory):
         raise IOError("%s exists and is not a directory." % directory)
     # This file has a file path that we can move.
     if hasattr(content, 'temporary_file_path'):
         file_move_safe(content.temporary_file_path(), full_path)
         content.close()
     # This is a normal uploadedfile that we can stream.
     else:
         # This fun binary flag incantation makes os.open throw an
         # OSError if the file already exists before we open it.
         fd = os.open(full_path, os.O_WRONLY | os.O_TRUNC \
                 | os.O_CREAT | getattr(os, 'O_BINARY', 0))
         try:
             locks.lock(fd, locks.LOCK_EX)
             for chunk in content.chunks():
                 os.write(fd, chunk)
         finally:
             locks.unlock(fd)
             os.close(fd)
     if settings.FILE_UPLOAD_PERMISSIONS is not None:
         os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
     return name
Beispiel #5
0
    def _save(self, name, content):
        full_path = self.path(name)
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            os.makedirs(directory)
        elif not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # This file has a file path that we can move.
        if hasattr(content, 'temporary_file_path'):
            file_move_safe(content.temporary_file_path(), full_path)
            content.close()

        # This is a normal uploadedfile that we can stream.
        else:
            # Open file for writting in binary mode
            fd = open(full_path.encode('utf-8'), "wb")
            try:
                locks.lock(fd, locks.LOCK_EX)
                for chunk in content.chunks():
                    fd.write(chunk)
            finally:
                locks.unlock(fd)
                fd.close()

        if settings.FILE_UPLOAD_PERMISSIONS is not None:
            os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)

        return name
def update_item(item, newname):
        file_root = os.path.join(settings.MEDIA_ROOT, get_storage_path(item, ''))
        path = item.file.path
        if not os.path.exists(path):
            print newname, ": files doesn't exist: ", path
            return
        root, name = os.path.split(path)
        base, ext = os.path.splitext(name)
        renamed = os.path.join(root, u'%s%s' % (newname, ext))

        cache = {}
        for size in MediaSizeCache().sizes.values():
            func = getattr(item, "get_%s_filename" % size.name, None)
            if func:
                sizename = func()
                if os.path.exists(sizename) and sizename.startswith(file_root):
                    cache[func] = sizename

        # Do the move and save
        move_file(item, path, renamed)
        # No need to regenerate caches
        item.prevent_cache_clear = True
        item.remove_deleted = False
        item.save()

        for key, value in cache.items():
            target = key()
            if not target.endswith('unconverted') and os.path.exists(value):
                file_move_safe(value, target)

        taken = item.date_taken
        taken = taken.astimezone(get_current_timezone())
        mtime = int(time.mktime(taken.timetuple()))
        os.utime(item.file.path, (mtime, mtime))
Beispiel #7
0
    def _save(self, name, content):
        full_path = self.path(name)

        # Create any intermediate directories that do not exist.
        # Note that there is a race between os.path.exists and os.makedirs:
        # if os.makedirs fails with EEXIST, the directory was created
        # concurrently, and we can continue normally. Refs #16082.
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                if self.directory_permissions_mode is not None:
                    # os.makedirs applies the global umask, so we reset it,
                    # for consistency with file_permissions_mode behavior.
                    old_umask = os.umask(0)
                    try:
                        os.makedirs(directory, self.directory_permissions_mode)
                    finally:
                        os.umask(old_umask)
                else:
                    os.makedirs(directory)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        tmp_file = tempfile.mktemp()
        filey = open(tmp_file, 'wb')
        filey.write(content.read())
        # make sure that all data is on disk
        filey.flush()
        os.fsync(filey.fileno())
        filey.close()
        file_move_safe(tmp_file, full_path, allow_overwrite=True)
        return name
Beispiel #8
0
def _upload_file(request):
    """
    Upload file to the server.
    """
    
    from django.core.files.move import file_move_safe
    
    if request.method == 'POST':
        folder = request.POST.get('folder')
        fb_uploadurl_re = re.compile(r'^.*(%s)' % reverse("fb_upload"))
        folder = fb_uploadurl_re.sub('', folder)
        abs_path = _check_access(request, folder)
        if request.FILES:
            filedata = request.FILES['Filedata']
            filedata.name = convert_filename(filedata.name)
            _check_access(request, abs_path, filedata.name)
            # PRE UPLOAD SIGNAL
            filebrowser_pre_upload.send(sender=request, path=request.POST.get('folder'), file=filedata)
            # HANDLE UPLOAD
            uploadedfile = handle_file_upload(abs_path, filedata)
            # MOVE UPLOADED FILE
            # if file already exists
            if os.path.isfile(smart_str(os.path.join(fb_settings.MEDIA_ROOT, fb_settings.DIRECTORY, folder, filedata.name))):
                old_file = smart_str(os.path.join(abs_path, filedata.name))
                new_file = smart_str(os.path.join(abs_path, uploadedfile))
                file_move_safe(new_file, old_file)
            # POST UPLOAD SIGNAL
            filebrowser_post_upload.send(sender=request, path=request.POST.get('folder'), file=FileObject(smart_str(os.path.join(fb_settings.DIRECTORY, folder, filedata.name))))
    return HttpResponse('True')
Beispiel #9
0
    def _save(self, name, content):
        """
        Lifted partially from django/core/files/storage.py
        """
        full_path = self.path(name)

        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            os.makedirs(directory)
        elif not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # Ensure that content is open
        content.open()

        if hasattr(content, 'temporary_file_path'):
            # Content has a file that we can move.
            temp_data_location = content.temporary_file_path()
            file_move_safe(temp_data_location, full_path, allow_overwrite=True)
        else:
            # Write the content stream to a temporary file and move it.
            fd, tmp_path = mkstemp()
            locks.lock(fd, locks.LOCK_EX)
            for chunk in content.chunks():
                os.write(fd, chunk)
            locks.unlock(fd)
            os.close(fd)
            file_move_safe(tmp_path, full_path, allow_overwrite=True)

        content.close()
        if settings.FILE_UPLOAD_PERMISSIONS is not None:
            os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)

        return name
Beispiel #10
0
 def save_video(self, filename):
     self.type = 2
     (self.width, self.height) = get_video_size(filename)
     self.length = get_media_duration(filename)
     super(LibraryFile, self).save() # Intermediate save to get new ID
     self.filename = "adex%s_%s" % (self.id, self.name)
     genVideoThumb(filename, settings.MEDIA_ROOT + "v/thumb/%s.jpg" % self.filename)
     file_move_safe(filename, settings.MEDIA_ROOT + "v/%s" % self.filename)
Beispiel #11
0
    def _save(self, name, content):
        full_path = self.path(name)

        # Create any intermediate directories that do not exist.
        # Note that there is a race between os.path.exists and os.makedirs:
        # if os.makedirs fails with EEXIST, the directory was created
        # concurrently, and we can continue normally. Refs #16082.
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                os.makedirs(directory)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # There's a potential race condition between get_available_name and
        # saving the file; it's possible that two threads might return the
        # same name, at which point all sorts of fun happens. So we need to
        # try to create the file, but if it already exists we have to go back
        # to get_available_name() and try again.

        while True:
            try:
                # This file has a file path that we can move.
                if hasattr(content, 'temporary_file_path'):
                    file_move_safe(content.temporary_file_path(), full_path)
                    content.close()

                # This is a normal uploadedfile that we can stream.
                else:
                    # This fun binary flag incantation makes os.open throw an
                    # OSError if the file already exists before we open it.
                    fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
                    try:
                        locks.lock(fd, locks.LOCK_EX)
                        for chunk in content.chunks():
                            os.write(fd, chunk)
                    finally:
                        locks.unlock(fd)
                        os.close(fd)
            except OSError as e:
                if e.errno == errno.EEXIST:
                    # Ooops, the file exists. We need a new file name.
                    name = self.get_available_name(name)
                    full_path = self.path(name)
                else:
                    raise
            else:
                # OK, the file save worked. Break out of the loop.
                break

        if settings.FILE_UPLOAD_PERMISSIONS is not None:
            os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)

        return name
Beispiel #12
0
    def test_file_move_overwrite(self):
        handle_a, self.file_a = tempfile.mkstemp(dir=os.environ['DJANGO_TEST_TEMP_DIR'])
        handle_b, self.file_b = tempfile.mkstemp(dir=os.environ['DJANGO_TEST_TEMP_DIR'])

        # file_move_safe should raise an IOError exception if destination file exists and allow_overwrite is False
        self.assertRaises(IOError, lambda: file_move_safe(self.file_a, self.file_b, allow_overwrite=False))

        # should allow it and continue on if allow_overwrite is True
        self.assertIsNone(file_move_safe(self.file_a, self.file_b, allow_overwrite=True))
Beispiel #13
0
 def save_image(self, filename):
     self.type = 1
     im = pil.open(filename)
     (self.width, self.height) = im.size
     super(LibraryFile, self).save() # Intermediate save to get new ID
     self.filename = "adex%s_%s" % (self.id, self.name)
     im.thumbnail((LIBRARYFILE_THUMB_WIDTH,LIBRARYFILE_THUMB_HEIGHT), pil.ANTIALIAS)
     im.save(settings.MEDIA_ROOT + "i/thumb/%s" % self.filename)
     file_move_safe(filename, settings.MEDIA_ROOT + "i/%s" % self.filename)
Beispiel #14
0
    def _save(self, name, content):
        """
        Lifted partially from django/core/files/storage.py
        """
        full_path = self.path(name)

        # Create any intermediate directories that do not exist.
        # Note that there is a race between os.path.exists and os.makedirs:
        # if os.makedirs fails with EEXIST, the directory was created
        # concurrently, and we can continue normally. Refs #16082.
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                os.makedirs(directory)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

            # There's a potential race condition between get_available_name and
            # saving the file; it's possible that two threads might return the
            # same name, at which point all sorts of fun happens. So we need to
            # try to create the file, but if it already exists we have to go back
            # to get_available_name() and try again.

            # This file has a file path that we can move.
        if hasattr(content, 'temporary_file_path'):
            temp_data_location = content.temporary_file_path()
        else:
            tmp_prefix = "tmp_%s" % (get_valid_filename(name), )
            temp_data_location = tempfile.mktemp(prefix=tmp_prefix,
                                                 dir=self.location)
            try:
                # This is a normal uploadedfile that we can stream.
                # This fun binary flag incantation makes os.open throw an
                # OSError if the file already exists before we open it.
                fd = os.open(temp_data_location,
                             os.O_WRONLY | os.O_CREAT |
                             os.O_EXCL | getattr(os, 'O_BINARY', 0))
                locks.lock(fd, locks.LOCK_EX)
                for chunk in content.chunks():
                    os.write(fd, chunk)
                locks.unlock(fd)
                os.close(fd)
            except Exception:
                if os.path.exists(temp_data_location):
                    os.remove(temp_data_location)
                raise

        file_move_safe(temp_data_location, full_path, allow_overwrite=True)
        content.close()

        if settings.FILE_UPLOAD_PERMISSIONS is not None:
            os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)

        return name
Beispiel #15
0
    def _save(self, name, content):
        # Prepare file name from the file content hash
        dir_name, file_full_name = os.path.split(name)
        file_name, file_ext = os.path.splitext(file_full_name)
        content_hash = hashlib.sha1()
        for chunk in content.chunks():
            content_hash.update(chunk)
        hash_name = self._split_name(content_hash.hexdigest())
            # + str(content.size) Can add size if anytime stuck into collisions.
        name = os.path.join(dir_name, hash_name + file_ext.lower())

        full_path = self.path(name)

        # Create any intermediate directories that do not exist.
        # Note that there is a race between os.path.exists and os.makedirs:
        # if os.makedirs fails with EEXIST, the directory was created
        # concurrently, and we can continue normally. Refs #16082.
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                os.makedirs(directory)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # There's a potential race condition: it's possible that two threads
        # might return the same name. In this case we just continue normally.
        # NOTE! This differs considerably from the default file storage behaviour.
        try:
            # This file has a file path that we can move.
            if hasattr(content, 'temporary_file_path'):
                file_move_safe(content.temporary_file_path(), full_path)
                content.close()

            # This is a normal uploadedfile that we can stream.
            else:
                # This fun binary flag incantation makes os.open throw an
                # OSError if the file already exists before we open it.
                fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
                try:
                    locks.lock(fd, locks.LOCK_EX)
                    for chunk in content.chunks():
                        os.write(fd, chunk)
                finally:
                    locks.unlock(fd)
                    os.close(fd)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

        if settings.FILE_UPLOAD_PERMISSIONS is not None:
            os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)

        return name
Beispiel #16
0
def write_hosts_file(vm,password):

	towrite = '[vm]' + '\n'
	towrite += str(vm) + " ansible_ssh_pass='******'"

	with open('hosts', 'wb') as f:
		locks.lock(f, locks.LOCK_EX)
		f.write(towrite)

	file_move_safe("hosts", "../Ansible/hosts", allow_overwrite = True)
Beispiel #17
0
def write_variable_file_2(user):

	towrite = '---' +'\n'
	towrite += '\n'
	towrite += ('user: "******"\n')

	with open('test2.yml', 'wb') as f:
		locks.lock(f, locks.LOCK_EX)
		f.write(towrite)

	file_move_safe("test2.yml", "../Ansible/variables2.yml", allow_overwrite = True)
Beispiel #18
0
    def test_file_move_overwrite(self):
        handle_a, self.file_a = tempfile.mkstemp()
        handle_b, self.file_b = tempfile.mkstemp()

        # file_move_safe should raise an IOError exception if destination file exists and allow_overwrite is False
        self.assertRaises(IOError, lambda: file_move_safe(self.file_a, self.file_b, allow_overwrite=False))

        # should allow it and continue on if allow_overwrite is True
        self.assertIsNone(file_move_safe(self.file_a, self.file_b, allow_overwrite=True))

        os.close(handle_a)
        os.close(handle_b)
 def safely_rename(self, url_request_file, new_name, clobber=False):
     """ Pass a URLRequestFile, with a new filename, to move or rename. """
     new_path = safe_join(
         dirname(self.path(url_request_file.name)),
         new_name)
     
     file_move_safe(
         self.path(url_request_file.name),
         new_path,
         allow_overwrite=clobber)
     
     url_request_file.name = new_name
Beispiel #20
0
 def save_flash(self, filename):
     self.type = 4
     super(LibraryFile, self).save() # Intermediate save to get new ID
     self.filename = "adex%s_%s" % (self.id, self.name)
     webthumb(
         'http://anondex.com/flashview/?'+settings.MEDIA_URL+'tmp/'+self.name,
         settings.MEDIA_ROOT+"f/thumb/%s.jpg"%self.filename,
         is_flash=True,
     )
     file_move_safe(filename, settings.MEDIA_ROOT + "f/%s" % self.filename)
     if settings.DEBUG:
         return
    def _save(self, name, content):
        '''
        Copied from super and lightly modified - Unfortunately, the default 
        race condition handling will lead to an infinite loop here, since 
        get_available_name() doesn't return unique names for identical hashes.
        '''
        full_path = self.path(name)
        if os.path.exists(full_path):
            return name

        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            # handle concurrency issue where the directory is created while
            # by another thread after the call to exists()
            try:
                os.makedirs(directory)
            except OSError as e:
                # ignore EEXIST, since it means that our goal here was already
                # accomplished
                if e.errno != errno.EEXIST:
                    raise
        elif not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)
        
        # There's a potential race condition when saving the file; it's 
        # possible that two threads might try to write the file simultaneously.
        # We need to try to create the file, but abort if we detect that it
        # already exists (in which case we assume that another thread is
        # writing it).
        try:
            # This file has a file path that we can move. 
            if hasattr(content, 'temporary_file_path'):
                file_move_safe(content.temporary_file_path(), full_path)
                content.close()

            # This is a normal uploadedfile that we can stream.
            else:
                # This fun binary flag incantation makes os.open throw an 
                # OSError if the file already exists before we open it.
                fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
                try:
                    locks.lock(fd, locks.LOCK_EX)
                    for chunk in content.chunks():
                        os.write(fd, chunk)
                finally:
                    locks.unlock(fd)
                    os.close(fd)
        except OSError, e:
            # abort and continue normally if we detect the existence of the 
            # file we're trying to write, otherwise raise normally
            if e.errno != errno.EEXIST:
                raise
Beispiel #22
0
 def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
     self._createdir()  # Cache dir can be deleted at any time.
     fname = self._key_to_file(key, version)
     self._cull()  # make some room if necessary
     fd, tmp_path = tempfile.mkstemp(dir=self._dir)
     renamed = False
     try:
         with open(fd, 'wb') as f:
             self._write_content(f, timeout, value)
         file_move_safe(tmp_path, fname, allow_overwrite=True)
         renamed = True
     finally:
         if not renamed:
             os.remove(tmp_path)
Beispiel #23
0
    def test_file_move_overwrite(self):
        handle_a, self.file_a = tempfile.mkstemp()
        handle_b, self.file_b = tempfile.mkstemp()

        # file_move_safe() raises OSError if the destination file exists and
        # allow_overwrite is False.
        with self.assertRaises(FileExistsError):
            file_move_safe(self.file_a, self.file_b, allow_overwrite=False)

        # should allow it and continue on if allow_overwrite is True
        self.assertIsNone(file_move_safe(self.file_a, self.file_b, allow_overwrite=True))

        os.close(handle_a)
        os.close(handle_b)
Beispiel #24
0
 def add_revision(self, content, instance, commit_msg, username):
     fname = self.get_filename(instance)
     full_path = os.path.join(self.location, fname)
     if hasattr(content, 'temporary_file_path'):
         # This file has a file path that we can move.
         file_move_safe(content.temporary_file_path(), full_path)
         content.close()
     else:
         # This is a normal uploadedfile that we can stream.
         with open(full_path, 'wb') as f:
             content.seek(0)
             f.write(content.read())
     if settings.FILE_UPLOAD_PERMISSIONS is not None:
         os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
     self._commit(fname, commit_msg, username, 'add')
Beispiel #25
0
    def save(self, *args, **kwargs):
        # temporary values for version, date
        self.version = 0
        self.date = datetime.date.today()
        super(CacheDatabase, self).save(*args, **kwargs)

        conn = sqlite3.connect(self.file.path)
        c = conn.cursor()
        
        c.execute("SELECT name, value FROM meta")
        date = None
        version = None
        for i in c.fetchall():
            if i[0] == 'version':
                version = i[1]
            if i[0] == 'date':
                date = i[1]
        
        try:
            if not version:
                raise ValueError
            self.version = int(version)
        except ValueError:
            raise models.IntegrityError('database does not list valid version')
        
        try:
            if not date:
                raise ValueError
            # MM/DD/YYYY (unfortunately... grr...)
            date = map(int, date.split('/', 2))
            if len(date) != 3:
                raise ValueError
            self.date = datetime.date(date[2], date[0], date[1])
        except ValueError:
            raise models.IntegrityError('database does not list valid date')
        
        c.close()
        conn.close()
        
        # move the file, if needed
        if self.file.name.startswith('databases/tmp.'):
            newpath = 'databases/' + str(self)
            newpathreal = self.file.storage.path(newpath)
            file_move_safe(self.file.path, newpathreal)
            self.file = newpath
        
        # resave
        super(CacheDatabase, self).save(*args, **kwargs)
Beispiel #26
0
 def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
     self._createdir()  # Cache dir can be deleted at any time.
     fname = self._key_to_file(key, version)
     self._cull()  # make some room if necessary
     fd, tmp_path = tempfile.mkstemp(dir=self._dir)
     renamed = False
     try:
         with io.open(fd, 'wb') as f:
             expiry = self.get_backend_timeout(timeout)
             f.write(pickle.dumps(expiry, pickle.HIGHEST_PROTOCOL))
             f.write(zlib.compress(pickle.dumps(value, pickle.HIGHEST_PROTOCOL)))
         file_move_safe(tmp_path, fname, allow_overwrite=True)
         renamed = True
     finally:
         if not renamed:
             os.remove(tmp_path)
    def handle_noargs(self, **options):
        processed = 0
        delete_orphans = options.get("delete_orphans", False)
        blank_orphans = options.get("blank_orphans", False)
        if blank_orphans and delete_orphans:
            raise Exception("Can't set both --blank-orphans and --delete-orphans")
            sys.exit()
        delete_orphans = not blank_orphans

        for item in PrivateFile.objects.all():
            if item.file is None:
                print("ignoring", item)
                continue

            itempath = item.file.path

            if not os.path.exists(itempath):
                if delete_orphans:
                    print("Deleting {}".format(item.file))
                    item.delete()

                if blank_orphans:
                    print("Blanking {}".format(item.file))
                    item.file = ""
                    item.save()

                continue

            if " " in itempath:

                ext = None
                dirname = os.path.dirname(itempath)
                filename = os.path.basename(itempath)
                if "." in filename:
                    filename, ext = filename.split(".")
                    filename = ".".join([slugify(filename), ext])
                else:
                    filename = slugify(filename)
                new_path = os.sep.join([dirname, filename])
                file_move_safe(item.file.path, new_path)
                item.file = new_path.replace(settings.MEDIA_ROOT + "/", "")
                item.save()

                processed += 1

        if options["verbosity"] > 0:
            print("Cleaned up {} objects".format(processed))
    def _save(self, name, content):
        full_path = self.path(name)

        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            os.makedirs(directory)
        elif not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # There's a potential race condition between get_available_name and
        # saving the file; it's possible that two threads might return the
        # same name, at which point all sorts of fun happens. So we need to
        # try to create the file, but if it already exists we have to go back
        # to get_available_name() and try again.

        while True:
            try:
                # This file has a file path that we can move.
                if hasattr(content, 'temporary_file_path'):
                    file_move_safe(content.temporary_file_path(), full_path)
                    content.close()

                # This is a normal uploadedfile that we can stream.
                else:
                    # This fun binary flag incantation makes os.open throw an
                    # OSError if the file already exists before we open it.
                    fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
                    try:
                        locks.lock(fd, locks.LOCK_EX)
                        for chunk in content.chunks():
                            os.write(fd, chunk)
                            # CHANGED: This un-hangs us long enough to keep things rolling.
                            eventlet.sleep(0)
                    finally:
                        locks.unlock(fd)
                        os.close(fd)
            except OSError, e:
                if e.errno == errno.EEXIST:
                    # Ooops, the file exists. We need a new file name.
                    name = self.get_available_name(name)
                    full_path = self.path(name)
                else:
                    raise
            else:
                # OK, the file save worked. Break out of the loop.
                break
Beispiel #29
0
    def _save(self, name, content):
        """
        Lifted partially from django/core/files/storage.py
        """
        full_path = self.path(name)

        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                os.makedirs(directory)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # This file has a file path that we can move.
        if hasattr(content, 'temporary_file_path'):
            temp_data_location = content.temporary_file_path()
        else:
            tmp_prefix = "tmp_%s" % (get_valid_filename(name), )
            temp_data_location = tempfile.mktemp(prefix=tmp_prefix,
                                                 dir=self.location)
            try:
                # This is a normal uploadedfile that we can stream.
                # This fun binary flag incantation makes os.open throw an
                # OSError if the file already exists before we open it.
                fd = os.open(temp_data_location,
                             os.O_WRONLY | os.O_CREAT |
                             os.O_EXCL | getattr(os, 'O_BINARY', 0))
                locks.lock(fd, locks.LOCK_EX)
                for chunk in content.chunks():
                    os.write(fd, chunk)
                locks.unlock(fd)
                os.close(fd)
            except Exception:
                if os.path.exists(temp_data_location):
                    os.remove(temp_data_location)
                raise

        file_move_safe(temp_data_location, full_path, allow_overwrite=True)
        content.close()

        if settings.FILE_UPLOAD_PERMISSIONS is not None:
            os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
        return name
    def _save(self, name, content):
        full_path = self.path(name)

        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            os.makedirs(directory)
        elif not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # There's a potential race condition between get_available_name and
        # saving the file; it's possible that two threads might return the
        # same name, at which point all sorts of fun happens. So we need to
        # try to create the file, but if it already exists we have to go back
        # to get_available_name() and try again.

        while True:
            try:
                # This file has a file path that we can move.
                if hasattr(content, 'temporary_file_path'):
                    file_move_safe(content.temporary_file_path(), full_path)
                    content.close()

                # This is a normal uploadedfile that we can stream.
                else:
                    # This fun binary flag incantation makes os.open throw an
                    # OSError if the file already exists before we open it.
                    try:
                        content.seek(0)
                        image = Image.open(content)
                    except IOError:
                        raise exceptions.InvalidImageUploadException()
                    image_size = (self.PHOTO_MAX_X, self.PHOTO_MAX_Y)
                    image.thumbnail(image_size, Image.ANTIALIAS)
                    image.save(full_path)
            except OSError, e:
                if e.errno == errno.EEXIST:
                    # Ooops, the file exists. We need a new file name.
                    name = self.get_available_name(name)
                    full_path = self.path(name)
                else:
                    raise
            else:
                # OK, the file save worked. Break out of the loop.
                break
Beispiel #31
0
def _upload_file(request):
    """
    Upload file to the server.
    """

    from django.core.files.move import file_move_safe

    if request.method == 'POST':
        folder = request.POST.get('folder')
        fb_uploadurl_re = re.compile(r'^(%s)' % reverse("fb_upload"))
        folder = fb_uploadurl_re.sub('', folder)
        abs_path = os.path.join(MEDIA_ROOT, DIRECTORY, folder)
        if request.FILES:
            filedata = request.FILES['Filedata']
            filedata.name = convert_filename(filedata.name)
            # PRE UPLOAD SIGNAL
            filebrowser_pre_upload.send(sender=request,
                                        path=request.POST.get('folder'),
                                        file=filedata)
            # HANDLE UPLOAD
            uploadedfile = handle_file_upload(abs_path, filedata)
            # MOVE UPLOADED FILE
            # if file already exists
            if os.path.isfile(
                    os.path.join(MEDIA_ROOT, DIRECTORY, folder,
                                 filedata.name)):
                old_file = os.path.join(abs_path, filedata.name)
                new_file = os.path.join(abs_path, uploadedfile)
                file_move_safe(new_file, old_file)
            # POST UPLOAD SIGNAL
            filebrowser_post_upload.send(
                sender=request,
                path=request.POST.get('folder'),
                file=FileObject(os.path.join(DIRECTORY, folder,
                                             filedata.name)))
    return HttpResponse('True')
Beispiel #32
0
def _upload_file(request):
    """
    Upload file to the server.
    """

    if request.method == 'POST':
        folder = request.GET.get('folder')
        fb_uploadurl_re = re.compile(r'^.*({0})'.format(reverse("fb_upload")))
        folder = fb_uploadurl_re.sub('', folder)
        abs_path = _check_access(request, folder)
        if request.FILES:
            filedata = request.FILES['file']
            filedata.name = convert_filename(filedata.name)
            _check_access(request, abs_path, filedata.name)
            # PRE UPLOAD SIGNAL
            filebrowser_pre_upload.send(sender=request,
                                        path=request.GET.get('folder'),
                                        file=filedata)
            # HANDLE UPLOAD
            uploadedfile = handle_file_upload(abs_path, filedata)
            # MOVE UPLOADED FILE
            # if file already exists
            if os.path.isfile(smart_str(
                    os.path.join(fb_settings.MEDIA_ROOT, fb_settings.DIRECTORY,
                                 folder, filedata.name))):
                old_file = smart_str(os.path.join(abs_path, filedata.name))
                new_file = smart_str(os.path.join(abs_path, uploadedfile))
                file_move_safe(new_file, old_file)
            # POST UPLOAD SIGNAL
            filebrowser_post_upload.send(sender=request,
                                         path=request.GET.get('folder'),
                                         file=FileObject(smart_str(
                                             os.path.join(
                                                 fb_settings.DIRECTORY, folder,
                                                 filedata.name))))
    return JsonResponse({'success': True})
Beispiel #33
0
class OverwriteStorage(FileSystemStorage):
    """ 
    File storage that allows overwriting of stored files.
    """
    def get_available_name(self, name):
        return name

    def _save(self, name, content):
        """
        Lifted partially from django/core/files/storage.py
        """
        full_path = self.path(name)

        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            os.makedirs(directory)
        elif not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # This file has a file path that we can move.
        if hasattr(content, 'temporary_file_path'):
            temp_data_location = content.temporary_file_path()
        else:
            tmp_prefix = "tmp_%s" % (get_valid_filename(name), )
            temp_data_location = tempfile.mktemp(prefix=tmp_prefix,
                                                 dir=self.location)
            try:
                # This is a normal uploadedfile that we can stream.
                # This fun binary flag incantation makes os.open throw an
                # OSError if the file already exists before we open it.
                fd = os.open(
                    temp_data_location, os.O_WRONLY | os.O_CREAT | os.O_EXCL
                    | getattr(os, 'O_BINARY', 0))
                locks.lock(fd, locks.LOCK_EX)
                for chunk in content.chunks():
                    os.write(fd, chunk)
                locks.unlock(fd)
                os.close(fd)
            except Exception, e:
                if os.path.exists(temp_data_location):
                    os.remove(temp_data_location)
                raise

        file_move_safe(temp_data_location, full_path)
        content.close()

        return name
Beispiel #34
0
    def _save(self, name, content):
        full_path = self.path(name)

        # Create any intermediate directories that do not exist.
        # Note that there is a race between os.path.exists and os.makedirs:
        # if os.makedirs fails with EEXIST, the directory was created
        # concurrently, and we can continue normally. Refs #16082.
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                if self.directory_permissions_mode is not None:
                    # os.makedirs applies the global umask, so we reset it,
                    # for consistency with file_permissions_mode behavior.
                    old_umask = os.umask(0)
                    try:
                        os.makedirs(directory, self.directory_permissions_mode)
                    finally:
                        os.umask(old_umask)
                else:
                    os.makedirs(directory)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # There's a potential race condition between get_available_name and
        # saving the file; it's possible that two threads might return the
        # same name, at which point all sorts of fun happens. So we need to
        # try to create the file, but if it already exists we have to go back
        # to get_available_name() and try again.

        while True:
            try:
                # This file has a file path that we can move.
                if hasattr(content, 'temporary_file_path'):
                    file_move_safe(content.temporary_file_path(), full_path)

                # This is a normal uploadedfile that we can stream.
                else:
                    # This fun binary flag incantation makes os.open throw an
                    # OSError if the file already exists before we open it.
                    flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL
                             | getattr(os, 'O_BINARY', 0))
                    # The current umask value is masked out by os.open!
                    fd = os.open(full_path, flags, 0o666)
                    _file = None
                    try:
                        locks.lock(fd, locks.LOCK_EX)
                        for chunk in content.chunks():
                            if _file is None:
                                mode = 'wb' if isinstance(chunk,
                                                          bytes) else 'wt'
                                _file = os.fdopen(fd, mode)
                            _file.write(chunk)
                    finally:
                        locks.unlock(fd)
                        if _file is not None:
                            _file.close()
                        else:
                            os.close(fd)
            except OSError as e:
                if e.errno == errno.EEXIST:
                    # Ooops, the file exists. We need a new file name.
                    name = self.get_available_name(name)
                    full_path = self.path(name)
                else:
                    raise
            else:
                # OK, the file save worked. Break out of the loop.
                break

        if self.file_permissions_mode is not None:
            os.chmod(full_path, self.file_permissions_mode)

        return name
Beispiel #35
0
    def _save(self, name, content):
        # print(f'about to save {name} to {self.path(name)}')

        if self.exists(name):
            self.delete(name)
            # print("deleted!!!")

        full_path = self.path(name)

        # Create any intermediate directories that do not exist.
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                if self.directory_permissions_mode is not None:
                    # os.makedirs applies the global umask, so we reset it,
                    # for consistency with file_permissions_mode behavior.
                    old_umask = os.umask(0)
                    try:
                        os.makedirs(directory, self.directory_permissions_mode)
                    finally:
                        os.umask(old_umask)
                else:
                    os.makedirs(directory)
            except FileExistsError:
                # There's a race between os.path.exists() and os.makedirs().
                # If os.makedirs() fails with FileExistsError, the directory
                # was created concurrently.
                pass
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)


        while True:
            try:
                # This file has a file path that we can move.
                if hasattr(content, 'temporary_file_path'):
                    file_move_safe(content.temporary_file_path(), full_path)

                # This is a normal uploadedfile that we can stream.
                else:
                    # The current umask value is masked out by os.open!
                    fd = os.open(full_path, self.OS_OPEN_FLAGS, 0o666)
                    _file = None
                    try:
                        locks.lock(fd, locks.LOCK_EX)
                        for chunk in content.chunks():
                            if _file is None:
                                mode = 'wb' if isinstance(chunk, bytes) else 'wt'
                                _file = os.fdopen(fd, mode)
                            _file.write(chunk)
                    finally:
                        locks.unlock(fd)
                        if _file is not None:
                            _file.close()
                        else:
                            os.close(fd)
            except FileExistsError:
                # A new name is needed if the file exists.
                pass
                name = self.get_available_name(name)
                full_path = self.path(name)
            else:
                # OK, the file save worked. Break out of the loop.
                break

        if self.file_permissions_mode is not None:
            os.chmod(full_path, self.file_permissions_mode)

        # Store filenames with forward slashes, even on Windows.
        return name.replace('\\', '/')
Beispiel #36
0
class FileSystemStorage(Storage):
    """
    Standard filesystem storage
    """

    def __init__(self, location=None, base_url=None):
        if location is None:
            location = settings.MEDIA_ROOT
        self.base_location = location
        self.location = abspathu(self.base_location)
        if base_url is None:
            base_url = settings.MEDIA_URL
        self.base_url = base_url

    def _open(self, name, mode='rb'):
        return File(open(self.path(name), mode))

    def _save(self, name, content):
        full_path = self.path(name)

        # Create any intermediate directories that do not exist.
        # Note that there is a race between os.path.exists and os.makedirs:
        # if os.makedirs fails with EEXIST, the directory was created
        # concurrently, and we can continue normally. Refs #16082.
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                os.makedirs(directory)
            except OSError, e:
                if e.errno != errno.EEXIST:
                    raise
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # There's a potential race condition between get_available_name and
        # saving the file; it's possible that two threads might return the
        # same name, at which point all sorts of fun happens. So we need to
        # try to create the file, but if it already exists we have to go back
        # to get_available_name() and try again.

        while True:
            try:
                # This file has a file path that we can move.
                if hasattr(content, 'temporary_file_path'):
                    file_move_safe(content.temporary_file_path(), full_path)
                    content.close()

                # This is a normal uploadedfile that we can stream.
                else:
                    # This fun binary flag incantation makes os.open throw an
                    # OSError if the file already exists before we open it.
                    fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
                    try:
                        locks.lock(fd, locks.LOCK_EX)
                        for chunk in content.chunks():
                            os.write(fd, chunk)
                    finally:
                        locks.unlock(fd)
                        os.close(fd)
            except OSError, e:
                if e.errno == errno.EEXIST:
                    # Ooops, the file exists. We need a new file name.
                    name = self.get_available_name(name)
                    full_path = self.path(name)
                else:
                    raise
            else:
                # OK, the file save worked. Break out of the loop.
                break
Beispiel #37
0
 def move(self, src_name, to_name):
     file_move_safe(self.path(src_name), self.path(to_name))
Beispiel #38
0
 def move(self, old_file_name, new_file_name, allow_overwrite=False):
     file_move_safe(self.path(old_file_name),
                    self.path(new_file_name),
                    allow_overwrite=True)
Beispiel #39
0
    def post(self, request, *args, **kwargs):
        form = self.form_class(request.POST, request.FILES)

        if form.is_valid():
            try:
                file_data = form.cleaned_data['zipfile']
                filepath = get_uploaded_filepath(file_data)
                solutions_zip = ZipFile(filepath)

                # Check for any corrupted files in the zip
                # testzip returns the list of corrupted ones
                if solutions_zip.testzip():
                    messages.error(
                        request, '"%s" in the .zip archive is corrupt.' %
                        solutions_zip.testzip())
                    raise Exception('Corrpted archive.')

                # We loop over all PDF files in the zip
                for filename in [
                        name for name in solutions_zip.namelist()
                        if name.endswith('.pdf')
                ]:

                    # Check that the name is of the form
                    # <score>-<username>-<problem_pk>.pdf
                    try:
                        parts = filename.rstrip('.pdf').split('-')

                        score = int(parts[0])
                        username = '******'.join(parts[1:-1])
                        problem_pk = int(parts[-1])

                    except (IndexError, ValueError, AssertionError):
                        messages.error(
                            request, '"%s" is not of the correct form '
                            '<score>-<username>-<problem_pk>.pdf' %
                            remove_accents(filename))
                        continue

                    # Find the UserSolution and modify it
                    try:
                        user = User.objects.get(username=username)
                        solution = UserSolution.objects.get(user=user,
                                                            problem=problem_pk)
                    except User.DoesNotExist:
                        messages.error('User %s does not exist' % username)
                        continue
                    except UserSolution.DoesNotExist:
                        messages.error(
                            request, 'Solution for user %s and problem '
                            '%d does not exist.' % (username, problem_pk))
                        continue

                    extracted_path = solutions_zip.extract(filename,
                                                           path='/tmp')
                    new_path = os.path.join(
                        settings.SENDFILE_ROOT,
                        solution.get_corrected_solution_path())
                    file_move_safe(extracted_path,
                                   new_path,
                                   allow_overwrite=True)

                    solution.score = score
                    solution.corrected_solution = solution.get_corrected_solution_path(
                    )
                    solution.save()

                    messages.success(
                        request,
                        _("%s assigned %d points") % (solution, score))

            except Exception, e:
                # If any exceptions happened, errors should be in messages
                messages.error(request, 'exception happened: %s' % e)
            finally:
Beispiel #40
0
def move(source_username, source_path, target_username, target_path):
    source_full_path = path_(source_username, source_path)
    target_full_path = path_(target_username, target_path)
    file_move_safe(source_full_path, target_full_path)
    return target_full_path
Beispiel #41
0
  def _save(self, name, content):
    #if isinstance(content, ContentFile) or isinstance(content, InMemoryUploadedFile):
    print("IN MOVER")
    print(type(content))
    print("INNER")
    print(type(content.file))
    print("START FILE CHECK")
    if not isinstance(content.file, File):
      return super()._save(name, content)

    full_path = self.path(name)

    # Create any intermediate directories that do not exist.
    directory = os.path.dirname(full_path)
    if not os.path.exists(directory):
        try:
            if self.directory_permissions_mode is not None:
                # os.makedirs applies the global umask, so we reset it,
                # for consistency with file_permissions_mode behavior.
                old_umask = os.umask(0)
                try:
                    os.makedirs(directory, self.directory_permissions_mode)
                finally:
                    os.umask(old_umask)
            else:
                os.makedirs(directory)
        except FileNotFoundError:
            # There's a race between os.path.exists() and os.makedirs().
            # If os.makedirs() fails with FileNotFoundError, the directory
            # was created concurrently.
            pass

    if not os.path.isdir(directory):
        raise IOError("%s exists and is not a directory." % directory)

    print("")
    print("MADE IT HERE")
    print("")
    # There's a potential race condition between get_available_name and
    # saving the file; it's possible that two threads might return the
    # same name, at which point all sorts of fun happens. So we need to
    # try to create the file, but if it already exists we have to go back
    # to get_available_name() and try again.
    while True:
      try:
        # This file has a file path that we can move.
        if hasattr(content, 'temporary_file_path'):
            file_move_safe(content.temporary_file_path(), full_path)

        else:
          input_file_path = content.file.name
          # Close input file before moving
          content.close()
          os.rename(input_file_path, full_path)
      except FileExistsError:
        # A new name is needed if the file exists.
        name = self.get_available_name(name)
        full_path = self.path(name)
      else:
        break

    if self.file_permissions_mode is not None:
      os.chmod(full_path, self.file_permissions_mode)

    # Store filenames with forward slashes, even on Windows.
    return name.replace('\\', '/')
Beispiel #42
0
    def _save_FIELD_file(self, field, filename, raw_field, save=True):
        # Create the upload directory if it doesn't already exist
        directory = os.path.join(settings.MEDIA_ROOT,
                                 field.get_directory_name())
        if not os.path.exists(directory):
            os.makedirs(directory)
        elif not os.path.isdir(directory):
            raise IOError('%s exists and is not a directory' % directory)

        # Check for old-style usage (files-as-dictionaries). Warn here first
        # since there are multiple locations where we need to support both new
        # and old usage.
        if isinstance(raw_field, dict):
            import warnings
            warnings.warn(
                message=
                "Representing uploaded files as dictionaries is deprecated. Use django.core.files.uploadedfile.SimpleUploadedFile instead.",
                category=DeprecationWarning,
                stacklevel=2)
            from django.core.files.uploadedfile import SimpleUploadedFile
            raw_field = SimpleUploadedFile.from_dict(raw_field)

        elif isinstance(raw_field, str):
            import warnings
            warnings.warn(
                message=
                "Representing uploaded files as strings is deprecated. Use django.core.files.uploadedfile.SimpleUploadedFile instead.",
                category=DeprecationWarning,
                stacklevel=2)
            from django.core.files.uploadedfile import SimpleUploadedFile
            raw_field = SimpleUploadedFile(filename, raw_field)

        if filename is None:
            filename = raw_field.file_name

        filename = field.get_filename(filename)

        # If the filename already exists, keep adding an underscore to the name
        # of the file until the filename doesn't exist.
        while os.path.exists(os.path.join(settings.MEDIA_ROOT, filename)):
            try:
                dot_index = filename.rindex('.')
            except ValueError:  # filename has no dot.
                filename += '_'
            else:
                filename = filename[:dot_index] + '_' + filename[dot_index:]

        # Save the file name on the object and write the file to disk.
        setattr(self, field.attname, filename)
        full_filename = self._get_FIELD_filename(field)
        if hasattr(raw_field, 'temporary_file_path'):
            # This file has a file path that we can move.
            raw_field.close()
            file_move_safe(raw_field.temporary_file_path(), full_filename)
        else:
            # This is a normal uploadedfile that we can stream.
            fp = open(full_filename, 'wb')
            locks.lock(fp, locks.LOCK_EX)
            for chunk in raw_field.chunks():
                fp.write(chunk)
            locks.unlock(fp)
            fp.close()

        # Save the width and/or height, if applicable.
        if isinstance(field, ImageField) and \
                (field.width_field or field.height_field):
            from django.utils.images import get_image_dimensions
            width, height = get_image_dimensions(full_filename)
            if field.width_field:
                setattr(self, field.width_field, width)
            if field.height_field:
                setattr(self, field.height_field, height)

        # Save the object because it has changed, unless save is False.
        if save:
            self.save()
Beispiel #43
0
    def _save(self, name, content, max_length=None):
        """
        Create dirs to the destination, move the file if already in MEDIA_ROOT, or copy otherwise.

        Args:
            name (str): Target path to which the file is copied.
            content (File): Source file object.
            max_length (int): Maximum supported length of file name.

        Returns:
            str: Final storage path.
        """
        full_path = self.path(name)

        # Create any intermediate directories that do not exist.
        directory = os.path.dirname(full_path)
        try:
            if self.directory_permissions_mode is not None:
                # os.makedirs applies the global umask, so we reset it,
                # for consistency with file_permissions_mode behavior.
                old_umask = os.umask(0)
                try:
                    os.makedirs(directory, self.directory_permissions_mode, exist_ok=True)
                finally:
                    os.umask(old_umask)
            else:
                os.makedirs(directory, exist_ok=True)
        except FileExistsError:
            raise FileExistsError('%s exists and is not a directory.' % directory)

        try:
            if hasattr(content, 'temporary_file_path') and \
                    content.temporary_file_path().startswith(settings.MEDIA_ROOT):
                file_move_safe(content.temporary_file_path(), full_path)
            else:
                # This is a normal uploaded file that we can stream.

                # The current umask value is masked out by os.open!
                fd = os.open(full_path, self.OS_OPEN_FLAGS, 0o666)
                _file = None
                try:
                    locks.lock(fd, locks.LOCK_EX)
                    for chunk in content.chunks():
                        if _file is None:
                            mode = 'wb' if isinstance(chunk, bytes) else 'wt'
                            _file = os.fdopen(fd, mode)
                        _file.write(chunk)
                finally:
                    locks.unlock(fd)
                    if _file is not None:
                        _file.close()
                    else:
                        os.close(fd)
        except FileExistsError:
            # It's a content addressable store so if the file is already in place we can do nothing
            pass

        if self.file_permissions_mode is not None:
            os.chmod(full_path, self.file_permissions_mode)

        # Store filenames with forward slashes, even on Windows.
        return str(name).replace('\\', '/')
Beispiel #44
0
    def _save(self, name, content):
        full_path = self.path(name)

        # Create any intermediate directories that do not exist.
        directory = os.path.dirname(full_path)
        if not os.path.exists(directory):
            try:
                if self.directory_permissions_mode is not None:
                    # os.makedirs applies the global umask, so we reset it,
                    # for consistency with file_permissions_mode behavior.
                    old_umask = os.umask(0)
                    try:
                        os.makedirs(directory, self.directory_permissions_mode)
                    finally:
                        os.umask(old_umask)
                else:
                    os.makedirs(directory)
            except FileNotFoundError:
                # There's a race between os.path.exists() and os.makedirs().
                # If os.makedirs() fails with FileNotFoundError, the directory
                # was created concurrently.
                pass
        if not os.path.isdir(directory):
            raise IOError("%s exists and is not a directory." % directory)

        # There's a potential race condition between get_available_name and
        # saving the file; it's possible that two threads might return the
        # same name, at which point all sorts of fun happens. So we need to
        # try to create the file, but if it already exists we have to go back
        # to get_available_name() and try again.

        while True:
            try:
                # This file has a file path that we can move.
                if hasattr(content, 'temporary_file_path'):
                    file_move_safe(content.temporary_file_path(), full_path)

                # This is a normal uploadedfile that we can stream.
                else:
                    # The current umask value is masked out by os.open!
                    fd = os.open(full_path, self.OS_OPEN_FLAGS, 0o666)
                    _file = None
                    try:
                        locks.lock(fd, locks.LOCK_EX)
                        for chunk in content.chunks():
                            if _file is None:
                                mode = 'wb' if isinstance(chunk,
                                                          bytes) else 'wt'
                                _file = os.fdopen(fd, mode)
                            _file.write(chunk)
                    finally:
                        locks.unlock(fd)
                        if _file is not None:
                            _file.close()
                        else:
                            os.close(fd)
            except FileExistsError:
                # A new name is needed if the file exists.
                name = self.get_available_name(name)
                full_path = self.path(name)
            else:
                # OK, the file save worked. Break out of the loop.
                break

        if self.file_permissions_mode is not None:
            os.chmod(full_path, self.file_permissions_mode)

        # Store filenames with forward slashes, even on Windows.
        return name.replace('\\', '/')
Beispiel #45
0
    def _save(self, name, content):

        # e.g. name = 'images/takeaway.jpg'
        #print('_save name: ' + str(name))
        collection_str = os.path.dirname(name)
        if (not collection_str):
            raise ImproperlyConfigured(
                "No collection string can be extracted from the file name '{}'. Has 'upload_to' not been declared in the Model field?"
                .format(name))
        #print('collection_str: ' + str(collection_str))
        #print('uoload_to: ' + str(self.upload_to))
        coll = self.db(collection_str)
        path = None
        with coll.auto_create_cb() as full_path:
            #print('full_path: ' + str(full_path))
            #path = full_path
            # path is storing the 'name' (relpath) for return.
            # One way to get this is by splitting the now-known
            # fullpath (or getting the pk then calling
            # coll.relpath(pk)?)
            # NB: BucketDB does not guarentee the separator (unlike Django)
            path_segs = full_path.split(os.path.sep)
            path = os.path.join(path_segs[-3], path_segs[-2], path_segs[-1])
            #print('relpath: ' + str(path))
            # This file has a file path that we can move.
            if hasattr(content, 'temporary_file_path'):
                file_move_safe(content.temporary_file_path(), full_path)

            # This is a normal uploadedfile that we can stream.
            else:
                # This fun binary flag incantation makes os.open throw an
                # OSError if the file already exists before we open it.
                #? unecessary
                flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL
                         | getattr(os, 'O_BINARY', 0))
                # The current umask value is masked out by os.open!
                fd = os.open(full_path, flags, 0o666)
                _file = None
                try:
                    #locks.lock(fd, locks.LOCK_EX)
                    for chunk in content.chunks():
                        if _file is None:
                            mode = 'wb' if isinstance(chunk, bytes) else 'wt'
                            _file = os.fdopen(fd, mode)
                        _file.write(chunk)
                finally:
                    #locks.unlock(fd)
                    if _file is not None:
                        _file.close()
                    else:
                        os.close(fd)

        # The original code returns a name (which is 'upload_to' +
        # name). We return the relpath (which is 'upload_to'
        # (collection) + bucket + pk).
        # In the field, the return is dumped on FieldFile.name.
        # self.name is used to construct filepaths, but not in
        # any other way (it is an id locator for the file).
        # Replacing with the relpath means attributes like 'size'
        # work without overriding.
        return path
Beispiel #46
0
    def _save(self, name, content):
        full_path = self.path(name)

        # Create any intermediate directories that do not exist.
        directory = os.path.dirname(full_path)
        try:
            if self.directory_permissions_mode is not None:
                # Set the umask because os.makedirs() doesn't apply the "mode"
                # argument to intermediate-level directories.
                old_umask = os.umask(0o777 & ~self.directory_permissions_mode)
                try:
                    os.makedirs(
                        directory, self.directory_permissions_mode, exist_ok=True
                    )
                finally:
                    os.umask(old_umask)
            else:
                os.makedirs(directory, exist_ok=True)
        except FileExistsError:
            raise FileExistsError("%s exists and is not a directory." % directory)

        # There's a potential race condition between get_available_name and
        # saving the file; it's possible that two threads might return the
        # same name, at which point all sorts of fun happens. So we need to
        # try to create the file, but if it already exists we have to go back
        # to get_available_name() and try again.

        while True:
            try:
                # This file has a file path that we can move.
                if hasattr(content, "temporary_file_path"):
                    file_move_safe(content.temporary_file_path(), full_path)

                # This is a normal uploadedfile that we can stream.
                else:
                    # The current umask value is masked out by os.open!
                    fd = os.open(full_path, self.OS_OPEN_FLAGS, 0o666)
                    _file = None
                    try:
                        locks.lock(fd, locks.LOCK_EX)
                        for chunk in content.chunks():
                            if _file is None:
                                mode = "wb" if isinstance(chunk, bytes) else "wt"
                                _file = os.fdopen(fd, mode)
                            _file.write(chunk)
                    finally:
                        locks.unlock(fd)
                        if _file is not None:
                            _file.close()
                        else:
                            os.close(fd)
            except FileExistsError:
                # A new name is needed if the file exists.
                name = self.get_available_name(name)
                full_path = self.path(name)
            else:
                # OK, the file save worked. Break out of the loop.
                break

        if self.file_permissions_mode is not None:
            os.chmod(full_path, self.file_permissions_mode)

        # Ensure the saved path is always relative to the storage root.
        name = os.path.relpath(full_path, self.location)
        # Store filenames with forward slashes, even on Windows.
        return str(name).replace("\\", "/")
Beispiel #47
0
    def upload_solutions_with_points(self, request, pk=None):
        if 'file' not in request.data:
            raise exceptions.ParseError(detail='No file attached')
        zfile = request.data['file']
        if not zipfile.is_zipfile(zfile):
            raise exceptions.ParseError(
                detail='Attached file is not a zip file')
        with zipfile.ZipFile(zfile) as zfile:
            if zfile.testzip():
                raise exceptions.ParseError(detail='Zip file is corrupted')
            pdf_files = [
                name for name in zfile.namelist() if name.endswith('.pdf')
            ]
            errors = []
            for filename in pdf_files:
                try:
                    parts = filename.rstrip('.pdf').split('-')
                    score = int(parts[0])
                    problem_pk = int(parts[-2])
                    registration_pk = int(parts[-1])
                    event_reg = EventRegistration.objects.get(
                        pk=registration_pk)
                    solution = Solution.objects.get(
                        semester_registration=event_reg, problem=problem_pk)
                except (IndexError, ValueError, AssertionError):
                    errors.append({
                        'filename': filename,
                        'status': 'Cannot parse file'
                    })
                    continue
                except EventRegistration.DoesNotExist:
                    errors.append({
                        'filename':
                        filename,
                        'status':
                        f'User registration with id {registration_pk} does not exist'
                    })
                    continue
                except Solution.DoesNotExist:
                    errors.append({
                        'filename':
                        filename,
                        'status':
                        f'Solution with registration id {registration_pk}'
                        f'and problem id {problem_pk} does not exist'
                    })
                    continue

                extracted_path = zfile.extract(filename, path='/tmp')
                new_path = os.path.join(
                    settings.MEDIA_ROOT, 'solutions',
                    solution.get_corrected_solution_file_name())
                file_move_safe(extracted_path, new_path, allow_overwrite=True)

                solution.score = score
                solution.corrected_solution = solution.get_corrected_solution_file_name(
                )
                solution.save()
                errors.append({
                    'filename': filename,
                    'status': f'OK - points: {score}'
                })
        return Response(json.dumps(errors))
Beispiel #48
0
    def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
        self._createdir()  # Cache dir can be deleted at any time.
        fname = self._key_to_file(key, version)
        self._cull()  # make some room if necessary
        fd, tmp_path = tempfile.mkstemp(dir=self._dir)
        renamed = False
        try:
<<<<<<< HEAD
            with io.open(fd, 'wb') as f:
=======
            with open(fd, 'wb') as f:
>>>>>>> 37c99181c9a6b95433d60f8c8ef9af5731096435
                expiry = self.get_backend_timeout(timeout)
                f.write(pickle.dumps(expiry, pickle.HIGHEST_PROTOCOL))
                f.write(zlib.compress(pickle.dumps(value, pickle.HIGHEST_PROTOCOL)))
            file_move_safe(tmp_path, fname, allow_overwrite=True)
            renamed = True
        finally:
            if not renamed:
                os.remove(tmp_path)

    def delete(self, key, version=None):
        self._delete(self._key_to_file(key, version))

    def _delete(self, fname):
        if not fname.startswith(self._dir) or not os.path.exists(fname):
            return
        try:
            os.remove(fname)
<<<<<<< HEAD
        except OSError as e: