예제 #1
0
파일: filecacher.py 프로젝트: Trenson/cms
    def load(self, digest):
        """Load the file with the given digest into the cache.

        Ask the backend to provide the file and, if it's available,
        copy its content into the file-system cache.

        digest (unicode): the digest of the file to load.

        raise (KeyError): if the backend cannot find the file.

        """
        ftmp_handle, temp_file_path = tempfile.mkstemp(dir=self.temp_dir,
                                                       text=False)
        ftmp = os.fdopen(ftmp_handle, 'w')
        cache_file_path = os.path.join(self.file_dir, digest)

        fobj = self.backend.get_file(digest)

        # Copy the file to a temporary position
        try:
            copyfileobj(fobj, ftmp, self.CHUNK_SIZE)
        finally:
            ftmp.close()
            fobj.close()

        # Then move it to its real location (this operation is atomic
        # by POSIX requirement)
        os.rename(temp_file_path, cache_file_path)
예제 #2
0
    def save(self, digest, desc=""):
        """Save the file with the given digest into the backend.

        Use to local copy, available in the file-system cache, to store
        the file in the backend, if it's not already there.

        digest (unicode): the digest of the file to load.
        desc (unicode): the (optional) description to associate to the
            file.

        """
        cache_file_path = os.path.join(self.file_dir, digest)

        fobj = self.backend.put_file(digest, desc)

        if fobj is None:
            return

        try:
            with io.open(cache_file_path, 'rb') as src:
                copyfileobj(src, fobj, self.CHUNK_SIZE)
        finally:
            fobj.close()
            if not isinstance(self.backend, NullBackend):
                os.unlink(cache_file_path)
예제 #3
0
    def save(self, digest, desc="", file_path=None):
        """Save the file with the given digest into the backend.

        Store the file in the backend, if it's not already there, using
        either the file given in the file_path argument or, if that argument
        is None, the copy of the file present in the cache.

        digest (unicode): the digest of the file to load.
        desc (unicode): the (optional) description to associate to the
            file.
        file_path (unicode): if it is None, use the copy of the file present
            in the cache. Otherwise, use the file pointed to by this path.

        """
        if file_path is None:
            file_path = os.path.join(self.file_dir, digest)

        fobj = self.backend.put_file(digest, desc)

        if fobj is None:
            return

        try:
            with io.open(file_path, 'rb') as src:
                copyfileobj(src, fobj, self.CHUNK_SIZE)
        finally:
            fobj.close()
예제 #4
0
    def save(self, digest, desc="", file_path=None):
        """Save the file with the given digest into the backend.

        Store the file in the backend, if it's not already there, using
        either the file given in the file_path argument or, if that argument
        is None, the copy of the file present in the cache.

        digest (unicode): the digest of the file to load.
        desc (unicode): the (optional) description to associate to the
            file.
        file_path (unicode): if it is None, use the copy of the file present
            in the cache. Otherwise, use the file pointed to by this path.

        """
        if file_path is None:
            file_path = os.path.join(self.file_dir, digest)

        fobj = self.backend.put_file(digest, desc)

        if fobj is None:
            return

        try:
            with io.open(file_path, 'rb') as src:
                copyfileobj(src, fobj, self.CHUNK_SIZE)
        finally:
            fobj.close()
예제 #5
0
파일: filecacher.py 프로젝트: PJeBeK/cms
    def save(self, digest, desc=""):
        """Save the file with the given digest into the backend.

        Use to local copy, available in the file-system cache, to store
        the file in the backend, if it's not already there.

        digest (unicode): the digest of the file to load.
        desc (unicode): the (optional) description to associate to the
            file.

        raise (TombstoneError): if the digest is the tombstone

        """
        if digest == FileCacher.TOMBSTONE_DIGEST:
            raise TombstoneError()
        cache_file_path = os.path.join(self.file_dir, digest)

        fobj = self.backend.put_file(digest, desc)

        if fobj is None:
            return

        try:
            with io.open(cache_file_path, 'rb') as src:
                copyfileobj(src, fobj, self.CHUNK_SIZE)
        finally:
            fobj.close()
예제 #6
0
    def save(self, digest, desc=""):
        """Save the file with the given digest into the backend.

        Use to local copy, available in the file-system cache, to store
        the file in the backend, if it's not already there.

        digest (unicode): the digest of the file to load.
        desc (unicode): the (optional) description to associate to the
            file.

        raise (TombstoneError): if the digest is the tombstone

        """
        if digest == FileCacher.TOMBSTONE_DIGEST:
            raise TombstoneError()
        cache_file_path = os.path.join(self.file_dir, digest)

        fobj = self.backend.create_file(digest)

        if fobj is None:
            return

        with io.open(cache_file_path, 'rb') as src:
            copyfileobj(src, fobj, self.CHUNK_SIZE)

        self.backend.commit_file(fobj, digest, desc)
예제 #7
0
파일: filecacher.py 프로젝트: renegat96/cms
    def load(self, digest, if_needed=False):
        """Load the file with the given digest into the cache.

        Ask the backend to provide the file and, if it's available,
        copy its content into the file-system cache.

        digest (unicode): the digest of the file to load.
        if_needed (bool): only load the file if it is not present in
            the local cache.

        raise (KeyError): if the backend cannot find the file.

        """
        cache_file_path = os.path.join(self.file_dir, digest)
        if if_needed and os.path.exists(cache_file_path):
            return

        ftmp_handle, temp_file_path = tempfile.mkstemp(dir=self.temp_dir,
                                                       text=False)
        ftmp = os.fdopen(ftmp_handle, 'w')

        fobj = self.backend.get_file(digest)

        # Copy the file to a temporary position
        try:
            copyfileobj(fobj, ftmp, self.CHUNK_SIZE)
        finally:
            ftmp.close()
            fobj.close()

        # Then move it to its real location (this operation is atomic
        # by POSIX requirement)
        os.rename(temp_file_path, cache_file_path)
예제 #8
0
파일: filecacher.py 프로젝트: cbolk/cms
    def put_file(self, description="", binary_data=None,
                 file_obj=None, path=None):
        """Put a file in the storage, and keep a copy locally. The
        caller has to provide exactly one among binary_data, file_obj
        and path.

        description (string): a human-readable description of the
                              content.
        binary_data (string): the content of the file to send.
        file_obj (file): the file-like object to send.
        path (string): the file to send.

        """
        temp_fd, temp_path = tempfile.mkstemp(dir=self.tmp_dir)
        os.close(temp_fd)

        # Input checking
        if [binary_data, file_obj, path].count(None) != 2:
            error_string = "No content (or too many) specified in put_file."
            logger.error(error_string)
            raise ValueError(error_string)

        logger.debug("Reading input file to store on the database.")

        # Copy the file content, whatever forms it arrives, into the
        # temporary file
        if path is not None:
            copy(path, temp_path)
        elif binary_data is not None:
            with open(temp_path, 'wb') as temp_file:
                temp_file.write(binary_data)
        else:  # file_obj is not None.
            with open(temp_path, 'wb') as temp_file:
                copyfileobj(file_obj, temp_file)

        hasher = hashlib.sha1()

        # Calculate the file SHA1 digest
        with open(temp_path, 'rb') as temp_file:
            buf = temp_file.read(self.CHUNK_SIZE)
            while buf != '':
                hasher.update(buf)
                buf = temp_file.read(self.CHUNK_SIZE)
        digest = hasher.hexdigest()

        logger.debug("File has digest %s." % digest)

        self.backend.put_file(digest, temp_path, description=description)

        # Move the temporary file in the cache
        move(temp_path,
             os.path.join(self.obj_dir, digest))

        return digest
예제 #9
0
파일: Sandbox.py 프로젝트: ioi-2017/cms
    def create_file_from_fileobj(self, path, file_obj, executable=False):
        """Write a file in the sandbox copying the content of an open
        file-like object.

        path (string): relative path of the file inside the sandbox.
        file_obj (file): where from read the file content.
        executable (bool): to set permissions.

        """
        dest = self.create_file(path, executable)
        copyfileobj(file_obj, dest)
        dest.close()
예제 #10
0
파일: Sandbox.py 프로젝트: beyondai/cms
    def create_file_from_fileobj(self, path, file_obj, executable=False):
        """Write a file in the sandbox copying the content of an open
        file-like object.

        path (string): relative path of the file inside the sandbox.
        file_obj (file): where from read the file content.
        executable (bool): to set permissions.

        """
        dest = self.create_file(path, executable)
        copyfileobj(file_obj, dest)
        dest.close()
예제 #11
0
파일: filecacher.py 프로젝트: renegat96/cms
    def get_file_to_fobj(self, digest, dst):
        """Retrieve a file from the storage.

        See `get_file'. This method will write the content of the file
        to the given file-object.

        digest (unicode): the digest of the file to get.
        dst (fileobj): a writable binary file-like object on which to
            write the contents of the file.

        raise (KeyError): if the file cannot be found.

        """
        with self.get_file(digest) as src:
            copyfileobj(src, dst, self.CHUNK_SIZE)
예제 #12
0
파일: filecacher.py 프로젝트: Trenson/cms
    def get_file_to_fobj(self, digest, dst):
        """Retrieve a file from the storage.

        See `get_file'. This method will write the content of the file
        to the given file-object.

        digest (unicode): the digest of the file to get.
        dst (fileobj): a writable binary file-like object on which to
            write the contents of the file.

        raise (KeyError): if the file cannot be found.

        """
        with self.get_file(digest) as src:
            copyfileobj(src, dst, self.CHUNK_SIZE)
예제 #13
0
파일: filecacher.py 프로젝트: Trenson/cms
    def get_file_to_path(self, digest, dst_path):
        """Retrieve a file from the storage.

        See `get_file'. This method will write the content of a file
        to the given file-system location.

        digest (unicode): the digest of the file to get.
        dst_path (string): an accessible location on the file-system on
            which to write the contents of the file.

        raise (KeyError): if the file cannot be found.

        """
        with self.get_file(digest) as src:
            with io.open(dst_path, 'wb') as dst:
                copyfileobj(src, dst, self.CHUNK_SIZE)
예제 #14
0
파일: filecacher.py 프로젝트: renegat96/cms
    def get_file_to_path(self, digest, dst_path):
        """Retrieve a file from the storage.

        See `get_file'. This method will write the content of a file
        to the given file-system location.

        digest (unicode): the digest of the file to get.
        dst_path (string): an accessible location on the file-system on
            which to write the contents of the file.

        raise (KeyError): if the file cannot be found.

        """
        with self.get_file(digest) as src:
            with io.open(dst_path, 'wb') as dst:
                copyfileobj(src, dst, self.CHUNK_SIZE)
예제 #15
0
    def load(self, digest):
        """Load the file with the given digest into the cache.

        Ask the backend to provide the file and, if it's available,
        copy its content into the file-system cache.

        digest (unicode): the digest of the file to load.

        raise (KeyError): if the backend cannot find the file.

        """
        cache_file_path = os.path.join(self.file_dir, digest)

        fobj = self.backend.get_file(digest)

        try:
            with io.open(cache_file_path, 'wb') as dst:
                copyfileobj(fobj, dst, self.CHUNK_SIZE)
        finally:
            fobj.close()
예제 #16
0
    def load(self, digest):
        """Load the file with the given digest into the cache.

        Ask the backend to provide the file and, if it's available,
        copy its content into the file-system cache.

        digest (unicode): the digest of the file to load.

        raise (KeyError): if the backend cannot find the file.

        """
        cache_file_path = os.path.join(self.file_dir, digest)

        fobj = self.backend.get_file(digest)

        try:
            with io.open(cache_file_path, 'wb') as dst:
                copyfileobj(fobj, dst, self.CHUNK_SIZE)
        finally:
            fobj.close()
예제 #17
0
파일: filecacher.py 프로젝트: renegat96/cms
    def save(self, digest, desc=""):
        """Save the file with the given digest into the backend.

        Use to local copy, available in the file-system cache, to store
        the file in the backend, if it's not already there.

        digest (unicode): the digest of the file to load.
        desc (unicode): the (optional) description to associate to the
            file.

        """
        cache_file_path = os.path.join(self.file_dir, digest)

        fobj = self.backend.put_file(digest, desc)

        if fobj is None:
            return

        try:
            with io.open(cache_file_path, 'rb') as src:
                copyfileobj(src, fobj, self.CHUNK_SIZE)
        finally:
            fobj.close()
예제 #18
0
파일: filecacher.py 프로젝트: PJeBeK/cms
    def load(self, digest, if_needed=False):
        """Load the file with the given digest into the cache.

        Ask the backend to provide the file and, if it's available,
        copy its content into the file-system cache.

        digest (unicode): the digest of the file to load.
        if_needed (bool): only load the file if it is not present in
            the local cache.

        raise (KeyError): if the backend cannot find the file.
        raise (TombstoneError): if the digest is the tombstone

        """
        if digest == FileCacher.TOMBSTONE_DIGEST:
            raise TombstoneError()
        cache_file_path = os.path.join(self.file_dir, digest)
        if if_needed and os.path.exists(cache_file_path):
            return

        ftmp_handle, temp_file_path = tempfile.mkstemp(dir=self.temp_dir,
                                                       text=False)
        ftmp = os.fdopen(ftmp_handle, 'w')

        fobj = self.backend.get_file(digest)

        # Copy the file to a temporary position
        try:
            copyfileobj(fobj, ftmp, self.CHUNK_SIZE)
        finally:
            ftmp.close()
            fobj.close()

        # Then move it to its real location (this operation is atomic
        # by POSIX requirement)
        os.rename(temp_file_path, cache_file_path)
예제 #19
0
파일: filecacher.py 프로젝트: cbolk/cms
    def get_file(self, digest, path=None, file_obj=None,
                 string=False, temp_path=False, temp_file_obj=False):
        """Get a file from the storage, possibly using the cache if
        the file is available there.

        digest (string): the sha1 sum of the file.
        path (string): a path where to save the file.
        file_obj (file): a handler where to save the file (that is not
                         closed at return).
        string (bool): True to return content as a string.
        temp_path (bool): True to return path of a temporary file with
                          that content. The file is reserved to the
                          caller, who has the duty to unlink it.
        temp_file-obj (bool): True to return a file object opened to a
                              temporary file with that content. The
                              file is reserved to the caller. Use this
                              method only for debugging purpose, as it
                              leave a file lying in the temporary
                              directory of FileCacher.

        """
        if [string, temp_path, temp_file_obj].count(True) > 1:
            raise ValueError("Ask for at most one amongst content, "
                             "temp path and temp file obj.")

        cache_path = os.path.join(self.obj_dir, digest)
        cache_exists = os.path.exists(cache_path)

        logger.debug("Getting file %s." % (digest))

        if not cache_exists:
            logger.debug("File %s not in cache, downloading "
                         "from database." % digest)

            # Receives the file from the database
            temp_file, temp_filename = tempfile.mkstemp(dir=self.tmp_dir)
            temp_file = os.fdopen(temp_file, "wb")
            self.backend.get_file(digest, temp_filename)

            # And move it in the cache. Warning: this is not atomic if
            # the temp and the cache dir are on different filesystems.
            move(temp_filename, cache_path)

            logger.debug("File %s downloaded." % digest)

        # Saving to path
        if path is not None:
            copy(cache_path, path)

        # Saving to file object
        if file_obj is not None:
            with open(cache_path, "rb") as file_:
                copyfileobj(file_, file_obj)

        # Returning string?
        if string:
            with open(cache_path, "rb") as cache_file:
                return cache_file.read()

        # Returning temporary file?
        elif temp_path:
            temp_file, temp_filename = tempfile.mkstemp(dir=self.tmp_dir)
            os.close(temp_file)
            copy(cache_path, temp_filename)
            return temp_filename

        # Returning temporary file object?
        elif temp_file_obj:
            temp_file, temp_filename = tempfile.mkstemp(dir=self.tmp_dir)
            os.close(temp_file)
            copy(cache_path, temp_filename)
            temp_file = open(temp_filename, "rb")
            return temp_file