Beispiel #1
0
    def __load_base_from_remote(self):
        """Download the data for the entry that we represent. This is probably 
        a file, but could also be a stub for -any- entry.
        """

        # If it's loaded and not-changed, don't do anything.
        if self.__is_loaded is True and self.__is_dirty is False:
            _logger.debug("Not syncing-down non-dirty file.")
            return

        if self.__fh is not None:
            self.__fh.close()
            self.__fh = None

        entry = self.__cache.get(self.__entry_id)

        _logger.debug("Ensuring local availability of [%s]: [%s]", 
                      entry, self.__temp_filepath)

        # Get the current version of the write-cache file, or note that we 
        # don't have it.

        _logger.info("Attempting local cache update of file [%s] for entry "
                     "[%s] and mime-type [%s].",
                     self.__temp_filepath, entry, self.mime_type)

        if entry.requires_mimetype:
            length = DisplacedFile.file_size

            d = DisplacedFile(entry)
            stub_data = d.deposit_file(self.mime_type)

            self.__fh = open(self.__temp_filepath, 'w+')
            self.__fh.write(stub_data)
        else:
            _logger.debug("Executing the download: [%s] => [%s]", 
                          entry.id, self.__temp_filepath)
            
            try:
# TODO(dustin): We need to inherit a file that we might've already cached by 
#               opening.
# TODO(dustin): Any call to download_to_local should use a local, temporarily 
#               file is already established. We can't use it in the reverse 
#               order though: It's one thing to already have a cache from 
#               having opened it, and it's a another thing to maintain a cache 
#               of every file that is copied.
                gd = get_gdrive()
                result = gd.download_to_local(
                            self.__temp_filepath,
                            entry,
                            self.mime_type)

                (length, cache_fault) = result
            except ExportFormatError:
                _logger.exception("There was an export-format error.")
                raise fuse.FuseOSError(ENOENT)

            self.__fh = open(self.__temp_filepath, 'r+')

            self.__is_dirty = False
            self.__is_loaded = True

        _logger.debug("Established base file-data for [%s]: [%s]", 
                      entry, self.__temp_filepath)
Beispiel #2
0
    def __load_base_from_remote(self):
        """Download the data for the entry that we represent. This is probably 
        a file, but could also be a stub for -any- entry.
        """

        # If it's loaded and not-changed, don't do anything.
        if self.__is_loaded is True and self.__is_dirty is False:
            _logger.debug("Not syncing-down non-dirty file.")
            return

        if self.__fh is not None:
            self.__fh.close()
            self.__fh = None

        entry = self.__cache.get(self.__entry_id)

        _logger.debug("Ensuring local availability of [%s]: [%s]", entry,
                      self.__temp_filepath)

        # Get the current version of the write-cache file, or note that we
        # don't have it.

        _logger.info(
            "Attempting local cache update of file [%s] for entry "
            "[%s] and mime-type [%s].", self.__temp_filepath, entry,
            self.mime_type)

        if entry.requires_mimetype:
            length = DisplacedFile.file_size

            d = DisplacedFile(entry)
            stub_data = d.deposit_file(self.mime_type)

            self.__fh = open(self.__temp_filepath, 'w+')
            self.__fh.write(stub_data)
        else:
            _logger.debug("Executing the download: [%s] => [%s]", entry.id,
                          self.__temp_filepath)

            try:
                # TODO(dustin): We need to inherit a file that we might've already cached by
                #               opening.
                # TODO(dustin): Any call to download_to_local should use a local, temporarily
                #               file is already established. We can't use it in the reverse
                #               order though: It's one thing to already have a cache from
                #               having opened it, and it's a another thing to maintain a cache
                #               of every file that is copied.
                gd = get_gdrive()
                result = gd.download_to_local(self.__temp_filepath, entry,
                                              self.mime_type)

                (length, cache_fault) = result
            except ExportFormatError:
                _logger.exception("There was an export-format error.")
                raise fuse.FuseOSError(ENOENT)

            self.__fh = open(self.__temp_filepath, 'r+')

            self.__is_dirty = False
            self.__is_loaded = True

        _logger.debug("Established base file-data for [%s]: [%s]", entry,
                      self.__temp_filepath)
Beispiel #3
0
    def __load_base_from_remote(self):
        """Download the data for the entry that we represent. This is probably 
        a file, but could also be a stub for -any- entry.
        """

        try:
            entry = self.__get_entry_or_raise()
        except:
            self.__log.exception("Could not get entry with ID [%s] for "
                                 "write-flush." % (self.__entry_id))
            raise

        self.__log.debug("Ensuring local availability of [%s]." % (entry))

        temp_file_path = get_temp_filepath(entry, self.mime_type)

        self.__log.debug("__load_base_from_remote about to download.")

        with self.__class__.__download_lock:
            # Get the current version of the write-cache file, or note that we 
            # don't have it.

            self.__log.info("Attempting local cache update of file [%s] for "
                            "entry [%s] and mime-type [%s]." % 
                            (temp_file_path, entry, self.mime_type))

            if entry.requires_mimetype:
                length = DisplacedFile.file_size

                try:
                    d = DisplacedFile(entry)
                    stub_data = d.deposit_file(self.mime_type)

                    with file(temp_file_path, 'w') as f:
                        f.write(stub_data)
                except:
                    self.__log.exception("Could not deposit to file [%s] from "
                                         "entry [%s]." % (temp_file_path, 
                                                          entry))
                    raise

# TODO: Accommodate the cache for displaced-files.
                cache_fault = True

            else:
                self.__log.info("Executing the download.")
                
                try:
# TODO(dustin): We're not inheriting an existing file (same mtime, same size).
                    result = drive_proxy('download_to_local', 
                                         output_file_path=temp_file_path,
                                         normalized_entry=entry,
                                         mime_type=self.mime_type)

                    (length, cache_fault) = result
                except ExportFormatError:
                    self.__log.exception("There was an export-format error.")
                    raise FuseOSError(ENOENT)
                except:
                    self.__log.exception("Could not localize file with entry "
                                         "[%s]." % (entry))
                    raise

            self.__log.info("Download complete.  cache_fault= [%s] "
                            "__is_loaded= [%s]" % 
                            (cache_fault, self.__is_loaded))

            # We've either not loaded it, yet, or it has changed.
            if cache_fault or not self.__is_loaded:
                with self.__class__.__update_lock:
                    self.__log.info("Checking queued items for fault.")

                    if cache_fault:
                        if self.__is_dirty:
                            self.__log.error("Entry [%s] has been changed. "
                                             "Forcing buffer updates, and "
                                             "clearing uncommitted updates." % 
                                             (entry))
                        else:
                            self.__log.debug("Entry [%s] has changed. "
                                             "Updating buffers." % (entry))

                    self.__log.debug("Loading buffers.")

                    with open(temp_file_path, 'rb') as f:
                        # Read the locally cached file in.

                        try:
# TODO(dustin): Our accounting is broken when it comes to loading and/or update-tracking. If we have a guarantee thawrites only appear in sequence and in increasing order, we can dump BufferSegments.

# TODO(dustin): This is the source of:
# 1) An enormous slowdown where we first have to write the data, and then have to read it back.
# 2) An enormous resource burden.
                            data = f.read()

                            read_blocksize = Conf.get('default_buffer_read_blocksize')
                            self.__buffer = BufferSegments(data, read_blocksize)
                        except:
                            self.__log.exception("Could not read current cached "
                                                 "file into buffer.")
                            raise

                        self.__is_dirty = False

                    self.__is_loaded = True

        self.__log.debug("__load_base_from_remote complete.")
        return cache_fault
Beispiel #4
0
    def __load_base_from_remote(self):
        """Download the data for the entry that we represent. This is probably 
        a file, but could also be a stub for -any- entry.
        """

        try:
            entry = self.__get_entry_or_raise()
        except:
            self.__log.exception("Could not get entry with ID [%s] for "
                                 "write-flush." % (self.__entry_id))
            raise

        self.__log.debug("Ensuring local availability of [%s]." % (entry))

        temp_file_path = get_temp_filepath(entry, self.mime_type)

        self.__log.debug("__load_base_from_remote about to download.")

        with self.__class__.__download_lock:
            # Get the current version of the write-cache file, or note that we
            # don't have it.

            self.__log.info("Attempting local cache update of file [%s] for "
                            "entry [%s] and mime-type [%s]." %
                            (temp_file_path, entry, self.mime_type))

            if entry.requires_mimetype:
                length = DisplacedFile.file_size

                try:
                    d = DisplacedFile(entry)
                    stub_data = d.deposit_file(self.mime_type)

                    with file(temp_file_path, 'w') as f:
                        f.write(stub_data)
                except:
                    self.__log.exception("Could not deposit to file [%s] from "
                                         "entry [%s]." %
                                         (temp_file_path, entry))
                    raise

# TODO: Accommodate the cache for displaced-files.
                cache_fault = True

            else:
                self.__log.info("Executing the download.")

                try:
                    # TODO(dustin): We're not inheriting an existing file (same mtime, same size).
                    result = drive_proxy('download_to_local',
                                         output_file_path=temp_file_path,
                                         normalized_entry=entry,
                                         mime_type=self.mime_type)

                    (length, cache_fault) = result
                except ExportFormatError:
                    self.__log.exception("There was an export-format error.")
                    raise FuseOSError(ENOENT)
                except:
                    self.__log.exception("Could not localize file with entry "
                                         "[%s]." % (entry))
                    raise

            self.__log.info("Download complete.  cache_fault= [%s] "
                            "__is_loaded= [%s]" %
                            (cache_fault, self.__is_loaded))

            # We've either not loaded it, yet, or it has changed.
            if cache_fault or not self.__is_loaded:
                with self.__class__.__update_lock:
                    self.__log.info("Checking queued items for fault.")

                    if cache_fault:
                        if self.__is_dirty:
                            self.__log.error("Entry [%s] has been changed. "
                                             "Forcing buffer updates, and "
                                             "clearing uncommitted updates." %
                                             (entry))
                        else:
                            self.__log.debug("Entry [%s] has changed. "
                                             "Updating buffers." % (entry))

                    self.__log.debug("Loading buffers.")

                    with open(temp_file_path, 'rb') as f:
                        # Read the locally cached file in.

                        try:
                            # TODO(dustin): This is the source of:
                            # 1) An enormous slowdown where we first have to write the data, and then have to read it back.
                            # 2) An enormous resource burden.
                            data = f.read()

                            read_blocksize = Conf.get(
                                'default_buffer_read_blocksize')
                            self.__buffer = BufferSegments(
                                data, read_blocksize)
                        except:
                            self.__log.exception(
                                "Could not read current cached "
                                "file into buffer.")
                            raise

                        self.__is_dirty = False

                    self.__is_loaded = True

        self.__log.debug("__load_base_from_remote complete.")
        return cache_fault