def __load_base_from_remote(self):
"""Download the data for the entry that we represent. This is probably
a file, but could also be a stub for -any- entry.
"""
# If it's loaded and not-changed, don't do anything.
if self.__is_loaded is True and self.__is_dirty is False:
_logger.debug("Not syncing-down non-dirty file.")
return
if self.__fh is not None:
self.__fh.close()
self.__fh = None
entry = self.__cache.get(self.__entry_id)
_logger.debug("Ensuring local availability of [%s]: [%s]",
entry, self.__temp_filepath)
# Get the current version of the write-cache file, or note that we
# don't have it.
_logger.info("Attempting local cache update of file [%s] for entry "
"[%s] and mime-type [%s].",
self.__temp_filepath, entry, self.mime_type)
if entry.requires_mimetype:
length = DisplacedFile.file_size
d = DisplacedFile(entry)
stub_data = d.deposit_file(self.mime_type)
self.__fh = open(self.__temp_filepath, 'w+')
self.__fh.write(stub_data)
else:
_logger.debug("Executing the download: [%s] => [%s]",
entry.id, self.__temp_filepath)
try:
# TODO(dustin): We need to inherit a file that we might've already cached by
# opening.
# TODO(dustin): Any call to download_to_local should use a local, temporarily
# file is already established. We can't use it in the reverse
# order though: It's one thing to already have a cache from
# having opened it, and it's a another thing to maintain a cache
# of every file that is copied.
gd = get_gdrive()
result = gd.download_to_local(
self.__temp_filepath,
entry,
self.mime_type)
(length, cache_fault) = result
except ExportFormatError:
_logger.exception("There was an export-format error.")
raise fuse.FuseOSError(ENOENT)
self.__fh = open(self.__temp_filepath, 'r+')
self.__is_dirty = False
self.__is_loaded = True
_logger.debug("Established base file-data for [%s]: [%s]",
entry, self.__temp_filepath)
def __load_base_from_remote(self):
"""Download the data for the entry that we represent. This is probably
a file, but could also be a stub for -any- entry.
"""
# If it's loaded and not-changed, don't do anything.
if self.__is_loaded is True and self.__is_dirty is False:
_logger.debug("Not syncing-down non-dirty file.")
return
if self.__fh is not None:
self.__fh.close()
self.__fh = None
entry = self.__cache.get(self.__entry_id)
_logger.debug("Ensuring local availability of [%s]: [%s]", entry,
self.__temp_filepath)
# Get the current version of the write-cache file, or note that we
# don't have it.
_logger.info(
"Attempting local cache update of file [%s] for entry "
"[%s] and mime-type [%s].", self.__temp_filepath, entry,
self.mime_type)
if entry.requires_mimetype:
length = DisplacedFile.file_size
d = DisplacedFile(entry)
stub_data = d.deposit_file(self.mime_type)
self.__fh = open(self.__temp_filepath, 'w+')
self.__fh.write(stub_data)
else:
_logger.debug("Executing the download: [%s] => [%s]", entry.id,
self.__temp_filepath)
try:
# TODO(dustin): We need to inherit a file that we might've already cached by
# opening.
# TODO(dustin): Any call to download_to_local should use a local, temporarily
# file is already established. We can't use it in the reverse
# order though: It's one thing to already have a cache from
# having opened it, and it's a another thing to maintain a cache
# of every file that is copied.
gd = get_gdrive()
result = gd.download_to_local(self.__temp_filepath, entry,
self.mime_type)
(length, cache_fault) = result
except ExportFormatError:
_logger.exception("There was an export-format error.")
raise fuse.FuseOSError(ENOENT)
self.__fh = open(self.__temp_filepath, 'r+')
self.__is_dirty = False
self.__is_loaded = True
_logger.debug("Established base file-data for [%s]: [%s]", entry,
self.__temp_filepath)
def __load_base_from_remote(self):
"""Download the data for the entry that we represent. This is probably
a file, but could also be a stub for -any- entry.
"""
try:
entry = self.__get_entry_or_raise()
except:
self.__log.exception("Could not get entry with ID [%s] for "
"write-flush." % (self.__entry_id))
raise
self.__log.debug("Ensuring local availability of [%s]." % (entry))
temp_file_path = get_temp_filepath(entry, self.mime_type)
self.__log.debug("__load_base_from_remote about to download.")
with self.__class__.__download_lock:
# Get the current version of the write-cache file, or note that we
# don't have it.
self.__log.info("Attempting local cache update of file [%s] for "
"entry [%s] and mime-type [%s]." %
(temp_file_path, entry, self.mime_type))
if entry.requires_mimetype:
length = DisplacedFile.file_size
try:
d = DisplacedFile(entry)
stub_data = d.deposit_file(self.mime_type)
with file(temp_file_path, 'w') as f:
f.write(stub_data)
except:
self.__log.exception("Could not deposit to file [%s] from "
"entry [%s]." % (temp_file_path,
entry))
raise
# TODO: Accommodate the cache for displaced-files.
cache_fault = True
else:
self.__log.info("Executing the download.")
try:
# TODO(dustin): We're not inheriting an existing file (same mtime, same size).
result = drive_proxy('download_to_local',
output_file_path=temp_file_path,
normalized_entry=entry,
mime_type=self.mime_type)
(length, cache_fault) = result
except ExportFormatError:
self.__log.exception("There was an export-format error.")
raise FuseOSError(ENOENT)
except:
self.__log.exception("Could not localize file with entry "
"[%s]." % (entry))
raise
self.__log.info("Download complete. cache_fault= [%s] "
"__is_loaded= [%s]" %
(cache_fault, self.__is_loaded))
# We've either not loaded it, yet, or it has changed.
if cache_fault or not self.__is_loaded:
with self.__class__.__update_lock:
self.__log.info("Checking queued items for fault.")
if cache_fault:
if self.__is_dirty:
self.__log.error("Entry [%s] has been changed. "
"Forcing buffer updates, and "
"clearing uncommitted updates." %
(entry))
else:
self.__log.debug("Entry [%s] has changed. "
"Updating buffers." % (entry))
self.__log.debug("Loading buffers.")
with open(temp_file_path, 'rb') as f:
# Read the locally cached file in.
try:
# TODO(dustin): Our accounting is broken when it comes to loading and/or update-tracking. If we have a guarantee thawrites only appear in sequence and in increasing order, we can dump BufferSegments.
# TODO(dustin): This is the source of:
# 1) An enormous slowdown where we first have to write the data, and then have to read it back.
# 2) An enormous resource burden.
data = f.read()
read_blocksize = Conf.get('default_buffer_read_blocksize')
self.__buffer = BufferSegments(data, read_blocksize)
except:
self.__log.exception("Could not read current cached "
"file into buffer.")
raise
self.__is_dirty = False
self.__is_loaded = True
self.__log.debug("__load_base_from_remote complete.")
return cache_fault
def __load_base_from_remote(self):
"""Download the data for the entry that we represent. This is probably
a file, but could also be a stub for -any- entry.
"""
try:
entry = self.__get_entry_or_raise()
except:
self.__log.exception("Could not get entry with ID [%s] for "
"write-flush." % (self.__entry_id))
raise
self.__log.debug("Ensuring local availability of [%s]." % (entry))
temp_file_path = get_temp_filepath(entry, self.mime_type)
self.__log.debug("__load_base_from_remote about to download.")
with self.__class__.__download_lock:
# Get the current version of the write-cache file, or note that we
# don't have it.
self.__log.info("Attempting local cache update of file [%s] for "
"entry [%s] and mime-type [%s]." %
(temp_file_path, entry, self.mime_type))
if entry.requires_mimetype:
length = DisplacedFile.file_size
try:
d = DisplacedFile(entry)
stub_data = d.deposit_file(self.mime_type)
with file(temp_file_path, 'w') as f:
f.write(stub_data)
except:
self.__log.exception("Could not deposit to file [%s] from "
"entry [%s]." %
(temp_file_path, entry))
raise
# TODO: Accommodate the cache for displaced-files.
cache_fault = True
else:
self.__log.info("Executing the download.")
try:
# TODO(dustin): We're not inheriting an existing file (same mtime, same size).
result = drive_proxy('download_to_local',
output_file_path=temp_file_path,
normalized_entry=entry,
mime_type=self.mime_type)
(length, cache_fault) = result
except ExportFormatError:
self.__log.exception("There was an export-format error.")
raise FuseOSError(ENOENT)
except:
self.__log.exception("Could not localize file with entry "
"[%s]." % (entry))
raise
self.__log.info("Download complete. cache_fault= [%s] "
"__is_loaded= [%s]" %
(cache_fault, self.__is_loaded))
# We've either not loaded it, yet, or it has changed.
if cache_fault or not self.__is_loaded:
with self.__class__.__update_lock:
self.__log.info("Checking queued items for fault.")
if cache_fault:
if self.__is_dirty:
self.__log.error("Entry [%s] has been changed. "
"Forcing buffer updates, and "
"clearing uncommitted updates." %
(entry))
else:
self.__log.debug("Entry [%s] has changed. "
"Updating buffers." % (entry))
self.__log.debug("Loading buffers.")
with open(temp_file_path, 'rb') as f:
# Read the locally cached file in.
try:
# TODO(dustin): This is the source of:
# 1) An enormous slowdown where we first have to write the data, and then have to read it back.
# 2) An enormous resource burden.
data = f.read()
read_blocksize = Conf.get(
'default_buffer_read_blocksize')
self.__buffer = BufferSegments(
data, read_blocksize)
except:
self.__log.exception(
"Could not read current cached "
"file into buffer.")
raise
self.__is_dirty = False
self.__is_loaded = True
self.__log.debug("__load_base_from_remote complete.")
return cache_fault