def _walk(self, walker):
"""
Traverses the filesystem by iterating over `walker`, checking with the database to ensure
that every encountered file has a corresponding record. If not, the file is unlinked.
"""
try:
for (path, files) in walker:
for filename in files:
try:
if not self._keep_file(filename):
_logger.warn("Discovered orphaned file '%(name)s'; unlinking..." % {
'name': filename,
})
try:
filesystem.unlink(path + '/' + filename)
except Exception as e:
_logger.warn("Unable to unlink file: %(error)s" % {
'error': str(e),
})
except Exception as e:
_logger.warn("Unable to query database: %(error)s" % {
'error': str(e),
})
except Exception as e:
_logger.warn("Unable to traverse filesystem: %(error)s" % {
'error': str(e),
})
def _process_record(self, record):
"""
Determines whether the given `record` is a candidate for deletion, removing it and the
associated file if it is.
"""
_logger.info("Unlinking record...")
filesystem = state.get_filesystem(record['physical']['family'])
try:
filesystem.unlink(record)
except Exception as e:
_logger.warn("Unable to unlink record: %(error)s" % {
'error': str(e),
})
return False
else:
database.drop_record(record['_id'])
return True
def _process_record(self, record):
"""
Determines whether the given `record` is a candidate for compression, compressing the
associated file and updating the record if it is.
"""
_logger.info("Compressing record '%(uid)s'..." % {
'uid': record['_id'],
})
current_compression = record['physical']['format'].get('comp')
target_compression = record['policy']['compress'].get('comp')
if current_compression == target_compression:
_logger.debug("File already compressed in target format")
record['policy']['compress'].clear() #Drop the compression policy
try:
database.update_record(record)
except Exception as e:
_logger.error("Unable to update record to reflect already-applied compression; compression routine will retry later: %(error)s" % {
'error': str(e),
})
return False
else:
return True
filesystem = state.get_filesystem(record['physical']['family'])
data = filesystem.get(record)
if current_compression: #Must be decompressed first
_logger.info("Decompressing file...")
data = compression.get_decompressor(current_compression)(data)
data = compression.get_compressor(target_compression)(data)
_logger.info("Updating entity...")
old_format = record['physical']['format'].copy()
record['physical']['format']['comp'] = target_compression
try:
filesystem.put(record, data, tempfile=True)
except Exception as e: #Harmless backout point
_logger.warn("Unable to write compressed file to disk; backing out with no consequences")
return False
else:
old_compression_policy = record['policy']['compress'].copy()
record['policy']['compress'].clear() #Drop the compression policy
try:
database.update_record(record)
except Exception as e: #Results in wasted space until the next attempt
_logger.error("Unable to update record; old file will be served, and new file will be replaced on a subsequent compression attempt: %(error)s" % {
'error': str(e),
})
return False
else:
try:
filesystem.make_permanent(record)
except Exception as e:
_logger.error("Unable to update on-disk file; rolling back database update: %(error)s" % {
'error': str(e),
})
record['policy']['compress'] = old_compression_policy
record['physical']['format'] = old_format
try:
database.update_record(record)
except Exception as e:
_logger.error("Unable to roll back database update; '%(uid)s' is inaccessible and must be manually decompressed from '%(comp)s' format: %(error)s" % {
'error': str(e),
'uid': record['_id'],
'comp': target_compression,
})
return False
record['physical']['format'] = old_format
try:
filesystem.unlink(record)
except Exception as e: #Results in wasted space, but non-fatal
_logger.error("Unable to unlink old file; space occupied by '%(uid)s' non-recoverable unless unlinked manually: %(family)r | %(file)s : %(error)s" % {
'family': record['physical']['family'],
'file': filesystem.resolve_path(record),
'uid': record['_id'],
'error': str(e),
})
return True
