def _walk(self, walker): """ Traverses the filesystem by iterating over `walker`, checking with the database to ensure that every encountered file has a corresponding record. If not, the file is unlinked. """ try: for (path, files) in walker: for filename in files: try: if not self._keep_file(filename): _logger.warn("Discovered orphaned file '%(name)s'; unlinking..." % { 'name': filename, }) try: filesystem.unlink(path + '/' + filename) except Exception as e: _logger.warn("Unable to unlink file: %(error)s" % { 'error': str(e), }) except Exception as e: _logger.warn("Unable to query database: %(error)s" % { 'error': str(e), }) except Exception as e: _logger.warn("Unable to traverse filesystem: %(error)s" % { 'error': str(e), })
def _process_record(self, record): """ Determines whether the given `record` is a candidate for deletion, removing it and the associated file if it is. """ _logger.info("Unlinking record...") filesystem = state.get_filesystem(record['physical']['family']) try: filesystem.unlink(record) except Exception as e: _logger.warn("Unable to unlink record: %(error)s" % { 'error': str(e), }) return False else: database.drop_record(record['_id']) return True
def _process_record(self, record): """ Determines whether the given `record` is a candidate for compression, compressing the associated file and updating the record if it is. """ _logger.info("Compressing record '%(uid)s'..." % { 'uid': record['_id'], }) current_compression = record['physical']['format'].get('comp') target_compression = record['policy']['compress'].get('comp') if current_compression == target_compression: _logger.debug("File already compressed in target format") record['policy']['compress'].clear() #Drop the compression policy try: database.update_record(record) except Exception as e: _logger.error("Unable to update record to reflect already-applied compression; compression routine will retry later: %(error)s" % { 'error': str(e), }) return False else: return True filesystem = state.get_filesystem(record['physical']['family']) data = filesystem.get(record) if current_compression: #Must be decompressed first _logger.info("Decompressing file...") data = compression.get_decompressor(current_compression)(data) data = compression.get_compressor(target_compression)(data) _logger.info("Updating entity...") old_format = record['physical']['format'].copy() record['physical']['format']['comp'] = target_compression try: filesystem.put(record, data, tempfile=True) except Exception as e: #Harmless backout point _logger.warn("Unable to write compressed file to disk; backing out with no consequences") return False else: old_compression_policy = record['policy']['compress'].copy() record['policy']['compress'].clear() #Drop the compression policy try: database.update_record(record) except Exception as e: #Results in wasted space until the next attempt _logger.error("Unable to update record; old file will be served, and new file will be replaced on a subsequent compression attempt: %(error)s" % { 'error': str(e), }) return False else: try: filesystem.make_permanent(record) except Exception as e: _logger.error("Unable to update on-disk file; rolling back database update: %(error)s" % { 'error': str(e), }) record['policy']['compress'] = old_compression_policy record['physical']['format'] = old_format try: database.update_record(record) except Exception as e: _logger.error("Unable to roll back database update; '%(uid)s' is inaccessible and must be manually decompressed from '%(comp)s' format: %(error)s" % { 'error': str(e), 'uid': record['_id'], 'comp': target_compression, }) return False record['physical']['format'] = old_format try: filesystem.unlink(record) except Exception as e: #Results in wasted space, but non-fatal _logger.error("Unable to unlink old file; space occupied by '%(uid)s' non-recoverable unless unlinked manually: %(family)r | %(file)s : %(error)s" % { 'family': record['physical']['family'], 'file': filesystem.resolve_path(record), 'uid': record['_id'], 'error': str(e), }) return True