def _post(self): request = _get_json(self.request.body) uid = request['uid'] _logger.info("Proceeding with unlink request for '%(uid)s'..." % { 'uid': uid, }) record = database.get_record(uid) if not record: self.send_error(404) return trust = _get_trust(record, request.get('keys'), self.request.remote_ip) if not trust.write: self.send_error(403) return fs = state.get_filesystem(record['physical']['family']) try: fs.unlink(record) except filesystem.FileNotFoundError as e: _logger.error("Database record exists for '%(uid)s', but filesystem entry does not" % { 'uid': uid, }) self.send_error(404) return else: database.drop_record(uid)
def run(self): """ Cycles through every database record in order, removing any records associated with files that do not exist. """ ctime = -1.0 while True: while not self._within_window(DATABASE_WINDOWS): _logger.debug("Not in execution window; sleeping" % { 'name': self.name, }) time.sleep(60) records_retrieved = False for record in database.enumerate_all(ctime): ctime = record['physical']['ctime'] records_retrieved = True filesystem = state.get_filesystem(record['physical']['family']) if not filesystem.file_exists(record): _logger.warn("Discovered database record for '%(uid)s' without matching file; dropping record..." % { 'uid': record['_id'], }) database.drop_record(record['_id']) if not records_retrieved: #Cycle complete _logger.debug("All records processed; sleeping") time.sleep(CONFIG.maintainer_database_sleep) ctime = -1.0
def _post(self): request = _get_json(self.request.body) uid = request['uid'] _logger.info("Proceeding with retrieval request for '%(uid)s'..." % { 'uid': uid, }) record = database.get_record(uid) if not record: self.send_error(404) return trust = _get_trust(record, request.get('keys'), self.request.remote_ip) if not trust.read: self.send_error(403) return current_time = int(time.time()) record['physical']['atime'] = current_time record['stats']['accesses'] += 1 for policy in ('delete', 'compress'): if 'stale' in record['policy'][policy]: record['policy'][policy]['staleTime'] = current_time + record['policy'][policy]['stale'] database.update_record(record) fs = state.get_filesystem(record['physical']['family']) try: data = fs.get(record) except filesystem.FileNotFoundError as e: _logger.error("Database record exists for '%(uid)s', but filesystem entry does not" % { 'uid': uid, }) self.send_error(404) return else: _logger.debug("Evaluating decompression requirements...") applied_compression = record['physical']['format'].get('comp') supported_compressions = (c.strip() for c in (self.request.headers.get('Media-Storage-Supported-Compression') or '').split(';')) if applied_compression and not applied_compression in supported_compressions: #Must be decompressed first data = compression.get_decompressor(applied_compression)(data) applied_compression = None _logger.debug("Returning entity...") self.set_header('Content-Type', record['physical']['format']['mime']) if applied_compression: self.set_header('Media-Storage-Applied-Compression', applied_compression) while True: chunk = data.read(_CHUNK_SIZE) if chunk: self.write(chunk) else: break
def run(self): """ Cycles through every filesystem entry in order, removing any files associated with database records that do not exist. """ while True: for family in state.get_families(): _logger.info("Processing family %(family)r..." % { 'family': family, }) filesystem = state.get_filesystem(family) self._walk(filesystem.walk()) _logger.debug("All records processed; sleeping") time.sleep(CONFIG.maintainer_filesystem_sleep)
def _post(self): (header, data) = self._get_payload() current_time = time.time() try: _logger.debug("Assembling database record...") record = { '_id': header.get('uid') or uuid.uuid1().hex, 'keys': self._build_keys(header), 'physical': { 'family': header['physical'].get('family'), 'ctime': current_time, 'minRes': CONFIG.storage_minute_resolution, 'atime': int(current_time), 'format': self._build_format(header), }, 'policy': self._build_policy(header), 'stats': { 'accesses': 0, }, 'meta': header.get('meta') or {}, } except (KeyError, TypeError, AttributeError) as e: _logger.error("Request received did not adhere to expected structure: %(error)s" % { 'error': str(e), }) self.send_error(409) return else: _logger.info("Proceeding with storage request for '%(uid)s'..." % { 'uid': record['_id'], }) _logger.debug("Evaluating compression requirements...") target_compression = record['physical']['format'].get('comp') if target_compression and self.request.headers.get('Media-Storage-Compress-On-Server') == 'yes': _logger.info("Compressing file...") data = compression.get_compressor(target_compression)(data) _logger.debug("Storing entity...") database.add_record(record) fs = state.get_filesystem(record['physical']['family']) fs.put(record, data) return { 'uid': record['_id'], 'keys': record['keys'], }
def _process_record(self, record): """ Determines whether the given `record` is a candidate for deletion, removing it and the associated file if it is. """ _logger.info("Unlinking record...") filesystem = state.get_filesystem(record['physical']['family']) try: filesystem.unlink(record) except Exception as e: _logger.warn("Unable to unlink record: %(error)s" % { 'error': str(e), }) return False else: database.drop_record(record['_id']) return True
def _post(self): request = _get_json(self.request.body) uid = request['uid'] _logger.info("Proceeding with description request for '%(uid)s'..." % { 'uid': uid, }) record = database.get_record(uid) if not record: self.send_error(404) return trust = _get_trust(record, request.get('keys'), self.request.remote_ip) if not trust.read: self.send_error(403) return _logger.debug("Describing entity...") record['physical']['exists'] = state.get_filesystem(record['physical']['family']).file_exists(record) record['uid'] = record['_id'] del record['_id'] del record['physical']['minRes'] del record['keys'] return record
def _post(self): request = _get_json(self.request.body) query = {} trust = _get_trust(None, None, self.request.remote_ip) if not trust.read: query['keys.read'] = None #Anonymous records only def _assemble_range_block(name, attribute): attribute_block = {} _min = request[name]['min'] _max = request[name]['max'] _block = {} if _min: attribute_block['$gte'] = _min if _max: attribute_block['$lte'] = _max if attribute_block: query[attribute] = attribute_block _assemble_range_block('ctime', 'physical.ctime') _assemble_range_block('atime', 'physical.atime') _assemble_range_block('accesses', 'stats.accesses') query['physical.family'] = request['family'] mime = request['mime'] if mime: if '/' in mime: query['physical.mime'] = mime else: query['physical.mime'] = {'$regex': '^' + mime} for (key, value) in request['meta'].items(): key = 'meta.' + key if type(value) in types.StringTypes: if value.startswith('::'): value = value[1:] else: match = _FILTER_RE.match(value) if match: filter = match.group('filter') query = match.group('query') if filter == 'range': (_min, _max) = (float(v) for v in query.split(':', 1)) value = {'$gte': _min, '$lte': _max} elif filter == 'lte': value = {'$lte': float(query)} elif filter == 'gte': value = {'$gte': float(query)} elif filter == 're': value = {'$regex': query} elif filter == 're': value = {'$regex': query} elif filter == 're.i': value = {'$regex': query, '$options': 'i'} elif filter == 'like': if query.count('%') == 1 and query.endswith('%'): value = {'$regex': '^' + query[:-1]} else: value = {'$regex': '^' + query.replace('%', '.*') + '$'} elif filter == 'ilike': value = {'$regex': '^' + query.replace('%', '.*') + '$', '$options': 'i'} else: raise ValueError("Unrecognised filter: %(filter)s" % { 'filter': filter, }) query[key] = value records = [] for record in database.enumerate_where(query): record['physical']['exists'] = state.get_filesystem(record['physical']['family']).file_exists(record) if not trust.read: del record['keys'] else: record['physical']['path'] = state.get_filesystem(request['family']).resolve_path(record) #Has to happen after the filesystem search record['uid'] = record['_id'] del record['_id'] del record['physical']['minRes'] records.append(record) return { 'records': records, }
def _process_record(self, record): """ Determines whether the given `record` is a candidate for compression, compressing the associated file and updating the record if it is. """ _logger.info("Compressing record '%(uid)s'..." % { 'uid': record['_id'], }) current_compression = record['physical']['format'].get('comp') target_compression = record['policy']['compress'].get('comp') if current_compression == target_compression: _logger.debug("File already compressed in target format") record['policy']['compress'].clear() #Drop the compression policy try: database.update_record(record) except Exception as e: _logger.error("Unable to update record to reflect already-applied compression; compression routine will retry later: %(error)s" % { 'error': str(e), }) return False else: return True filesystem = state.get_filesystem(record['physical']['family']) data = filesystem.get(record) if current_compression: #Must be decompressed first _logger.info("Decompressing file...") data = compression.get_decompressor(current_compression)(data) data = compression.get_compressor(target_compression)(data) _logger.info("Updating entity...") old_format = record['physical']['format'].copy() record['physical']['format']['comp'] = target_compression try: filesystem.put(record, data, tempfile=True) except Exception as e: #Harmless backout point _logger.warn("Unable to write compressed file to disk; backing out with no consequences") return False else: old_compression_policy = record['policy']['compress'].copy() record['policy']['compress'].clear() #Drop the compression policy try: database.update_record(record) except Exception as e: #Results in wasted space until the next attempt _logger.error("Unable to update record; old file will be served, and new file will be replaced on a subsequent compression attempt: %(error)s" % { 'error': str(e), }) return False else: try: filesystem.make_permanent(record) except Exception as e: _logger.error("Unable to update on-disk file; rolling back database update: %(error)s" % { 'error': str(e), }) record['policy']['compress'] = old_compression_policy record['physical']['format'] = old_format try: database.update_record(record) except Exception as e: _logger.error("Unable to roll back database update; '%(uid)s' is inaccessible and must be manually decompressed from '%(comp)s' format: %(error)s" % { 'error': str(e), 'uid': record['_id'], 'comp': target_compression, }) return False record['physical']['format'] = old_format try: filesystem.unlink(record) except Exception as e: #Results in wasted space, but non-fatal _logger.error("Unable to unlink old file; space occupied by '%(uid)s' non-recoverable unless unlinked manually: %(family)r | %(file)s : %(error)s" % { 'family': record['physical']['family'], 'file': filesystem.resolve_path(record), 'uid': record['_id'], 'error': str(e), }) return True