def run(self): """ Queries the database, looking for any records that match policy criteria. If any matches are found, they're processed, and the operation continues until no new records are discovered. """ while True: while not self._within_window(self._windows): _logger.debug("Not in execution window; sleeping") time.sleep(60) while True: records_processed = False for record in database.enumerate_where({ '$or': [ {self._fixed_field: {'$lt': int(time.time())}}, {self._stale_query: {'$lt': int(time.time())}}, ], }): _logger.info("Discovered candidate record: %(uid)s" % { 'uid': record['_id'], }) #Some records may fail to be processed for a variety of reasons; they shouldn't be considered active records_processed = self._process_record(record) or records_processed if not records_processed: #Nothing left to do break _logger.debug("All records processed; sleeping") time.sleep(self._sleep_period)
def _post(self): request = _get_json(self.request.body) query = {} trust = _get_trust(None, None, self.request.remote_ip) if not trust.read: query['keys.read'] = None #Anonymous records only def _assemble_range_block(name, attribute): attribute_block = {} _min = request[name]['min'] _max = request[name]['max'] _block = {} if _min: attribute_block['$gte'] = _min if _max: attribute_block['$lte'] = _max if attribute_block: query[attribute] = attribute_block _assemble_range_block('ctime', 'physical.ctime') _assemble_range_block('atime', 'physical.atime') _assemble_range_block('accesses', 'stats.accesses') query['physical.family'] = request['family'] mime = request['mime'] if mime: if '/' in mime: query['physical.mime'] = mime else: query['physical.mime'] = {'$regex': '^' + mime} for (key, value) in request['meta'].items(): key = 'meta.' + key if type(value) in types.StringTypes: if value.startswith('::'): value = value[1:] else: match = _FILTER_RE.match(value) if match: filter = match.group('filter') query = match.group('query') if filter == 'range': (_min, _max) = (float(v) for v in query.split(':', 1)) value = {'$gte': _min, '$lte': _max} elif filter == 'lte': value = {'$lte': float(query)} elif filter == 'gte': value = {'$gte': float(query)} elif filter == 're': value = {'$regex': query} elif filter == 're': value = {'$regex': query} elif filter == 're.i': value = {'$regex': query, '$options': 'i'} elif filter == 'like': if query.count('%') == 1 and query.endswith('%'): value = {'$regex': '^' + query[:-1]} else: value = {'$regex': '^' + query.replace('%', '.*') + '$'} elif filter == 'ilike': value = {'$regex': '^' + query.replace('%', '.*') + '$', '$options': 'i'} else: raise ValueError("Unrecognised filter: %(filter)s" % { 'filter': filter, }) query[key] = value records = [] for record in database.enumerate_where(query): record['physical']['exists'] = state.get_filesystem(record['physical']['family']).file_exists(record) if not trust.read: del record['keys'] else: record['physical']['path'] = state.get_filesystem(request['family']).resolve_path(record) #Has to happen after the filesystem search record['uid'] = record['_id'] del record['_id'] del record['physical']['minRes'] records.append(record) return { 'records': records, }