def cleanup(force=None): """ Delete queries until EXPIRE config is satisfied: 1 - Delete anything older than EXPIRE_TIME seconds 2 - Delete the oldest queries until we have at least EXPIRE_SPACE bytes free """ period = parse_duration(Config.get('CLEANUP_PERIOD', 0)) now = datetime.utcnow() global _LAST_CLEANED if not force and period and _LAST_CLEANED + period < now: Config.logger.debug("Cleaned recently, aborting: {}".format( _LAST_CLEANED.strftime(Config.get('DATE_FORMAT')))) return _LAST_CLEANED = datetime.utcnow() from heapq import heapify, heappop Config.logger.info("Running Cleanup: {}".format(now.strftime(ISOFORMAT))) ids = Query.get_unexpired() ordered = [(file_modified(Query.job_path_for_id(i)), i) for i in ids] heapify(ordered) Config.logger.info("Cleaning: {}".format( {v: o.strftime(ISOFORMAT) for o, v in ordered})) if EXPIRE_TIME: expiring = [] while ordered and ordered[0][0] + EXPIRE_TIME < now: _, q_id = heappop(ordered) expiring.append(q_id) Config.logger.info("Deleting old queries: {}".format(expiring)) expiring = Query.expire_now(expiring) if expiring: Config.logger.error("Couldn't delete: {}".format(expiring)) if EXPIRE_SPACE > 0: free_bytes = spool_space().bytes while ordered and free_bytes < EXPIRE_SPACE: _, q_id = heappop(ordered) Config.logger.info("Deleting for space: {}".format(q_id)) Query.expire_now(q_id) free_bytes = spool_space().bytes
def get(self, api=None, selected=None): """ inspect parameters and call the right Query method """ Config.logger.info("API request: {}".format(_get_request_nfo())) if type(selected) is str: selected = self.__class__.delims.split(selected) if api == "ids" or api == "urls": ids = Query.get_unexpired(selected) if api == "ids": return ids urls = {} for i in ids: if os.path.exists(Query.pcap_path_for_id(i)): urls[i] = Query.pcap_url_for_id(i) return urls elif api == "stats": from tasks import get_stats stats = get_stats(selected_sensors=selected) freespace = spool_space() stats['docket'] = {'Free space': freespace} return stats elif api == "status": r = Response(response=dumps(Query.status_for_ids( Query.get_unexpired(selected)), default=json_serial), mimetype="application/json") return r elif api == "clean" or api == 'cleanup': from tasks import cleanup cleanup.apply_async(queue='io', kwargs={'force': True}) return "Cleanup queued" elif api == "jobs": return Query.find({}) return "Unrecognized request: try /stats, /ids, /urls, /jobs or POST a json encoded stenographer query"