def cleanup(force=None):
""" Delete queries until EXPIRE config is satisfied:
1 - Delete anything older than EXPIRE_TIME seconds
2 - Delete the oldest queries until we have at least EXPIRE_SPACE bytes free
"""
period = parse_duration(Config.get('CLEANUP_PERIOD', 0))
now = datetime.utcnow()
global _LAST_CLEANED
if not force and period and _LAST_CLEANED + period < now:
Config.logger.debug("Cleaned recently, aborting: {}".format(
_LAST_CLEANED.strftime(Config.get('DATE_FORMAT'))))
return
_LAST_CLEANED = datetime.utcnow()
from heapq import heapify, heappop
Config.logger.info("Running Cleanup: {}".format(now.strftime(ISOFORMAT)))
ids = Query.get_unexpired()
ordered = [(file_modified(Query.job_path_for_id(i)), i) for i in ids]
heapify(ordered)
Config.logger.info("Cleaning: {}".format(
{v: o.strftime(ISOFORMAT)
for o, v in ordered}))
if EXPIRE_TIME:
expiring = []
while ordered and ordered[0][0] + EXPIRE_TIME < now:
_, q_id = heappop(ordered)
expiring.append(q_id)
Config.logger.info("Deleting old queries: {}".format(expiring))
expiring = Query.expire_now(expiring)
if expiring:
Config.logger.error("Couldn't delete: {}".format(expiring))
if EXPIRE_SPACE > 0:
free_bytes = spool_space().bytes
while ordered and free_bytes < EXPIRE_SPACE:
_, q_id = heappop(ordered)
Config.logger.info("Deleting for space: {}".format(q_id))
Query.expire_now(q_id)
free_bytes = spool_space().bytes
def get(self, api=None, selected=None):
""" inspect parameters and call the right Query method """
Config.logger.info("API request: {}".format(_get_request_nfo()))
if type(selected) is str:
selected = self.__class__.delims.split(selected)
if api == "ids" or api == "urls":
ids = Query.get_unexpired(selected)
if api == "ids":
return ids
urls = {}
for i in ids:
if os.path.exists(Query.pcap_path_for_id(i)):
urls[i] = Query.pcap_url_for_id(i)
return urls
elif api == "stats":
from tasks import get_stats
stats = get_stats(selected_sensors=selected)
freespace = spool_space()
stats['docket'] = {'Free space': freespace}
return stats
elif api == "status":
r = Response(response=dumps(Query.status_for_ids(
Query.get_unexpired(selected)),
default=json_serial),
mimetype="application/json")
return r
elif api == "clean" or api == 'cleanup':
from tasks import cleanup
cleanup.apply_async(queue='io', kwargs={'force': True})
return "Cleanup queued"
elif api == "jobs":
return Query.find({})
return "Unrecognized request: try /stats, /ids, /urls, /jobs or POST a json encoded stenographer query"