def perform_maintenance(limit = None, legacy_limit = None):
logging.info("Starting maintenance job...")
if limit is None:
now = time.time()
limit = now - config.expiry_days * 24 * 3600
if legacy_limit is None:
now = time.time()
legacy_limit = now - config.legacy_expiry_days * 24 * 3600
reports_deleted = 0
crackers_deleted = 0
legacy_deleted = 0
batch_size = 1000
while True:
old_reports = yield Report.find(where=["latest_report_time<?", limit], limit=batch_size)
if len(old_reports) == 0:
break
logging.debug("Removing batch of {} old reports".format(len(old_reports)))
for report in old_reports:
cracker = yield report.cracker.get()
yield utils.wait_and_lock_host(cracker.ip_address)
try:
logging.debug("Maintenance: removing report from {} for cracker {}".format(report.ip_address, cracker.ip_address))
yield report.cracker.clear()
yield report.delete()
reports_deleted += 1
current_reports = yield cracker.reports.get(group='ip_address')
cracker.current_reports = len(current_reports)
yield cracker.save()
if cracker.current_reports == 0:
logging.debug("Maintenance: removing cracker {}".format(cracker.ip_address))
yield cracker.delete()
crackers_deleted += 1
finally:
utils.unlock_host(cracker.ip_address)
logging.debug("Maintenance on report from {} for cracker {} done".format(report.ip_address, cracker.ip_address))
legacy_reports = yield Legacy.find(where=["retrieved_time<?", legacy_limit])
if legacy_reports is not None:
for legacy in legacy_reports:
yield legacy.delete()
legacy_deleted += 1
logging.info("Done maintenance job")
logging.info("Expired {} reports and {} hosts, plus {} hosts from the legacy list".format(reports_deleted, crackers_deleted, legacy_deleted))
returnValue(0)
def add_report_to_cracker(cracker, client_ip, when=None):
if when is None:
when = time.time()
reports = yield Report.find(
where=["cracker_id=? AND ip_address=?", cracker.id, client_ip],
orderby='latest_report_time ASC'
)
if len(reports) == 0:
report = Report(ip_address=client_ip, first_report_time=when, latest_report_time=when)
yield report.save()
cracker.current_reports += 1
yield report.cracker.set(cracker)
elif len(reports) == 1:
report = reports[0]
# Add second report after 24 hours
if when > report.latest_report_time + 24*3600:
report = Report(ip_address=client_ip, first_report_time=when, latest_report_time=when)
yield report.save()
yield report.cracker.set(cracker)
elif len(reports) == 2:
latest_report = reports[1]
# Add third report after again 24 hours
if when > latest_report.latest_report_time + 24*3600:
report = Report(ip_address=client_ip, first_report_time=when, latest_report_time=when)
yield report.save()
yield report.cracker.set(cracker)
else:
latest_report = reports[-1]
latest_report.latest_report_time = when
yield latest_report.save()
cracker.total_reports += 1
cracker.latest_time = when
cracker.resiliency = when - cracker.first_time
yield cracker.save()
