def download_from_legacy_server():
if config.legacy_server is None or config.legacy_server == "":
returnValue(0)
logging.info("Downloading hosts from legacy server...")
rows = yield database.run_query('SELECT `value` FROM info WHERE `key`="last_legacy_sync"')
last_legacy_sync_time = int(rows[0][0])
try:
server = yield deferToThread(xmlrpclib.ServerProxy, config.legacy_server)
response = yield deferToThread(server.get_new_hosts,
last_legacy_sync_time, config.legacy_threshold, [],
config.legacy_resiliency)
try:
last_legacy_sync_time = int(response["timestamp"])
except:
logging.ERROR("Illegal timestamp {} from legacy server".format(response["timestamp"]))
#Registry.DBPOOL.runOperation('UPDATE info SET `value`=%s WHERE `key`="last_legacy_sync"', (str(last_legacy_sync_time),))
database.run_operation('UPDATE info SET `value`=? WHERE `key`="last_legacy_sync"', str(last_legacy_sync_time))
now = time.time()
logging.debug("Got {} hosts from legacy server".format(len(response["hosts"])))
for host in response["hosts"]:
legacy = yield Legacy.find(where=["ip_address=?",host], limit=1)
if legacy is None:
logging.debug("New host from legacy server: {}".format(host))
legacy = Legacy(ip_address=host, retrieved_time=now)
else:
logging.debug("Known host from legacy server: {}".format(host))
legacy.retrieved_time = now
yield legacy.save()
except Exception, e:
logging.error("Error retrieving info from legacy server: {}".format(e))
def perform_maintenance(limit = None, legacy_limit = None):
logging.info("Starting maintenance job...")
if limit is None:
now = time.time()
limit = now - config.expiry_days * 24 * 3600
if legacy_limit is None:
now = time.time()
legacy_limit = now - config.legacy_expiry_days * 24 * 3600
reports_deleted = 0
crackers_deleted = 0
legacy_deleted = 0
batch_size = 1000
while True:
old_reports = yield Report.find(where=["latest_report_time<?", limit], limit=batch_size)
if len(old_reports) == 0:
break
logging.debug("Removing batch of {} old reports".format(len(old_reports)))
for report in old_reports:
cracker = yield report.cracker.get()
yield utils.wait_and_lock_host(cracker.ip_address)
try:
logging.debug("Maintenance: removing report from {} for cracker {}".format(report.ip_address, cracker.ip_address))
yield report.cracker.clear()
yield report.delete()
reports_deleted += 1
current_reports = yield cracker.reports.get(group='ip_address')
cracker.current_reports = len(current_reports)
yield cracker.save()
if cracker.current_reports == 0:
logging.debug("Maintenance: removing cracker {}".format(cracker.ip_address))
yield cracker.delete()
crackers_deleted += 1
finally:
utils.unlock_host(cracker.ip_address)
logging.debug("Maintenance on report from {} for cracker {} done".format(report.ip_address, cracker.ip_address))
legacy_reports = yield Legacy.find(where=["retrieved_time<?", legacy_limit])
if legacy_reports is not None:
for legacy in legacy_reports:
yield legacy.delete()
legacy_deleted += 1
logging.info("Done maintenance job")
logging.info("Expired {} reports and {} hosts, plus {} hosts from the legacy list".format(reports_deleted, crackers_deleted, legacy_deleted))
returnValue(0)
def get_qualifying_crackers(min_reports, min_resilience, previous_timestamp,
max_crackers, latest_added_hosts):
# Thank to Anne Bezemer for the algorithm in this function.
# See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=622697
# This query takes care of conditions (a) and (b)
# cracker_ids = yield database.runGetPossibleQualifyingCrackerQuery(min_reports, min_resilience, previous_timestamp)
cracker_ids = yield database.run_query("""
SELECT DISTINCT c.id, c.ip_address
FROM crackers c
WHERE (c.current_reports >= ?)
AND (c.resiliency >= ?)
AND (c.latest_time >= ?)
ORDER BY c.first_time DESC
""", min_reports, min_resilience, previous_timestamp)
if cracker_ids is None:
returnValue([])
# Now look for conditions (c) and (d)
result = []
for c in cracker_ids:
cracker_id = c[0]
if c[1] in latest_added_hosts:
logging.debug("Skipping {}, just reported by client".format(c[1]))
continue
cracker = yield Cracker.find(cracker_id)
if cracker is None:
continue
logging.debug("Examining cracker:")
logging.debug(cracker)
reports = yield cracker.reports.get(orderby="first_report_time ASC")
#logging.debug("reports:")
#for r in reports:
# logging.debug(" "+str(r))
logging.debug("r[m-1].first, prev: {}, {}".format(reports[min_reports-1].first_report_time, previous_timestamp))
if (len(reports)>=min_reports and
reports[min_reports-1].first_report_time >= previous_timestamp):
# condition (c) satisfied
logging.debug("c")
result.append(cracker.ip_address)
else:
logging.debug("checking (d)...")
satisfied = False
for report in reports:
#logging.debug(" "+str(report))
if (not satisfied and
report.latest_report_time>=previous_timestamp and
report.latest_report_time-cracker.first_time>=min_resilience):
logging.debug(" d1")
satisfied = True
if (report.latest_report_time<=previous_timestamp and
report.latest_report_time-cracker.first_time>=min_resilience):
logging.debug(" d2 failed")
satisfied = False
break
if satisfied:
logging.debug("Appending {}".format(cracker.ip_address))
result.append(cracker.ip_address)
else:
logging.debug(" skipping")
if len(result)>=max_crackers:
break
if len(result) < max_crackers:
# Add results from legacy server
extras = yield Legacy.find(where=["retrieved_time>?", previous_timestamp],
orderby="retrieved_time DESC", limit=max_crackers-len(result))
result = result + [extra.ip_address for extra in extras]
logging.debug("Returning {} hosts".format(len(result)))
returnValue(result)
