def download_from_legacy_server(): if config.legacy_server is None or config.legacy_server == "": returnValue(0) logging.info("Downloading hosts from legacy server...") rows = yield database.run_query('SELECT `value` FROM info WHERE `key`="last_legacy_sync"') last_legacy_sync_time = int(rows[0][0]) try: server = yield deferToThread(xmlrpclib.ServerProxy, config.legacy_server) response = yield deferToThread(server.get_new_hosts, last_legacy_sync_time, config.legacy_threshold, [], config.legacy_resiliency) try: last_legacy_sync_time = int(response["timestamp"]) except: logging.ERROR("Illegal timestamp {} from legacy server".format(response["timestamp"])) #Registry.DBPOOL.runOperation('UPDATE info SET `value`=%s WHERE `key`="last_legacy_sync"', (str(last_legacy_sync_time),)) database.run_operation('UPDATE info SET `value`=? WHERE `key`="last_legacy_sync"', str(last_legacy_sync_time)) now = time.time() logging.debug("Got {} hosts from legacy server".format(len(response["hosts"]))) for host in response["hosts"]: legacy = yield Legacy.find(where=["ip_address=?",host], limit=1) if legacy is None: logging.debug("New host from legacy server: {}".format(host)) legacy = Legacy(ip_address=host, retrieved_time=now) else: logging.debug("Known host from legacy server: {}".format(host)) legacy.retrieved_time = now yield legacy.save() except Exception, e: logging.error("Error retrieving info from legacy server: {}".format(e))
def perform_maintenance(limit = None, legacy_limit = None): logging.info("Starting maintenance job...") if limit is None: now = time.time() limit = now - config.expiry_days * 24 * 3600 if legacy_limit is None: now = time.time() legacy_limit = now - config.legacy_expiry_days * 24 * 3600 reports_deleted = 0 crackers_deleted = 0 legacy_deleted = 0 batch_size = 1000 while True: old_reports = yield Report.find(where=["latest_report_time<?", limit], limit=batch_size) if len(old_reports) == 0: break logging.debug("Removing batch of {} old reports".format(len(old_reports))) for report in old_reports: cracker = yield report.cracker.get() yield utils.wait_and_lock_host(cracker.ip_address) try: logging.debug("Maintenance: removing report from {} for cracker {}".format(report.ip_address, cracker.ip_address)) yield report.cracker.clear() yield report.delete() reports_deleted += 1 current_reports = yield cracker.reports.get(group='ip_address') cracker.current_reports = len(current_reports) yield cracker.save() if cracker.current_reports == 0: logging.debug("Maintenance: removing cracker {}".format(cracker.ip_address)) yield cracker.delete() crackers_deleted += 1 finally: utils.unlock_host(cracker.ip_address) logging.debug("Maintenance on report from {} for cracker {} done".format(report.ip_address, cracker.ip_address)) legacy_reports = yield Legacy.find(where=["retrieved_time<?", legacy_limit]) if legacy_reports is not None: for legacy in legacy_reports: yield legacy.delete() legacy_deleted += 1 logging.info("Done maintenance job") logging.info("Expired {} reports and {} hosts, plus {} hosts from the legacy list".format(reports_deleted, crackers_deleted, legacy_deleted)) returnValue(0)
def get_qualifying_crackers(min_reports, min_resilience, previous_timestamp, max_crackers, latest_added_hosts): # Thank to Anne Bezemer for the algorithm in this function. # See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=622697 # This query takes care of conditions (a) and (b) # cracker_ids = yield database.runGetPossibleQualifyingCrackerQuery(min_reports, min_resilience, previous_timestamp) cracker_ids = yield database.run_query(""" SELECT DISTINCT c.id, c.ip_address FROM crackers c WHERE (c.current_reports >= ?) AND (c.resiliency >= ?) AND (c.latest_time >= ?) ORDER BY c.first_time DESC """, min_reports, min_resilience, previous_timestamp) if cracker_ids is None: returnValue([]) # Now look for conditions (c) and (d) result = [] for c in cracker_ids: cracker_id = c[0] if c[1] in latest_added_hosts: logging.debug("Skipping {}, just reported by client".format(c[1])) continue cracker = yield Cracker.find(cracker_id) if cracker is None: continue logging.debug("Examining cracker:") logging.debug(cracker) reports = yield cracker.reports.get(orderby="first_report_time ASC") #logging.debug("reports:") #for r in reports: # logging.debug(" "+str(r)) logging.debug("r[m-1].first, prev: {}, {}".format(reports[min_reports-1].first_report_time, previous_timestamp)) if (len(reports)>=min_reports and reports[min_reports-1].first_report_time >= previous_timestamp): # condition (c) satisfied logging.debug("c") result.append(cracker.ip_address) else: logging.debug("checking (d)...") satisfied = False for report in reports: #logging.debug(" "+str(report)) if (not satisfied and report.latest_report_time>=previous_timestamp and report.latest_report_time-cracker.first_time>=min_resilience): logging.debug(" d1") satisfied = True if (report.latest_report_time<=previous_timestamp and report.latest_report_time-cracker.first_time>=min_resilience): logging.debug(" d2 failed") satisfied = False break if satisfied: logging.debug("Appending {}".format(cracker.ip_address)) result.append(cracker.ip_address) else: logging.debug(" skipping") if len(result)>=max_crackers: break if len(result) < max_crackers: # Add results from legacy server extras = yield Legacy.find(where=["retrieved_time>?", previous_timestamp], orderby="retrieved_time DESC", limit=max_crackers-len(result)) result = result + [extra.ip_address for extra in extras] logging.debug("Returning {} hosts".format(len(result))) returnValue(result)