def xmlrpc_test_bulk_insert(self, count, same_crackers = False):
if same_crackers and len(self._crackers) < count:
logging.debug("Filling static crackers from {} to {}".format(len(self._crackers), count))
for i in xrange(len(self._crackers), count):
self._crackers.append(self.random_ip_address())
for i in xrange(count):
reporter = self.random_ip_address()
if same_crackers:
cracker_ip = self._crackers[i]
else:
cracker_ip = self.random_ip_address()
logging.debug("Adding report for {} from {}".format(cracker_ip, reporter))
yield utils.wait_and_lock_host(cracker_ip)
cracker = yield Cracker.find(where=['ip_address=?', cracker_ip], limit=1)
if cracker is None:
now = time.time()
cracker = Cracker(ip_address=cracker_ip, first_time=now, latest_time=now, total_reports=0, current_reports=0)
yield cracker.save()
yield controllers.add_report_to_cracker(cracker, reporter)
utils.unlock_host(cracker_ip)
logging.debug("Done adding report for {} from {}".format(cracker_ip,reporter))
total = yield Cracker.count()
total_reports = yield Report.count()
returnValue((total,total_reports))
def xmlrpc_add_hosts(self, request, hosts):
try:
x_real_ip = request.received_headers.get("X-Real-IP")
remote_ip = x_real_ip or request.getClientIP()
logging.info("add_hosts({}) from {}".format(hosts, remote_ip))
for cracker_ip in hosts:
if not self.is_valid_ip_address(cracker_ip):
logging.warning("Illegal host ip address {} from {}".format(cracker_ip, remote_ip))
raise xmlrpc.Fault(101, "Illegal IP address \"{}\".".format(cracker_ip))
logging.debug("Adding report for {} from {}".format(cracker_ip, remote_ip))
yield utils.wait_and_lock_host(cracker_ip)
try:
cracker = yield Cracker.find(where=['ip_address=?', cracker_ip], limit=1)
if cracker is None:
now = time.time()
cracker = Cracker(ip_address=cracker_ip, first_time=now,
latest_time=now, resiliency=0, total_reports=0, current_reports=0)
yield cracker.save()
yield controllers.add_report_to_cracker(cracker, remote_ip)
finally:
utils.unlock_host(cracker_ip)
logging.debug("Done adding report for {} from {}".format(cracker_ip,remote_ip))
except xmlrpc.Fault, e:
raise e
def xmlrpc_test_bulk_insert(self, count, same_crackers = False, when=None):
if same_crackers and len(self._crackers) < count:
logging.debug("Filling static crackers from {} to {}".format(len(self._crackers), count))
for i in xrange(len(self._crackers), count):
self._crackers.append(self.random_ip_address())
if when is None:
when = time.time()
for i in xrange(count):
reporter = self.random_ip_address()
if same_crackers:
cracker_ip = self._crackers[i]
else:
cracker_ip = self.random_ip_address()
logging.debug("Adding report for {} from {} at {}".format(cracker_ip, reporter, when))
yield utils.wait_and_lock_host(cracker_ip)
cracker = yield Cracker.find(where=['ip_address=?', cracker_ip], limit=1)
if cracker is None:
cracker = Cracker(ip_address=cracker_ip, first_time=when, latest_time=when, total_reports=0, current_reports=0)
yield cracker.save()
yield controllers.add_report_to_cracker(cracker, reporter, when=when)
utils.unlock_host(cracker_ip)
logging.debug("Done adding report for {} from {}".format(cracker_ip,reporter))
total = yield Cracker.count()
total_reports = yield Report.count()
returnValue((total,total_reports))
def perform_maintenance(limit = None, legacy_limit = None):
logging.info("Starting maintenance job...")
if limit is None:
now = time.time()
limit = now - config.expiry_days * 24 * 3600
if legacy_limit is None:
now = time.time()
legacy_limit = now - config.legacy_expiry_days * 24 * 3600
reports_deleted = 0
crackers_deleted = 0
legacy_deleted = 0
batch_size = 1000
while True:
old_reports = yield Report.find(where=["latest_report_time<?", limit], limit=batch_size)
if len(old_reports) == 0:
break
logging.debug("Removing batch of {} old reports".format(len(old_reports)))
for report in old_reports:
cracker = yield report.cracker.get()
yield utils.wait_and_lock_host(cracker.ip_address)
try:
logging.debug("Maintenance: removing report from {} for cracker {}".format(report.ip_address, cracker.ip_address))
yield report.cracker.clear()
yield report.delete()
reports_deleted += 1
current_reports = yield cracker.reports.get(group='ip_address')
cracker.current_reports = len(current_reports)
yield cracker.save()
if cracker.current_reports == 0:
logging.debug("Maintenance: removing cracker {}".format(cracker.ip_address))
yield cracker.delete()
crackers_deleted += 1
finally:
utils.unlock_host(cracker.ip_address)
logging.debug("Maintenance on report from {} for cracker {} done".format(report.ip_address, cracker.ip_address))
legacy_reports = yield Legacy.find(where=["retrieved_time<?", legacy_limit])
if legacy_reports is not None:
for legacy in legacy_reports:
yield legacy.delete()
legacy_deleted += 1
logging.info("Done maintenance job")
logging.info("Expired {} reports and {} hosts, plus {} hosts from the legacy list".format(reports_deleted, crackers_deleted, legacy_deleted))
returnValue(0)
def handle_report_from_client(client_ip, timestamp, hosts):
for cracker_ip in hosts:
if not utils.is_valid_ip_address(cracker_ip):
logging.warning("Illegal host ip address {} from {}".format(cracker_ip, client_ip))
raise Exception("Illegal IP address \"{}\".".format(cracker_ip))
logging.debug("Adding report for {} from {}".format(cracker_ip, client_ip))
yield utils.wait_and_lock_host(cracker_ip)
try:
cracker = yield Cracker.find(where=['ip_address=?', cracker_ip], limit=1)
if cracker is None:
cracker = Cracker(ip_address=cracker_ip, first_time=timestamp,
latest_time=timestamp, resiliency=0, total_reports=0, current_reports=0)
yield cracker.save()
yield add_report_to_cracker(cracker, client_ip, when=timestamp)
finally:
utils.unlock_host(cracker_ip)
logging.debug("Done adding report for {} from {}".format(cracker_ip,client_ip))
def xmlrpc_add_hosts(self, request, hosts):
try:
x_real_ip = request.received_headers.get("X-Real-IP")
remote_ip = x_real_ip or request.getClientIP()
logging.info("add_hosts({}) from {}".format(hosts, remote_ip))
for cracker_ip in hosts:
if not self.is_valid_ip_address(cracker_ip):
logging.warning(
"Illegal host ip address {} from {}".format(
cracker_ip, remote_ip))
raise xmlrpc.Fault(
101, "Illegal IP address \"{}\".".format(cracker_ip))
logging.debug("Adding report for {} from {}".format(
cracker_ip, remote_ip))
yield utils.wait_and_lock_host(cracker_ip)
try:
cracker = yield Cracker.find(
where=['ip_address=?', cracker_ip], limit=1)
if cracker is None:
now = time.time()
cracker = Cracker(ip_address=cracker_ip,
first_time=now,
latest_time=now,
resiliency=0,
total_reports=0,
current_reports=0)
yield cracker.save()
yield controllers.add_report_to_cracker(cracker, remote_ip)
finally:
utils.unlock_host(cracker_ip)
logging.debug("Done adding report for {} from {}".format(
cracker_ip, remote_ip))
except xmlrpc.Fault, e:
raise e