def xmlrpc_test_bulk_insert(self, count, same_crackers = False):
if same_crackers and len(self._crackers) < count:
logging.debug("Filling static crackers from {} to {}".format(len(self._crackers), count))
for i in xrange(len(self._crackers), count):
self._crackers.append(self.random_ip_address())
for i in xrange(count):
reporter = self.random_ip_address()
if same_crackers:
cracker_ip = self._crackers[i]
else:
cracker_ip = self.random_ip_address()
logging.debug("Adding report for {} from {}".format(cracker_ip, reporter))
yield utils.wait_and_lock_host(cracker_ip)
cracker = yield Cracker.find(where=['ip_address=?', cracker_ip], limit=1)
if cracker is None:
now = time.time()
cracker = Cracker(ip_address=cracker_ip, first_time=now, latest_time=now, total_reports=0, current_reports=0)
yield cracker.save()
yield controllers.add_report_to_cracker(cracker, reporter)
utils.unlock_host(cracker_ip)
logging.debug("Done adding report for {} from {}".format(cracker_ip,reporter))
total = yield Cracker.count()
total_reports = yield Report.count()
returnValue((total,total_reports))
def xmlrpc_add_hosts(self, request, hosts):
try:
x_real_ip = request.received_headers.get("X-Real-IP")
remote_ip = x_real_ip or request.getClientIP()
logging.info("add_hosts({}) from {}".format(hosts, remote_ip))
for cracker_ip in hosts:
if not self.is_valid_ip_address(cracker_ip):
logging.warning("Illegal host ip address {} from {}".format(cracker_ip, remote_ip))
raise xmlrpc.Fault(101, "Illegal IP address \"{}\".".format(cracker_ip))
logging.debug("Adding report for {} from {}".format(cracker_ip, remote_ip))
yield utils.wait_and_lock_host(cracker_ip)
try:
cracker = yield Cracker.find(where=['ip_address=?', cracker_ip], limit=1)
if cracker is None:
now = time.time()
cracker = Cracker(ip_address=cracker_ip, first_time=now,
latest_time=now, resiliency=0, total_reports=0, current_reports=0)
yield cracker.save()
yield controllers.add_report_to_cracker(cracker, remote_ip)
finally:
utils.unlock_host(cracker_ip)
logging.debug("Done adding report for {} from {}".format(cracker_ip,remote_ip))
except xmlrpc.Fault, e:
raise e
def xmlrpc_test_bulk_insert(self, count, same_crackers = False, when=None):
if same_crackers and len(self._crackers) < count:
logging.debug("Filling static crackers from {} to {}".format(len(self._crackers), count))
for i in xrange(len(self._crackers), count):
self._crackers.append(self.random_ip_address())
if when is None:
when = time.time()
for i in xrange(count):
reporter = self.random_ip_address()
if same_crackers:
cracker_ip = self._crackers[i]
else:
cracker_ip = self.random_ip_address()
logging.debug("Adding report for {} from {} at {}".format(cracker_ip, reporter, when))
yield utils.wait_and_lock_host(cracker_ip)
cracker = yield Cracker.find(where=['ip_address=?', cracker_ip], limit=1)
if cracker is None:
cracker = Cracker(ip_address=cracker_ip, first_time=when, latest_time=when, total_reports=0, current_reports=0)
yield cracker.save()
yield controllers.add_report_to_cracker(cracker, reporter, when=when)
utils.unlock_host(cracker_ip)
logging.debug("Done adding report for {} from {}".format(cracker_ip,reporter))
total = yield Cracker.count()
total_reports = yield Report.count()
returnValue((total,total_reports))
def handle_report_from_client(client_ip, timestamp, hosts):
for cracker_ip in hosts:
if not utils.is_valid_ip_address(cracker_ip):
logging.warning("Illegal host ip address {} from {}".format(cracker_ip, client_ip))
raise Exception("Illegal IP address \"{}\".".format(cracker_ip))
logging.debug("Adding report for {} from {}".format(cracker_ip, client_ip))
yield utils.wait_and_lock_host(cracker_ip)
try:
cracker = yield Cracker.find(where=['ip_address=?', cracker_ip], limit=1)
if cracker is None:
cracker = Cracker(ip_address=cracker_ip, first_time=timestamp,
latest_time=timestamp, resiliency=0, total_reports=0, current_reports=0)
yield cracker.save()
yield add_report_to_cracker(cracker, client_ip, when=timestamp)
finally:
utils.unlock_host(cracker_ip)
logging.debug("Done adding report for {} from {}".format(cracker_ip,client_ip))
def xmlrpc_add_hosts(self, request, hosts):
try:
x_real_ip = request.received_headers.get("X-Real-IP")
remote_ip = x_real_ip or request.getClientIP()
logging.info("add_hosts({}) from {}".format(hosts, remote_ip))
for cracker_ip in hosts:
if not self.is_valid_ip_address(cracker_ip):
logging.warning(
"Illegal host ip address {} from {}".format(
cracker_ip, remote_ip))
raise xmlrpc.Fault(
101, "Illegal IP address \"{}\".".format(cracker_ip))
logging.debug("Adding report for {} from {}".format(
cracker_ip, remote_ip))
yield utils.wait_and_lock_host(cracker_ip)
try:
cracker = yield Cracker.find(
where=['ip_address=?', cracker_ip], limit=1)
if cracker is None:
now = time.time()
cracker = Cracker(ip_address=cracker_ip,
first_time=now,
latest_time=now,
resiliency=0,
total_reports=0,
current_reports=0)
yield cracker.save()
yield controllers.add_report_to_cracker(cracker, remote_ip)
finally:
utils.unlock_host(cracker_ip)
logging.debug("Done adding report for {} from {}".format(
cracker_ip, remote_ip))
except xmlrpc.Fault, e:
raise e
def get_qualifying_crackers(min_reports, min_resilience, previous_timestamp,
max_crackers, latest_added_hosts):
# Thank to Anne Bezemer for the algorithm in this function.
# See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=622697
# This query takes care of conditions (a) and (b)
# cracker_ids = yield database.runGetPossibleQualifyingCrackerQuery(min_reports, min_resilience, previous_timestamp)
cracker_ids = yield database.run_query("""
SELECT DISTINCT c.id, c.ip_address
FROM crackers c
WHERE (c.current_reports >= ?)
AND (c.resiliency >= ?)
AND (c.latest_time >= ?)
ORDER BY c.first_time DESC
""", min_reports, min_resilience, previous_timestamp)
if cracker_ids is None:
returnValue([])
# Now look for conditions (c) and (d)
result = []
for c in cracker_ids:
cracker_id = c[0]
if c[1] in latest_added_hosts:
logging.debug("Skipping {}, just reported by client".format(c[1]))
continue
cracker = yield Cracker.find(cracker_id)
if cracker is None:
continue
logging.debug("Examining cracker:")
logging.debug(cracker)
reports = yield cracker.reports.get(orderby="first_report_time ASC")
#logging.debug("reports:")
#for r in reports:
# logging.debug(" "+str(r))
logging.debug("r[m-1].first, prev: {}, {}".format(reports[min_reports-1].first_report_time, previous_timestamp))
if (len(reports)>=min_reports and
reports[min_reports-1].first_report_time >= previous_timestamp):
# condition (c) satisfied
logging.debug("c")
result.append(cracker.ip_address)
else:
logging.debug("checking (d)...")
satisfied = False
for report in reports:
#logging.debug(" "+str(report))
if (not satisfied and
report.latest_report_time>=previous_timestamp and
report.latest_report_time-cracker.first_time>=min_resilience):
logging.debug(" d1")
satisfied = True
if (report.latest_report_time<=previous_timestamp and
report.latest_report_time-cracker.first_time>=min_resilience):
logging.debug(" d2 failed")
satisfied = False
break
if satisfied:
logging.debug("Appending {}".format(cracker.ip_address))
result.append(cracker.ip_address)
else:
logging.debug(" skipping")
if len(result)>=max_crackers:
break
if len(result) < max_crackers:
# Add results from legacy server
extras = yield Legacy.find(where=["retrieved_time>?", previous_timestamp],
orderby="retrieved_time DESC", limit=max_crackers-len(result))
result = result + [extra.ip_address for extra in extras]
logging.debug("Returning {} hosts".format(len(result)))
returnValue(result)
def get_cracker(ip_address):
return Cracker.find(where=["ip_address=?",ip_address], limit=1)
def xmlrpc_list_all_hosts(self):
crackers = yield Cracker.all()
returnValue([c.ip_address for c in crackers])
def xmlrpc_list_all_hosts(self):
crackers = yield Cracker.all()
returnValue([c.ip_address for c in crackers])
