def xmlrpc_test_bulk_insert(self, count, same_crackers = False):
        if same_crackers and len(self._crackers) < count:
            logging.debug("Filling static crackers from {} to {}".format(len(self._crackers), count))
            for i in xrange(len(self._crackers), count):
                self._crackers.append(self.random_ip_address())

        for i in xrange(count):
            reporter = self.random_ip_address()
            if same_crackers:
                cracker_ip = self._crackers[i]
            else:
                cracker_ip = self.random_ip_address()

            logging.debug("Adding report for {} from {}".format(cracker_ip, reporter))

            yield utils.wait_and_lock_host(cracker_ip)
            
            cracker = yield Cracker.find(where=['ip_address=?', cracker_ip], limit=1)
            if cracker is None:
                now = time.time()
                cracker = Cracker(ip_address=cracker_ip, first_time=now, latest_time=now, total_reports=0, current_reports=0)
                yield cracker.save()
            yield controllers.add_report_to_cracker(cracker, reporter)
            
            utils.unlock_host(cracker_ip)
            logging.debug("Done adding report for {} from {}".format(cracker_ip,reporter))
        total = yield Cracker.count()
        total_reports = yield Report.count()
        returnValue((total,total_reports))
    def xmlrpc_add_hosts(self, request, hosts):
        try:
            x_real_ip = request.received_headers.get("X-Real-IP")
            remote_ip = x_real_ip or request.getClientIP()

            logging.info("add_hosts({}) from {}".format(hosts, remote_ip))
            for cracker_ip in hosts:
                if not self.is_valid_ip_address(cracker_ip):
                    logging.warning("Illegal host ip address {} from {}".format(cracker_ip, remote_ip))
                    raise xmlrpc.Fault(101, "Illegal IP address \"{}\".".format(cracker_ip))

                logging.debug("Adding report for {} from {}".format(cracker_ip, remote_ip))
                yield utils.wait_and_lock_host(cracker_ip)
                try:
                    cracker = yield Cracker.find(where=['ip_address=?', cracker_ip], limit=1)
                    if cracker is None:
                        now = time.time()
                        cracker = Cracker(ip_address=cracker_ip, first_time=now,
                            latest_time=now, resiliency=0, total_reports=0, current_reports=0)
                        yield cracker.save()
                    yield controllers.add_report_to_cracker(cracker, remote_ip)
                finally:
                    utils.unlock_host(cracker_ip)
                logging.debug("Done adding report for {} from {}".format(cracker_ip,remote_ip))
        except xmlrpc.Fault, e:
            raise e
    def xmlrpc_test_bulk_insert(self, count, same_crackers = False, when=None):
        if same_crackers and len(self._crackers) < count:
            logging.debug("Filling static crackers from {} to {}".format(len(self._crackers), count))
            for i in xrange(len(self._crackers), count):
                self._crackers.append(self.random_ip_address())

        if when is None:
            when = time.time()

        for i in xrange(count):
            reporter = self.random_ip_address()
            if same_crackers:
                cracker_ip = self._crackers[i]
            else:
                cracker_ip = self.random_ip_address()

            logging.debug("Adding report for {} from {} at {}".format(cracker_ip, reporter, when))

            yield utils.wait_and_lock_host(cracker_ip)
            
            cracker = yield Cracker.find(where=['ip_address=?', cracker_ip], limit=1)
            if cracker is None:
                cracker = Cracker(ip_address=cracker_ip, first_time=when, latest_time=when, total_reports=0, current_reports=0)
                yield cracker.save()
            yield controllers.add_report_to_cracker(cracker, reporter, when=when)
            
            utils.unlock_host(cracker_ip)
            logging.debug("Done adding report for {} from {}".format(cracker_ip,reporter))
        total = yield Cracker.count()
        total_reports = yield Report.count()
        returnValue((total,total_reports))
Exemple #4
0
def perform_maintenance(limit = None, legacy_limit = None):
    logging.info("Starting maintenance job...")
    
    if limit is None:
        now = time.time()
        limit = now - config.expiry_days * 24 * 3600

    if legacy_limit is None:
        now = time.time()
        legacy_limit = now - config.legacy_expiry_days * 24 * 3600

    reports_deleted = 0
    crackers_deleted = 0
    legacy_deleted = 0

    batch_size = 1000
  
    while True:
        old_reports = yield Report.find(where=["latest_report_time<?", limit], limit=batch_size)
        if len(old_reports) == 0:
            break
        logging.debug("Removing batch of {} old reports".format(len(old_reports)))
        for report in old_reports:
            cracker = yield report.cracker.get()
            yield utils.wait_and_lock_host(cracker.ip_address)
            try:
                logging.debug("Maintenance: removing report from {} for cracker {}".format(report.ip_address, cracker.ip_address))
                yield report.cracker.clear()
                yield report.delete()
                reports_deleted += 1

                current_reports = yield cracker.reports.get(group='ip_address')
                cracker.current_reports = len(current_reports)
                yield cracker.save()

                if cracker.current_reports == 0:
                    logging.debug("Maintenance: removing cracker {}".format(cracker.ip_address))
                    yield cracker.delete()
                    crackers_deleted += 1
            finally:
                utils.unlock_host(cracker.ip_address)
            logging.debug("Maintenance on report from {} for cracker {} done".format(report.ip_address, cracker.ip_address))

    legacy_reports = yield Legacy.find(where=["retrieved_time<?", legacy_limit])
    if legacy_reports is not None:
        for legacy in legacy_reports:
            yield legacy.delete()
            legacy_deleted += 1

    logging.info("Done maintenance job")
    logging.info("Expired {} reports and {} hosts, plus {} hosts from the legacy list".format(reports_deleted, crackers_deleted, legacy_deleted))
    returnValue(0)
Exemple #5
0
def handle_report_from_client(client_ip, timestamp, hosts):
    for cracker_ip in hosts:
        if not utils.is_valid_ip_address(cracker_ip):
            logging.warning("Illegal host ip address {} from {}".format(cracker_ip, client_ip))
            raise Exception("Illegal IP address \"{}\".".format(cracker_ip))

        logging.debug("Adding report for {} from {}".format(cracker_ip, client_ip))
        yield utils.wait_and_lock_host(cracker_ip)
        try:
            cracker = yield Cracker.find(where=['ip_address=?', cracker_ip], limit=1)
            if cracker is None:
                cracker = Cracker(ip_address=cracker_ip, first_time=timestamp,
                    latest_time=timestamp, resiliency=0, total_reports=0, current_reports=0)
                yield cracker.save()
            yield add_report_to_cracker(cracker, client_ip, when=timestamp)
        finally:
            utils.unlock_host(cracker_ip)
        logging.debug("Done adding report for {} from {}".format(cracker_ip,client_ip))
    def xmlrpc_add_hosts(self, request, hosts):
        try:
            x_real_ip = request.received_headers.get("X-Real-IP")
            remote_ip = x_real_ip or request.getClientIP()

            logging.info("add_hosts({}) from {}".format(hosts, remote_ip))
            for cracker_ip in hosts:
                if not self.is_valid_ip_address(cracker_ip):
                    logging.warning(
                        "Illegal host ip address {} from {}".format(
                            cracker_ip, remote_ip))
                    raise xmlrpc.Fault(
                        101, "Illegal IP address \"{}\".".format(cracker_ip))

                logging.debug("Adding report for {} from {}".format(
                    cracker_ip, remote_ip))
                yield utils.wait_and_lock_host(cracker_ip)
                try:
                    cracker = yield Cracker.find(
                        where=['ip_address=?', cracker_ip], limit=1)
                    if cracker is None:
                        now = time.time()
                        cracker = Cracker(ip_address=cracker_ip,
                                          first_time=now,
                                          latest_time=now,
                                          resiliency=0,
                                          total_reports=0,
                                          current_reports=0)
                        yield cracker.save()
                    yield controllers.add_report_to_cracker(cracker, remote_ip)
                finally:
                    utils.unlock_host(cracker_ip)
                logging.debug("Done adding report for {} from {}".format(
                    cracker_ip, remote_ip))
        except xmlrpc.Fault, e:
            raise e