def build(self): with pebble.ProcessPool(conf.get("workers")) as pool: instance = pool.map(resolv, sorted(self.threats.keys()), timeout=conf.get("queryTimeout")) iterator = instance.result() for index, element in enumerate(sorted(self.threats.keys()), start=1): try: self.threats.update({element: next(iterator)}) except: self.threats.update({element: []}) if index % round(len(self.threats) / 100) == 0 or index == len( self.threats): log.info( str("{}% done... ({}/{})").format( int(100 / len(self.threats) * index), index, len(self.threats))) try: next(iterator) log.warning( "Process pool is not empty (iterator object is still iterable)" ) except StopIteration: pass log.info( str("[!] BUILD part 1/1 done ({} threats)").format( len(self.threats))) return 0
def fetch(self): for element in conf.get("feeds"): if element not in self.feeds.keys(): log.info(str("Subscribing to '{}'").format(element)) module = getattr( importlib.import_module(str("feeds.{}").format(element)), element) self.feeds.update({ element: module(log, conf.get("groupRange"), conf.get("queryUserAgent"), conf.get("queryTimeout")) }) for element in sorted(self.feeds.keys()): if element not in conf.get("feeds"): log.warning(str("Unsubscribing from '{}'").format(element)) self.feeds.pop(element) for element in self.feeds.values(): self.threats.update(element.refresh()) if len(self.threats.keys()): db.engine.dispose() log.info( str("[!] FETCH part 1/1 done ({} threats)").format( len(self.threats))) return 0
def start(self): while self.idx >= 0: try: conf.reload() except Exception as error: log.critical(error) return 1 if os.path.exists(conf.get("monitor")): self.watch(conf.get("monitor")) else: log.error(str("[!] Could not find '{}' to monitor network events").format(conf.get("monitor"))) time.sleep(1) return 0
def clean(self): for element in conf.get("exemptions"): if re.search("[.][a-z]+$", element): db.session_append( db.models.exemptions(ts=int(time.time()), domain=element.lower())) else: db.session_append( db.models.exemptions(ts=int(time.time()), ipaddr=element.lower())) try: db.models.exemptions().metadata.drop_all(db.engine) db.models.exemptions().metadata.create_all(db.engine) db.session_commit() except Exception as error: log.error(error) log.info( str("[!] CLEAN part 1/2 done ({} threats)").format( len(self.threats))) for row in db.session.query(db.models.exemptions).order_by( db.models.exemptions.id).yield_per(db.chunk): regex = [] if row.domain: pattern = row.domain if row.ipaddr: pattern = row.ipaddr for index, node in enumerate(reversed(pattern.split("."))): if len(node) != 0 and index == 0 and node == "tld": regex.append("((co|com|net|org|edu|gov)[.])?([a-z]+)") if len(node) != 0 and index == 0 and node != "tld": regex.append(str("({})").format(node)) if len(node) != 0 and index != 0 and node != "*?": regex.append(str("({})[.]").format(node)) if len(node) != 0 and index != 0 and node == "*?": regex.append("(([^.]+)[.])?") for element, revlookup in sorted(self.threats.items()): for record in revlookup: if re.search( str("^{}$").format(str().join(reversed(regex))), record): log.warning( str("Ignoring '{}' -> '{}' -> '{}'").format( element, record, str().join(reversed(regex)))) self.threats.pop(element) if element in self.threats: if re.search( str("^{}$").format(str().join(reversed(regex))), element): log.warning( str("Ignoring '{}' -> '{}'").format( element, str().join(reversed(regex)))) self.threats.pop(element) log.info( str("[!] CLEAN part 2/2 done ({} threats)").format( len(self.threats))) return 0
def write(self): with open(conf.get("publish"), "w+") as fp: for element, revlookup in sorted(self.threats.items()): fp.write( str("{};{}").format(element, str(",").join(revlookup)) + "\n") log.info( str("[!] WRITE part 1/1 done ({} threats)").format( len(self.threats))) return 0
def refresh(self, counter): if counter == 0 and conf.get("mode") in [ "server", "standalone", "client" ]: log.info(str("[!] Starting RESET...")) self.reset() if counter >= 0 and conf.get("mode") in [ "server", "standalone", "client" ]: log.info(str("[!] Starting FETCH...")) self.fetch() if counter >= 0 and conf.get("mode") in ["server", "standalone"]: log.info(str("[!] Starting BUILD...")) self.build() if counter >= 0 and conf.get("mode") in ["server", "standalone"]: log.info(str("[!] Starting CLEAN...")) self.clean() if counter >= 0 and conf.get("mode") in ["server"]: log.info(str("[!] Starting WRITE...")) self.write() if counter >= 0 and conf.get("mode") in [ "server", "standalone", "client" ]: log.info(str("[!] Starting MERGE...")) self.merge() return 0
def start(self): counter = 0 timestamp = 0 while timestamp >= 0: try: conf.reload() except Exception as error: log.critical(error) return 1 if int(time.time()) - timestamp >= conf.get("refreshDelay"): timestamp = int(time.time()) self.refresh(counter) counter = counter + 1 else: time.sleep(60) return 0