def fetch(self):
for element in conf.get("feeds"):
if element not in self.feeds.keys():
log.info(str("Subscribing to '{}'").format(element))
module = getattr(
importlib.import_module(str("feeds.{}").format(element)),
element)
self.feeds.update({
element:
module(log, conf.get("groupRange"),
conf.get("queryUserAgent"),
conf.get("queryTimeout"))
})
for element in sorted(self.feeds.keys()):
if element not in conf.get("feeds"):
log.warning(str("Unsubscribing from '{}'").format(element))
self.feeds.pop(element)
for element in self.feeds.values():
self.threats.update(element.refresh())
if len(self.threats.keys()):
db.engine.dispose()
log.info(
str("[!] FETCH part 1/1 done ({} threats)").format(
len(self.threats)))
return 0
def build(self):
with pebble.ProcessPool(conf.get("workers")) as pool:
instance = pool.map(resolv,
sorted(self.threats.keys()),
timeout=conf.get("queryTimeout"))
iterator = instance.result()
for index, element in enumerate(sorted(self.threats.keys()),
start=1):
try:
self.threats.update({element: next(iterator)})
except:
self.threats.update({element: []})
if index % round(len(self.threats) / 100) == 0 or index == len(
self.threats):
log.info(
str("{}% done... ({}/{})").format(
int(100 / len(self.threats) * index), index,
len(self.threats)))
try:
next(iterator)
log.warning(
"Process pool is not empty (iterator object is still iterable)"
)
except StopIteration:
pass
log.info(
str("[!] BUILD part 1/1 done ({} threats)").format(
len(self.threats)))
return 0
def clean(self):
for element in conf.get("exemptions"):
if re.search("[.][a-z]+$", element):
db.session_append(
db.models.exemptions(ts=int(time.time()),
domain=element.lower()))
else:
db.session_append(
db.models.exemptions(ts=int(time.time()),
ipaddr=element.lower()))
try:
db.models.exemptions().metadata.drop_all(db.engine)
db.models.exemptions().metadata.create_all(db.engine)
db.session_commit()
except Exception as error:
log.error(error)
log.info(
str("[!] CLEAN part 1/2 done ({} threats)").format(
len(self.threats)))
for row in db.session.query(db.models.exemptions).order_by(
db.models.exemptions.id).yield_per(db.chunk):
regex = []
if row.domain:
pattern = row.domain
if row.ipaddr:
pattern = row.ipaddr
for index, node in enumerate(reversed(pattern.split("."))):
if len(node) != 0 and index == 0 and node == "tld":
regex.append("((co|com|net|org|edu|gov)[.])?([a-z]+)")
if len(node) != 0 and index == 0 and node != "tld":
regex.append(str("({})").format(node))
if len(node) != 0 and index != 0 and node != "*?":
regex.append(str("({})[.]").format(node))
if len(node) != 0 and index != 0 and node == "*?":
regex.append("(([^.]+)[.])?")
for element, revlookup in sorted(self.threats.items()):
for record in revlookup:
if re.search(
str("^{}$").format(str().join(reversed(regex))),
record):
log.warning(
str("Ignoring '{}' -> '{}' -> '{}'").format(
element, record,
str().join(reversed(regex))))
self.threats.pop(element)
if element in self.threats:
if re.search(
str("^{}$").format(str().join(reversed(regex))),
element):
log.warning(
str("Ignoring '{}' -> '{}'").format(
element,
str().join(reversed(regex))))
self.threats.pop(element)
log.info(
str("[!] CLEAN part 2/2 done ({} threats)").format(
len(self.threats)))
return 0
def watch(self, file):
self.idx = 0
if not self.ino:
self.idx = os.stat(file).st_size
self.ino = os.stat(file).st_ino
log.info(str("[!] Subscribing to file '{}'").format(file))
with open(file) as fp:
fp.seek(self.idx)
try:
while self.ino == os.stat(file).st_ino:
for element in [x for x in fp.read().splitlines() if x.strip()]:
self.parse(element)
time.sleep(1)
except:
pass
log.warning(str("[!] Unsubscribing from file '{}' (probably because of log rotation)").format(file))
return 0