def process(self, i, router):
if i.itype == 'fqdn':
r = resolve_ns(i.indicator, t='CNAME')
self.logger.debug('CNAME: {}'.format(r))
for rr in r:
fqdn = Indicator(**i.__dict__)
fqdn.indicator = str(rr).rstrip('.')
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
x = router.indicators_create(fqdn)
self.logger.debug(x)
if i.is_subdomain():
fqdn = Indicator(**i.__dict__)
fqdn.indicator = i.is_subdomain()
fqdn.confidence = (int(fqdn.confidence) / 3)
x = router.indicators_create(fqdn)
self.logger.debug(x)
r = resolve_ns(i.indicator)
self.logger.debug(r)
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr)
ip.itype = 'ipv4'
ip.confidence = (int(ip.confidence) / 4)
x = router.indicators_create(ip)
self.logger.debug(x)
r = resolve_ns(i.indicator, t='NS')
self.logger.debug('NS: {}'.format(r))
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.confidence = (int(ip.confidence) / 5)
x = router.indicators_create(ip)
self.logger.debug(x)
r = resolve_ns(i.indicator, t='MX')
self.logger.debug('MX: {}'.format(r))
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.confidence = (int(ip.confidence) / 6)
x = router.indicators_create(ip)
self.logger.debug(x)
def process(self, i, router):
if i.itype == 'fqdn':
r = resolve_ns(i.indicator, t='CNAME')
self.logger.debug('CNAME: {}'.format(r))
for rr in r:
fqdn = Indicator(**i.__dict__)
fqdn.indicator = str(rr).rstrip('.')
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
x = router.indicators_create(fqdn)
self.logger.debug(x)
if i.is_subdomain():
fqdn = Indicator(**i.__dict__)
fqdn.indicator = i.is_subdomain()
fqdn.confidence = (int(fqdn.confidence) / 3)
x = router.indicators_create(fqdn)
self.logger.debug(x)
r = resolve_ns(i.indicator)
self.logger.debug(r)
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr)
ip.itype = 'ipv4'
ip.confidence = (int(ip.confidence) / 4)
x = router.indicators_create(ip)
self.logger.debug(x)
r = resolve_ns(i.indicator, t='NS')
self.logger.debug('NS: {}'.format(r))
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.confidence = (int(ip.confidence) / 5)
x = router.indicators_create(ip)
self.logger.debug(x)
r = resolve_ns(i.indicator, t='MX')
self.logger.debug('MX: {}'.format(r))
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.confidence = (int(ip.confidence) / 6)
x = router.indicators_create(ip)
self.logger.debug(x)
def process(self, i, router, **kwargs):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator, t='NS')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
return
for rr in r:
rr = str(rr).rstrip('.')
if rr in ["", 'localhost', '0.0.0.0']:
continue
i_ns = Indicator(**i.__dict__())
i_ns.indicator = rr
try:
i_ns_itype = resolve_itype(i_ns.indicator)
except InvalidIndicator as e:
self.logger.error(i_ns)
self.logger.error(e)
else:
i_ns.lasttime = i_ns.reporttime = arrow.utcnow()
i_ns.itype = i_ns_itype
i_ns.rdata = "{} nameserver".format(i.indicator)
if 'hunter' not in i_ns.tags:
i_ns.tags.append('hunter')
i_ns.confidence = (i_ns.confidence - 4) if i_ns.confidence >= 4 else 0
router.indicators_create(i_ns)
self.logger.debug("FQDN NS Hunter: {}".format(i_ns))
def process(self, i, router):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator, t='MX')
except Timeout:
self.logger.info('timeout trying to resolve MX for: {}'.format(i.indicator))
return
for rr in r:
rr = re.sub(r'^\d+ ', '', str(rr))
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr.rstrip('.')
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
if not str(e).startswith('unknown itype for "localhost"'):
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.itype = 'fqdn'
fqdn.rdata = i.indicator
fqdn.confidence = (int(fqdn.confidence) / 6)
router.indicators_create(fqdn)
def process(self, i, router):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator)
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
return
for rr in r:
if str(rr).rstrip('.') in ["", 'localhost']:
continue
ip = Indicator(**i.__dict__())
ip.indicator = str(rr)
ip.lasttime = arrow.utcnow()
try:
resolve_itype(ip.indicator)
except InvalidIndicator as e:
self.logger.error(ip)
self.logger.error(e)
else:
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = (ip.confidence - 4) if ip.confidence >= 4 else 0
router.indicators_create(ip)
def process(self, i, router, **kwargs):
if i.itype != 'url':
return
if 'search' in i.tags:
return
# prevent recursion with fqdn_wl hunter
if ('whitelist') in i.tags and (i.rdata is not None or i.rdata != ''):
return
u = urlparse(i.indicator)
if not u.hostname:
return
try:
resolve_itype(u.hostname)
except InvalidIndicator as e:
self.logger.error(u.hostname)
self.logger.error(e)
else:
fqdn = Indicator(**i.__dict__())
fqdn.lasttime = fqdn.reporttime = arrow.utcnow()
fqdn.indicator = u.hostname
fqdn.itype = 'fqdn'
if 'hunter' not in fqdn.tags:
fqdn.tags.append('hunter')
fqdn.confidence = (int(fqdn.confidence) / 2)
fqdn.rdata = i.indicator
self.logger.debug('[Hunter: Url] sending to router {}'.format(fqdn))
router.indicators_create(fqdn)
def process(self, i, router):
if i.itype != 'url':
return
if 'search' in i.tags:
return
u = urlparse(i.indicator)
if not u.hostname:
return
try:
resolve_itype(u.hostname)
except InvalidIndicator as e:
self.logger.error(u.hostname)
self.logger.error(e)
else:
fqdn = Indicator(**i.__dict__())
fqdn.lasttime = arrow.utcnow()
fqdn.indicator = u.hostname
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
fqdn.rdata = i.indicator
self.logger.debug('sending to router: {}'.format(fqdn))
router.indicators_create(fqdn)
def process(self, i, router):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator)
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
return
for rr in r:
if str(rr).rstrip('.') in ["", 'localhost']:
continue
ip = Indicator(**i.__dict__())
ip.indicator = str(rr)
ip.lasttime = arrow.utcnow()
try:
resolve_itype(ip.indicator)
except InvalidIndicator as e:
self.logger.error(ip)
self.logger.error(e)
else:
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = (ip.confidence -
4) if ip.confidence >= 4 else 0
router.indicators_create(ip)
def process(self, i, router):
if i.itype != 'fqdn':
return
try:
r = resolve_ns(i.indicator, t='CNAME')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
r = []
for rr in r:
# http://serverfault.com/questions/44618/is-a-wildcard-cname-dns-record-valid
rr = str(rr).rstrip('.').lstrip('*.')
if rr in ['', 'localhost']:
continue
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr
fqdn.lasttime = arrow.utcnow()
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
return
fqdn.itype = 'fqdn'
fqdn.confidence = (fqdn.confidence - 1)
router.indicators_create(fqdn)
def process(self, i, router, **kwargs):
if i.itype != 'fqdn':
return
if 'whitelist' not in i.tags:
return
urls = []
for p in ['http://', 'https://']:
urls.append('{}{}'.format(p, i.indicator))
if not i.indicator.startswith('www.'):
urls.append('{}www.{}'.format(p, i.indicator))
for u in urls:
url = Indicator(**i.__dict__())
url.indicator = u
try:
resolve_itype(url.indicator)
except InvalidIndicator as e:
self.logger.error(url)
self.logger.error(e)
else:
url.tags = ['whitelist', 'hunter']
url.itype = 'url'
url.rdata = i.indicator
url.lasttime = url.reporttime = arrow.utcnow()
router.indicators_create(url)
def process(self, i, router):
if i.itype != 'fqdn':
return
try:
r = resolve_ns(i.indicator, t='CNAME')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
r = []
for rr in r:
fqdn = Indicator(**i.__dict__())
fqdn.indicator = str(rr).rstrip('.')
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
return
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
router.indicators_create(fqdn)
def process(self, i, router):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator)
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
return
for rr in r:
ip = Indicator(**i.__dict__())
ip.indicator = str(rr)
try:
resolve_itype(ip.indicator)
except InvalidIndicator as e:
self.logger.error(ip)
self.logger.error(e)
else:
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = (int(ip.confidence) / 4)
router.indicators_create(ip)
# also create a passive dns tag
ip.tags = 'pdns'
ip.confidence = 10
router.indicators_create(ip)
def process(self, i, router):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator, t='CNAME')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
r = []
for rr in r:
# http://serverfault.com/questions/44618/is-a-wildcard-cname-dns-record-valid
rr = str(rr).rstrip('.').lstrip('*.')
if rr in ['', 'localhost', '0.0.0.0']:
continue
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr
fqdn.lasttime = arrow.utcnow()
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
return
fqdn.itype = 'fqdn'
fqdn.confidence = (fqdn.confidence - 1)
router.indicators_create(fqdn)
def process(self, i, router, **kwargs):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator)
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
return
for rr in r:
rr = str(rr)
if rr in ["", 'localhost', '0.0.0.0']:
continue
ip = Indicator(**i.__dict__())
ip.lasttime = ip.reporttime = arrow.utcnow()
ip.indicator = rr
try:
resolve_itype(ip.indicator)
except InvalidIndicator as e:
self.logger.error(ip)
self.logger.error(e)
else:
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.tags = ['pdns', 'hunter']
ip.confidence = 10
router.indicators_create(ip)
self.logger.debug("FQDN Hunter: {}".format(ip))
def process(self, i, router):
if i.itype == 'url':
u = urlparse(i.indicator)
if u.netloc:
fqdn = Indicator(**i.__dict__)
fqdn.indicator = u.netloc
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
fqdn.rdata = i.indicator
self.logger.debug('sending to router..')
x = router.indicators_create(fqdn)
def process(self, i, router):
if i.itype == 'url':
u = urlparse(i.indicator)
if u.netloc:
fqdn = Indicator(**i.__dict__)
fqdn.indicator = u.netloc
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
fqdn.rdata = i.indicator
self.logger.debug('sending to router..')
x = router.indicators_create(fqdn)
def text_to_list(text, known_only=True):
separator = find_seperator(text)
t_tokens = top_tokens(text)
top = set()
for t in range(0, 9):
top.add(t_tokens[t])
if known_only:
if separator not in KNOWN_SEPERATORS:
pprint(top)
raise SystemError('separator not in known list: {}'.format(separator))
ret = []
for l in text.split("\n"):
if l == '':
continue
if l.startswith('#') or l.startswith(';'):
continue
cols = l.split(separator)
cols = [x.strip() for x in cols]
indicator = Indicator()
for e in cols:
if e:
try:
i = resolve_itype(e)
if i:
indicator.indicator = e
indicator.itype = i
except NotImplementedError:
pass
try:
ts = arrow.get(e)
if ts:
indicator.lasttime = ts.datetime
except (arrow.parser.ParserError, UnicodeDecodeError):
pass
if e in top:
indicator.tags = [e]
if indicator.itype and indicator.indicator:
ret.append(indicator)
return ret
def text_to_list(text, known_only=True):
separator = find_seperator(text)
t_tokens = top_tokens(text)
top = set()
for t in range(0, 9):
top.add(t_tokens[t])
if known_only:
if separator not in KNOWN_SEPERATORS:
raise SystemError(
'separator not in known list: {}'.format(separator))
ret = []
for l in text.split("\n"):
if l == '':
continue
if l.startswith('#') or l.startswith(';'):
continue
cols = l.split(separator)
cols = [x.strip() for x in cols]
indicator = Indicator()
for e in cols:
if e:
try:
i = resolve_itype(e)
if i:
indicator.indicator = e
indicator.itype = i
except TypeError:
pass
try:
ts = arrow.get(e)
if ts:
indicator.lasttime = ts.datetime
except (arrow.parser.ParserError, UnicodeDecodeError):
pass
if e in top:
indicator.tags = [e]
if indicator.itype and indicator.indicator:
ret.append(indicator)
return ret
def process(self, i, router, **kwargs):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator, t='MX')
except Timeout:
self.logger.info('timeout trying to resolve MX for: {}'.format(
i.indicator))
return
try:
for rr in r:
rr = re.sub(r'^\d+ ', '', str(rr))
rr = str(rr).rstrip('.')
if rr in ["", 'localhost', '0.0.0.0']:
continue
elif re.match('^\d+$', rr) or re.match(r'^.{0,3}$', rr):
# exclude spurious entries like those too short to be real
continue
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr.rstrip('.')
fqdn.lasttime = fqdn.reporttime = arrow.utcnow()
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.info(fqdn)
self.logger.info(e)
else:
fqdn.itype = 'fqdn'
if 'hunter' not in fqdn.tags:
fqdn.tags.append('hunter')
fqdn.rdata = '{} mx'.format(i.indicator)
fqdn.confidence = (fqdn.confidence -
5) if fqdn.confidence >= 5 else 0
router.indicators_create(fqdn)
self.logger.debug("FQDN MX Hunter: {}".format(fqdn))
except Exception as e:
self.logger.error(
'[Hunter: FqdnMx] {}: giving up on rr {} from indicator {}'.
format(e, rr, i))
def process(self, i, router):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator, t='MX')
except Timeout:
self.logger.info('timeout trying to resolve MX for: {}'.format(
i.indicator))
return
try:
for rr in r:
rr = re.sub(r'^\d+ ', '', str(rr))
rr = str(rr).rstrip('.')
if rr in ["", 'localhost', '0.0.0.0']:
continue
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr.rstrip('.')
fqdn.lasttime = fqdn.reporttime = arrow.utcnow()
# 10
if re.match('^\d+$', rr):
return
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.info(fqdn)
self.logger.info(e)
else:
fqdn.itype = 'fqdn'
if 'hunter' not in fqdn.tags:
fqdn.tags.append('hunter')
fqdn.rdata = i.indicator
fqdn.confidence = (fqdn.confidence -
5) if fqdn.confidence >= 5 else 0
router.indicators_create(fqdn)
except Exception as e:
self.logger.error(
'[Hunter: FqdnMx] {}: giving up on indicator {}'.format(e, rr))
def process(i):
if not ENABLED:
return
if i.itype != 'fqdn':
return
try:
r = resolve_ns(i.indicator)
if not r:
return
except Timeout:
return
rv = []
for rr in r:
rr = str(rr)
if rr in ["", 'localhost']:
continue
ip = Indicator(**i.__dict__())
ip.lasttime = arrow.utcnow()
ip.indicator = rr
try:
resolve_itype(ip.indicator)
except:
continue
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = 1
ip.probability = 0
rv.append(ip)
pdns = Indicator(**copy.deepcopy(i.__dict__()))
# also create a passive dns tag
pdns.tags = 'pdns'
pdns.confidence = 4
pdns.probability = i.probability
pdns.indicator = ip.indicator
pdns.rdata = i.indicator
rv.append(pdns)
return rv
def process(i):
return
if not ENABLED:
return
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator, t='MX')
if not r:
return
except Timeout:
return
rv = []
for rr in r:
rr = re.sub(r'^\d+ ', '', str(rr))
rr = str(rr).rstrip('.')
if rr in ["", 'localhost']:
continue
# 10
if re.match('^\d+$', rr):
continue
fqdn = Indicator(**i.__dict__())
fqdn.probability = 0
fqdn.indicator = rr.rstrip('.')
fqdn.lasttime = arrow.utcnow()
try:
resolve_itype(fqdn.indicator)
except:
continue
fqdn.itype = 'fqdn'
fqdn.rdata = i.indicator
fqdn.confidence = 0
rv.append(fqdn)
return rv
def process(self, i, router):
if i.itype == 'url':
u = urlparse(i.indicator)
if u.hostname:
try:
resolve_itype(u.hostname)
except InvalidIndicator as e:
self.logger.error(u.hostname)
self.logger.error(e)
else:
fqdn = Indicator(**i.__dict__())
fqdn.indicator = u.hostname
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
fqdn.rdata = i.indicator
self.logger.debug('sending to router..')
router.indicators_create(fqdn)
def process(self, i, router):
if i.itype == 'url':
u = urlparse(i.indicator)
if u.netloc:
try:
resolve_itype(u.netloc)
except InvalidIndicator as e:
self.logger.error(u.netloc)
self.logger.error(e)
else:
fqdn = Indicator(**i.__dict__())
fqdn.indicator = u.netloc
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
fqdn.rdata = i.indicator
self.logger.debug('sending to router..')
router.indicators_create(fqdn)
def process(self, i, router, **kwargs):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator, t='CNAME')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
r = []
for rr in r:
# http://serverfault.com/questions/44618/is-a-wildcard-cname-dns-record-valid
rr = str(rr).rstrip('.').lstrip('*.')
if rr in ['', 'localhost', '0.0.0.0']:
continue
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr
fqdn.lasttime = fqdn.reporttime = arrow.utcnow()
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
return
fqdn.itype = 'fqdn'
fqdn.rdata = '{} cname'.format(i.indicator)
if 'hunter' not in fqdn.tags:
fqdn.tags.append('hunter')
if fqdn.confidence < 8:
fqdn.confidence -= 1
else:
fqdn.confidence = 7
router.indicators_create(fqdn)
self.logger.debug("FQDN CNAME Hunter: {}".format(fqdn))
def process(i):
return
if not ENABLED:
return
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator)
if not r:
return
except Timeout:
return
rv = []
for rr in r:
if str(rr).rstrip('.') in ["", 'localhost']:
continue
ip = Indicator(**i.__dict__())
ip.probability = 0
ip.indicator = str(rr)
ip.lasttime = arrow.utcnow()
try:
resolve_itype(ip.indicator)
except:
continue
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = 0
rv.append(ip)
return rv
def process(self, i, router):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator)
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
return
for rr in r:
rr = str(rr)
if rr in ["", 'localhost', '0.0.0.0']:
continue
ip = Indicator(**i.__dict__())
ip.lasttime = arrow.utcnow()
ip.indicator = rr
try:
resolve_itype(ip.indicator)
except InvalidIndicator as e:
self.logger.error(ip)
self.logger.error(e)
else:
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = (ip.confidence -
2) if ip.confidence >= 2 else 0
router.indicators_create(ip)
# also create a passive dns tag
ip.tags = 'pdns'
ip.confidence = 10
router.indicators_create(ip)
def process(i):
if not ENABLED:
return
if i.itype != 'fqdn':
return
try:
r = resolve_ns(i.indicator, t='CNAME')
if not r:
return
except Timeout:
return
rv = []
for rr in r:
# http://serverfault.com/questions/44618/is-a-wildcard-cname-dns-record-valid
rr = str(rr).rstrip('.').lstrip('*.')
if rr in ['', 'localhost']:
continue
fqdn = Indicator(**i.__dict__())
fqdn.probability = 0
fqdn.indicator = rr
fqdn.lasttime = arrow.utcnow()
try:
resolve_itype(fqdn.indicator)
except:
return
fqdn.itype = 'fqdn'
# keep avoid recursive cname lookups
fqdn.confidence = int(fqdn.confidence /
2) if fqdn.confidence >= 2 else 0
rv.append(fqdn)
return rv
def process(self, i, router):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator, t='MX')
except Timeout:
self.logger.info('timeout trying to resolve MX for: {}'.format(i.indicator))
return
for rr in r:
rr = re.sub(r'^\d+ ', '', str(rr))
rr = str(rr).rstrip('.')
if rr in ["", 'localhost']:
continue
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr.rstrip('.')
fqdn.lasttime = arrow.utcnow()
# 10
if re.match('^\d+$', rr):
return
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.info(fqdn)
self.logger.info(e)
else:
fqdn.itype = 'fqdn'
fqdn.rdata = i.indicator
fqdn.confidence = (fqdn.confidence - 5) if fqdn.confidence >= 5 else 0
router.indicators_create(fqdn)
def process(self, i, router):
if i.itype != 'url':
return
u = urlparse(i.indicator)
if not u.hostname:
return
try:
resolve_itype(u.hostname)
except InvalidIndicator as e:
self.logger.error(u.hostname)
self.logger.error(e)
else:
fqdn = Indicator(**i.__dict__())
fqdn.lasttime = arrow.utcnow()
fqdn.indicator = u.hostname
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
fqdn.rdata = i.indicator
self.logger.debug('sending to router: {}'.format(fqdn))
router.indicators_create(fqdn)
def process(i):
if i.itype != 'url':
return
u = urlparse(i.indicator)
if not u.hostname:
return
try:
resolve_itype(u.hostname)
except TypeError as e:
logger.error(u.hostname)
logger.error(e)
return
fqdn = Indicator(**i.__dict__())
fqdn.lasttime = arrow.utcnow()
fqdn.indicator = u.hostname
fqdn.itype = 'fqdn'
fqdn.confidence = 2
fqdn.rdata = i.indicator
fqdn.probability = 0
return fqdn
def process(self, i, router):
if i.itype == 'fqdn':
try:
r = resolve_ns(i.indicator, t='CNAME')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
r = []
for rr in r:
fqdn = Indicator(**i.__dict__())
fqdn.indicator = str(rr).rstrip('.')
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
router.indicators_create(fqdn)
if i.is_subdomain():
fqdn = Indicator(**i.__dict__())
fqdn.indicator = i.is_subdomain()
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.confidence = (int(fqdn.confidence) / 3)
router.indicators_create(fqdn)
try:
r = resolve_ns(i.indicator)
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
r = []
for rr in r:
ip = Indicator(**i.__dict__())
ip.indicator = str(rr)
try:
resolve_itype(ip.indicator)
except InvalidIndicator as e:
self.logger.error(ip)
self.logger.error(e)
else:
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = (int(ip.confidence) / 4)
router.indicators_create(ip)
try:
r = resolve_ns(i.indicator, t='NS')
except Timeout:
self.logger.info('timeout trying to resolve NS for: {}'.format(i.indicator))
r = []
for rr in r:
fqdn = Indicator(**i.__dict__())
fqdn.indicator = str(rr).rstrip('.')
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.itype = 'fqdn'
fqdn.rdata = i.indicator
fqdn.confidence = (int(fqdn.confidence) / 5)
router.indicators_create(fqdn)
try:
r = resolve_ns(i.indicator, t='MX')
except Timeout:
self.logger.info('timeout trying to resolve MX for: {}'.format(i.indicator))
r = []
for rr in r:
rr = re.sub(r'^\d+ ', '', str(rr))
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr.rstrip('.')
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
if not str(e).startswith('unknown itype for "localhost"'):
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.itype = 'fqdn'
fqdn.rdata = i.indicator
fqdn.confidence = (int(fqdn.confidence) / 6)
router.indicators_create(fqdn)
def process(self, i, router):
if i.itype == 'fqdn':
try:
r = resolve_ns(i.indicator, t='CNAME')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
r = []
for rr in r:
fqdn = Indicator(**i.__dict__)
fqdn.indicator = str(rr).rstrip('.')
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
x = router.indicators_create(fqdn)
if i.is_subdomain():
fqdn = Indicator(**i.__dict__)
fqdn.indicator = i.is_subdomain()
fqdn.confidence = (int(fqdn.confidence) / 3)
x = router.indicators_create(fqdn)
try:
r = resolve_ns(i.indicator)
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
r = []
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr)
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = (int(ip.confidence) / 4)
x = router.indicators_create(ip)
self.logger.debug(x)
try:
r = resolve_ns(i.indicator, t='NS')
except Timeout:
self.logger.info('timeout trying to resolve NS for: {}'.format(
i.indicator))
r = []
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.rdata = i.indicator
ip.confidence = (int(ip.confidence) / 5)
x = router.indicators_create(ip)
self.logger.debug(x)
try:
r = resolve_ns(i.indicator, t='MX')
except Timeout:
self.logger.info('timeout trying to resolve MX for: {}'.format(
i.indicator))
r = []
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.rdata = i.indicator
ip.confidence = (int(ip.confidence) / 6)
x = router.indicators_create(ip)
self.logger.debug(x)
def process(self, i, router):
if i.itype == 'fqdn':
try:
r = resolve_ns(i.indicator, t='CNAME')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
r = []
for rr in r:
fqdn = Indicator(**i.__dict__())
fqdn.indicator = str(rr).rstrip('.')
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
router.indicators_create(fqdn)
if i.is_subdomain():
fqdn = Indicator(**i.__dict__())
fqdn.indicator = i.is_subdomain()
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.confidence = (int(fqdn.confidence) / 3)
router.indicators_create(fqdn)
try:
r = resolve_ns(i.indicator)
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
r = []
for rr in r:
ip = Indicator(**i.__dict__())
ip.indicator = str(rr)
try:
resolve_itype(ip.indicator)
except InvalidIndicator as e:
self.logger.error(ip)
self.logger.error(e)
else:
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = (int(ip.confidence) / 4)
router.indicators_create(ip)
try:
r = resolve_ns(i.indicator, t='NS')
except Timeout:
self.logger.info('timeout trying to resolve NS for: {}'.format(
i.indicator))
r = []
for rr in r:
fqdn = Indicator(**i.__dict__())
fqdn.indicator = str(rr).rstrip('.')
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.itype = 'fqdn'
fqdn.rdata = i.indicator
fqdn.confidence = (int(fqdn.confidence) / 5)
router.indicators_create(fqdn)
try:
r = resolve_ns(i.indicator, t='MX')
except Timeout:
self.logger.info('timeout trying to resolve MX for: {}'.format(
i.indicator))
r = []
for rr in r:
rr = re.sub(r'^\d+ ', '', str(rr))
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr.rstrip('.')
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
if not str(e).startswith('unknown itype for "localhost"'):
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.itype = 'fqdn'
fqdn.rdata = i.indicator
fqdn.confidence = (int(fqdn.confidence) / 6)
router.indicators_create(fqdn)
def process(self, i, router):
if i.itype == 'fqdn':
try:
r = resolve_ns(i.indicator, t='CNAME')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
r = []
for rr in r:
fqdn = Indicator(**i.__dict__)
fqdn.indicator = str(rr).rstrip('.')
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
x = router.indicators_create(fqdn)
if i.is_subdomain():
fqdn = Indicator(**i.__dict__)
fqdn.indicator = i.is_subdomain()
fqdn.confidence = (int(fqdn.confidence) / 3)
x = router.indicators_create(fqdn)
try:
r = resolve_ns(i.indicator)
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
r = []
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr)
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = (int(ip.confidence) / 4)
x = router.indicators_create(ip)
self.logger.debug(x)
try:
r = resolve_ns(i.indicator, t='NS')
except Timeout:
self.logger.info('timeout trying to resolve NS for: {}'.format(i.indicator))
r = []
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.rdata = i.indicator
ip.confidence = (int(ip.confidence) / 5)
x = router.indicators_create(ip)
self.logger.debug(x)
try:
r = resolve_ns(i.indicator, t='MX')
except Timeout:
self.logger.info('timeout trying to resolve MX for: {}'.format(i.indicator))
r = []
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.rdata = i.indicator
ip.confidence = (int(ip.confidence) / 6)
x = router.indicators_create(ip)
self.logger.debug(x)
