def process(self, indicator):
if indicator.itype == 'ipv4' and not indicator.is_private():
i = indicator.indicator
match = re.search('^(\S+)\/\d+$', i)
if match:
i = match.group(1)
i = '.'.join(reversed(i.split('.')))
answers = resolve_ns('{}.{}'.format(i, 'origin.asn.cymru.com'), t='TXT')
if len(answers) > 0:
# Separate fields and order by netmask length
# 23028 | 216.90.108.0/24 | US | arin | 1998-09-25
# 701 1239 3549 3561 7132 | 216.90.108.0/24 | US | arin | 1998-09-25
# i.asn_desc ????
self.logger.debug(answers[0])
bits = str(answers[0]).replace('"', '').strip().split(' | ')
asns = bits[0].split(' ')
indicator.asn = asns[0]
indicator.prefix = bits[1]
indicator.cc = bits[2]
indicator.rir = bits[3]
answers = resolve_ns('as{}.{}'.format(asns[0], 'asn.cymru.com'), t='TXT')
bits = str(answers[0]).replace('"', '').strip().split(' | ')
if len(bits) > 4:
indicator.asn_desc = bits[4]
# send back to router
return indicator
def process(self, i, router):
if i.itype == 'fqdn':
r = resolve_ns(i.indicator, t='CNAME')
self.logger.debug('CNAME: {}'.format(r))
for rr in r:
fqdn = Indicator(**i.__dict__)
fqdn.indicator = str(rr).rstrip('.')
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
x = router.indicators_create(fqdn)
self.logger.debug(x)
if i.is_subdomain():
fqdn = Indicator(**i.__dict__)
fqdn.indicator = i.is_subdomain()
fqdn.confidence = (int(fqdn.confidence) / 3)
x = router.indicators_create(fqdn)
self.logger.debug(x)
r = resolve_ns(i.indicator)
self.logger.debug(r)
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr)
ip.itype = 'ipv4'
ip.confidence = (int(ip.confidence) / 4)
x = router.indicators_create(ip)
self.logger.debug(x)
r = resolve_ns(i.indicator, t='NS')
self.logger.debug('NS: {}'.format(r))
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.confidence = (int(ip.confidence) / 5)
x = router.indicators_create(ip)
self.logger.debug(x)
r = resolve_ns(i.indicator, t='MX')
self.logger.debug('MX: {}'.format(r))
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.confidence = (int(ip.confidence) / 6)
x = router.indicators_create(ip)
self.logger.debug(x)
def process(self, i, router):
if i.itype == 'fqdn':
r = resolve_ns(i.indicator, t='CNAME')
self.logger.debug('CNAME: {}'.format(r))
for rr in r:
fqdn = Indicator(**i.__dict__)
fqdn.indicator = str(rr).rstrip('.')
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
x = router.indicators_create(fqdn)
self.logger.debug(x)
if i.is_subdomain():
fqdn = Indicator(**i.__dict__)
fqdn.indicator = i.is_subdomain()
fqdn.confidence = (int(fqdn.confidence) / 3)
x = router.indicators_create(fqdn)
self.logger.debug(x)
r = resolve_ns(i.indicator)
self.logger.debug(r)
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr)
ip.itype = 'ipv4'
ip.confidence = (int(ip.confidence) / 4)
x = router.indicators_create(ip)
self.logger.debug(x)
r = resolve_ns(i.indicator, t='NS')
self.logger.debug('NS: {}'.format(r))
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.confidence = (int(ip.confidence) / 5)
x = router.indicators_create(ip)
self.logger.debug(x)
r = resolve_ns(i.indicator, t='MX')
self.logger.debug('MX: {}'.format(r))
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.confidence = (int(ip.confidence) / 6)
x = router.indicators_create(ip)
self.logger.debug(x)
def process(self, i, router, **kwargs):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator, t='NS')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
return
for rr in r:
rr = str(rr).rstrip('.')
if rr in ["", 'localhost', '0.0.0.0']:
continue
i_ns = Indicator(**i.__dict__())
i_ns.indicator = rr
try:
i_ns_itype = resolve_itype(i_ns.indicator)
except InvalidIndicator as e:
self.logger.error(i_ns)
self.logger.error(e)
else:
i_ns.lasttime = i_ns.reporttime = arrow.utcnow()
i_ns.itype = i_ns_itype
i_ns.rdata = "{} nameserver".format(i.indicator)
if 'hunter' not in i_ns.tags:
i_ns.tags.append('hunter')
i_ns.confidence = (i_ns.confidence - 4) if i_ns.confidence >= 4 else 0
router.indicators_create(i_ns)
self.logger.debug("FQDN NS Hunter: {}".format(i_ns))
def process(self, i, router, **kwargs):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator)
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
return
for rr in r:
rr = str(rr)
if rr in ["", 'localhost', '0.0.0.0']:
continue
ip = Indicator(**i.__dict__())
ip.lasttime = ip.reporttime = arrow.utcnow()
ip.indicator = rr
try:
resolve_itype(ip.indicator)
except InvalidIndicator as e:
self.logger.error(ip)
self.logger.error(e)
else:
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.tags = ['pdns', 'hunter']
ip.confidence = 10
router.indicators_create(ip)
self.logger.debug("FQDN Hunter: {}".format(ip))
def process(self, indicator):
if indicator.itype == 'ipv4' and not indicator.is_private():
i = str(indicator.indicator)
match = re.search('^(\S+)\/\d+$', i)
if match:
i = match.group(1)
i = '.'.join(reversed(i.split('.')))
answers = self._resolve(i)
if len(answers) > 0:
# Separate fields and order by netmask length
# 23028 | 216.90.108.0/24 | US | arin | 1998-09-25
# 701 1239 3549 3561 7132 | 216.90.108.0/24 | US | arin | 1998-09-25
# i.asn_desc ????
self.logger.debug(answers[0])
bits = str(answers[0]).replace('"', '').strip().split(' | ')
asns = bits[0].split(' ')
indicator.asn = asns[0]
indicator.prefix = bits[1]
indicator.cc = bits[2]
indicator.rir = bits[3]
answers = resolve_ns('as{}.{}'.format(asns[0],
'asn.cymru.com'),
t='TXT')
bits = str(answers[0]).replace('"', '').strip().split(' | ')
if len(bits) > 4:
indicator.asn_desc = bits[4]
# send back to router
return indicator
def process(self, i, router):
if i.itype != 'fqdn':
return
try:
r = resolve_ns(i.indicator, t='CNAME')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
r = []
for rr in r:
# http://serverfault.com/questions/44618/is-a-wildcard-cname-dns-record-valid
rr = str(rr).rstrip('.').lstrip('*.')
if rr in ['', 'localhost']:
continue
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr
fqdn.lasttime = arrow.utcnow()
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
return
fqdn.itype = 'fqdn'
fqdn.confidence = (fqdn.confidence - 1)
router.indicators_create(fqdn)
def process(self, i, router):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator)
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
return
for rr in r:
if str(rr).rstrip('.') in ["", 'localhost']:
continue
ip = Indicator(**i.__dict__())
ip.indicator = str(rr)
ip.lasttime = arrow.utcnow()
try:
resolve_itype(ip.indicator)
except InvalidIndicator as e:
self.logger.error(ip)
self.logger.error(e)
else:
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = (ip.confidence -
4) if ip.confidence >= 4 else 0
router.indicators_create(ip)
def process(self, i, router):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator)
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
return
for rr in r:
if str(rr).rstrip('.') in ["", 'localhost']:
continue
ip = Indicator(**i.__dict__())
ip.indicator = str(rr)
ip.lasttime = arrow.utcnow()
try:
resolve_itype(ip.indicator)
except InvalidIndicator as e:
self.logger.error(ip)
self.logger.error(e)
else:
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = (ip.confidence - 4) if ip.confidence >= 4 else 0
router.indicators_create(ip)
def process(self, i, router):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator)
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
return
for rr in r:
ip = Indicator(**i.__dict__())
ip.indicator = str(rr)
try:
resolve_itype(ip.indicator)
except InvalidIndicator as e:
self.logger.error(ip)
self.logger.error(e)
else:
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = (int(ip.confidence) / 4)
router.indicators_create(ip)
# also create a passive dns tag
ip.tags = 'pdns'
ip.confidence = 10
router.indicators_create(ip)
def process(self, i, router):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator, t='CNAME')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
r = []
for rr in r:
# http://serverfault.com/questions/44618/is-a-wildcard-cname-dns-record-valid
rr = str(rr).rstrip('.').lstrip('*.')
if rr in ['', 'localhost', '0.0.0.0']:
continue
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr
fqdn.lasttime = arrow.utcnow()
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
return
fqdn.itype = 'fqdn'
fqdn.confidence = (fqdn.confidence - 1)
router.indicators_create(fqdn)
def process(self, indicator):
if indicator.itype == 'ipv4' and not indicator.is_private:
i = '.'.join(reversed(indicator.indicator.split('.')))
answers = resolve_ns('{}.{}'.format(i, 'peer.asn.cymru.com'), t='TXT')
if len(answers) > 0:
if not indicator.peers:
indicator.peers = []
# Separate fields and order by netmask length
# 23028 | 216.90.108.0/24 | US | arin | 1998-09-25
# 701 1239 3549 3561 7132 | 216.90.108.0/24 | US | arin | 1998-09-25
for p in answers:
self.logger.debug(p)
bits = str(p).replace('"', '').strip().split(' | ')
asn = bits[0]
prefix = bits[1]
cc = bits[2]
rir = bits[3]
asns = asn.split(' ')
for a in asns:
indicator.peers.append({
'asn': a,
'prefix': prefix,
'cc': cc,
'rir': rir
})
return indicator
def process(self, i, router):
if i.itype != 'fqdn':
return
try:
r = resolve_ns(i.indicator, t='CNAME')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
r = []
for rr in r:
fqdn = Indicator(**i.__dict__())
fqdn.indicator = str(rr).rstrip('.')
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
return
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
router.indicators_create(fqdn)
def process(self, i, router):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator, t='MX')
except Timeout:
self.logger.info('timeout trying to resolve MX for: {}'.format(i.indicator))
return
for rr in r:
rr = re.sub(r'^\d+ ', '', str(rr))
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr.rstrip('.')
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
if not str(e).startswith('unknown itype for "localhost"'):
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.itype = 'fqdn'
fqdn.rdata = i.indicator
fqdn.confidence = (int(fqdn.confidence) / 6)
router.indicators_create(fqdn)
def process(self, indicator):
if indicator.itype == 'ipv4' and not indicator.is_private():
i = '.'.join(reversed(indicator.indicator.split('.')))
answers = resolve_ns('{}.{}'.format(i, 'peer.asn.cymru.com'), t='TXT')
if len(answers) > 0:
if not indicator.peers:
indicator.peers = []
# Separate fields and order by netmask length
# 23028 | 216.90.108.0/24 | US | arin | 1998-09-25
# 701 1239 3549 3561 7132 | 216.90.108.0/24 | US | arin | 1998-09-25
for p in answers:
self.logger.debug(p)
bits = str(p).replace('"', '').strip().split(' | ')
asn = bits[0]
prefix = bits[1]
cc = bits[2]
rir = bits[3]
asns = asn.split(' ')
for a in asns:
indicator.peers.append({
'asn': a,
'prefix': prefix,
'cc': cc,
'rir': rir
})
return indicator
def process(self, i, router):
if i.itype == 'fqdn':
r = resolve_ns(i.indicator)
self.logger.debug(r)
for rr in r:
ip = copy.deepcopy(i)
ip.indicator = str(rr)
ip.itype = 'ipv4'
ip.confidence = (int(ip.confidence) / 2)
x = router.submit(ip)
self.logger.debug(x)
r = resolve_ns(i.indicator, t='CNAME')
self.logger.debug(r)
for rr in r:
ip = copy.deepcopy(i)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.confidence = (int(ip.confidence) / 2)
x = router.submit(ip)
self.logger.debug(x)
r = resolve_ns(i.indicator, t='NS')
self.logger.debug(r)
for rr in r:
ip = copy.deepcopy(i)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.confidence = (int(ip.confidence) / 3)
x = router.submit(ip)
self.logger.debug(x)
r = resolve_ns(i.indicator, t='MX')
for rr in r:
ip = copy.deepcopy(i)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.confidence = (int(ip.confidence) / 4)
x = router.submit(ip)
self.logger.debug(x)
def process(self, i, router):
if i.itype == 'fqdn':
r = resolve_ns(i.indicator)
self.logger.debug(r)
for rr in r:
ip = copy.deepcopy(i)
ip.indicator = str(rr)
ip.itype = 'ipv4'
ip.confidence = (int(ip.confidence) / 2)
x = router.submit(ip)
self.logger.debug(x)
r = resolve_ns(i.indicator, t='CNAME')
self.logger.debug(r)
for rr in r:
ip = copy.deepcopy(i)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.confidence = (int(ip.confidence) / 2)
x = router.submit(ip)
self.logger.debug(x)
r = resolve_ns(i.indicator, t='NS')
self.logger.debug(r)
for rr in r:
ip = copy.deepcopy(i)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.confidence = (int(ip.confidence) / 3)
x = router.submit(ip)
self.logger.debug(x)
r = resolve_ns(i.indicator, t='MX')
for rr in r:
ip = copy.deepcopy(i)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.confidence = (int(ip.confidence) / 4)
x = router.submit(ip)
self.logger.debug(x)
def process(self, i, router, **kwargs):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator, t='MX')
except Timeout:
self.logger.info('timeout trying to resolve MX for: {}'.format(
i.indicator))
return
try:
for rr in r:
rr = re.sub(r'^\d+ ', '', str(rr))
rr = str(rr).rstrip('.')
if rr in ["", 'localhost', '0.0.0.0']:
continue
elif re.match('^\d+$', rr) or re.match(r'^.{0,3}$', rr):
# exclude spurious entries like those too short to be real
continue
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr.rstrip('.')
fqdn.lasttime = fqdn.reporttime = arrow.utcnow()
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.info(fqdn)
self.logger.info(e)
else:
fqdn.itype = 'fqdn'
if 'hunter' not in fqdn.tags:
fqdn.tags.append('hunter')
fqdn.rdata = '{} mx'.format(i.indicator)
fqdn.confidence = (fqdn.confidence -
5) if fqdn.confidence >= 5 else 0
router.indicators_create(fqdn)
self.logger.debug("FQDN MX Hunter: {}".format(fqdn))
except Exception as e:
self.logger.error(
'[Hunter: FqdnMx] {}: giving up on rr {} from indicator {}'.
format(e, rr, i))
def process(self, indicator):
if indicator.itype == 'ipv4' and not indicator.is_private():
i = str(indicator.indicator)
match = re.search('^(\S+)\/\d+$', i)
if match:
i = match.group(1)
i = '.'.join(reversed(i.split('.')))
answers = self._resolve(i)
if len(answers) > 0:
# Separate fields and order by netmask length
# 23028 | 216.90.108.0/24 | US | arin | 1998-09-25
# 701 1239 3549 3561 7132 | 216.90.108.0/24 | US | arin | 1998-09-25
# i.asn_desc ????
self.logger.debug(answers[0])
bits = str(answers[0]).replace('"', '').strip().split(' | ')
asns = bits[0].split(' ')
indicator.asn = asns[0]
indicator.prefix = bits[1]
indicator.cc = bits[2]
indicator.rir = bits[3]
answers = resolve_ns('as{}.{}'.format(asns[0],
'asn.cymru.com'),
t='TXT',
timeout=15)
## TODO - not fixed yet
try:
tmp = str(answers[0])
except UnicodeDecodeError:
# requires fix latin-1 fix _escapeify to dnspython > 1.14
return indicator
except IndexError:
from pprint import pprint
pprint(answers)
return indicator
bits = tmp.replace('"', '').strip().split(' | ')
if len(bits) > 4:
indicator.asn_desc = bits[4]
# send back to router
return indicator
def process(self, i, router):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator, t='MX')
except Timeout:
self.logger.info('timeout trying to resolve MX for: {}'.format(
i.indicator))
return
try:
for rr in r:
rr = re.sub(r'^\d+ ', '', str(rr))
rr = str(rr).rstrip('.')
if rr in ["", 'localhost', '0.0.0.0']:
continue
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr.rstrip('.')
fqdn.lasttime = fqdn.reporttime = arrow.utcnow()
# 10
if re.match('^\d+$', rr):
return
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.info(fqdn)
self.logger.info(e)
else:
fqdn.itype = 'fqdn'
if 'hunter' not in fqdn.tags:
fqdn.tags.append('hunter')
fqdn.rdata = i.indicator
fqdn.confidence = (fqdn.confidence -
5) if fqdn.confidence >= 5 else 0
router.indicators_create(fqdn)
except Exception as e:
self.logger.error(
'[Hunter: FqdnMx] {}: giving up on indicator {}'.format(e, rr))
def process(self, indicator):
if indicator.itype == 'ipv4' and not indicator.is_private():
i = str(indicator.indicator)
match = re.search('^(\S+)\/\d+$', i)
if match:
i = match.group(1)
i = '.'.join(reversed(i.split('.')))
answers = self._resolve(i)
if len(answers) > 0:
# Separate fields and order by netmask length
# 23028 | 216.90.108.0/24 | US | arin | 1998-09-25
# 701 1239 3549 3561 7132 | 216.90.108.0/24 | US | arin | 1998-09-25
# i.asn_desc ????
self.logger.debug(answers[0])
bits = str(answers[0]).replace('"', '').strip().split(' | ')
asns = bits[0].split(' ')
indicator.asn = asns[0]
indicator.prefix = bits[1]
indicator.cc = bits[2]
indicator.rir = bits[3]
answers = resolve_ns('as{}.{}'.format(asns[0], 'asn.cymru.com'), t='TXT', timeout=15)
## TODO - not fixed yet
try:
tmp = str(answers[0])
except UnicodeDecodeError:
# requires fix latin-1 fix _escapeify to dnspython > 1.14
return indicator
except IndexError:
from pprint import pprint
pprint(answers)
return indicator
bits = tmp.replace('"', '').strip().split(' | ')
if len(bits) > 4:
indicator.asn_desc = bits[4]
# send back to router
return indicator
def process(self, i, router, **kwargs):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator, t='CNAME')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
r = []
for rr in r:
# http://serverfault.com/questions/44618/is-a-wildcard-cname-dns-record-valid
rr = str(rr).rstrip('.').lstrip('*.')
if rr in ['', 'localhost', '0.0.0.0']:
continue
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr
fqdn.lasttime = fqdn.reporttime = arrow.utcnow()
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
return
fqdn.itype = 'fqdn'
fqdn.rdata = '{} cname'.format(i.indicator)
if 'hunter' not in fqdn.tags:
fqdn.tags.append('hunter')
if fqdn.confidence < 8:
fqdn.confidence -= 1
else:
fqdn.confidence = 7
router.indicators_create(fqdn)
self.logger.debug("FQDN CNAME Hunter: {}".format(fqdn))
def process(self, i, router):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator)
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
return
for rr in r:
rr = str(rr)
if rr in ["", 'localhost', '0.0.0.0']:
continue
ip = Indicator(**i.__dict__())
ip.lasttime = arrow.utcnow()
ip.indicator = rr
try:
resolve_itype(ip.indicator)
except InvalidIndicator as e:
self.logger.error(ip)
self.logger.error(e)
else:
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = (ip.confidence -
2) if ip.confidence >= 2 else 0
router.indicators_create(ip)
# also create a passive dns tag
ip.tags = 'pdns'
ip.confidence = 10
router.indicators_create(ip)
def process(self, i, router):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator, t='MX')
except Timeout:
self.logger.info('timeout trying to resolve MX for: {}'.format(i.indicator))
return
for rr in r:
rr = re.sub(r'^\d+ ', '', str(rr))
rr = str(rr).rstrip('.')
if rr in ["", 'localhost']:
continue
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr.rstrip('.')
fqdn.lasttime = arrow.utcnow()
# 10
if re.match('^\d+$', rr):
return
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.info(fqdn)
self.logger.info(e)
else:
fqdn.itype = 'fqdn'
fqdn.rdata = i.indicator
fqdn.confidence = (fqdn.confidence - 5) if fqdn.confidence >= 5 else 0
router.indicators_create(fqdn)
def _resolve(self, data):
return resolve_ns('{}.{}'.format(data, 'peer.asn.cymru.com', timeout=15), t='TXT')
def process(self, indicator):
if not self.enabled:
return
if indicator.is_private():
return
# TODO ipv6
if indicator.itype != 'ipv4':
return
i = str(indicator.indicator)
match = re.search('^(\S+)\/\d+$', i)
if match:
i = match.group(1)
if self.asn_fast:
bits = self._resolve_fast(indicator.indicator)
for k in bits:
if bits[k] == 'NA':
bits[k] = False
if bits['asn']:
bits['asn'] = str(bits['asn'])
indicator.asn = bits['asn']
indicator.prefix = bits['prefix']
indicator.asn_desc = bits['owner']
indicator.cc = bits['cc']
return indicator
# cache it to the /24
# 115.87.213.115
# 0.213.87.115
i = list(reversed(i.split('.')))
i = '0.{}.{}.{}'.format(i[1], i[2], i[3])
answers = self._resolve(i)
if len(answers) > 0:
# Separate fields and order by netmask length
# 23028 | 216.90.108.0/24 | US | arin | 1998-09-25
# 701 1239 3549 3561 7132 | 216.90.108.0/24 | US | arin | 1998-09-25
# i.asn_desc ????
self.logger.debug(answers[0])
bits = str(answers[0]).replace('"', '').strip().split(' | ')
asns = bits[0].split(' ')
indicator.asn = asns[0]
indicator.prefix = bits[1]
indicator.cc = bits[2]
indicator.rir = bits[3]
answers = resolve_ns('as{}.{}'.format(asns[0], 'asn.cymru.com'),
t='TXT',
timeout=15)
try:
tmp = str(answers[0])
except UnicodeDecodeError as e:
# requires fix latin-1 fix _escapeify to dnspython > 1.14
self.logger.debug(e)
return indicator
except IndexError:
from pprint import pprint
pprint(answers)
return indicator
bits = tmp.replace('"', '').strip().split(' | ')
if len(bits) > 4:
indicator.asn_desc = bits[4]
# send back to router
return indicator
def _resolve(self, data):
return resolve_ns('{}.{}'.format(data,
'peer.asn.cymru.com',
timeout=15),
t='TXT')
def process(self, i, router):
if i.itype == 'fqdn':
try:
r = resolve_ns(i.indicator, t='CNAME')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
r = []
for rr in r:
fqdn = Indicator(**i.__dict__())
fqdn.indicator = str(rr).rstrip('.')
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
router.indicators_create(fqdn)
if i.is_subdomain():
fqdn = Indicator(**i.__dict__())
fqdn.indicator = i.is_subdomain()
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.confidence = (int(fqdn.confidence) / 3)
router.indicators_create(fqdn)
try:
r = resolve_ns(i.indicator)
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
r = []
for rr in r:
ip = Indicator(**i.__dict__())
ip.indicator = str(rr)
try:
resolve_itype(ip.indicator)
except InvalidIndicator as e:
self.logger.error(ip)
self.logger.error(e)
else:
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = (int(ip.confidence) / 4)
router.indicators_create(ip)
try:
r = resolve_ns(i.indicator, t='NS')
except Timeout:
self.logger.info('timeout trying to resolve NS for: {}'.format(i.indicator))
r = []
for rr in r:
fqdn = Indicator(**i.__dict__())
fqdn.indicator = str(rr).rstrip('.')
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.itype = 'fqdn'
fqdn.rdata = i.indicator
fqdn.confidence = (int(fqdn.confidence) / 5)
router.indicators_create(fqdn)
try:
r = resolve_ns(i.indicator, t='MX')
except Timeout:
self.logger.info('timeout trying to resolve MX for: {}'.format(i.indicator))
r = []
for rr in r:
rr = re.sub(r'^\d+ ', '', str(rr))
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr.rstrip('.')
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
if not str(e).startswith('unknown itype for "localhost"'):
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.itype = 'fqdn'
fqdn.rdata = i.indicator
fqdn.confidence = (int(fqdn.confidence) / 6)
router.indicators_create(fqdn)
def _resolve(self, data):
return resolve_ns('{}.{}'.format(data, 'origin.asn.cymru.com'),
t='TXT')
def _resolve(self, data):
data = reversed(data.split('.'))
data = '{}.zen.spamhaus.org'.format('.'.join(data))
data = resolve_ns(data)
if data and data[0]:
return data[0]
def process(self, i, router):
if i.itype == 'fqdn':
try:
r = resolve_ns(i.indicator, t='CNAME')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
r = []
for rr in r:
fqdn = Indicator(**i.__dict__)
fqdn.indicator = str(rr).rstrip('.')
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
x = router.indicators_create(fqdn)
if i.is_subdomain():
fqdn = Indicator(**i.__dict__)
fqdn.indicator = i.is_subdomain()
fqdn.confidence = (int(fqdn.confidence) / 3)
x = router.indicators_create(fqdn)
try:
r = resolve_ns(i.indicator)
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
r = []
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr)
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = (int(ip.confidence) / 4)
x = router.indicators_create(ip)
self.logger.debug(x)
try:
r = resolve_ns(i.indicator, t='NS')
except Timeout:
self.logger.info('timeout trying to resolve NS for: {}'.format(
i.indicator))
r = []
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.rdata = i.indicator
ip.confidence = (int(ip.confidence) / 5)
x = router.indicators_create(ip)
self.logger.debug(x)
try:
r = resolve_ns(i.indicator, t='MX')
except Timeout:
self.logger.info('timeout trying to resolve MX for: {}'.format(
i.indicator))
r = []
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.rdata = i.indicator
ip.confidence = (int(ip.confidence) / 6)
x = router.indicators_create(ip)
self.logger.debug(x)
def process(self, indicator):
if not self.enabled:
return
if indicator.is_private():
return
# TODO ipv6
if indicator.itype != 'ipv4':
return
i = str(indicator.indicator)
match = re.search('^(\S+)\/\d+$', i)
if match:
i = match.group(1)
if self.asn_fast:
bits = self._resolve_fast(indicator.indicator)
for k in bits:
if bits[k] == 'NA':
bits[k] = False
if bits['asn']:
bits['asn'] = str(bits['asn'])
indicator.asn = bits['asn']
indicator.prefix = bits['prefix']
indicator.asn_desc = bits['owner']
indicator.cc = bits['cc']
return indicator
# cache it to the /24
# 115.87.213.115
# 0.213.87.115
i = list(reversed(i.split('.')))
i = '0.{}.{}.{}'.format(i[1], i[2], i[3])
answers = self._resolve(i)
if len(answers) > 0:
# Separate fields and order by netmask length
# 23028 | 216.90.108.0/24 | US | arin | 1998-09-25
# 701 1239 3549 3561 7132 | 216.90.108.0/24 | US | arin | 1998-09-25
# i.asn_desc ????
self.logger.debug(answers[0])
bits = str(answers[0]).replace('"', '').strip().split(' | ')
asns = bits[0].split(' ')
indicator.asn = asns[0]
indicator.prefix = bits[1]
indicator.cc = bits[2]
indicator.rir = bits[3]
answers = resolve_ns('as{}.{}'.format(asns[0], 'asn.cymru.com'), t='TXT', timeout=15)
try:
tmp = str(answers[0])
except UnicodeDecodeError as e:
# requires fix latin-1 fix _escapeify to dnspython > 1.14
self.logger.debug(e)
return indicator
except IndexError:
from pprint import pprint
pprint(answers)
return indicator
bits = tmp.replace('"', '').strip().split(' | ')
if len(bits) > 4:
indicator.asn_desc = bits[4]
# send back to router
return indicator
def process(self, i, router):
if i.itype == 'fqdn':
try:
r = resolve_ns(i.indicator, t='CNAME')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
r = []
for rr in r:
fqdn = Indicator(**i.__dict__())
fqdn.indicator = str(rr).rstrip('.')
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
router.indicators_create(fqdn)
if i.is_subdomain():
fqdn = Indicator(**i.__dict__())
fqdn.indicator = i.is_subdomain()
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.confidence = (int(fqdn.confidence) / 3)
router.indicators_create(fqdn)
try:
r = resolve_ns(i.indicator)
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
r = []
for rr in r:
ip = Indicator(**i.__dict__())
ip.indicator = str(rr)
try:
resolve_itype(ip.indicator)
except InvalidIndicator as e:
self.logger.error(ip)
self.logger.error(e)
else:
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = (int(ip.confidence) / 4)
router.indicators_create(ip)
try:
r = resolve_ns(i.indicator, t='NS')
except Timeout:
self.logger.info('timeout trying to resolve NS for: {}'.format(
i.indicator))
r = []
for rr in r:
fqdn = Indicator(**i.__dict__())
fqdn.indicator = str(rr).rstrip('.')
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.itype = 'fqdn'
fqdn.rdata = i.indicator
fqdn.confidence = (int(fqdn.confidence) / 5)
router.indicators_create(fqdn)
try:
r = resolve_ns(i.indicator, t='MX')
except Timeout:
self.logger.info('timeout trying to resolve MX for: {}'.format(
i.indicator))
r = []
for rr in r:
rr = re.sub(r'^\d+ ', '', str(rr))
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr.rstrip('.')
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
if not str(e).startswith('unknown itype for "localhost"'):
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.itype = 'fqdn'
fqdn.rdata = i.indicator
fqdn.confidence = (int(fqdn.confidence) / 6)
router.indicators_create(fqdn)
def _resolve(self, data):
data = '{}.dbl.spamhaus.org'.format(data)
data = resolve_ns(data)
if data and data[0]:
return data[0]
def _resolve(self, data):
data = '{}.dbl.spamhaus.org'.format(data)
data = resolve_ns(data)
if data and data[0]:
return data[0]
def process(self, i, router):
if i.itype == 'fqdn':
try:
r = resolve_ns(i.indicator, t='CNAME')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
r = []
for rr in r:
fqdn = Indicator(**i.__dict__)
fqdn.indicator = str(rr).rstrip('.')
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
x = router.indicators_create(fqdn)
if i.is_subdomain():
fqdn = Indicator(**i.__dict__)
fqdn.indicator = i.is_subdomain()
fqdn.confidence = (int(fqdn.confidence) / 3)
x = router.indicators_create(fqdn)
try:
r = resolve_ns(i.indicator)
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
r = []
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr)
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = (int(ip.confidence) / 4)
x = router.indicators_create(ip)
self.logger.debug(x)
try:
r = resolve_ns(i.indicator, t='NS')
except Timeout:
self.logger.info('timeout trying to resolve NS for: {}'.format(i.indicator))
r = []
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.rdata = i.indicator
ip.confidence = (int(ip.confidence) / 5)
x = router.indicators_create(ip)
self.logger.debug(x)
try:
r = resolve_ns(i.indicator, t='MX')
except Timeout:
self.logger.info('timeout trying to resolve MX for: {}'.format(i.indicator))
r = []
for rr in r:
ip = Indicator(**i.__dict__)
ip.indicator = str(rr).rstrip('.')
ip.itype = 'fqdn'
ip.rdata = i.indicator
ip.confidence = (int(ip.confidence) / 6)
x = router.indicators_create(ip)
self.logger.debug(x)
def _resolve(self, data):
data = reversed(data.split('.'))
data = '{}.zen.spamhaus.org'.format('.'.join(data))
data = resolve_ns(data)
if data and data[0]:
return data[0]
def _resolve(self, data):
return resolve_ns('{}.{}'.format(data, 'origin.asn.cymru.com'), t='TXT')
