def _log_search(self, t, data):
if not data.get('indicator'):
return
if data.get('nolog') in ['1', 'True', 1, True]:
return
if '*' in data.get('indicator'):
return
if '%' in data.get('indicator'):
return
ts = arrow.utcnow().format('YYYY-MM-DDTHH:mm:ss.SSZ')
s = Indicator(
indicator=data['indicator'],
tlp='amber',
confidence=10,
tags='search',
provider=t['username'],
firsttime=ts,
lasttime=ts,
reporttime=ts,
group=t['groups'][0],
count=1,
)
self.store.indicators.upsert(t, [s.__dict__()])
def indicator():
return Indicator(indicator='example.com',
tags='botnet',
provider='csirtg.io',
group='everyone',
lasttime=arrow.utcnow().datetime,
reporttime=arrow.utcnow().datetime)
def indicator_malware():
return Indicator(indicator='d52380918a07322c50f1bfa2b43af3bb54cb33db',
tags='malware',
provider='csirtg.io',
group='everyone',
lasttime=arrow.utcnow().datetime,
reporttime=arrow.utcnow().datetime)
def indicator_ipv6():
return Indicator(indicator='2001:4860:4860::8888',
tags='botnet',
provider='csirtg.io',
group='everyone',
lasttime=arrow.utcnow().datetime,
reporttime=arrow.utcnow().datetime)
def indicator_url():
return Indicator(indicator='http://pwmsteel.com/dhYtebv3',
tags='exploit',
provider='csirtg.io',
group='everyone',
lasttime=arrow.utcnow().datetime,
reporttime=arrow.utcnow().datetime)
def test_copy():
i1 = Indicator('128.205.1.1', tags='malware')
i2 = i1.copy(tags='pdns', reported_at=arrow.utcnow())
assert i1 != i2
assert i1.tags != i2.tags
assert i1.uuid != i2.uuid
def indicator_ipv4():
return Indicator(indicator='1.2.3.4',
tags='botnet',
provider='csirtg.io',
group='everyone',
lasttime=arrow.utcnow().datetime,
reporttime=arrow.utcnow().datetime)
def process(self, i, router):
if i.itype != 'ipv4':
return
if 'whitelist' not in i.tags:
return
if i.indicator.endswith('/24'):
return
prefix = i.indicator.split('.')
prefix = prefix[:3]
prefix.append('0/24')
prefix = '.'.join(prefix)
try:
ii = Indicator(**i.__dict__())
except InvalidIndicator as e:
self.logger.error(e)
return
ii.lasttime = arrow.utcnow()
ii.indicator = prefix
ii.tags = ['whitelist']
ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0
router.indicators_create(ii)
def process(self, i, router):
if i.itype != 'ipv4':
return
if 'whitelist' not in i.tags:
return
# only run this hunter if it's a single address (no CIDRs)
if ipaddress.IPv4Network(i.indicator).prefixlen != 32:
return
prefix = i.indicator.split('.')
prefix = prefix[:3]
prefix.append('0/24')
prefix = '.'.join(prefix)
try:
ii = Indicator(**i.__dict__())
except InvalidIndicator as e:
self.logger.error(e)
return
ii.lasttime = arrow.utcnow()
ii.indicator = prefix
ii.tags = ['whitelist', 'hunter']
ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0
router.indicators_create(ii)
def process(self, i, router, **kwargs):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
if not i.is_subdomain():
return
fqdn = Indicator(**i.__dict__())
fqdn.indicator = i.is_subdomain()
fqdn.lasttime = fqdn.reporttime = arrow.utcnow()
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.confidence = (fqdn.confidence -
3) if fqdn.confidence >= 3 else 0
fqdn.rdata = '{} subdomain'.format(i.indicator)
if 'hunter' not in fqdn.tags:
fqdn.tags.append('hunter')
router.indicators_create(fqdn)
self.logger.debug("FQDN Subdomain Hunter: {}".format(fqdn))
def process(self, i, router):
if i.itype != 'fqdn':
return
try:
r = resolve_ns(i.indicator, t='CNAME')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
r = []
for rr in r:
# http://serverfault.com/questions/44618/is-a-wildcard-cname-dns-record-valid
rr = str(rr).rstrip('.').lstrip('*.')
if rr in ['', 'localhost']:
continue
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
return
fqdn.itype = 'fqdn'
fqdn.confidence = (int(fqdn.confidence) / 2)
router.indicators_create(fqdn)
def process(self, i, router):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator, t='MX')
except Timeout:
self.logger.info('timeout trying to resolve MX for: {}'.format(
i.indicator))
return
for rr in r:
rr = re.sub(r'^\d+ ', '', str(rr))
rr = str(rr).rstrip('.')
if rr in ["", 'localhost']:
continue
fqdn = Indicator(**i.__dict__())
fqdn.indicator = rr.rstrip('.')
try:
resolve_itype(fqdn.indicator)
except InvalidIndicator as e:
self.logger.error(fqdn)
self.logger.error(e)
else:
fqdn.itype = 'fqdn'
fqdn.rdata = i.indicator
fqdn.confidence = (int(fqdn.confidence) / 6)
router.indicators_create(fqdn)
def clean_indicator(self, i, rule):
# check for de-fang'd feed
if rule.replace:
for e in i:
if not rule.replace.get(e):
continue
for k, v in rule.replace[e].items():
i[e] = i[e].replace(k, v)
i = normalize_itype(i)
if isinstance(i, dict):
i = Indicator(**i)
if not i.firsttime:
i.firsttime = i.lasttime
if not i.reporttime:
i.reporttime = arrow.utcnow().datetime
if not i.group:
i.group = 'everyone'
return i
def post(self):
if not session['write']:
return redirect('/u/search', code=401)
i = dict(request.form)
for k in i:
i[k] = i[k][0]
i['provider'] = session['username']
try:
i = Indicator(**i)
except InvalidIndicator as e:
logger.error(e)
flash(e, 'error')
return render_template('submit.html', error='Invalid itype')
logger.debug(i)
try:
r = Client(remote, session['token']).indicators_create(i)
except Exception as e:
logger.error(e)
flash(e, 'error')
response = render_template('submit.html', error='submit failed')
else:
flash('submission successful', 'success')
response = render_template('submit.html', groups=session['groups'])
return response
def process(self, i, router, **kwargs):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator, t='NS')
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(i.indicator))
return
for rr in r:
rr = str(rr).rstrip('.')
if rr in ["", 'localhost', '0.0.0.0']:
continue
i_ns = Indicator(**i.__dict__())
i_ns.indicator = rr
try:
i_ns_itype = resolve_itype(i_ns.indicator)
except InvalidIndicator as e:
self.logger.error(i_ns)
self.logger.error(e)
else:
i_ns.lasttime = i_ns.reporttime = arrow.utcnow()
i_ns.itype = i_ns_itype
i_ns.rdata = "{} nameserver".format(i.indicator)
if 'hunter' not in i_ns.tags:
i_ns.tags.append('hunter')
i_ns.confidence = (i_ns.confidence - 4) if i_ns.confidence >= 4 else 0
router.indicators_create(i_ns)
self.logger.debug("FQDN NS Hunter: {}".format(i_ns))
def process(self, i, router):
if 'search' in i.tags:
return
if i.itype == 'fqdn' and i.provider != 'spamhaus.org':
try:
r = self._resolve(i.indicator)
try:
r = CODES.get(str(r), None)
except Exception as e:
# https://www.spamhaus.org/faq/section/DNSBL%20Usage
self.logger.error(e)
self.logger.info('check spamhaus return codes')
r = None
if r:
confidence = CONFIDENCE
if ' legit ' in r['description']:
confidence = 6
f = Indicator(**i.__dict__())
f.tags = [r['tags']]
f.description = r['description']
f.confidence = confidence
f.provider = PROVIDER
f.reference_tlp = 'white'
f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(
f.indicator)
f.lasttime = arrow.utcnow()
x = router.indicators_create(f)
self.logger.debug(x)
except KeyError as e:
self.logger.error(e)
def process(self, i, router):
if i.itype != 'fqdn':
return
if 'search' in i.tags:
return
try:
r = resolve_ns(i.indicator)
except Timeout:
self.logger.info('timeout trying to resolve: {}'.format(
i.indicator))
return
for rr in r:
if str(rr).rstrip('.') in ["", 'localhost', '0.0.0.0']:
continue
ip = Indicator(**i.__dict__())
ip.indicator = str(rr)
ip.lasttime = arrow.utcnow()
try:
resolve_itype(ip.indicator)
except InvalidIndicator as e:
self.logger.error(ip)
self.logger.error(e)
else:
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = (ip.confidence -
4) if ip.confidence >= 4 else 0
router.indicators_create(ip)
def process(i):
if not ENABLED:
return
if i.itype != 'fqdn':
return
try:
r = resolve_ns(i.indicator)
if not r:
return
except Timeout:
return
rv = []
for rr in r:
rr = str(rr)
if rr in ["", 'localhost']:
continue
ip = Indicator(**i.__dict__())
ip.lasttime = arrow.utcnow()
ip.indicator = rr
try:
resolve_itype(ip.indicator)
except:
continue
ip.itype = 'ipv4'
ip.rdata = i.indicator
ip.confidence = 1
ip.probability = 0
rv.append(ip)
pdns = Indicator(**copy.deepcopy(i.__dict__()))
# also create a passive dns tag
pdns.tags = 'pdns'
pdns.confidence = 4
pdns.probability = i.probability
pdns.indicator = ip.indicator
pdns.rdata = i.indicator
rv.append(pdns)
return rv
def indicator4():
return Indicator(indicator='example.com',
tags='botnet',
provider='test-provider',
group='everyone',
lasttime=arrow.utcnow().datetime,
reporttime=arrow.utcnow().datetime,
confidence=8.0)
def main():
g = Geo()
i = sys.argv[1]
i = Indicator(i)
i = g.process(i)
pprint(i)
def main():
from csirtg_indicator import Indicator
i = Indicator('71.6.146.130')
import logging
logger = logging.getLogger('')
logger.setLevel(logging.DEBUG)
print(process(i))
def main():
from csirtg_indicator import Indicator
i = Indicator('ns2.ndxylfpxuwowlhycfh.pw')
import logging
logger = logging.getLogger('')
logger.setLevel(logging.DEBUG)
print(process(i))
def indicator_email():
return Indicator(indicator='*****@*****.**',
tags='botnet',
provider='csirtg.io',
group='everyone',
lasttime=arrow.utcnow().datetime,
reporttime=arrow.utcnow().datetime,
tlp='green')
def indicator_broken_multi_tag_el():
return Indicator(
indicator='d52380918a07322c50f1bfa2b43af3bb54cb33db',
tags=['malware,exploit'], # this is intentionally bad for the test
provider='csirtg.io',
group='everyone',
lasttime=arrow.utcnow().datetime,
reporttime=arrow.utcnow().datetime)
def test_indicator_dest():
i = Indicator(indicator='192.168.1.1',
dest='10.0.0.1',
portlist="23",
protocol="tcp",
dest_portlist='21,22-23')
assert i.dest
assert i.dest_portlist
def process(i):
return
try:
for ii in i.csirtg():
yield Indicator(**ii, resolve_geo=True)
except Exception as e:
pass
def process(self, i, router):
if not self.token:
return
if i.itype != 'ipv4':
return
if 'search' not in i.tags:
return
if i.confidence and i.confidence < 9:
return
if re.search('^(\S+)\/(\d+)$', i.indicator):
return
max = MAX_QUERY_RESULTS
try:
for r in self.client.search(i.indicator):
first = arrow.get(r.get('time_first') or r.get('zone_time_first'))
first = first.datetime
last = arrow.get(r.get('time_last') or r.get('zone_time_last'))
last = last.datetime
reporttime = arrow.utcnow().datetime
r['rrname'] = r['rrname'].rstrip('.')
try:
ii = Indicator(
indicator=r['rdata'],
rdata=r['rrname'].rstrip('.'),
count=r['count'],
tags='pdns',
confidence=10,
firsttime=first,
lasttime=last,
reporttime=reporttime,
provider=PROVIDER,
tlp='amber',
group='everyone'
)
except InvalidIndicator as e:
self.logger.error(e)
return
router.indicators_create(ii)
max -= 1
if max == 0:
break
except QuotaLimit:
logger.warn('farsight quota limit reached... skipping')
except Exception as e:
logger.exception(e)
return
def test_urls_ok():
data = [
'http://192.168.1.1/1.html', 'http://www41.xzmnt.com',
'http://get.ahoybest.com/n/3.6.16/12205897/microsoft lync server 2010.exe'
]
for d in data:
d = Indicator(d)
assert d.itype is 'url'
def test_indicator_timezones():
t = '2017-03-06T11:41:48-06:00'
a = arrow.get('2017-03-06T17:41:48Z').datetime
i = Indicator('example.com', firsttime=t, lasttime=t, reporttime=t)
assert i.firsttime == a
assert i.lasttime == a
assert i.reporttime == a
def _create(cli, args, options):
print("submitting {0}".format(options.get("submit")))
i = Indicator(indicator=args.indicator,
tags=args.tags,
confidence=args.confidence)
rv = cli.indicators_create(i)
print('success id: {}\n'.format(rv))
