def _log_search(self, t, data): if not data.get('indicator'): return if data.get('nolog') in ['1', 'True', 1, True]: return if '*' in data.get('indicator'): return if '%' in data.get('indicator'): return ts = arrow.utcnow().format('YYYY-MM-DDTHH:mm:ss.SSZ') s = Indicator( indicator=data['indicator'], tlp='amber', confidence=10, tags='search', provider=t['username'], firsttime=ts, lasttime=ts, reporttime=ts, group=t['groups'][0], count=1, ) self.store.indicators.upsert(t, [s.__dict__()])
def indicator(): return Indicator(indicator='example.com', tags='botnet', provider='csirtg.io', group='everyone', lasttime=arrow.utcnow().datetime, reporttime=arrow.utcnow().datetime)
def indicator_malware(): return Indicator(indicator='d52380918a07322c50f1bfa2b43af3bb54cb33db', tags='malware', provider='csirtg.io', group='everyone', lasttime=arrow.utcnow().datetime, reporttime=arrow.utcnow().datetime)
def indicator_ipv6(): return Indicator(indicator='2001:4860:4860::8888', tags='botnet', provider='csirtg.io', group='everyone', lasttime=arrow.utcnow().datetime, reporttime=arrow.utcnow().datetime)
def indicator_url(): return Indicator(indicator='http://pwmsteel.com/dhYtebv3', tags='exploit', provider='csirtg.io', group='everyone', lasttime=arrow.utcnow().datetime, reporttime=arrow.utcnow().datetime)
def test_copy(): i1 = Indicator('128.205.1.1', tags='malware') i2 = i1.copy(tags='pdns', reported_at=arrow.utcnow()) assert i1 != i2 assert i1.tags != i2.tags assert i1.uuid != i2.uuid
def indicator_ipv4(): return Indicator(indicator='1.2.3.4', tags='botnet', provider='csirtg.io', group='everyone', lasttime=arrow.utcnow().datetime, reporttime=arrow.utcnow().datetime)
def process(self, i, router): if i.itype != 'ipv4': return if 'whitelist' not in i.tags: return if i.indicator.endswith('/24'): return prefix = i.indicator.split('.') prefix = prefix[:3] prefix.append('0/24') prefix = '.'.join(prefix) try: ii = Indicator(**i.__dict__()) except InvalidIndicator as e: self.logger.error(e) return ii.lasttime = arrow.utcnow() ii.indicator = prefix ii.tags = ['whitelist'] ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0 router.indicators_create(ii)
def process(self, i, router): if i.itype != 'ipv4': return if 'whitelist' not in i.tags: return # only run this hunter if it's a single address (no CIDRs) if ipaddress.IPv4Network(i.indicator).prefixlen != 32: return prefix = i.indicator.split('.') prefix = prefix[:3] prefix.append('0/24') prefix = '.'.join(prefix) try: ii = Indicator(**i.__dict__()) except InvalidIndicator as e: self.logger.error(e) return ii.lasttime = arrow.utcnow() ii.indicator = prefix ii.tags = ['whitelist', 'hunter'] ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0 router.indicators_create(ii)
def process(self, i, router, **kwargs): if i.itype != 'fqdn': return if 'search' in i.tags: return if not i.is_subdomain(): return fqdn = Indicator(**i.__dict__()) fqdn.indicator = i.is_subdomain() fqdn.lasttime = fqdn.reporttime = arrow.utcnow() try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.error(fqdn) self.logger.error(e) else: fqdn.confidence = (fqdn.confidence - 3) if fqdn.confidence >= 3 else 0 fqdn.rdata = '{} subdomain'.format(i.indicator) if 'hunter' not in fqdn.tags: fqdn.tags.append('hunter') router.indicators_create(fqdn) self.logger.debug("FQDN Subdomain Hunter: {}".format(fqdn))
def process(self, i, router): if i.itype != 'fqdn': return try: r = resolve_ns(i.indicator, t='CNAME') except Timeout: self.logger.info('timeout trying to resolve: {}'.format( i.indicator)) r = [] for rr in r: # http://serverfault.com/questions/44618/is-a-wildcard-cname-dns-record-valid rr = str(rr).rstrip('.').lstrip('*.') if rr in ['', 'localhost']: continue fqdn = Indicator(**i.__dict__()) fqdn.indicator = rr try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.error(fqdn) self.logger.error(e) return fqdn.itype = 'fqdn' fqdn.confidence = (int(fqdn.confidence) / 2) router.indicators_create(fqdn)
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator, t='MX') except Timeout: self.logger.info('timeout trying to resolve MX for: {}'.format( i.indicator)) return for rr in r: rr = re.sub(r'^\d+ ', '', str(rr)) rr = str(rr).rstrip('.') if rr in ["", 'localhost']: continue fqdn = Indicator(**i.__dict__()) fqdn.indicator = rr.rstrip('.') try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.error(fqdn) self.logger.error(e) else: fqdn.itype = 'fqdn' fqdn.rdata = i.indicator fqdn.confidence = (int(fqdn.confidence) / 6) router.indicators_create(fqdn)
def clean_indicator(self, i, rule): # check for de-fang'd feed if rule.replace: for e in i: if not rule.replace.get(e): continue for k, v in rule.replace[e].items(): i[e] = i[e].replace(k, v) i = normalize_itype(i) if isinstance(i, dict): i = Indicator(**i) if not i.firsttime: i.firsttime = i.lasttime if not i.reporttime: i.reporttime = arrow.utcnow().datetime if not i.group: i.group = 'everyone' return i
def post(self): if not session['write']: return redirect('/u/search', code=401) i = dict(request.form) for k in i: i[k] = i[k][0] i['provider'] = session['username'] try: i = Indicator(**i) except InvalidIndicator as e: logger.error(e) flash(e, 'error') return render_template('submit.html', error='Invalid itype') logger.debug(i) try: r = Client(remote, session['token']).indicators_create(i) except Exception as e: logger.error(e) flash(e, 'error') response = render_template('submit.html', error='submit failed') else: flash('submission successful', 'success') response = render_template('submit.html', groups=session['groups']) return response
def process(self, i, router, **kwargs): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator, t='NS') except Timeout: self.logger.info('timeout trying to resolve: {}'.format(i.indicator)) return for rr in r: rr = str(rr).rstrip('.') if rr in ["", 'localhost', '0.0.0.0']: continue i_ns = Indicator(**i.__dict__()) i_ns.indicator = rr try: i_ns_itype = resolve_itype(i_ns.indicator) except InvalidIndicator as e: self.logger.error(i_ns) self.logger.error(e) else: i_ns.lasttime = i_ns.reporttime = arrow.utcnow() i_ns.itype = i_ns_itype i_ns.rdata = "{} nameserver".format(i.indicator) if 'hunter' not in i_ns.tags: i_ns.tags.append('hunter') i_ns.confidence = (i_ns.confidence - 4) if i_ns.confidence >= 4 else 0 router.indicators_create(i_ns) self.logger.debug("FQDN NS Hunter: {}".format(i_ns))
def process(self, i, router): if 'search' in i.tags: return if i.itype == 'fqdn' and i.provider != 'spamhaus.org': try: r = self._resolve(i.indicator) try: r = CODES.get(str(r), None) except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: confidence = CONFIDENCE if ' legit ' in r['description']: confidence = 6 f = Indicator(**i.__dict__()) f.tags = [r['tags']] f.description = r['description'] f.confidence = confidence f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format( f.indicator) f.lasttime = arrow.utcnow() x = router.indicators_create(f) self.logger.debug(x) except KeyError as e: self.logger.error(e)
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator) except Timeout: self.logger.info('timeout trying to resolve: {}'.format( i.indicator)) return for rr in r: if str(rr).rstrip('.') in ["", 'localhost', '0.0.0.0']: continue ip = Indicator(**i.__dict__()) ip.indicator = str(rr) ip.lasttime = arrow.utcnow() try: resolve_itype(ip.indicator) except InvalidIndicator as e: self.logger.error(ip) self.logger.error(e) else: ip.itype = 'ipv4' ip.rdata = i.indicator ip.confidence = (ip.confidence - 4) if ip.confidence >= 4 else 0 router.indicators_create(ip)
def process(i): if not ENABLED: return if i.itype != 'fqdn': return try: r = resolve_ns(i.indicator) if not r: return except Timeout: return rv = [] for rr in r: rr = str(rr) if rr in ["", 'localhost']: continue ip = Indicator(**i.__dict__()) ip.lasttime = arrow.utcnow() ip.indicator = rr try: resolve_itype(ip.indicator) except: continue ip.itype = 'ipv4' ip.rdata = i.indicator ip.confidence = 1 ip.probability = 0 rv.append(ip) pdns = Indicator(**copy.deepcopy(i.__dict__())) # also create a passive dns tag pdns.tags = 'pdns' pdns.confidence = 4 pdns.probability = i.probability pdns.indicator = ip.indicator pdns.rdata = i.indicator rv.append(pdns) return rv
def indicator4(): return Indicator(indicator='example.com', tags='botnet', provider='test-provider', group='everyone', lasttime=arrow.utcnow().datetime, reporttime=arrow.utcnow().datetime, confidence=8.0)
def main(): g = Geo() i = sys.argv[1] i = Indicator(i) i = g.process(i) pprint(i)
def main(): from csirtg_indicator import Indicator i = Indicator('71.6.146.130') import logging logger = logging.getLogger('') logger.setLevel(logging.DEBUG) print(process(i))
def main(): from csirtg_indicator import Indicator i = Indicator('ns2.ndxylfpxuwowlhycfh.pw') import logging logger = logging.getLogger('') logger.setLevel(logging.DEBUG) print(process(i))
def indicator_email(): return Indicator(indicator='*****@*****.**', tags='botnet', provider='csirtg.io', group='everyone', lasttime=arrow.utcnow().datetime, reporttime=arrow.utcnow().datetime, tlp='green')
def indicator_broken_multi_tag_el(): return Indicator( indicator='d52380918a07322c50f1bfa2b43af3bb54cb33db', tags=['malware,exploit'], # this is intentionally bad for the test provider='csirtg.io', group='everyone', lasttime=arrow.utcnow().datetime, reporttime=arrow.utcnow().datetime)
def test_indicator_dest(): i = Indicator(indicator='192.168.1.1', dest='10.0.0.1', portlist="23", protocol="tcp", dest_portlist='21,22-23') assert i.dest assert i.dest_portlist
def process(i): return try: for ii in i.csirtg(): yield Indicator(**ii, resolve_geo=True) except Exception as e: pass
def process(self, i, router): if not self.token: return if i.itype != 'ipv4': return if 'search' not in i.tags: return if i.confidence and i.confidence < 9: return if re.search('^(\S+)\/(\d+)$', i.indicator): return max = MAX_QUERY_RESULTS try: for r in self.client.search(i.indicator): first = arrow.get(r.get('time_first') or r.get('zone_time_first')) first = first.datetime last = arrow.get(r.get('time_last') or r.get('zone_time_last')) last = last.datetime reporttime = arrow.utcnow().datetime r['rrname'] = r['rrname'].rstrip('.') try: ii = Indicator( indicator=r['rdata'], rdata=r['rrname'].rstrip('.'), count=r['count'], tags='pdns', confidence=10, firsttime=first, lasttime=last, reporttime=reporttime, provider=PROVIDER, tlp='amber', group='everyone' ) except InvalidIndicator as e: self.logger.error(e) return router.indicators_create(ii) max -= 1 if max == 0: break except QuotaLimit: logger.warn('farsight quota limit reached... skipping') except Exception as e: logger.exception(e) return
def test_urls_ok(): data = [ 'http://192.168.1.1/1.html', 'http://www41.xzmnt.com', 'http://get.ahoybest.com/n/3.6.16/12205897/microsoft lync server 2010.exe' ] for d in data: d = Indicator(d) assert d.itype is 'url'
def test_indicator_timezones(): t = '2017-03-06T11:41:48-06:00' a = arrow.get('2017-03-06T17:41:48Z').datetime i = Indicator('example.com', firsttime=t, lasttime=t, reporttime=t) assert i.firsttime == a assert i.lasttime == a assert i.reporttime == a
def _create(cli, args, options): print("submitting {0}".format(options.get("submit"))) i = Indicator(indicator=args.indicator, tags=args.tags, confidence=args.confidence) rv = cli.indicators_create(i) print('success id: {}\n'.format(rv))