def process(self, i, router): if i.itype == 'fqdn': r = resolve_ns(i.indicator, t='CNAME') self.logger.debug('CNAME: {}'.format(r)) for rr in r: fqdn = Indicator(**i.__dict__) fqdn.indicator = str(rr).rstrip('.') fqdn.itype = 'fqdn' fqdn.confidence = (int(fqdn.confidence) / 2) x = router.indicators_create(fqdn) self.logger.debug(x) if i.is_subdomain(): fqdn = Indicator(**i.__dict__) fqdn.indicator = i.is_subdomain() fqdn.confidence = (int(fqdn.confidence) / 3) x = router.indicators_create(fqdn) self.logger.debug(x) r = resolve_ns(i.indicator) self.logger.debug(r) for rr in r: ip = Indicator(**i.__dict__) ip.indicator = str(rr) ip.itype = 'ipv4' ip.confidence = (int(ip.confidence) / 4) x = router.indicators_create(ip) self.logger.debug(x) r = resolve_ns(i.indicator, t='NS') self.logger.debug('NS: {}'.format(r)) for rr in r: ip = Indicator(**i.__dict__) ip.indicator = str(rr).rstrip('.') ip.itype = 'fqdn' ip.confidence = (int(ip.confidence) / 5) x = router.indicators_create(ip) self.logger.debug(x) r = resolve_ns(i.indicator, t='MX') self.logger.debug('MX: {}'.format(r)) for rr in r: ip = Indicator(**i.__dict__) ip.indicator = str(rr).rstrip('.') ip.itype = 'fqdn' ip.confidence = (int(ip.confidence) / 6) x = router.indicators_create(ip) self.logger.debug(x)
def process(self, i, router, **kwargs): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator, t='NS') except Timeout: self.logger.info('timeout trying to resolve: {}'.format(i.indicator)) return for rr in r: rr = str(rr).rstrip('.') if rr in ["", 'localhost', '0.0.0.0']: continue i_ns = Indicator(**i.__dict__()) i_ns.indicator = rr try: i_ns_itype = resolve_itype(i_ns.indicator) except InvalidIndicator as e: self.logger.error(i_ns) self.logger.error(e) else: i_ns.lasttime = i_ns.reporttime = arrow.utcnow() i_ns.itype = i_ns_itype i_ns.rdata = "{} nameserver".format(i.indicator) if 'hunter' not in i_ns.tags: i_ns.tags.append('hunter') i_ns.confidence = (i_ns.confidence - 4) if i_ns.confidence >= 4 else 0 router.indicators_create(i_ns) self.logger.debug("FQDN NS Hunter: {}".format(i_ns))
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator) except Timeout: self.logger.info('timeout trying to resolve: {}'.format( i.indicator)) return for rr in r: ip = Indicator(**i.__dict__()) ip.indicator = str(rr) try: resolve_itype(ip.indicator) except InvalidIndicator as e: self.logger.error(ip) self.logger.error(e) else: ip.itype = 'ipv4' ip.rdata = i.indicator ip.confidence = (int(ip.confidence) / 4) router.indicators_create(ip) # also create a passive dns tag ip.tags = 'pdns' ip.confidence = 10 router.indicators_create(ip)
def process(self, i, router): if i.itype != 'ipv4': return if 'whitelist' not in i.tags: return if i.indicator.endswith('/24'): return prefix = i.indicator.split('.') prefix = prefix[:3] prefix.append('0/24') prefix = '.'.join(prefix) try: ii = Indicator(**i.__dict__()) except InvalidIndicator as e: self.logger.error(e) return ii.lasttime = arrow.utcnow() ii.indicator = prefix ii.tags = ['whitelist'] ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0 router.indicators_create(ii)
def test_get_set(): i = Indicator('localhost.com') try: i.indicator = 'localhost' except InvalidIndicator: pass i.indicator = 'localhost.org' assert i.itype == 'fqdn' i.indicator = 'https://192.168.1.1' assert i.itype == 'url' assert str(i) print(i)
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator) except Timeout: self.logger.info('timeout trying to resolve: {}'.format(i.indicator)) return for rr in r: if str(rr).rstrip('.') in ["", 'localhost']: continue ip = Indicator(**i.__dict__()) ip.indicator = str(rr) ip.lasttime = arrow.utcnow() try: resolve_itype(ip.indicator) except InvalidIndicator as e: self.logger.error(ip) self.logger.error(e) else: ip.itype = 'ipv4' ip.rdata = i.indicator ip.confidence = (ip.confidence - 4) if ip.confidence >= 4 else 0 router.indicators_create(ip)
def process(self, i, router, **kwargs): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator) except Timeout: self.logger.info('timeout trying to resolve: {}'.format( i.indicator)) return for rr in r: rr = str(rr) if rr in ["", 'localhost', '0.0.0.0']: continue ip = Indicator(**i.__dict__()) ip.lasttime = ip.reporttime = arrow.utcnow() ip.indicator = rr try: resolve_itype(ip.indicator) except InvalidIndicator as e: self.logger.error(ip) self.logger.error(e) else: ip.itype = 'ipv4' ip.rdata = i.indicator ip.tags = ['pdns', 'hunter'] ip.confidence = 10 router.indicators_create(ip) self.logger.debug("FQDN Hunter: {}".format(ip))
def process(self, i, router): if i.itype != 'ipv4': return if 'whitelist' not in i.tags: return # only run this hunter if it's a single address (no CIDRs) if ipaddress.IPv4Network(i.indicator).prefixlen != 32: return prefix = i.indicator.split('.') prefix = prefix[:3] prefix.append('0/24') prefix = '.'.join(prefix) try: ii = Indicator(**i.__dict__()) except InvalidIndicator as e: self.logger.error(e) return ii.lasttime = ii.reporttime = arrow.utcnow() ii.indicator = prefix ii.tags = ['whitelist', 'hunter'] ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0 router.indicators_create(ii)
def process(self, i, router, **kwargs): if i.itype != 'fqdn': return if 'whitelist' not in i.tags: return urls = [] for p in ['http://', 'https://']: urls.append('{}{}'.format(p, i.indicator)) if not i.indicator.startswith('www.'): urls.append('{}www.{}'.format(p, i.indicator)) for u in urls: url = Indicator(**i.__dict__()) url.indicator = u try: resolve_itype(url.indicator) except InvalidIndicator as e: self.logger.error(url) self.logger.error(e) else: url.tags = ['whitelist', 'hunter'] url.itype = 'url' url.rdata = i.indicator url.lasttime = url.reporttime = arrow.utcnow() router.indicators_create(url)
def process(self, i, router, **kwargs): if i.itype != 'url': return if 'search' in i.tags: return # prevent recursion with fqdn_wl hunter if ('whitelist') in i.tags and (i.rdata is not None or i.rdata != ''): return u = urlparse(i.indicator) if not u.hostname: return try: resolve_itype(u.hostname) except InvalidIndicator as e: self.logger.error(u.hostname) self.logger.error(e) else: fqdn = Indicator(**i.__dict__()) fqdn.lasttime = fqdn.reporttime = arrow.utcnow() fqdn.indicator = u.hostname fqdn.itype = 'fqdn' if 'hunter' not in fqdn.tags: fqdn.tags.append('hunter') fqdn.confidence = (int(fqdn.confidence) / 2) fqdn.rdata = i.indicator self.logger.debug('[Hunter: Url] sending to router {}'.format(fqdn)) router.indicators_create(fqdn)
def process(self, i, router): if i.itype != 'fqdn': return try: r = resolve_ns(i.indicator, t='CNAME') except Timeout: self.logger.info('timeout trying to resolve: {}'.format( i.indicator)) r = [] for rr in r: fqdn = Indicator(**i.__dict__()) fqdn.indicator = str(rr).rstrip('.') try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.error(fqdn) self.logger.error(e) return fqdn.itype = 'fqdn' fqdn.confidence = (int(fqdn.confidence) / 2) router.indicators_create(fqdn)
def process(self, i, router): if i.itype != 'url': return if 'search' in i.tags: return u = urlparse(i.indicator) if not u.hostname: return try: resolve_itype(u.hostname) except InvalidIndicator as e: self.logger.error(u.hostname) self.logger.error(e) else: fqdn = Indicator(**i.__dict__()) fqdn.lasttime = arrow.utcnow() fqdn.indicator = u.hostname fqdn.itype = 'fqdn' fqdn.confidence = (int(fqdn.confidence) / 2) fqdn.rdata = i.indicator self.logger.debug('sending to router: {}'.format(fqdn)) router.indicators_create(fqdn)
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return if not i.is_subdomain(): return fqdn = Indicator(**i.__dict__()) fqdn.indicator = i.is_subdomain() fqdn.lasttime = fqdn.reporttime = arrow.utcnow() try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.error(fqdn) self.logger.error(e) else: fqdn.confidence = (fqdn.confidence - 3) if fqdn.confidence >= 3 else 0 if 'hunter' not in fqdn.tags: fqdn.tags.append('hunter') router.indicators_create(fqdn)
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator) except Timeout: self.logger.info('timeout trying to resolve: {}'.format( i.indicator)) return for rr in r: if str(rr).rstrip('.') in ["", 'localhost']: continue ip = Indicator(**i.__dict__()) ip.indicator = str(rr) ip.lasttime = arrow.utcnow() try: resolve_itype(ip.indicator) except InvalidIndicator as e: self.logger.error(ip) self.logger.error(e) else: ip.itype = 'ipv4' ip.rdata = i.indicator ip.confidence = (ip.confidence - 4) if ip.confidence >= 4 else 0 router.indicators_create(ip)
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator, t='CNAME') except Timeout: self.logger.info('timeout trying to resolve: {}'.format(i.indicator)) r = [] for rr in r: # http://serverfault.com/questions/44618/is-a-wildcard-cname-dns-record-valid rr = str(rr).rstrip('.').lstrip('*.') if rr in ['', 'localhost', '0.0.0.0']: continue fqdn = Indicator(**i.__dict__()) fqdn.indicator = rr fqdn.lasttime = arrow.utcnow() try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.error(fqdn) self.logger.error(e) return fqdn.itype = 'fqdn' fqdn.confidence = (fqdn.confidence - 1) router.indicators_create(fqdn)
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator, t='MX') except Timeout: self.logger.info('timeout trying to resolve MX for: {}'.format(i.indicator)) return for rr in r: rr = re.sub(r'^\d+ ', '', str(rr)) fqdn = Indicator(**i.__dict__()) fqdn.indicator = rr.rstrip('.') try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: if not str(e).startswith('unknown itype for "localhost"'): self.logger.error(fqdn) self.logger.error(e) else: fqdn.itype = 'fqdn' fqdn.rdata = i.indicator fqdn.confidence = (int(fqdn.confidence) / 6) router.indicators_create(fqdn)
def process(self, i, router): if i.itype != 'fqdn': return try: r = resolve_ns(i.indicator, t='CNAME') except Timeout: self.logger.info('timeout trying to resolve: {}'.format(i.indicator)) r = [] for rr in r: # http://serverfault.com/questions/44618/is-a-wildcard-cname-dns-record-valid rr = str(rr).rstrip('.').lstrip('*.') if rr in ['', 'localhost']: continue fqdn = Indicator(**i.__dict__()) fqdn.indicator = rr fqdn.lasttime = arrow.utcnow() try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.error(fqdn) self.logger.error(e) return fqdn.itype = 'fqdn' fqdn.confidence = (fqdn.confidence - 1) router.indicators_create(fqdn)
def process(i): if not ENABLED: return if i.itype != 'fqdn': return try: r = resolve_ns(i.indicator) if not r: return except Timeout: return rv = [] for rr in r: rr = str(rr) if rr in ["", 'localhost']: continue ip = Indicator(**i.__dict__()) ip.lasttime = arrow.utcnow() ip.indicator = rr try: resolve_itype(ip.indicator) except: continue ip.itype = 'ipv4' ip.rdata = i.indicator ip.confidence = 1 ip.probability = 0 rv.append(ip) pdns = Indicator(**copy.deepcopy(i.__dict__())) # also create a passive dns tag pdns.tags = 'pdns' pdns.confidence = 4 pdns.probability = i.probability pdns.indicator = ip.indicator pdns.rdata = i.indicator rv.append(pdns) return rv
def process(self, i, router): if i.itype == 'url': u = urlparse(i.indicator) if u.netloc: fqdn = Indicator(**i.__dict__) fqdn.indicator = u.netloc fqdn.itype = 'fqdn' fqdn.confidence = (int(fqdn.confidence) / 2) fqdn.rdata = i.indicator self.logger.debug('sending to router..') x = router.indicators_create(fqdn)
def _get_indicator(i): i2 = Indicator() timestamps = [] ports = [] # prioritize the various elements.. for e in i: if i[e] == 'CC': i2.cc = e continue if i[e] == 'indicator': if i2.indicator: i2.reference = e else: i2.indicator = e continue if i[e] == 'timestamp': timestamps.append(get_ts(e)) continue if i[e] == 'float': i2.asn = e continue if i[e] == 'int': ports.append(e) continue if i[e] == 'description': i2.description = e continue if i[e] == 'string': if re.match(r'[0-9A-Za-z\.\s\/]+', e) and i2.asn: i2.asn_desc = e continue if 4 <= len(e) <= 10 and re.match('[a-z-A-Z]+,?', e) \ and e not in ['ipv4', 'fqdn', 'url', 'ipv6']: i2.tags = [e] continue if ' ' in e and 5 <= len(e) and not i2.asn_desc: i2.description = e continue _calc_timestamps(i2, timestamps) _calc_ports(i2, ports) return i2
def text_to_list(text, known_only=True): separator = find_seperator(text) t_tokens = top_tokens(text) top = set() for t in range(0, 9): top.add(t_tokens[t]) if known_only: if separator not in KNOWN_SEPERATORS: pprint(top) raise SystemError('separator not in known list: {}'.format(separator)) ret = [] for l in text.split("\n"): if l == '': continue if l.startswith('#') or l.startswith(';'): continue cols = l.split(separator) cols = [x.strip() for x in cols] indicator = Indicator() for e in cols: if e: try: i = resolve_itype(e) if i: indicator.indicator = e indicator.itype = i except NotImplementedError: pass try: ts = arrow.get(e) if ts: indicator.lasttime = ts.datetime except (arrow.parser.ParserError, UnicodeDecodeError): pass if e in top: indicator.tags = [e] if indicator.itype and indicator.indicator: ret.append(indicator) return ret
def process(self, i, router, **kwargs): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator, t='MX') except Timeout: self.logger.info('timeout trying to resolve MX for: {}'.format( i.indicator)) return try: for rr in r: rr = re.sub(r'^\d+ ', '', str(rr)) rr = str(rr).rstrip('.') if rr in ["", 'localhost', '0.0.0.0']: continue elif re.match('^\d+$', rr) or re.match(r'^.{0,3}$', rr): # exclude spurious entries like those too short to be real continue fqdn = Indicator(**i.__dict__()) fqdn.indicator = rr.rstrip('.') fqdn.lasttime = fqdn.reporttime = arrow.utcnow() try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.info(fqdn) self.logger.info(e) else: fqdn.itype = 'fqdn' if 'hunter' not in fqdn.tags: fqdn.tags.append('hunter') fqdn.rdata = '{} mx'.format(i.indicator) fqdn.confidence = (fqdn.confidence - 5) if fqdn.confidence >= 5 else 0 router.indicators_create(fqdn) self.logger.debug("FQDN MX Hunter: {}".format(fqdn)) except Exception as e: self.logger.error( '[Hunter: FqdnMx] {}: giving up on rr {} from indicator {}'. format(e, rr, i))
def text_to_list(text, known_only=True): separator = find_seperator(text) t_tokens = top_tokens(text) top = set() for t in range(0, 9): top.add(t_tokens[t]) if known_only: if separator not in KNOWN_SEPERATORS: raise SystemError( 'separator not in known list: {}'.format(separator)) ret = [] for l in text.split("\n"): if l == '': continue if l.startswith('#') or l.startswith(';'): continue cols = l.split(separator) cols = [x.strip() for x in cols] indicator = Indicator() for e in cols: if e: try: i = resolve_itype(e) if i: indicator.indicator = e indicator.itype = i except TypeError: pass try: ts = arrow.get(e) if ts: indicator.lasttime = ts.datetime except (arrow.parser.ParserError, UnicodeDecodeError): pass if e in top: indicator.tags = [e] if indicator.itype and indicator.indicator: ret.append(indicator) return ret
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator, t='MX') except Timeout: self.logger.info('timeout trying to resolve MX for: {}'.format( i.indicator)) return try: for rr in r: rr = re.sub(r'^\d+ ', '', str(rr)) rr = str(rr).rstrip('.') if rr in ["", 'localhost', '0.0.0.0']: continue fqdn = Indicator(**i.__dict__()) fqdn.indicator = rr.rstrip('.') fqdn.lasttime = fqdn.reporttime = arrow.utcnow() # 10 if re.match('^\d+$', rr): return try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.info(fqdn) self.logger.info(e) else: fqdn.itype = 'fqdn' if 'hunter' not in fqdn.tags: fqdn.tags.append('hunter') fqdn.rdata = i.indicator fqdn.confidence = (fqdn.confidence - 5) if fqdn.confidence >= 5 else 0 router.indicators_create(fqdn) except Exception as e: self.logger.error( '[Hunter: FqdnMx] {}: giving up on indicator {}'.format(e, rr))
def process(i): return if not ENABLED: return if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator, t='MX') if not r: return except Timeout: return rv = [] for rr in r: rr = re.sub(r'^\d+ ', '', str(rr)) rr = str(rr).rstrip('.') if rr in ["", 'localhost']: continue # 10 if re.match('^\d+$', rr): continue fqdn = Indicator(**i.__dict__()) fqdn.probability = 0 fqdn.indicator = rr.rstrip('.') fqdn.lasttime = arrow.utcnow() try: resolve_itype(fqdn.indicator) except: continue fqdn.itype = 'fqdn' fqdn.rdata = i.indicator fqdn.confidence = 0 rv.append(fqdn) return rv
def process(self, i, router): if i.itype == 'url': u = urlparse(i.indicator) if u.hostname: try: resolve_itype(u.hostname) except InvalidIndicator as e: self.logger.error(u.hostname) self.logger.error(e) else: fqdn = Indicator(**i.__dict__()) fqdn.indicator = u.hostname fqdn.itype = 'fqdn' fqdn.confidence = (int(fqdn.confidence) / 2) fqdn.rdata = i.indicator self.logger.debug('sending to router..') router.indicators_create(fqdn)
def process(self, i, router): if i.itype == 'url': u = urlparse(i.indicator) if u.netloc: try: resolve_itype(u.netloc) except InvalidIndicator as e: self.logger.error(u.netloc) self.logger.error(e) else: fqdn = Indicator(**i.__dict__()) fqdn.indicator = u.netloc fqdn.itype = 'fqdn' fqdn.confidence = (int(fqdn.confidence) / 2) fqdn.rdata = i.indicator self.logger.debug('sending to router..') router.indicators_create(fqdn)
def process(self, i, router): if i.itype not in ['ipv4', 'ipv6']: return if 'whitelist' not in i.tags: return prefix = i.indicator.split('.') prefix = prefix[:3] prefix.append('0/24') prefix = '.'.join(prefix) ii = Indicator(**i.__dict__()) ii.lasttime = arrow.utcnow() ii.indicator = prefix ii.tags = ['whitelist'] ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0 router.indicators_create(ii)
def process(i): if i.itype != 'fqdn': return if not i.is_subdomain(): return fqdn = Indicator(**i.__dict__()) fqdn.probability = 0 fqdn.indicator = i.is_subdomain() fqdn.lasttime = arrow.utcnow() try: resolve_itype(fqdn.indicator) except: return fqdn.confidence = 1 return fqdn
def process(i): if i.itype not in ['ipv4', 'ipv6']: return if 'whitelist' not in i.tags: return prefix = i.indicator.split('.') prefix = prefix[:3] prefix.append('0/24') prefix = '.'.join(prefix) ii = Indicator(**i.__dict__()) ii.probability = 0 ii.lasttime = arrow.utcnow() ii.indicator = prefix ii.tags = ['whitelist'] ii.confidence = 2 return ii
def process(self, i, router, **kwargs): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator, t='CNAME') except Timeout: self.logger.info('timeout trying to resolve: {}'.format( i.indicator)) r = [] for rr in r: # http://serverfault.com/questions/44618/is-a-wildcard-cname-dns-record-valid rr = str(rr).rstrip('.').lstrip('*.') if rr in ['', 'localhost', '0.0.0.0']: continue fqdn = Indicator(**i.__dict__()) fqdn.indicator = rr fqdn.lasttime = fqdn.reporttime = arrow.utcnow() try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.error(fqdn) self.logger.error(e) return fqdn.itype = 'fqdn' fqdn.rdata = '{} cname'.format(i.indicator) if 'hunter' not in fqdn.tags: fqdn.tags.append('hunter') if fqdn.confidence < 8: fqdn.confidence -= 1 else: fqdn.confidence = 7 router.indicators_create(fqdn) self.logger.debug("FQDN CNAME Hunter: {}".format(fqdn))
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return if not i.is_subdomain(): return fqdn = Indicator(**i.__dict__()) fqdn.indicator = i.is_subdomain() try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.error(fqdn) self.logger.error(e) else: fqdn.confidence = (int(fqdn.confidence) / 3) router.indicators_create(fqdn)
def process(i): return if not ENABLED: return if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator) if not r: return except Timeout: return rv = [] for rr in r: if str(rr).rstrip('.') in ["", 'localhost']: continue ip = Indicator(**i.__dict__()) ip.probability = 0 ip.indicator = str(rr) ip.lasttime = arrow.utcnow() try: resolve_itype(ip.indicator) except: continue ip.itype = 'ipv4' ip.rdata = i.indicator ip.confidence = 0 rv.append(ip) return rv
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return if not i.is_subdomain(): return fqdn = Indicator(**i.__dict__()) fqdn.indicator = i.is_subdomain() fqdn.lasttime = arrow.utcnow() try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.error(fqdn) self.logger.error(e) else: fqdn.confidence = (fqdn.confidence - 3) if fqdn.confidence >= 3 else 0 router.indicators_create(fqdn)
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator) except Timeout: self.logger.info('timeout trying to resolve: {}'.format( i.indicator)) return for rr in r: rr = str(rr) if rr in ["", 'localhost', '0.0.0.0']: continue ip = Indicator(**i.__dict__()) ip.lasttime = arrow.utcnow() ip.indicator = rr try: resolve_itype(ip.indicator) except InvalidIndicator as e: self.logger.error(ip) self.logger.error(e) else: ip.itype = 'ipv4' ip.rdata = i.indicator ip.confidence = (ip.confidence - 2) if ip.confidence >= 2 else 0 router.indicators_create(ip) # also create a passive dns tag ip.tags = 'pdns' ip.confidence = 10 router.indicators_create(ip)
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator, t='MX') except Timeout: self.logger.info('timeout trying to resolve MX for: {}'.format(i.indicator)) return for rr in r: rr = re.sub(r'^\d+ ', '', str(rr)) rr = str(rr).rstrip('.') if rr in ["", 'localhost']: continue fqdn = Indicator(**i.__dict__()) fqdn.indicator = rr.rstrip('.') fqdn.lasttime = arrow.utcnow() # 10 if re.match('^\d+$', rr): return try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.info(fqdn) self.logger.info(e) else: fqdn.itype = 'fqdn' fqdn.rdata = i.indicator fqdn.confidence = (fqdn.confidence - 5) if fqdn.confidence >= 5 else 0 router.indicators_create(fqdn)
def process(self, i, router): if i.itype != 'url': return u = urlparse(i.indicator) if not u.hostname: return try: resolve_itype(u.hostname) except InvalidIndicator as e: self.logger.error(u.hostname) self.logger.error(e) else: fqdn = Indicator(**i.__dict__()) fqdn.lasttime = arrow.utcnow() fqdn.indicator = u.hostname fqdn.itype = 'fqdn' fqdn.confidence = (int(fqdn.confidence) / 2) fqdn.rdata = i.indicator self.logger.debug('sending to router: {}'.format(fqdn)) router.indicators_create(fqdn)
def process(self, i, router): if i.itype == 'fqdn': try: r = resolve_ns(i.indicator, t='CNAME') except Timeout: self.logger.info('timeout trying to resolve: {}'.format(i.indicator)) r = [] for rr in r: fqdn = Indicator(**i.__dict__()) fqdn.indicator = str(rr).rstrip('.') try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.error(fqdn) self.logger.error(e) else: fqdn.itype = 'fqdn' fqdn.confidence = (int(fqdn.confidence) / 2) router.indicators_create(fqdn) if i.is_subdomain(): fqdn = Indicator(**i.__dict__()) fqdn.indicator = i.is_subdomain() try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.error(fqdn) self.logger.error(e) else: fqdn.confidence = (int(fqdn.confidence) / 3) router.indicators_create(fqdn) try: r = resolve_ns(i.indicator) except Timeout: self.logger.info('timeout trying to resolve: {}'.format(i.indicator)) r = [] for rr in r: ip = Indicator(**i.__dict__()) ip.indicator = str(rr) try: resolve_itype(ip.indicator) except InvalidIndicator as e: self.logger.error(ip) self.logger.error(e) else: ip.itype = 'ipv4' ip.rdata = i.indicator ip.confidence = (int(ip.confidence) / 4) router.indicators_create(ip) try: r = resolve_ns(i.indicator, t='NS') except Timeout: self.logger.info('timeout trying to resolve NS for: {}'.format(i.indicator)) r = [] for rr in r: fqdn = Indicator(**i.__dict__()) fqdn.indicator = str(rr).rstrip('.') try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.error(fqdn) self.logger.error(e) else: fqdn.itype = 'fqdn' fqdn.rdata = i.indicator fqdn.confidence = (int(fqdn.confidence) / 5) router.indicators_create(fqdn) try: r = resolve_ns(i.indicator, t='MX') except Timeout: self.logger.info('timeout trying to resolve MX for: {}'.format(i.indicator)) r = [] for rr in r: rr = re.sub(r'^\d+ ', '', str(rr)) fqdn = Indicator(**i.__dict__()) fqdn.indicator = rr.rstrip('.') try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: if not str(e).startswith('unknown itype for "localhost"'): self.logger.error(fqdn) self.logger.error(e) else: fqdn.itype = 'fqdn' fqdn.rdata = i.indicator fqdn.confidence = (int(fqdn.confidence) / 6) router.indicators_create(fqdn)
def process(self, i, router): if i.itype == 'fqdn': try: r = resolve_ns(i.indicator, t='CNAME') except Timeout: self.logger.info('timeout trying to resolve: {}'.format(i.indicator)) r = [] for rr in r: fqdn = Indicator(**i.__dict__) fqdn.indicator = str(rr).rstrip('.') fqdn.itype = 'fqdn' fqdn.confidence = (int(fqdn.confidence) / 2) x = router.indicators_create(fqdn) if i.is_subdomain(): fqdn = Indicator(**i.__dict__) fqdn.indicator = i.is_subdomain() fqdn.confidence = (int(fqdn.confidence) / 3) x = router.indicators_create(fqdn) try: r = resolve_ns(i.indicator) except Timeout: self.logger.info('timeout trying to resolve: {}'.format(i.indicator)) r = [] for rr in r: ip = Indicator(**i.__dict__) ip.indicator = str(rr) ip.itype = 'ipv4' ip.rdata = i.indicator ip.confidence = (int(ip.confidence) / 4) x = router.indicators_create(ip) self.logger.debug(x) try: r = resolve_ns(i.indicator, t='NS') except Timeout: self.logger.info('timeout trying to resolve NS for: {}'.format(i.indicator)) r = [] for rr in r: ip = Indicator(**i.__dict__) ip.indicator = str(rr).rstrip('.') ip.itype = 'fqdn' ip.rdata = i.indicator ip.confidence = (int(ip.confidence) / 5) x = router.indicators_create(ip) self.logger.debug(x) try: r = resolve_ns(i.indicator, t='MX') except Timeout: self.logger.info('timeout trying to resolve MX for: {}'.format(i.indicator)) r = [] for rr in r: ip = Indicator(**i.__dict__) ip.indicator = str(rr).rstrip('.') ip.itype = 'fqdn' ip.rdata = i.indicator ip.confidence = (int(ip.confidence) / 6) x = router.indicators_create(ip) self.logger.debug(x)