def process(self, i, router): if i.itype != 'ipv4': return if 'whitelist' not in i.tags: return if i.indicator.endswith('/24'): return prefix = i.indicator.split('.') prefix = prefix[:3] prefix.append('0/24') prefix = '.'.join(prefix) try: ii = Indicator(**i.__dict__()) except InvalidIndicator as e: self.logger.error(e) return ii.lasttime = arrow.utcnow() ii.indicator = prefix ii.tags = ['whitelist'] ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0 router.indicators_create(ii)
def process(self, i, router): if i.itype != 'fqdn': return try: r = resolve_ns(i.indicator, t='CNAME') except Timeout: self.logger.info('timeout trying to resolve: {}'.format(i.indicator)) r = [] for rr in r: # http://serverfault.com/questions/44618/is-a-wildcard-cname-dns-record-valid rr = str(rr).rstrip('.').lstrip('*.') if rr in ['', 'localhost']: continue fqdn = Indicator(**i.__dict__()) fqdn.indicator = rr fqdn.lasttime = arrow.utcnow() try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.error(fqdn) self.logger.error(e) return fqdn.itype = 'fqdn' fqdn.confidence = (fqdn.confidence - 1) router.indicators_create(fqdn)
def process(i): if not ENABLED: return if i.itype != 'url': return if i.probability: return for t in i.tags: if t == 'predicted': return if not predict(i.indicator): return i = Indicator(**i.__dict__()) i.lasttime = arrow.utcnow() i.confidence = 4 i.probability = 84 i.provider = 'csirtgadgets.com' i.reference = 'https://github.com/csirtgadgets/csirtg-urlsml-py' + '#' + VERSION tags = set(i.tags) tags.add('predicted') i.tags = list(tags) return i
def process(self, i, router): if i.itype != 'ipv4': return if 'whitelist' not in i.tags: return # only run this hunter if it's a single address (no CIDRs) if ipaddress.IPv4Network(i.indicator).prefixlen != 32: return prefix = i.indicator.split('.') prefix = prefix[:3] prefix.append('0/24') prefix = '.'.join(prefix) try: ii = Indicator(**i.__dict__()) except InvalidIndicator as e: self.logger.error(e) return ii.lasttime = arrow.utcnow() ii.indicator = prefix ii.tags = ['whitelist', 'hunter'] ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0 router.indicators_create(ii)
def process(i): if not ENABLED: return if i.itype not in ['ipv4', 'ipv6']: return if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator): return try: r = _resolve(i.indicator) except Exception as e: return r = CODES.get(str(r), None) if not r: return f = Indicator(**i.__dict__()) f.tags = [r['tags']] f.description = r['description'] f.confidence = CONFIDENCE f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(f.indicator) f.lasttime = arrow.utcnow() f.probability = 0 return f
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator) except Timeout: self.logger.info('timeout trying to resolve: {}'.format(i.indicator)) return for rr in r: if str(rr).rstrip('.') in ["", 'localhost']: continue ip = Indicator(**i.__dict__()) ip.indicator = str(rr) ip.lasttime = arrow.utcnow() try: resolve_itype(ip.indicator) except InvalidIndicator as e: self.logger.error(ip) self.logger.error(e) else: ip.itype = 'ipv4' ip.rdata = i.indicator ip.confidence = (ip.confidence - 4) if ip.confidence >= 4 else 0 router.indicators_create(ip)
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator, t='CNAME') except Timeout: self.logger.info('timeout trying to resolve: {}'.format(i.indicator)) r = [] for rr in r: # http://serverfault.com/questions/44618/is-a-wildcard-cname-dns-record-valid rr = str(rr).rstrip('.').lstrip('*.') if rr in ['', 'localhost', '0.0.0.0']: continue fqdn = Indicator(**i.__dict__()) fqdn.indicator = rr fqdn.lasttime = arrow.utcnow() try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.error(fqdn) self.logger.error(e) return fqdn.itype = 'fqdn' fqdn.confidence = (fqdn.confidence - 1) router.indicators_create(fqdn)
def process(self, i, router): if i.itype != 'ipv4' and i.itype != 'ipv6': return if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator): return try: r = self._resolve(i.indicator) try: r = CODES.get(str(r), None) except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: f = Indicator(**i.__dict__()) f.tags = [r['tags']] f.description = r['description'] f.confidence = CONFIDENCE f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format( f.indicator) f.lasttime = arrow.utcnow() x = router.indicators_create(f) except Exception as e: self.logger.error(e) import traceback traceback.print_exc()
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator) except Timeout: self.logger.info('timeout trying to resolve: {}'.format( i.indicator)) return for rr in r: if str(rr).rstrip('.') in ["", 'localhost']: continue ip = Indicator(**i.__dict__()) ip.indicator = str(rr) ip.lasttime = arrow.utcnow() try: resolve_itype(ip.indicator) except InvalidIndicator as e: self.logger.error(ip) self.logger.error(e) else: ip.itype = 'ipv4' ip.rdata = i.indicator ip.confidence = (ip.confidence - 4) if ip.confidence >= 4 else 0 router.indicators_create(ip)
def process(self, i, router): if i.itype == 'fqdn' and i.provider != 'spamhaus.org': try: r = self._resolve(i.indicator) try: r = CODES.get(str(r), None) except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: confidence = CONFIDENCE if ' legit ' in r['description']: confidence = 6 f = Indicator(**i.__dict__()) f.tags = [r['tags']] f.description = r['description'] f.confidence = confidence f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format(f.indicator) f.lasttime = arrow.utcnow() x = router.indicators_create(f) self.logger.debug(x) except KeyError as e: self.logger.error(e)
def process(i): if not ENABLED: return if i.itype != 'fqdn': return if i.provider == 'spamhaus.org': return r = _resolve(i.indicator) r = CODES.get(str(r), None) if not r: return confidence = CONFIDENCE if ' legit ' in r['description']: confidence = 1 f = Indicator(**i.__dict__()) f.tags = [r['tags']] f.description = r['description'] f.confidence = confidence f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format( f.indicator) f.lasttime = arrow.utcnow() f.probability = 0 return f
def process(self, i, router): if i.itype != 'ipv4' and i.itype != 'ipv6': return if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator): return try: r = self._resolve(i.indicator) try: r = CODES.get(str(r), None) except Exception as e: # https://www.spamhaus.org/faq/section/DNSBL%20Usage self.logger.error(e) self.logger.info('check spamhaus return codes') r = None if r: f = Indicator(**i.__dict__()) f.tags = [r['tags']] f.description = r['description'] f.confidence = CONFIDENCE f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(f.indicator) f.lasttime = arrow.utcnow() x = router.indicators_create(f) except Exception as e: self.logger.error(e) import traceback traceback.print_exc()
def process(self, i, router): if i.itype != 'url': return if 'search' in i.tags: return u = urlparse(i.indicator) if not u.hostname: return try: resolve_itype(u.hostname) except InvalidIndicator as e: self.logger.error(u.hostname) self.logger.error(e) else: fqdn = Indicator(**i.__dict__()) fqdn.lasttime = arrow.utcnow() fqdn.indicator = u.hostname fqdn.itype = 'fqdn' fqdn.confidence = (int(fqdn.confidence) / 2) fqdn.rdata = i.indicator self.logger.debug('sending to router: {}'.format(fqdn)) router.indicators_create(fqdn)
def text_to_list(text, known_only=True): separator = find_seperator(text) t_tokens = top_tokens(text) top = set() for t in range(0, 9): top.add(t_tokens[t]) if known_only: if separator not in KNOWN_SEPERATORS: pprint(top) raise SystemError('separator not in known list: {}'.format(separator)) ret = [] for l in text.split("\n"): if l == '': continue if l.startswith('#') or l.startswith(';'): continue cols = l.split(separator) cols = [x.strip() for x in cols] indicator = Indicator() for e in cols: if e: try: i = resolve_itype(e) if i: indicator.indicator = e indicator.itype = i except NotImplementedError: pass try: ts = arrow.get(e) if ts: indicator.lasttime = ts.datetime except (arrow.parser.ParserError, UnicodeDecodeError): pass if e in top: indicator.tags = [e] if indicator.itype and indicator.indicator: ret.append(indicator) return ret
def text_to_list(text, known_only=True): separator = find_seperator(text) t_tokens = top_tokens(text) top = set() for t in range(0, 9): top.add(t_tokens[t]) if known_only: if separator not in KNOWN_SEPERATORS: raise SystemError( 'separator not in known list: {}'.format(separator)) ret = [] for l in text.split("\n"): if l == '': continue if l.startswith('#') or l.startswith(';'): continue cols = l.split(separator) cols = [x.strip() for x in cols] indicator = Indicator() for e in cols: if e: try: i = resolve_itype(e) if i: indicator.indicator = e indicator.itype = i except TypeError: pass try: ts = arrow.get(e) if ts: indicator.lasttime = ts.datetime except (arrow.parser.ParserError, UnicodeDecodeError): pass if e in top: indicator.tags = [e] if indicator.itype and indicator.indicator: ret.append(indicator) return ret
def process(i): if not ENABLED: return if i.itype != 'fqdn': return try: r = resolve_ns(i.indicator) if not r: return except Timeout: return rv = [] for rr in r: rr = str(rr) if rr in ["", 'localhost']: continue ip = Indicator(**i.__dict__()) ip.lasttime = arrow.utcnow() ip.indicator = rr try: resolve_itype(ip.indicator) except: continue ip.itype = 'ipv4' ip.rdata = i.indicator ip.confidence = 1 ip.probability = 0 rv.append(ip) pdns = Indicator(**copy.deepcopy(i.__dict__())) # also create a passive dns tag pdns.tags = 'pdns' pdns.confidence = 4 pdns.probability = i.probability pdns.indicator = ip.indicator pdns.rdata = i.indicator rv.append(pdns) return rv
def process(i): return if not ENABLED: return if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator, t='MX') if not r: return except Timeout: return rv = [] for rr in r: rr = re.sub(r'^\d+ ', '', str(rr)) rr = str(rr).rstrip('.') if rr in ["", 'localhost']: continue # 10 if re.match('^\d+$', rr): continue fqdn = Indicator(**i.__dict__()) fqdn.probability = 0 fqdn.indicator = rr.rstrip('.') fqdn.lasttime = arrow.utcnow() try: resolve_itype(fqdn.indicator) except: continue fqdn.itype = 'fqdn' fqdn.rdata = i.indicator fqdn.confidence = 0 rv.append(fqdn) return rv
def process(i): if i.itype != 'fqdn': return if not i.is_subdomain(): return fqdn = Indicator(**i.__dict__()) fqdn.probability = 0 fqdn.indicator = i.is_subdomain() fqdn.lasttime = arrow.utcnow() try: resolve_itype(fqdn.indicator) except: return fqdn.confidence = 1 return fqdn
def process(self, i, router): if i.itype not in ['ipv4', 'ipv6']: return if 'whitelist' not in i.tags: return prefix = i.indicator.split('.') prefix = prefix[:3] prefix.append('0/24') prefix = '.'.join(prefix) ii = Indicator(**i.__dict__()) ii.lasttime = arrow.utcnow() ii.indicator = prefix ii.tags = ['whitelist'] ii.confidence = (ii.confidence - 2) if ii.confidence >= 2 else 0 router.indicators_create(ii)
def process(i): if i.itype not in ['ipv4', 'ipv6']: return if 'whitelist' not in i.tags: return prefix = i.indicator.split('.') prefix = prefix[:3] prefix.append('0/24') prefix = '.'.join(prefix) ii = Indicator(**i.__dict__()) ii.probability = 0 ii.lasttime = arrow.utcnow() ii.indicator = prefix ii.tags = ['whitelist'] ii.confidence = 2 return ii
def process(i): return if not ENABLED: return if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator) if not r: return except Timeout: return rv = [] for rr in r: if str(rr).rstrip('.') in ["", 'localhost']: continue ip = Indicator(**i.__dict__()) ip.probability = 0 ip.indicator = str(rr) ip.lasttime = arrow.utcnow() try: resolve_itype(ip.indicator) except: continue ip.itype = 'ipv4' ip.rdata = i.indicator ip.confidence = 0 rv.append(ip) return rv
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator, t='MX') except Timeout: self.logger.info('timeout trying to resolve MX for: {}'.format( i.indicator)) return for rr in r: rr = re.sub(r'^\d+ ', '', str(rr)) rr = str(rr).rstrip('.') if rr in ["", 'localhost', '0.0.0.0']: continue fqdn = Indicator(**i.__dict__()) fqdn.indicator = rr.rstrip('.') fqdn.lasttime = arrow.utcnow() # 10 if re.match('^\d+$', rr): return try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.info(fqdn) self.logger.info(e) else: fqdn.itype = 'fqdn' fqdn.rdata = i.indicator fqdn.confidence = (fqdn.confidence - 5) if fqdn.confidence >= 5 else 0 router.indicators_create(fqdn)
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return if not i.is_subdomain(): return fqdn = Indicator(**i.__dict__()) fqdn.indicator = i.is_subdomain() fqdn.lasttime = arrow.utcnow() try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.error(fqdn) self.logger.error(e) else: fqdn.confidence = (fqdn.confidence - 3) if fqdn.confidence >= 3 else 0 router.indicators_create(fqdn)
def process(i): if not ENABLED: return if i.itype != 'fqdn': return try: r = resolve_ns(i.indicator, t='CNAME') if not r: return except Timeout: return rv = [] for rr in r: # http://serverfault.com/questions/44618/is-a-wildcard-cname-dns-record-valid rr = str(rr).rstrip('.').lstrip('*.') if rr in ['', 'localhost']: continue fqdn = Indicator(**i.__dict__()) fqdn.probability = 0 fqdn.indicator = rr fqdn.lasttime = arrow.utcnow() try: resolve_itype(fqdn.indicator) except: return fqdn.itype = 'fqdn' # keep avoid recursive cname lookups fqdn.confidence = int(fqdn.confidence / 2) if fqdn.confidence >= 2 else 0 rv.append(fqdn) return rv
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator) except Timeout: self.logger.info('timeout trying to resolve: {}'.format( i.indicator)) return for rr in r: rr = str(rr) if rr in ["", 'localhost', '0.0.0.0']: continue ip = Indicator(**i.__dict__()) ip.lasttime = arrow.utcnow() ip.indicator = rr try: resolve_itype(ip.indicator) except InvalidIndicator as e: self.logger.error(ip) self.logger.error(e) else: ip.itype = 'ipv4' ip.rdata = i.indicator ip.confidence = (ip.confidence - 2) if ip.confidence >= 2 else 0 router.indicators_create(ip) # also create a passive dns tag ip.tags = 'pdns' ip.confidence = 10 router.indicators_create(ip)
def process(self, i, router): if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator, t='MX') except Timeout: self.logger.info('timeout trying to resolve MX for: {}'.format(i.indicator)) return for rr in r: rr = re.sub(r'^\d+ ', '', str(rr)) rr = str(rr).rstrip('.') if rr in ["", 'localhost']: continue fqdn = Indicator(**i.__dict__()) fqdn.indicator = rr.rstrip('.') fqdn.lasttime = arrow.utcnow() # 10 if re.match('^\d+$', rr): return try: resolve_itype(fqdn.indicator) except InvalidIndicator as e: self.logger.info(fqdn) self.logger.info(e) else: fqdn.itype = 'fqdn' fqdn.rdata = i.indicator fqdn.confidence = (fqdn.confidence - 5) if fqdn.confidence >= 5 else 0 router.indicators_create(fqdn)
def process(self, i, router): if i.itype != 'url': return u = urlparse(i.indicator) if not u.hostname: return try: resolve_itype(u.hostname) except InvalidIndicator as e: self.logger.error(u.hostname) self.logger.error(e) else: fqdn = Indicator(**i.__dict__()) fqdn.lasttime = arrow.utcnow() fqdn.indicator = u.hostname fqdn.itype = 'fqdn' fqdn.confidence = (int(fqdn.confidence) / 2) fqdn.rdata = i.indicator self.logger.debug('sending to router: {}'.format(fqdn)) router.indicators_create(fqdn)
def process(i): if i.itype != 'url': return u = urlparse(i.indicator) if not u.hostname: return try: resolve_itype(u.hostname) except TypeError as e: logger.error(u.hostname) logger.error(e) return fqdn = Indicator(**i.__dict__()) fqdn.lasttime = arrow.utcnow() fqdn.indicator = u.hostname fqdn.itype = 'fqdn' fqdn.confidence = 2 fqdn.rdata = i.indicator fqdn.probability = 0 return fqdn
def process(i): if not ENABLED: return if i.itype != 'fqdn': return if i.probability: return if not predict(i.indicator): return fqdn = Indicator(**i.__dict__()) fqdn.lasttime = arrow.utcnow() fqdn.confidence = 4 fqdn.probability = 84 fqdn.provider = 'csirtgadgets.com' fqdn.reference = 'https://github.com/csirtgadgets/csirtg-domainsml-py' + '#' + VERSION tags = set(fqdn.tags) tags.add('predicted') fqdn.tags = list(tags) return fqdn
def get_indicator(l): i = {} # step 1, detect datatypes for e in l: if isinstance(e, int): i[e] = 'int' continue t = None try: t = resolve_itype(e) if t: i[e] = 'indicator' continue except Exception: pass if is_timestamp(e): i[e] = 'timestamp' continue if isinstance(e, basestring): i[e] = 'string' i2 = Indicator() timestamps = [] ports = [] for e in i: if i[e] == 'indicator': i2.indicator = e continue if i[e] == 'timestamp': timestamps.append(e) continue if i[e] == 'int': ports.append(e) continue if i[e] == 'string': if ' ' in e: i2.description = e continue if len(e) < 10: i2.tags = [e] continue timestamps = sorted(timestamps, reverse=True) if len(timestamps) > 0: i2.lasttime = timestamps[0] if len(timestamps) > 1: i2.firsttime = timestamps[1] if len(ports) > 0: if len(ports) == 1: i2.portlist = ports[0] else: if ports[0] > ports[1]: i2.portlist = ports[0] i2.dest_portlist = ports[1] else: i2.portlist = ports[1] i2.dest_portlist = ports[0] return i2