def process(i): if not ENABLED: return if i.itype not in ['ipv4', 'ipv6']: return if i.provider == 'spamhaus.org' and not is_ipv4_net(i.indicator): return try: r = _resolve(i.indicator) except Exception as e: return r = CODES.get(str(r), None) if not r: return f = Indicator(**i.__dict__()) f.tags = [r['tags']] f.description = r['description'] f.confidence = CONFIDENCE f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/bl?ip={}'.format(f.indicator) f.lasttime = arrow.utcnow() f.probability = 0 return f
def process(i): if not ENABLED: return if i.itype != 'url': return if i.probability: return for t in i.tags: if t == 'predicted': return if not predict(i.indicator): return i = Indicator(**i.__dict__()) i.lasttime = arrow.utcnow() i.confidence = 4 i.probability = 84 i.provider = 'csirtgadgets.com' i.reference = 'https://github.com/csirtgadgets/csirtg-urlsml-py' + '#' + VERSION tags = set(i.tags) tags.add('predicted') i.tags = list(tags) return i
def process(i): if not ENABLED: return if i.itype != 'fqdn': return if i.provider == 'spamhaus.org': return r = _resolve(i.indicator) r = CODES.get(str(r), None) if not r: return confidence = CONFIDENCE if ' legit ' in r['description']: confidence = 1 f = Indicator(**i.__dict__()) f.tags = [r['tags']] f.description = r['description'] f.confidence = confidence f.provider = PROVIDER f.reference_tlp = 'white' f.reference = 'http://www.spamhaus.org/query/dbl?domain={}'.format( f.indicator) f.lasttime = arrow.utcnow() f.probability = 0 return f
def process(i): if not ENABLED: return if i.itype != 'fqdn': return try: r = resolve_ns(i.indicator) if not r: return except Timeout: return rv = [] for rr in r: rr = str(rr) if rr in ["", 'localhost']: continue ip = Indicator(**i.__dict__()) ip.lasttime = arrow.utcnow() ip.indicator = rr try: resolve_itype(ip.indicator) except: continue ip.itype = 'ipv4' ip.rdata = i.indicator ip.confidence = 1 ip.probability = 0 rv.append(ip) pdns = Indicator(**copy.deepcopy(i.__dict__())) # also create a passive dns tag pdns.tags = 'pdns' pdns.confidence = 4 pdns.probability = i.probability pdns.indicator = ip.indicator pdns.rdata = i.indicator rv.append(pdns) return rv
def process(i): return if not ENABLED: return if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator, t='MX') if not r: return except Timeout: return rv = [] for rr in r: rr = re.sub(r'^\d+ ', '', str(rr)) rr = str(rr).rstrip('.') if rr in ["", 'localhost']: continue # 10 if re.match('^\d+$', rr): continue fqdn = Indicator(**i.__dict__()) fqdn.probability = 0 fqdn.indicator = rr.rstrip('.') fqdn.lasttime = arrow.utcnow() try: resolve_itype(fqdn.indicator) except: continue fqdn.itype = 'fqdn' fqdn.rdata = i.indicator fqdn.confidence = 0 rv.append(fqdn) return rv
def process(i): if i.itype != 'fqdn': return if not i.is_subdomain(): return fqdn = Indicator(**i.__dict__()) fqdn.probability = 0 fqdn.indicator = i.is_subdomain() fqdn.lasttime = arrow.utcnow() try: resolve_itype(fqdn.indicator) except: return fqdn.confidence = 1 return fqdn
def process(i): if i.itype not in ['ipv4', 'ipv6']: return if 'whitelist' not in i.tags: return prefix = i.indicator.split('.') prefix = prefix[:3] prefix.append('0/24') prefix = '.'.join(prefix) ii = Indicator(**i.__dict__()) ii.probability = 0 ii.lasttime = arrow.utcnow() ii.indicator = prefix ii.tags = ['whitelist'] ii.confidence = 2 return ii
def process(i): return if not ENABLED: return if i.itype != 'fqdn': return if 'search' in i.tags: return try: r = resolve_ns(i.indicator) if not r: return except Timeout: return rv = [] for rr in r: if str(rr).rstrip('.') in ["", 'localhost']: continue ip = Indicator(**i.__dict__()) ip.probability = 0 ip.indicator = str(rr) ip.lasttime = arrow.utcnow() try: resolve_itype(ip.indicator) except: continue ip.itype = 'ipv4' ip.rdata = i.indicator ip.confidence = 0 rv.append(ip) return rv
def process(i): if not ENABLED: return if i.itype != 'fqdn': return try: r = resolve_ns(i.indicator, t='CNAME') if not r: return except Timeout: return rv = [] for rr in r: # http://serverfault.com/questions/44618/is-a-wildcard-cname-dns-record-valid rr = str(rr).rstrip('.').lstrip('*.') if rr in ['', 'localhost']: continue fqdn = Indicator(**i.__dict__()) fqdn.probability = 0 fqdn.indicator = rr fqdn.lasttime = arrow.utcnow() try: resolve_itype(fqdn.indicator) except: return fqdn.itype = 'fqdn' # keep avoid recursive cname lookups fqdn.confidence = int(fqdn.confidence / 2) if fqdn.confidence >= 2 else 0 rv.append(fqdn) return rv
def process(i): if not ENABLED: return if i.itype != 'fqdn': return if i.probability: return if not predict(i.indicator): return fqdn = Indicator(**i.__dict__()) fqdn.lasttime = arrow.utcnow() fqdn.confidence = 4 fqdn.probability = 84 fqdn.provider = 'csirtgadgets.com' fqdn.reference = 'https://github.com/csirtgadgets/csirtg-domainsml-py' + '#' + VERSION tags = set(fqdn.tags) tags.add('predicted') fqdn.tags = list(tags) return fqdn
def process(i): if i.itype != 'url': return u = urlparse(i.indicator) if not u.hostname: return try: resolve_itype(u.hostname) except TypeError as e: logger.error(u.hostname) logger.error(e) return fqdn = Indicator(**i.__dict__()) fqdn.lasttime = arrow.utcnow() fqdn.indicator = u.hostname fqdn.itype = 'fqdn' fqdn.confidence = 2 fqdn.rdata = i.indicator fqdn.probability = 0 return fqdn