def get_dns(self, **kwargs): client = DnsRequest(self.username, self.apikey) keys = ['query', 'end', 'start', 'timeout', 'sources'] params = self._cleanup_params(keys, **kwargs) if kwargs.get('unique'): return client.get_unique_resolutions(**params) else: return client.get_passive_dns(**params)
def load_client(context): """Get an instance of a loaded client.""" username = context.getTransformSetting('username') api_key = context.getTransformSetting('aKey') test_status = context.getTransformSetting('test_local') if test_status and test_status == 'True': server = context.getTransformSetting('server') version = context.getTransformSetting('version') return DnsRequest(username, api_key, server, version) else: return DnsRequest(username, api_key, headers=gen_debug(request))
def main(): """Perform a passive DNS lookup and save the output.""" if len(sys.argv) <= 1: print("Usage: python show_latest <query>") sys.exit(1) query = sys.argv[1] current_day = datetime.datetime.now().strftime("%Y-%m-%d") client = DnsRequest(PT_USERNAME, PT_API_KEY) results = client.get_passive_dns(query=query, start=current_day) unique = list() for record in results.get('results', list()): resolve = record['resolve'] if resolve in unique: continue unique.append(resolve) print resolve
def get_pt_domains_single_ip(ip): client = DnsRequest.from_config() while True: try: raw_results = client.get_passive_dns(query=ip) except requests.exceptions.RequestException: eprint('Request timeout, retrying') continue break domains = pyjq.all('.[].resolve', raw_results['results']) return domains
def __init__(self): try: self.clients = { 'ssl': SslRequest.from_config(), 'dns': DnsRequest.from_config(), 'enrichment': EnrichmentRequest.from_config(), 'whois': WhoisRequest.from_config(), 'attribute': AttributeRequest.from_config(), } except Exception: self.clients = None
def run(self): Analyzer.run(self) data = self.getData() try: # enrichment service if self.service == 'enrichment': enrichment_request = EnrichmentRequest(username=self.username, api_key=self.api_key) result = enrichment_request.get_enrichment(query=data) self.report(result) # malware service elif self.service == 'malware': enrichment_request = EnrichmentRequest(username=self.username, api_key=self.api_key) result = enrichment_request.get_malware(query=data) self.report(result) # osint service elif self.service == 'osint': enrichment_request = EnrichmentRequest(username=self.username, api_key=self.api_key) result = enrichment_request.get_osint(query=data) self.report(result) # passive dns service elif self.service == 'passive_dns': dns_request = DnsRequest(username=self.username, api_key=self.api_key) result = dns_request.get_passive_dns(query=data) self.report(result) # ssl certificate details service elif self.service == 'ssl_certificate_details': ssl_request = SslRequest(username=self.username, api_key=self.api_key) result = ssl_request.get_ssl_certificate_details(query=data) self.report(result) # ssl certificate history service elif self.service == 'ssl_certificate_history': ssl_request = SslRequest(username=self.username, api_key=self.api_key) result = ssl_request.get_ssl_certificate_history(query=data) self.report(result) # unique resolutions service elif self.service == 'unique_resolutions': dns_request = DnsRequest(username=self.username, api_key=self.api_key) result = dns_request.get_unique_resolutions(query=data) self.report(result) # whois details service elif self.service == 'whois_details': whois_request = WhoisRequest(username=self.username, api_key=self.api_key) result = whois_request.get_whois_details(query=data) self.report(result) else: self.error('Unknown PassiveTotal service') except Exception as e: self.unexpectedError(e)
class DnsTestCase(unittest.TestCase): """Test case for DNS methods.""" formats = ['json'] def setup_class(self): self.patcher = patch('passivetotal.api.Client._get', fake_request) self.patcher.start() self.client = DnsRequest('--No-User--', '--No-Key--') def teardown_class(self): self.patcher.stop() def test_dns_passive(self): """Test getting passive DNS records.""" payload = {'query': 'passivetotal.org'} response = self.client.get_passive_dns(**payload) assert (response.get('queryValue')) == 'passivetotal.org' def test_process_dns_passive(self): """Test processing passive DNS records.""" payload = {'query': 'passivetotal.org'} response = self.client.get_passive_dns(**payload) wrapped = Response(response) assert (wrapped.queryValue) == 'passivetotal.org' assert ( Response(wrapped.results.pop(0)).recordHash ) == '6d24bc7754af023afeaaa05ac689ac36e96656aa6519ba435b301b14916b27d3' def test_dns_passive_unique(self): """Test getting unique passive DNS records.""" payload = {'query': 'passivetotal.org'} response = self.client.get_unique_resolutions(**payload) wrapped = Response(response) assert (wrapped.queryValue) == 'passivetotal.org' record = wrapped.frequency.pop(0) assert (record[0]) == '107.170.89.121' assert (record[1]) == 2
class DnsTestCase(unittest.TestCase): """Test case for DNS methods.""" formats = ['json'] def setup_class(self): self.patcher = patch('passivetotal.api.Client._get', fake_request) self.patcher.start() self.client = DnsRequest('--No-User--', '--No-Key--') def teardown_class(self): self.patcher.stop() def test_dns_passive(self): """Test getting passive DNS records.""" payload = {'query': 'passivetotal.org'} response = self.client.get_passive_dns(**payload) assert (response.get('queryValue')) == 'passivetotal.org' def test_process_dns_passive(self): """Test processing passive DNS records.""" payload = {'query': 'passivetotal.org'} response = self.client.get_passive_dns(**payload) wrapped = Response(response) assert (wrapped.queryValue) == 'passivetotal.org' assert (Response(wrapped.results.pop(0)).recordHash) == '6d24bc7754af023afeaaa05ac689ac36e96656aa6519ba435b301b14916b27d3' def test_dns_passive_unique(self): """Test getting unique passive DNS records.""" payload = {'query': 'passivetotal.org'} response = self.client.get_unique_resolutions(**payload) wrapped = Response(response) assert (wrapped.queryValue) == 'passivetotal.org' record = wrapped.frequency.pop(0) assert (record[0]) == '107.170.89.121' assert (record[1]) == 2
def call_dns(args): """Abstract call to DNS-based queries.""" client = DnsRequest.from_config() pruned = prune_args(query=args.query, end=args.end, start=args.start, timeout=args.timeout, sources=args.sources) if args.unique: data = client.get_unique_resolutions(**pruned) else: data = client.get_passive_dns(**pruned) return data
def call_dns(args): """Abstract call to DNS-based queries.""" client = DnsRequest.from_config() pruned = prune_args( query=args.query, end=args.end, start=args.start, timeout=args.timeout, sources=args.sources ) if args.unique: data = client.get_unique_resolutions(**pruned) else: data = client.get_passive_dns(**pruned) return data
def main(): """Perform a passive DNS lookup and save the output.""" if len(sys.argv) <= 1: print "Usage: python pdns_multiput <query>" sys.exit(1) query = sys.argv[1] output_formats = ['json', 'xml', 'stix', 'csv', 'table'] client = DnsRequest.from_config() raw_results = client.get_passive_dns(query=query) pdns_results = DnsResponse(raw_results) for format_type in output_formats: save_location = "/tmp/%s.pdns.%s" % (query, format_type) tmp = open(save_location, "w") tmp.write(getattr(pdns_results, format_type)) tmp.close() print "Saved results inside of /tmp/%s" % (query)
def run(self, conf, args, plugins): if 'subcommand' in args: if args.subcommand == 'info': if not is_ip(unbracket(args.IP)): print("Invalid IP address") sys.exit(1) # FIXME: move code here in a library ip = unbracket(args.IP) try: ipy = IP(ip) except ValueError: print('Invalid IP format, quitting...') return ipinfo = self.ipinfo(ip) print('MaxMind: Located in %s, %s' % (ipinfo['city'], ipinfo['country'])) if ipinfo['asn'] == 0: print("MaxMind: IP not found in the ASN database") else: print('MaxMind: ASN%i, %s' % (ipinfo['asn'], ipinfo['asn_name'])) print('CAIDA Type: %s' % ipinfo['asn_type']) try: asndb2 = pyasn.pyasn(self.asncidr) res = asndb2.lookup(ip) except OSError: print("Configuration files are not available") print("Please run harpoon update before using harpoon") sys.exit(1) if res[1] is None: print("IP not found in ASN database") else: # Search for name f = open(self.asnname, 'r') found = False line = f.readline() name = '' while not found and line != '': s = line.split('|') if s[0] == str(res[0]): name = s[1].strip() found = True line = f.readline() print('ASN %i - %s (range %s)' % (res[0], name, res[1])) if ipinfo['hostname'] != '': print('Hostname: %s' % ipinfo['hostname']) if ipinfo['specific'] != '': print("Specific: %s" % ipinfo['specific']) if ipy.iptype() == "PRIVATE": "Private IP" print("") if ipy.version() == 4: print("Censys:\t\thttps://censys.io/ipv4/%s" % ip) print("Shodan:\t\thttps://www.shodan.io/host/%s" % ip) print("IP Info:\thttp://ipinfo.io/%s" % ip) print("BGP HE:\t\thttps://bgp.he.net/ip/%s" % ip) print( "IP Location:\thttps://www.iplocation.net/?query=%s" % ip) elif args.subcommand == "intel": if not is_ip(unbracket(args.IP)): print("Invalid IP address") sys.exit(1) # Start with MISP and OTX to get Intelligence Reports print('###################### %s ###################' % unbracket(args.IP)) passive_dns = [] urls = [] malware = [] files = [] # MISP misp_e = plugins['misp'].test_config(conf) if misp_e: print('[+] Downloading MISP information...') server = ExpandedPyMISP(conf['Misp']['url'], conf['Misp']['key']) misp_results = server.search('attributes', value=unbracket(args.IP)) # Binary Edge be_e = plugins['binaryedge'].test_config(conf) if be_e: try: print('[+] Downloading BinaryEdge information...') be = BinaryEdge(conf['BinaryEdge']['key']) # FIXME: this only get the first page res = be.domain_ip(unbracket(args.IP)) for d in res["events"]: passive_dns.append({ "domain": d['domain'], "first": parse(d['updated_at']).astimezone(pytz.utc), "last": parse(d['updated_at']).astimezone(pytz.utc), "source": "BinaryEdge" }) except BinaryEdgeException: print( 'BinaryEdge request failed, you need a paid subscription' ) # OTX otx_e = plugins['otx'].test_config(conf) if otx_e: print('[+] Downloading OTX information....') otx = OTXv2(conf["AlienVaultOtx"]["key"]) res = otx.get_indicator_details_full( IndicatorTypes.IPv4, unbracket(args.IP)) otx_pulses = res["general"]["pulse_info"]["pulses"] # Get Passive DNS if "passive_dns" in res: for r in res["passive_dns"]["passive_dns"]: passive_dns.append({ "domain": r['hostname'], "first": parse(r["first"]).astimezone(pytz.utc), "last": parse(r["last"]).astimezone(pytz.utc), "source": "OTX" }) if "url_list" in res: for r in res["url_list"]["url_list"]: if "result" in r: urls.append({ "date": parse(r["date"]).astimezone(pytz.utc), "url": r["url"], "ip": r["result"]["urlworker"]["ip"] if "ip" in r["result"]["urlworker"] else "", "source": "OTX" }) else: urls.append({ "date": parse(r["date"]).astimezone(pytz.utc), "url": r["url"], "ip": "", "source": "OTX" }) # RobTex print('[+] Downloading Robtex information....') rob = Robtex() try: res = rob.get_ip_info(unbracket(args.IP)) except RobtexError: print("Error with Robtex") else: for d in ["pas", "pash", "act", "acth"]: if d in res: for a in res[d]: passive_dns.append({ 'first': a['date'].astimezone(pytz.utc), 'last': a['date'].astimezone(pytz.utc), 'domain': a['o'], 'source': 'Robtex' }) # PT pt_e = plugins['pt'].test_config(conf) if pt_e: out_pt = False print('[+] Downloading Passive Total information....') client = DnsRequest(conf['PassiveTotal']['username'], conf['PassiveTotal']['key']) try: raw_results = client.get_passive_dns( query=unbracket(args.IP)) if "results" in raw_results: for res in raw_results["results"]: passive_dns.append({ "first": parse(res["firstSeen"]).astimezone( pytz.utc), "last": parse(res["lastSeen"]).astimezone( pytz.utc), "domain": res["resolve"], "source": "PT" }) if "message" in raw_results: if "quota_exceeded" in raw_results["message"]: print("Quota exceeded for Passive Total") out_pt = True pt_osint = {} except requests.exceptions.ReadTimeout: print("Timeout on Passive Total requests") if not out_pt: try: client2 = EnrichmentRequest( conf["PassiveTotal"]["username"], conf["PassiveTotal"]['key']) # Get OSINT # TODO: add PT projects here pt_osint = client2.get_osint( query=unbracket(args.IP)) # Get malware raw_results = client2.get_malware( query=unbracket(args.IP)) if "results" in raw_results: for r in raw_results["results"]: malware.append({ 'hash': r["sample"], 'date': parse(r['collectionDate']), 'source': 'PT (%s)' % r["source"] }) except requests.exceptions.ReadTimeout: print("Timeout on Passive Total requests") # Urlhaus uh_e = plugins['urlhaus'].test_config(conf) if uh_e: print("[+] Checking urlhaus data...") try: urlhaus = UrlHaus(conf["UrlHaus"]["key"]) res = urlhaus.get_host(unbracket(args.IP)) except UrlHausError: print("Error with the query") else: if "urls" in res: for r in res['urls']: urls.append({ "date": parse(r["date_added"]).astimezone( pytz.utc), "url": r["url"], "source": "UrlHaus" }) # VT vt_e = plugins['vt'].test_config(conf) if vt_e: if conf["VirusTotal"]["type"] != "public": print('[+] Downloading VT information....') vt = PrivateApi(conf["VirusTotal"]["key"]) res = vt.get_ip_report(unbracket(args.IP)) if "results" in res: if "resolutions" in res['results']: for r in res["results"]["resolutions"]: passive_dns.append({ "first": parse(r["last_resolved"]).astimezone( pytz.utc), "last": parse(r["last_resolved"]).astimezone( pytz.utc), "domain": r["hostname"], "source": "VT" }) if "undetected_downloaded_samples" in res[ 'results']: for r in res['results'][ 'undetected_downloaded_samples']: files.append({ 'hash': r['sha256'], 'date': parse(r['date']), 'source': 'VT' }) if "undetected_referrer_samples" in res['results']: for r in res['results'][ 'undetected_referrer_samples']: if 'date' in r: files.append({ 'hash': r['sha256'], 'date': parse(r['date']), 'source': 'VT' }) else: #FIXME : should consider data without dates files.append({ 'hash': r['sha256'], 'date': datetime.datetime(1970, 1, 1), 'source': 'VT' }) if "detected_downloaded_samples" in res['results']: for r in res['results'][ 'detected_downloaded_samples']: malware.append({ 'hash': r['sha256'], 'date': parse(r['date']), 'source': 'VT' }) if "detected_referrer_samples" in res['results']: for r in res['results'][ 'detected_referrer_samples']: if "date" in r: malware.append({ 'hash': r['sha256'], 'date': parse(r['date']), 'source': 'VT' }) else: vt_e = False print('[+] Downloading GreyNoise information....') gn = GreyNoise() try: greynoise = gn.query_ip(unbracket(args.IP)) except GreyNoiseError: greynoise = [] tg_e = plugins['threatgrid'].test_config(conf) if tg_e: print('[+] Downloading Threat Grid....') try: tg = ThreatGrid(conf['ThreatGrid']['key']) res = tg.search_samples(unbracket(args.IP), type='ip') already = [] if 'items' in res: for r in res['items']: if r['sample_sha256'] not in already: d = parse(r['ts']) d = d.replace(tzinfo=None) malware.append({ 'hash': r["sample_sha256"], 'date': d, 'source': 'TG' }) already.append(r['sample_sha256']) except ThreatGridError as e: print("Error with threat grid: {}".format(e.message)) # ThreatMiner print('[+] Downloading ThreatMiner....') tm = ThreatMiner() response = tm.get_report(unbracket(args.IP)) if response['status_code'] == '200': tmm = response['results'] else: tmm = [] if response['status_code'] != '404': print("Request to ThreatMiner failed: {}".format( response['status_message'])) response = tm.get_related_samples(unbracket(args.IP)) if response['status_code'] == '200': for r in response['results']: malware.append({ 'hash': r, 'date': None, 'source': 'ThreatMiner' }) print('----------------- Intelligence Report') ctor = CommandTor() tor_list = ctor.get_list() if tor_list: if unbracket(args.IP) in tor_list: print("{} is a Tor Exit node".format(unbracket( args.IP))) else: print("Impossible to reach the Tor Exit Node list") if otx_e: if len(otx_pulses): print('OTX:') for p in otx_pulses: print('- %s (%s - %s)' % (p['name'], p['created'][:10], "https://otx.alienvault.com/pulse/" + p['id'])) else: print('OTX: Not found in any pulse') if misp_e: if len(misp_results['Attribute']) > 0: print('MISP:') for event in misp_results['Attribute']: print("- {} - {}".format(event['Event']['id'], event['Event']['info'])) if len(greynoise) > 0: print("GreyNoise: IP identified as") for r in greynoise: print("\t%s (%s -> %s)" % (r["name"], r["first_seen"], r["last_updated"])) else: print("GreyNoise: Not found") if pt_e: if "results" in pt_osint: if len(pt_osint["results"]): if len(pt_osint["results"]) == 1: if "name" in pt_osint["results"][0]: print( "PT: %s %s" % (pt_osint["results"][0]["name"], pt_osint["results"][0]["sourceUrl"])) else: print("PT: %s" % pt_osint["results"][0]["sourceUrl"]) else: print("PT:") for r in pt_osint["results"]: if "name" in r: print("-%s %s" % (r["name"], r["sourceUrl"])) else: print("-%s" % r["sourceUrl"]) else: print("PT: Nothing found!") else: print("PT: Nothing found!") # ThreatMiner if len(tmm) > 0: print("ThreatMiner:") for r in tmm: print("- {} {} - {}".format(r['year'], r['filename'], r['URL'])) if len(malware) > 0: print('----------------- Malware') for r in malware: print("[%s] %s %s" % (r["source"], r["hash"], r["date"].strftime("%Y-%m-%d") if r["date"] else "")) if len(files) > 0: print('----------------- Files') for r in sorted(files, key=lambda x: x["date"]): print("[%s] %s %s" % (r["source"], r["hash"], r["date"].strftime("%Y-%m-%d"))) if len(passive_dns) > 0: print('----------------- Passive DNS') for r in sorted(passive_dns, key=lambda x: x["first"], reverse=True): print("[+] %-40s (%s -> %s)(%s)" % (r["domain"], r["first"].strftime("%Y-%m-%d"), r["last"].strftime("%Y-%m-%d"), r["source"])) if len(urls) > 0: print('----------------- Urls') for r in sorted(urls, key=lambda x: x["date"], reverse=True): print("[%s] %s - %s" % (r["source"], r["url"], r["date"].strftime("%Y-%m-%d"))) else: self.parser.print_help() else: self.parser.print_help()
There are times when it's difficult to tell which items have been tagged as something malicious or suspicious. This script will take an initial starting point and print out any tagged items along with their tags. """ __author__ = 'Brandon Dixon ([email protected])' __version__ = '1.0.0' __description__ = "Surface tagged items from a passive DNS query" __keywords__ = ['pdns', 'tags', 'triage', 'analysis'] import sys from passivetotal.libs.dns import DnsRequest from passivetotal.libs.enrichment import EnrichmentRequest query = sys.argv[1] client = DnsRequest.from_config() enricher = EnrichmentRequest.from_config() def main(): """Take an initial seed and identify OSINT tags.""" initial_seed = client.get_unique_resolutions(query=query) all_records = initial_seed.get('results', list()) all_records += query for item in all_records: tmp = enricher.get_enrichment(query=item) tags = tmp.get('tags', list()) if len(tags) > 0: print("%s - %s" % (item, ', '.join(tags)))
def run(self, conf, args, plugins): if 'subcommand' in args: if args.subcommand == "intel": # Start with MISP and OTX to get Intelligence Reports print('###################### %s ###################' % args.DOMAIN) passive_dns = [] urls = [] malware = [] files = [] # MISP misp_e = plugins['misp'].test_config(conf) if misp_e: print('[+] Downloading MISP information...') server = ExpandedPyMISP(conf['Misp']['url'], conf['Misp']['key']) misp_results = server.search('attributes', value=unbracket(args.DOMAIN)) # OTX otx_e = plugins['otx'].test_config(conf) if otx_e: print('[+] Downloading OTX information....') try: otx = OTXv2(conf["AlienVaultOtx"]["key"]) res = otx.get_indicator_details_full(IndicatorTypes.DOMAIN, unbracket(args.DOMAIN)) otx_pulses = res["general"]["pulse_info"]["pulses"] # Get Passive DNS if "passive_dns" in res: for r in res["passive_dns"]["passive_dns"]: passive_dns.append({ "ip": r['hostname'], "first": parse(r["first"]).astimezone(pytz.utc), "last": parse(r["last"]).astimezone(pytz.utc), "source" : "OTX" }) if "url_list" in res: for r in res["url_list"]["url_list"]: if "result" in r: urls.append({ "date": parse(r["date"]).astimezone(pytz.utc), "url": r["url"], "ip": r["result"]["urlworker"]["ip"] if "ip" in r["result"]["urlworker"] else "" , "source": "OTX" }) else: urls.append({ "date": parse(r["date"]).astimezone(pytz.utc), "url": r["url"], "ip": "", "source": "OTX" }) except AttributeError: print('OTX crashed ¯\_(ツ)_/¯') # UrlScan us = UrlScan() print('[+] Downloading UrlScan information....') res = us.search(args.DOMAIN) for r in res['results']: urls.append({ "date": parse(r["task"]["time"]).astimezone(pytz.utc), "url": r["page"]["url"], "ip": r["page"]["ip"] if "ip" in r["page"] else "", "source": "UrlScan" }) # UrlHaus uh_e = plugins['urlhaus'].test_config(conf) if uh_e: print("[+] Checking urlhaus...") try: urlhaus = UrlHaus(conf["UrlHaus"]["key"]) res = urlhaus.get_host(unbracket(args.DOMAIN)) except UrlHausError: print("Error with the query") else: if "urls" in res: for r in res['urls']: urls.append({ "date": parse(r["date_added"]).astimezone(pytz.utc), "url": r["url"], "ip":"", "source": "UrlHaus" }) # CIRCL circl_e = plugins['circl'].test_config(conf) if circl_e: print('[+] Downloading CIRCL passive DNS information....') x = pypdns.PyPDNS( basic_auth=( conf['Circl']['user'], conf['Circl']['pass'] ) ) res = x.query(unbracket(args.DOMAIN)) for answer in res: passive_dns.append({ "ip": answer['rdata'], "first": answer['time_first'].astimezone(pytz.utc), "last": answer['time_last'].astimezone(pytz.utc), "source" : "CIRCL" }) # BinaryEdge be_e = plugins['binaryedge'].test_config(conf) if be_e: print('[+] Downloading BinaryEdge information....') try: be = BinaryEdge(conf['BinaryEdge']['key']) res = be.domain_dns(unbracket(args.DOMAIN)) for d in res['events']: if "A" in d: for a in d['A']: passive_dns.append({ "ip": a, "first": parse(d['updated_at']).astimezone(pytz.utc), "last": parse(d['updated_at']).astimezone(pytz.utc), "source" : "BinaryEdge" }) except BinaryEdgeException: print('You need a paid BinaryEdge subscription for this request') # RobTex print('[+] Downloading Robtex information....') try: rob = Robtex() res = rob.get_pdns_domain(args.DOMAIN) for d in res: if d['rrtype'] in ['A', 'AAAA']: passive_dns.append({ 'first': d['time_first_o'].astimezone(pytz.utc), 'last': d['time_last_o'].astimezone(pytz.utc), 'ip': d['rrdata'], 'source': 'Robtex' }) except RobtexError: print("Robtex query failed") # PT pt_e = plugins['pt'].test_config(conf) if pt_e: try: pt_osint = {} ptout = False print('[+] Downloading Passive Total information....') client = DnsRequest(conf['PassiveTotal']['username'], conf['PassiveTotal']['key']) raw_results = client.get_passive_dns(query=unbracket(args.DOMAIN)) if "results" in raw_results: for res in raw_results["results"]: passive_dns.append({ "first": parse(res["firstSeen"]).astimezone(pytz.utc), "last": parse(res["lastSeen"]).astimezone(pytz.utc), "ip": res["resolve"], "source": "PT" }) if "message" in raw_results: if "quota_exceeded" in raw_results["message"]: print("PT quota exceeded") ptout = True if not ptout: client2 = EnrichmentRequest(conf["PassiveTotal"]["username"], conf["PassiveTotal"]['key']) # Get OSINT # TODO: add PT projects here pt_osint = client2.get_osint(query=unbracket(args.DOMAIN)) # Get malware raw_results = client2.get_malware(query=unbracket(args.DOMAIN)) if "results" in raw_results: for r in raw_results["results"]: malware.append({ 'hash': r["sample"], 'date': parse(r['collectionDate']).astimezone(pytz.utc), 'source' : 'PT (%s)' % r["source"] }) except requests.exceptions.ReadTimeout: print("PT: Time Out") # VT vt_e = plugins['vt'].test_config(conf) if vt_e: if conf["VirusTotal"]["type"] != "public": print('[+] Downloading VT information....') vt = PrivateApi(conf["VirusTotal"]["key"]) res = vt.get_domain_report(unbracket(args.DOMAIN)) if "results" in res: if "resolutions" in res['results']: for r in res["results"]["resolutions"]: passive_dns.append({ "first": parse(r["last_resolved"]).astimezone(pytz.utc), "last": parse(r["last_resolved"]).astimezone(pytz.utc), "ip": r["ip_address"], "source": "VT" }) if "undetected_downloaded_samples" in res['results']: for r in res['results']['undetected_downloaded_samples']: files.append({ 'hash': r['sha256'], 'date': parse(r['date']).astimezone(pytz.utc) if 'date' in r else '', 'source' : 'VT' }) if "undetected_referrer_samples" in res['results']: for r in res['results']['undetected_referrer_samples']: files.append({ 'hash': r['sha256'], 'date': parse(r['date']).astimezone(pytz.utc) if 'date' in r else '', 'source' : 'VT' }) if "detected_downloaded_samples" in res['results']: for r in res['results']['detected_downloaded_samples']: malware.append({ 'hash': r['sha256'], 'date': parse(r['date']).astimezone(pytz.utc), 'source' : 'VT' }) if "detected_referrer_samples" in res['results']: for r in res['results']['detected_referrer_samples']: if "date" in r: malware.append({ 'hash': r['sha256'], 'date': parse(r['date']).astimezone(pytz.utc), 'source' : 'VT' }) if "detected_urls" in res['results']: for r in res['results']['detected_urls']: urls.append({ 'date': parse(r['scan_date']).astimezone(pytz.utc), 'url': r['url'], 'ip': '', 'source': 'VT' }) else: vt_e = False tg_e = plugins['threatgrid'].test_config(conf) if tg_e: try: print('[+] Downloading Threat Grid....') tg = ThreatGrid(conf['ThreatGrid']['key']) res = tg.search_samples(unbracket(args.DOMAIN), type='domain') already = [] if 'items' in res: for r in res['items']: if r['sample_sha256'] not in already: d = parse(r['ts']).astimezone(pytz.utc) malware.append({ 'hash': r["sample_sha256"], 'date': d, 'source' : 'ThreatGrid' }) already.append(r['sample_sha256']) except ThreatGridError as e: print("Failed to connect to Threat Grid: %s" % e.message) print('[+] Downloading ThreatMiner....') tm = ThreatMiner() response = tm.get_report(unbracket(args.DOMAIN)) if response['status_code'] == '200': tmm = response['results'] else: tmm = [] if response['status_code'] == '404': print("Request to ThreatMiner failed: {}".format(response['status_message'])) response = tm.get_related_samples(unbracket(args.DOMAIN)) if response['status_code'] == '200': for r in response['results']: malware.append({ 'hash': r, 'date': None, 'source': 'ThreatMiner' }) print('----------------- Intelligence Report') if misp_e: if len(misp_results['Attribute']) > 0: print('MISP:') for event in misp_results['Attribute']: print("- {} - {}".format( event['Event']['id'], event['Event']['info'] )) if otx_e: if len(otx_pulses): print('OTX:') for p in otx_pulses: print('- %s (%s - %s)' % ( p['name'], p['created'][:10], "https://otx.alienvault.com/pulse/" + p['id'] ) ) else: print('OTX: Not found in any pulse') if pt_e: if "results" in pt_osint: if len(pt_osint["results"]): if len(pt_osint["results"]) == 1: if "name" in pt_osint["results"][0]: print("PT: %s %s" % (pt_osint["results"][0]["name"], pt_osint["results"][0]["sourceUrl"])) else: print("PT: %s" % (pt_osint["results"][0]["sourceUrl"])) else: print("PT:") for r in pt_osint["results"]: if "name" in r: print("- %s %s" % (r["name"], r["sourceUrl"])) else: print("- %s" % (r["sourceUrl"])) else: print("PT: Nothing found!") else: print("PT: Nothing found!") # ThreatMiner if len(tmm) > 0: print("ThreatMiner:") for r in tmm: print("- {} {} - {}".format( r['year'], r['filename'], r['URL'] )) if len(malware) > 0: print('----------------- Malware') for r in malware: print("[%s] %s %s" % ( r["source"], r["hash"], r["date"].strftime("%Y-%m-%d") if r["date"] else "" ) ) if len(files) > 0: print('----------------- Files') for r in files: if r['date'] != '': print("[%s] %s (%s)" % ( r["source"], r["hash"], r["date"].strftime("%Y-%m-%d") ) ) else: print("[%s] %s" % ( r["source"], r["hash"], ) ) if len(urls) > 0: print('----------------- Urls') for r in sorted(urls, key=lambda x: x["date"], reverse=True): print("[%s] %s - %s %s" % ( r["source"], r["url"], r["ip"], r["date"].strftime("%Y-%m-%d") ) ) # TODO: add ASN + location info here if len(passive_dns) > 0: print('----------------- Passive DNS') for r in sorted(passive_dns, key=lambda x: x["first"], reverse=True): print("[+] %-40s (%s -> %s)(%s)" % ( r["ip"], r["first"].strftime("%Y-%m-%d"), r["last"].strftime("%Y-%m-%d"), r["source"] ) ) else: self.parser.print_help() else: self.parser.print_help()
def run(self, conf, args, plugins): if 'subcommand' in args: if args.subcommand == 'whois': client = WhoisRequest(conf['PassiveTotal']['username'], conf['PassiveTotal']['key']) if args.domain: raw_results = client.search_whois_by_field(query=unbracket( args.domain.strip()), field="domain") print( json.dumps(raw_results, sort_keys=True, indent=4, separators=(',', ': '))) elif args.file: with open(args.file, 'r') as infile: data = infile.read().split() print( "Domain|Date|Registrar|name|email|Phone|organization|Street|City|Postal Code|State|Country" ) for d in data: do = unbracket(d.strip()) # FIXME: bulk request here raw_results = client.search_whois_by_field( query=do, field="domain") if "results" not in raw_results: print("%s|||||||||||" % bracket(do)) else: if len(raw_results["results"]) == 0: print("%s|||||||||||" % bracket(do)) else: r = raw_results["results"][0] if "registered" in r: dd = datetime.datetime.strptime( r["registered"], "%Y-%m-%dT%H:%M:%S.%f%z") ddo = dd.strftime("%m/%d/%Y %H:%M:%S") else: ddo = "" print( "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s" % (bracket(do), ddo, r["registrar"] if "registrar" in r else "", r["registrant"]["name"] if "name" in r["registrant"] else "", r["registrant"]["email"] if "email" in r["registrant"] else "", r["registrant"]["telephone"] if "telephone" in r["registrant"] else "", r["registrant"]["organization"] if "organization" in r["registrant"] else "", r["registrant"]["street"] if "street" in r["registrant"] else "", r["registrant"]["city"] if "city" in r["registrant"] else "", r["registrant"]["postalCode"] if "postalCode" in r["registrant"] else "", r["registrant"]["state"] if "state" in r["registrant"] else "", r["registrant"]["country"] if "country" in r["registrant"] else "")) elif args.email: raw_results = client.search_whois_by_field( query=args.email.strip(), field="email") print( json.dumps(raw_results, sort_keys=True, indent=4, separators=(',', ': '))) else: self.parser.print_help() elif args.subcommand == "dns": client = DnsRequest(conf['PassiveTotal']['username'], conf['PassiveTotal']['key']) raw_results = client.get_passive_dns(query=unbracket( args.DOMAIN), ) print( json.dumps(raw_results, sort_keys=True, indent=4, separators=(',', ': '))) elif args.subcommand == "malware": client = EnrichmentRequest(conf["PassiveTotal"]["username"], conf["PassiveTotal"]['key']) if args.domain: raw_results = client.get_malware(query=args.domain) print( json.dumps(raw_results, sort_keys=True, indent=4, separators=(',', ': '))) elif args.file: with open(args.file, 'r') as infile: data = infile.read().split() domain_list = list(set([a.strip() for a in data])) if len(domain_list) < 51: raw_results = client.get_bulk_malware( query=domain_list) if "results" not in raw_results or not raw_results[ "success"]: print("Request failed") print( json.dumps(raw_results, sort_keys=True, indent=4, separators=(',', ': '))) sys.exit(1) else: results = raw_results["results"] else: results = {} bulk_size = 50 i = 0 while i * bulk_size < len(domain_list): raw_results = client.get_bulk_malware( query=domain_list[i * bulk_size:(i + 1) * bulk_size]) if "results" not in raw_results or not raw_results[ "success"]: print("Request failed") print( json.dumps(raw_results, sort_keys=True, indent=4, separators=(',', ': '))) sys.exit(1) else: results.update(raw_results["results"]) i += 1 if args.raw: print( json.dumps(results, sort_keys=True, indent=4, separators=(',', ': '))) else: print("Domain|Date|Sample|Source|Source URL") for domain in results: if "results" in results[domain]: for sample in results[domain]["results"]: print("%s|%s|%s|%s|%s" % (domain, sample["collectionDate"], sample["sample"], sample["source"], sample["sourceUrl"])) else: self.parser.print_help() elif args.subcommand == "osint": # FIXME: add research of projects client = EnrichmentRequest(conf["PassiveTotal"]["username"], conf["PassiveTotal"]['key']) if args.domain: raw_results = client.get_osint(query=args.domain) print( json.dumps(raw_results, sort_keys=True, indent=4, separators=(',', ': '))) elif args.file: with open(args.file, 'r') as infile: data = infile.read().split() domain_list = list(set([a.strip() for a in data])) if len(domain_list) < 51: raw_results = client.get_bulk_osint(query=domain_list) if "results" not in raw_results or not raw_results[ "success"]: print("Request failed") print( json.dumps(raw_results, sort_keys=True, indent=4, separators=(',', ': '))) sys.exit(1) else: results = raw_results["results"] else: results = {} bulk_size = 50 i = 0 while i * bulk_size < len(domain_list): raw_results = client.get_bulk_osint( query=domain_list[i * bulk_size:(i + 1) * bulk_size]) if "results" not in raw_results or not raw_results[ "success"]: print("Request failed") print( json.dumps(raw_results, sort_keys=True, indent=4, separators=(',', ': '))) sys.exit(1) else: results.update(raw_results["results"]) i += 1 if args.raw: print( json.dumps(results, sort_keys=True, indent=4, separators=(',', ': '))) else: print("Domain|Source|URL|Tags") for domain in results: if "results" in results[domain]: for report in results[domain]["results"]: print("%s|%s|%s|%s" % (domain, report["source"], report["source_url"], " / ".join( report["tags"]))) else: self.parser.print_help() else: self.parser.print_help() else: self.parser.print_help()
def intel(self, type, query, data, conf): if type == "domain": print("[+] Checking Passive Total...") try: pt_osint = {} ptout = False client = DnsRequest( conf["PassiveTotal"]["username"], conf["PassiveTotal"]["key"], ) raw_results = client.get_passive_dns(query=query) if "results" in raw_results: for res in raw_results["results"]: data["passive_dns"].append({ "first": parse(res["firstSeen"]).astimezone(pytz.utc), "last": parse(res["lastSeen"]).astimezone(pytz.utc), "ip": res["resolve"], "source": "PT", }) if "message" in raw_results: if "quota_exceeded" in raw_results["message"]: print("PT quota exceeded") ptout = True if not ptout: client2 = EnrichmentRequest( conf["PassiveTotal"]["username"], conf["PassiveTotal"]["key"], ) # Get OSINT pt_osint = client2.get_osint(query=query) if "results" in pt_osint: for r in pt_osint["results"]: data["reports"].append({ "date": "", "title": r["name"] if "name" in r else "", "url": r["sourceUrl"], "source": "PT" }) # Get malware raw_results = client2.get_malware(query=query) if "results" in raw_results: for r in raw_results["results"]: data["malware"].append({ "hash": r["sample"], "date": parse(r["collectionDate"]).astimezone( pytz.utc), "source": "PT (%s)" % r["source"], }) except requests.exceptions.ReadTimeout: print("PT: Time Out") elif type == "ip": print("[+] Checking Passive Total...") try: pt_osint = {} ptout = False client = DnsRequest( conf["PassiveTotal"]["username"], conf["PassiveTotal"]["key"], ) raw_results = client.get_passive_dns(query=query) if "results" in raw_results: for res in raw_results["results"]: data["passive_dns"].append({ "first": parse(res["firstSeen"]).astimezone(pytz.utc), "last": parse(res["lastSeen"]).astimezone(pytz.utc), "domain": res["resolve"], "source": "PT", }) if "message" in raw_results: if "quota_exceeded" in raw_results["message"]: print("PT quota exceeded") ptout = True if not ptout: client2 = EnrichmentRequest( conf["PassiveTotal"]["username"], conf["PassiveTotal"]["key"], ) # Get OSINT pt_osint = client2.get_osint(query=query) if "results" in pt_osint: for r in pt_osint["results"]: data["reports"].append({ "date": "", "title": r["name"] if "name" in r else "", "url": r["sourceUrl"], "source": "PT" }) # Get malware raw_results = client2.get_malware(query=query) if "results" in raw_results: for r in raw_results["results"]: data["malware"].append({ "hash": r["sample"], "date": parse(r["collectionDate"]).astimezone( pytz.utc), "source": "PT (%s)" % r["source"], }) except requests.exceptions.ReadTimeout: print("PT: Time Out")
import sys from passivetotal.libs.dns import DnsRequest from passivetotal.libs.dns import DnsUniqueResponse from passivetotal.libs.whois import WhoisRequest from passivetotal.libs.whois import WhoisResponse from passivetotal.common.utilities import is_ip query = sys.argv[1] if not is_ip(query): raise Exception("This script only accepts valid IP addresses!") sys.exit(1) # look up the unique resolutions client = DnsRequest.from_config() raw_results = client.get_unique_resolutions(query=query) loaded = DnsUniqueResponse(raw_results) whois_client = WhoisRequest.from_config() for record in loaded.get_records()[:3]: raw_whois = whois_client.get_whois_details(query=record.resolve) whois = WhoisResponse(raw_whois) print record.resolve, whois.contactEmail
def run(self, conf, args, plugins): if 'subcommand' in args: if args.subcommand == 'info': print("Not implemented yet") elif args.subcommand == "intel": # Start with MISP and OTX to get Intelligence Reports print('###################### %s ###################' % args.DOMAIN) passive_dns = [] urls = [] malware = [] files = [] # OTX otx_e = plugins['otx'].test_config(conf) if otx_e: print('[+] Downloading OTX information....') otx = OTXv2(conf["AlienVaultOtx"]["key"]) res = otx.get_indicator_details_full(IndicatorTypes.DOMAIN, unbracket(args.DOMAIN)) otx_pulses = res["general"]["pulse_info"]["pulses"] # Get Passive DNS if "passive_dns" in res: for r in res["passive_dns"]["passive_dns"]: passive_dns.append({ "ip": r['hostname'], "first": parse(r["first"]), "last": parse(r["last"]), "source" : "OTX" }) if "url_list" in res: for r in res["url_list"]["url_list"]: if "result" in r: urls.append({ "date": parse(r["date"]), "url": r["url"], "ip": r["result"]["urlworker"]["ip"] if "ip" in r["result"]["urlworker"] else "" , "source": "OTX" }) else: urls.append({ "date": parse(r["date"]), "url": r["url"], "ip": "", "source": "OTX" }) # CIRCL circl_e = plugins['circl'].test_config(conf) if circl_e: print('[+] Downloading CIRCL passive DNS information....') x = pypdns.PyPDNS( basic_auth=( conf['Circl']['user'], conf['Circl']['pass'] ) ) res = x.query(unbracket(args.DOMAIN)) for answer in res: passive_dns.append({ "ip": answer['rdata'], "first": answer['time_first'], "last": answer['time_last'], "source" : "CIRCL" }) # BinaryEdge be_e = plugins['binaryedge'].test_config(conf) if be_e: print('[+] Downloading BinaryEdge information....') be = BinaryEdge(conf['BinaryEdge']['key']) res = be.domain_dns(unbracket(args.DOMAIN)) for d in res['events']: if "A" in d: for a in d['A']: passive_dns.append({ "ip": a, "first": parse(d['updated_at']), "last": parse(d['updated_at']), "source" : "BinaryEdge" }) # RobTex print('[+] Downloading Robtex information....') rob = Robtex() res = rob.get_pdns_domain(args.DOMAIN) for d in res: if d['rrtype'] in ['A', 'AAAA']: passive_dns.append({ 'first': d['time_first_o'], 'last': d['time_last_o'], 'ip': d['rrdata'], 'source': 'Robtex' }) # PT pt_e = plugins['pt'].test_config(conf) if pt_e: try: pt_osint = {} ptout = False print('[+] Downloading Passive Total information....') client = DnsRequest(conf['PassiveTotal']['username'], conf['PassiveTotal']['key']) raw_results = client.get_passive_dns(query=unbracket(args.DOMAIN)) if "results" in raw_results: for res in raw_results["results"]: passive_dns.append({ "first": parse(res["firstSeen"]), "last": parse(res["lastSeen"]), "ip": res["resolve"], "source": "PT" }) if "message" in raw_results: if "quota_exceeded" in raw_results["message"]: print("PT quota exceeded") ptout = True if not ptout: client2 = EnrichmentRequest(conf["PassiveTotal"]["username"], conf["PassiveTotal"]['key']) # Get OSINT # TODO: add PT projects here pt_osint = client2.get_osint(query=unbracket(args.DOMAIN)) # Get malware raw_results = client2.get_malware(query=unbracket(args.DOMAIN)) if "results" in raw_results: for r in raw_results["results"]: malware.append({ 'hash': r["sample"], 'date': parse(r['collectionDate']), 'source' : 'PT (%s)' % r["source"] }) except requests.exceptions.ReadTimeout: print("PT: Time Out") # VT vt_e = plugins['vt'].test_config(conf) if vt_e: if conf["VirusTotal"]["type"] != "public": print('[+] Downloading VT information....') vt = PrivateApi(conf["VirusTotal"]["key"]) res = vt.get_domain_report(unbracket(args.DOMAIN)) if "results" in res: if "resolutions" in res['results']: for r in res["results"]["resolutions"]: passive_dns.append({ "first": parse(r["last_resolved"]), "last": parse(r["last_resolved"]), "ip": r["ip_address"], "source": "VT" }) if "undetected_downloaded_samples" in res['results']: for r in res['results']['undetected_downloaded_samples']: files.append({ 'hash': r['sha256'], 'date': parse(r['date']) if 'date' in r else '', 'source' : 'VT' }) if "undetected_referrer_samples" in res['results']: for r in res['results']['undetected_referrer_samples']: files.append({ 'hash': r['sha256'], 'date': parse(r['date']) if 'date' in r else '', 'source' : 'VT' }) if "detected_downloaded_samples" in res['results']: for r in res['results']['detected_downloaded_samples']: malware.append({ 'hash': r['sha256'], 'date': parse(r['date']), 'source' : 'VT' }) if "detected_referrer_samples" in res['results']: for r in res['results']['detected_referrer_samples']: if "date" in r: malware.append({ 'hash': r['sha256'], 'date': parse(r['date']), 'source' : 'VT' }) if "detected_urls" in res['results']: for r in res['results']['detected_urls']: urls.append({ 'date': parse(r['scan_date']), 'url': r['url'], 'ip': '', 'source': 'VT' }) else: vt_e = False tg_e = plugins['threatgrid'].test_config(conf) if tg_e: try: print('[+] Downloading Threat Grid....') tg = ThreatGrid(conf['ThreatGrid']['key']) res = tg.search_samples(unbracket(args.DOMAIN), type='domain') already = [] if 'items' in res: for r in res['items']: if r['sample_sha256'] not in already: d = parse(r['ts']) d = d.replace(tzinfo=None) malware.append({ 'hash': r["sample_sha256"], 'date': d, 'source' : 'ThreatGrid' }) already.append(r['sample_sha256']) except ThreatGridError as e: print("Failed to connect to Threat Grid: %s" % e.message) # TODO: Add MISP print('----------------- Intelligence Report') if otx_e: if len(otx_pulses): print('OTX:') for p in otx_pulses: print(' -%s (%s - %s)' % ( p['name'], p['created'][:10], "https://otx.alienvault.com/pulse/" + p['id'] ) ) else: print('OTX: Not found in any pulse') if pt_e: if "results" in pt_osint: if len(pt_osint["results"]): if len(pt_osint["results"]) == 1: if "name" in pt_osint["results"][0]: print("PT: %s %s" % (pt_osint["results"][0]["name"], pt_osint["results"][0]["sourceUrl"])) else: print("PT: %s" % (pt_osint["results"][0]["sourceUrl"])) else: print("PT:") for r in pt_osint["results"]: if "name" in r: print("-%s %s" % (r["name"], r["sourceUrl"])) else: print("-%s" % (r["sourceUrl"])) else: print("PT: Nothing found!") else: print("PT: Nothing found!") if len(malware) > 0: print('----------------- Malware') for r in sorted(malware, key=lambda x: x["date"]): print("[%s] %s %s" % ( r["source"], r["hash"], r["date"].strftime("%Y-%m-%d") ) ) if len(files) > 0: print('----------------- Files') for r in files: if r['date'] != '': print("[%s] %s (%s)" % ( r["source"], r["hash"], r["date"].strftime("%Y-%m-%d") ) ) else: print("[%s] %s" % ( r["source"], r["hash"], ) ) if len(urls) > 0: print('----------------- Urls') for r in sorted(urls, key=lambda x: x["date"], reverse=True): print("[%s] %s - %s %s" % ( r["source"], r["url"], r["ip"], r["date"].strftime("%Y-%m-%d") ) ) # TODO: add ASN + location info here if len(passive_dns) > 0: print('----------------- Passive DNS') for r in sorted(passive_dns, key=lambda x: x["first"], reverse=True): print("[+] %-40s (%s -> %s)(%s)" % ( r["ip"], r["first"].strftime("%Y-%m-%d"), r["last"].strftime("%Y-%m-%d"), r["source"] ) ) else: self.parser.print_help() else: self.parser.print_help()
start = datetime.datetime.fromtimestamp(int(earliest)) kwords['start'] = start.strftime("%Y-%m-%d") if latest and (latest.isdigit() or latest == 'now'): if latest == 'now': end = datetime.datetime.now() else: end = datetime.datetime.fromtimestamp(int(latest)) kwords['end'] = end.strftime("%Y-%m-%d") kwords['query'] = query_value logger.info("Query target: %s" % query_value) logger.debug("Raw options: %s" % str(options)) configuration = get_config("passivetotal", "api-setup") username = configuration.get('username', None) api_key = configuration.get('apikey', None) output_events = list() pdns = DnsRequest(username, api_key, headers=build_headers()).get_unique_resolutions(**kwords) if 'error' in pdns: raise Exception("Whoa there, looks like you reached your quota for today! Please come back tomorrow to resume your investigation or contact support for details on enterprise plans.") for result in pdns.get("frequency", []): tmp = {'resolve': result[0], 'count': result[1]} output_events.append(tmp) splunk.Intersplunk.outputResults(output_events) except Exception, e: stack = traceback.format_exc() splunk.Intersplunk.generateErrorResults(str(e)) logger.error(str(e) + ". Traceback: " + str(stack))
def setup_class(self): self.patcher = patch('passivetotal.api.Client._get', fake_request) self.patcher.start() self.client = DnsRequest('--No-User--', '--No-Key--')
kwords['start'] = start.strftime("%Y-%m-%d") if latest and (latest.isdigit() or latest == 'now'): if latest == 'now': end = datetime.datetime.now() else: end = datetime.datetime.fromtimestamp(int(latest)) kwords['end'] = end.strftime("%Y-%m-%d") kwords['query'] = query_value logger.info("Query target: %s" % query_value) logger.debug("Raw options: %s" % str(options)) configuration = get_config("passivetotal", "api-setup") username = configuration.get('username', None) api_key = configuration.get('apikey', None) output_events = list() pdns = DnsRequest(username, api_key, headers=build_headers()).get_passive_dns(**kwords) if 'error' in pdns: raise Exception("Whoa there, looks like you reached your quota for today! Please come back tomorrow to resume your investigation or contact support for details on enterprise plans.") for result in pdns.get("results", []): result = remove_keys(result, ['value', 'recordHash', 'collected']) result['count'] = pdns.get('totalRecords', 0) output_events.append(result) splunk.Intersplunk.outputResults(output_events) except Exception, e: stack = traceback.format_exc() splunk.Intersplunk.generateErrorResults(str(e)) logger.error(str(e) + ". Traceback: " + str(stack))
def get_config(): conf_file = os.path.join(os.path.expanduser("~"), ".config/passivetotal/api_config.json") if os.path.isfile(conf_file): with open(conf_file, 'r') as f: conf = json.loads(f.read()) else: print('No config file') sys.exit(1) return conf if __name__ == '__main__': parser = argparse.ArgumentParser( description='Extract all domains from an IP address') parser.add_argument('IP', help='an IP address') args = parser.parse_args() conf = get_config() client = DnsRequest(conf['username'], conf['api_key']) raw_results = client.get_passive_dns(query=args.IP) print("{} domains identified".format(len(raw_results["results"]))) csvout = open("csv.out", "w+") csvout.write("Domain,First,Last,Type\n") for r in raw_results["results"]: csvout.write("{},{},{},{}\n".format(r['resolve'], r['firstSeen'], r['lastSeen'], r['recordType'])) print("extracted in csv.out")
if latest == 'now': end = datetime.datetime.now() else: end = datetime.datetime.fromtimestamp(int(latest)) kwords['end'] = end.strftime("%Y-%m-%d") kwords['query'] = query_value logger.info("Query target: %s" % query_value) logger.debug("Raw options: %s" % str(options)) configuration = get_config("passivetotal", "api-setup") username = configuration.get('username', None) api_key = configuration.get('apikey', None) output_events = list() pdns = DnsRequest(username, api_key, headers=build_headers()).get_passive_dns(**kwords) if 'error' in pdns: raise Exception( "Whoa there, looks like you reached your quota for today! Please come back tomorrow to resume your investigation or contact support for details on enterprise plans." ) for result in pdns.get("results", []): result = remove_keys(result, ['value', 'recordHash', 'collected']) result['count'] = pdns.get('totalRecords', 0) output_events.append(result) splunk.Intersplunk.outputResults(output_events) except Exception, e: stack = traceback.format_exc() splunk.Intersplunk.generateErrorResults(str(e)) logger.error(str(e) + ". Traceback: " + str(stack))
def run(self): data = self.get_data() try: # enrichment service if self.service == 'enrichment': enrichment_request = EnrichmentRequest(username=self.username, api_key=self.api_key) result = enrichment_request.get_enrichment(query=data) self.report(result) # malware service elif self.service == 'malware': enrichment_request = EnrichmentRequest(username=self.username, api_key=self.api_key) result = enrichment_request.get_malware(query=data) self.report(result) # osint service elif self.service == 'osint': enrichment_request = EnrichmentRequest(username=self.username, api_key=self.api_key) result = enrichment_request.get_osint(query=data) self.report(result) # passive dns service elif self.service == 'passive_dns': dns_request = DnsRequest(username=self.username, api_key=self.api_key) result = dns_request.get_passive_dns(query=data) self.report(result) # ssl certificate details service elif self.service == 'ssl_certificate_details': ssl_request = SslRequest(username=self.username, api_key=self.api_key) result = ssl_request.get_ssl_certificate_details(query=data) self.report(result) # ssl certificate history service elif self.service == 'ssl_certificate_history': ssl_request = SslRequest(username=self.username, api_key=self.api_key) result = ssl_request.get_ssl_certificate_history(query=data) print(len(result['results'])) if len(result['results'] ) == 1 and result['results'][0]['ipAddresses'] == 'N/A': print("ok") self.report({'results': []}) else: self.report(result) # unique resolutions service elif self.service == 'unique_resolutions': dns_request = DnsRequest(username=self.username, api_key=self.api_key) result = dns_request.get_unique_resolutions(query=data) self.report(result) # whois details service elif self.service == 'whois_details': whois_request = WhoisRequest(username=self.username, api_key=self.api_key) result = whois_request.get_whois_details(query=data) self.report(result) # components service elif self.service == 'components': host_attr_request = HostAttributeRequest( username=self.username, api_key=self.api_key) result = host_attr_request.get_components(query=data) self.report(result) # trackers service elif self.service == 'trackers': host_attr_request = HostAttributeRequest( username=self.username, api_key=self.api_key) result = host_attr_request.get_trackers(query=data) self.report(result) # host pairs service elif self.service == 'host_pairs': host_attr_request = HostAttributeRequest( username=self.username, api_key=self.api_key) result = host_attr_request.get_host_pairs(query=data, direction='parents') children = host_attr_request.get_host_pairs( query=data, direction='children') result['totalRecords'] += children['totalRecords'] result['results'] = result['results'] + children['results'] self.report(result) else: self.error('Unknown PassiveTotal service') except Exception as e: self.unexpectedError(e)
def run(self, conf, args, plugins): if 'subcommand' in args: if args.subcommand == 'info': if not is_ip(unbracket(args.IP)): print("Invalid IP address") sys.exit(1) # FIXME: move code here in a library ip = unbracket(args.IP) try: ipy = IP(ip) except ValueError: print('Invalid IP format, quitting...') return ipinfo = self.ipinfo(ip) print('MaxMind: Located in %s, %s' % (ipinfo['city'], ipinfo['country'])) if ipinfo['asn'] == 0: print("MaxMind: IP not found in the ASN database") else: print('MaxMind: ASN%i, %s' % (ipinfo['asn'], ipinfo['asn_name'])) asndb2 = pyasn.pyasn(self.asncidr) res = asndb2.lookup(ip) if res[1] is None: print("IP not found in ASN database") else: # Search for name f = open(self.asnname, 'r') found = False line = f.readline() name = '' while not found and line != '': s = line.split('|') if s[0] == str(res[0]): name = s[1].strip() found = True line = f.readline() print('ASN %i - %s (range %s)' % (res[0], name, res[1])) if ipinfo['hostname'] != '': print('Hostname: %s' % ipinfo['hostname']) if ipinfo['specific'] != '': print("Specific: %s" % ipinfo['specific']) if ipy.iptype() == "PRIVATE": "Private IP" print("") if ipy.version() == 4: print("Censys:\t\thttps://censys.io/ipv4/%s" % ip) print("Shodan:\t\thttps://www.shodan.io/host/%s" % ip) print("IP Info:\thttp://ipinfo.io/%s" % ip) print("BGP HE:\t\thttps://bgp.he.net/ip/%s" % ip) print( "IP Location:\thttps://www.iplocation.net/?query=%s" % ip) elif args.subcommand == "intel": if not is_ip(unbracket(args.IP)): print("Invalid IP address") sys.exit(1) # Start with MISP and OTX to get Intelligence Reports print('###################### %s ###################' % unbracket(args.IP)) passive_dns = [] urls = [] malware = [] files = [] # OTX otx_e = plugins['otx'].test_config(conf) if otx_e: print('[+] Downloading OTX information....') otx = OTXv2(conf["AlienVaultOtx"]["key"]) res = otx.get_indicator_details_full( IndicatorTypes.IPv4, unbracket(args.IP)) otx_pulses = res["general"]["pulse_info"]["pulses"] # Get Passive DNS if "passive_dns" in res: for r in res["passive_dns"]["passive_dns"]: passive_dns.append({ "domain": r['hostname'], "first": parse(r["first"]), "last": parse(r["last"]), "source": "OTX" }) if "url_list" in res: for r in res["url_list"]["url_list"]: urls.append(r) # RobTex print('[+] Downloading Robtex information....') rob = Robtex() res = rob.get_ip_info(unbracket(args.IP)) for d in ["pas", "pash", "act", "acth"]: if d in res: for a in res[d]: passive_dns.append({ 'first': a['date'], 'last': a['date'], 'domain': a['o'], 'source': 'Robtex' }) # PT pt_e = plugins['pt'].test_config(conf) if pt_e: out_pt = False print('[+] Downloading Passive Total information....') client = DnsRequest(conf['PassiveTotal']['username'], conf['PassiveTotal']['key']) raw_results = client.get_passive_dns( query=unbracket(args.IP)) if "results" in raw_results: for res in raw_results["results"]: passive_dns.append({ "first": parse(res["firstSeen"]), "last": parse(res["lastSeen"]), "domain": res["resolve"], "source": "PT" }) if "message" in raw_results: if "quota_exceeded" in raw_results["message"]: print("Quota exceeded for Passive Total") out_pt = True pt_osint = {} if not out_pt: client2 = EnrichmentRequest( conf["PassiveTotal"]["username"], conf["PassiveTotal"]['key']) # Get OSINT # TODO: add PT projects here pt_osint = client2.get_osint(query=unbracket(args.IP)) # Get malware raw_results = client2.get_malware( query=unbracket(args.IP)) if "results" in raw_results: for r in raw_results["results"]: malware.append({ 'hash': r["sample"], 'date': parse(r['collectionDate']), 'source': 'PT (%s)' % r["source"] }) # VT vt_e = plugins['vt'].test_config(conf) if vt_e: if conf["VirusTotal"]["type"] != "public": print('[+] Downloading VT information....') vt = PrivateApi(conf["VirusTotal"]["key"]) res = vt.get_ip_report(unbracket(args.IP)) if "results" in res: if "resolutions" in res['results']: for r in res["results"]["resolutions"]: passive_dns.append({ "first": parse(r["last_resolved"]), "last": parse(r["last_resolved"]), "domain": r["hostname"], "source": "VT" }) if "undetected_downloaded_samples" in res[ 'results']: for r in res['results'][ 'undetected_downloaded_samples']: files.append({ 'hash': r['sha256'], 'date': parse(r['date']), 'source': 'VT' }) if "undetected_referrer_samples" in res['results']: for r in res['results'][ 'undetected_referrer_samples']: files.append({ 'hash': r['sha256'], 'date': parse(r['date']), 'source': 'VT' }) if "detected_downloaded_samples" in res['results']: for r in res['results'][ 'detected_downloaded_samples']: malware.append({ 'hash': r['sha256'], 'date': parse(r['date']), 'source': 'VT' }) if "detected_referrer_samples" in res['results']: for r in res['results'][ 'detected_referrer_samples']: if "date" in r: malware.append({ 'hash': r['sha256'], 'date': parse(r['date']), 'source': 'VT' }) else: vt_e = False print('[+] Downloading GreyNoise information....') gn = GreyNoise() try: greynoise = gn.query_ip(unbracket(args.IP)) except GreyNoiseError: greynoise = [] tg_e = plugins['threatgrid'].test_config(conf) if tg_e: print('[+] Downloading Threat Grid....') tg = ThreatGrid(conf['ThreatGrid']['key']) res = tg.search_samples(unbracket(args.IP), type='ip') already = [] if 'items' in res: for r in res['items']: if r['sample_sha256'] not in already: d = parse(r['ts']) d = d.replace(tzinfo=None) malware.append({ 'hash': r["sample_sha256"], 'date': d, 'source': 'TG' }) already.append(r['sample_sha256']) # TODO: Add MISP print('----------------- Intelligence Report') if otx_e: if len(otx_pulses): print('OTX:') for p in otx_pulses: print(' -%s (%s - %s)' % (p['name'], p['created'][:10], "https://otx.alienvault.com/pulse/" + p['id'])) else: print('OTX: Not found in any pulse') if len(greynoise) > 0: print("GreyNoise: IP identified as") for r in greynoise: print("\t%s (%s -> %s)" % (r["name"], r["first_seen"], r["last_updated"])) else: print("GreyNoise: Not found") if pt_e: if "results" in pt_osint: if len(pt_osint["results"]): if len(pt_osint["results"]) == 1: if "name" in pt_osint["results"][0]: print( "PT: %s %s" % (pt_osint["results"][0]["name"], pt_osint["results"][0]["sourceUrl"])) else: print("PT: %s" % pt_osint["results"][0]["sourceUrl"]) else: print("PT:") for r in pt_osint["results"]: if "name" in r: print("-%s %s" % (r["name"], r["sourceUrl"])) else: print("-%s" % r["sourceUrl"]) else: print("PT: Nothing found!") else: print("PT: Nothing found!") if len(malware) > 0: print('----------------- Malware') for r in sorted(malware, key=lambda x: x["date"]): print("[%s] %s %s" % (r["source"], r["hash"], r["date"].strftime("%Y-%m-%d"))) if len(files) > 0: print('----------------- Files') for r in sorted(files, key=lambda x: x["date"]): print("[%s] %s %s" % (r["source"], r["hash"], r["date"].strftime("%Y-%m-%d"))) if len(passive_dns) > 0: print('----------------- Passive DNS') for r in sorted(passive_dns, key=lambda x: x["first"], reverse=True): print("[+] %-40s (%s -> %s)(%s)" % (r["domain"], r["first"].strftime("%Y-%m-%d"), r["last"].strftime("%Y-%m-%d"), r["source"])) else: self.parser.print_help() else: self.parser.print_help()