def analyze(observable, results): links = set() lookup_results = MacAddressIoApi.get(observable.value, results.settings) results.update(raw=json.dumps(lookup_results, indent=2)) if lookup_results["blockDetails"]["dateCreated"]: date_created = \ datetime.strptime(lookup_results["blockDetails"]["dateCreated"], "%Y-%m-%d") else: date_created = None if lookup_results["blockDetails"]["dateUpdated"]: date_updated = \ datetime.strptime(lookup_results["blockDetails"]["dateUpdated"], "%Y-%m-%d") else: date_updated = None try: if lookup_results["vendorDetails"]["companyName"] != "": vendor = \ Company.get_or_create(name=lookup_results["vendorDetails"]["companyName"]) links.update( observable.link_to(vendor, 'Vendor', MacAddressIoApi.__MODULE_GROUP__, date_created, date_updated)) except KeyError: pass MacAddressIo.add_context_to_observable(observable, lookup_results) return list(links)
def analyze(ip, results): links = set() result = ShodanApi.fetch(ip, results.settings['shodan_api_key']) results.update(raw=pformat(result)) if 'tags' in result and result['tags'] is not None: ip.tag(result['tags']) if 'asn' in result and result['asn'] is not None: o_asn = Text.get_or_create(value=result['asn']) links.update(ip.active_link_to(o_asn, 'asn#', 'Shodan Query')) if 'hostnames' in result and result['hostnames'] is not None: for hostname in result['hostnames']: h = Hostname.get_or_create(value=hostname) links.update(h.active_link_to(ip, 'A record', 'Shodan Query')) if 'isp' in result and result['isp'] is not None: o_isp = Company.get_or_create(name=result['isp']) links.update(ip.active_link_to(o_isp, 'hosting', 'Shodan Query')) for context in ip.context: if context['source'] == 'shodan_query': break else: # Remove the data part (Shodan Crawler Data, etc.) result.pop("data", None) result['source'] = 'shodan_query' ip.add_context(result) return list(links)
def analyze(observable, results): links = set() parts = extract(observable.value) if parts.subdomain == '': data = DomainToolsApi.get("/{}/whois/history".format(observable.value), results.settings) results.update(raw=json.dumps(data, indent=2)) for record in data['response']['history']: created = datetime.strptime(record['whois']['registration']['created'], "%Y-%m-%d") expires = datetime.strptime(record['whois']['registration']['expires'], "%Y-%m-%d") registrar = Company.get_or_create(name=record['whois']['registration']['registrar']) registrant = Text.get_or_create(value=record['whois']['registrant']) links.update(observable.link_to(registrar, 'Registrar', 'DomainTools', created, expires)) links.update(observable.link_to(registrant, 'Registrant', 'DomainTools', created, expires)) parsed = parse_raw_whois([record['whois']['record']], normalized=True) email = get_value_at(parsed, 'contacts.registrant.email') if email: email = Email.get_or_create(value=email) links.update(observable.link_to(email, 'Registrant Email', 'DomainTools', created, expires)) return list(links)
def analyze(ip, results): links = set() r = IPWhois(ip.value) result = r.lookup_whois() results.update(raw=pformat(result)) # Let's focus on the most specific information # Which should be in the smallest subnet n = 0 smallest_subnet = None for network in result['nets']: cidr_bits = int(network['cidr'].split('/')[1].split(',')[0]) if cidr_bits > n: n = cidr_bits smallest_subnet = network if smallest_subnet: # Create the company company = Company.get_or_create( name=smallest_subnet['description'].split("\n")[0]) links.update(ip.active_link_to(company, 'hosting', 'Network Whois')) # Link it to every email address referenced if smallest_subnet['emails']: for email_address in smallest_subnet['emails']: email = Email.get_or_create(value=email_address) links.update(company.link_to(email, None, 'Network Whois')) # Copy the subnet info into the main dict for key in smallest_subnet: if smallest_subnet[key]: result["net_{}".format(key)] = smallest_subnet[key] # Add the network whois to the context if not already present for context in ip.context: if context['source'] == 'network_whois': break else: # Remove the nets info (the main one was copied) result.pop("nets", None) result.pop("raw", None) result.pop("raw_referral", None) result.pop("referral", None) result.pop("query", None) result['source'] = 'network_whois' ip.add_context(result) return list(links)
def analyze(observable, results): links = set() json_result = VirustotalApi.fetch(observable, results.settings['virutotal_api_key']) json_string = json.dumps(json_result, sort_keys=True, indent=4, separators=(',', ': ')) results.update(raw=json_string) result = {'raw': json_string} if isinstance(observable, Ip): # Parse results for ip if json_result.get('as_owner'): result['Owner'] = json_result['as_owner'] o_isp = Company.get_or_create(name=json_result['as_owner']) links.update(observable.active_link_to(o_isp, 'hosting', 'virustotal_query')) if json_result.get('detected_urls'): result['detected_urls'] = json_result['detected_urls'] for detected_url in json_result['detected_urls']: o_url = Url.get_or_create(value=detected_url['url']) links.update(o_url.active_link_to(o_url, 'hostname', 'virustotal_query')) elif isinstance(observable, Hostname): if json_result.get('permalink'): result['permalink'] = json_result['permalink'] result['positives'] = json_result.get('positives', 0) if json_result.get('total'): result['total'] = json_result['total'] elif isinstance(observable, Hash): result['positives'] = json_result['positives'] if 'permalink' in json_result: result['permalink'] = json_result['permalink'] if 'total' in json_result: result['total'] = json_result['total'] hashes ={ 'md5': json_result['md5'], 'sha1': json_result['sha1'], 'sha256': json_result['sha256']} create_hashes = [(k, v) for k,v in hashes.items() if v != observable.value] for k, v in create_hashes: new_hash = Hash.get_or_create(value=v) new_hash.tag(observable.get_tags()) links.update(new_hash.active_link_to(observable, k, 'virustotal_query')) result['source'] = 'virustotal_query' observable.add_context(result) return list(links)
def analyze(observable, results): links = set() parts = tldextract_parser(observable.value) if parts.subdomain == "": data = DomainToolsApi.get( "/{}/whois/history".format(observable.value), results.settings ) results.update(raw=json.dumps(data, indent=2)) for record in data["response"]["history"]: created = datetime.strptime( record["whois"]["registration"]["created"], "%Y-%m-%d" ) expires = datetime.strptime( record["whois"]["registration"]["expires"], "%Y-%m-%d" ) registrar = Company.get_or_create( name=record["whois"]["registration"]["registrar"] ) registrant = Text.get_or_create(value=record["whois"]["registrant"]) links.update( observable.link_to( registrar, "Registrar", "DomainTools", created, expires ) ) links.update( observable.link_to( registrant, "Registrant", "DomainTools", created, expires ) ) parsed = parse_raw_whois([record["whois"]["record"]], normalized=True) email = get_value_at(parsed, "contacts.registrant.email") if email: email = Email.get_or_create(value=email) links.update( observable.link_to( email, "Registrant Email", "DomainTools", created, expires ) ) return list(links)
def analyze(observable, results): links = set() json_result = VirustotalApi.fetch( observable, results.settings['virutotal_api_key']) json_string = json.dumps(json_result, sort_keys=True, indent=4, separators=(',', ': ')) results.update(raw=json_string) result = {'raw': json_string} if isinstance(observable, Ip): # Parse results for ip if json_result.get('as_owner'): result['Owner'] = json_result['as_owner'] o_isp = Company.get_or_create(name=json_result['as_owner']) links.update( observable.active_link_to(o_isp, 'hosting', 'virustotal_query')) if json_result.get('detected_urls'): result['detected_urls'] = json_result['detected_urls'] for detected_url in json_result['detected_urls']: o_url = Url.get_or_create(value=detected_url['url']) links.update( o_url.active_link_to(o_url, 'hostname', 'virustotal_query')) elif isinstance(observable, Hostname): if json_result.get('permalink'): result['permalink'] = json_result['permalink'] result['positives'] = json_result.get('positives', 0) if json_result.get('total'): result['total'] = json_result['total'] result['source'] = 'virustotal_query' observable.add_context(result) return list(links)
def analyze(ip): links = [] results = IPWhois(ip.value) results = results.lookup_rdap() for entity in results['objects']: entity = results['objects'][entity] if entity['contact']['kind'] != 'individual': # Create the company company = Company.get_or_create(name=entity['contact']['name'], rdap=entity) link = Link.connect(ip, company) link.add_history('hosting') links.append(link) # Link it to every email address referenced for email_info in entity['contact']['email']: email = Email.get_or_create(value=email_info['value']) link = Link.connect(company, email) links.append(link) return links
def analyze(ip, results): links = set() result = ShodanApi.fetch(ip, results.settings['shodan_api_key']) json_string = json.dumps(result, sort_keys=True, indent=4, separators=(',', ': ')) results.update(raw=json_string) if 'tags' in result and result['tags'] is not None: ip.tag(result['tags']) if 'asn' in result and result['asn'] is not None: o_asn = AutonomousSystem.get_or_create( value=result['asn'].replace("AS", "")) links.update(ip.active_link_to(o_asn, 'asn#', 'Shodan Query')) if 'hostnames' in result and result['hostnames'] is not None: for hostname in result['hostnames']: h = Hostname.get_or_create(value=hostname) links.update(h.active_link_to(ip, 'A record', 'Shodan Query')) if 'isp' in result and result['isp'] is not None: o_isp = Company.get_or_create(name=result['isp']) links.update(ip.active_link_to(o_isp, 'hosting', 'Shodan Query')) for context in ip.context: if context['source'] == 'shodan_query': break else: # Remove the data part (Shodan Crawler Data, etc.) result.pop("data", None) result['source'] = 'shodan_query' ip.add_context(result) return list(links)
def analyze(ip, results): links = set() result = ShodanApi.fetch(ip, results.settings["shodan_api_key"]) json_string = json.dumps( result, sort_keys=True, indent=4, separators=(",", ": ") ) results.update(raw=json_string) if "tags" in result and result["tags"] is not None: ip.tag(result["tags"]) if "asn" in result and result["asn"] is not None: o_asn = AutonomousSystem.get_or_create( value=result["asn"].replace("AS", "") ) links.update(o_asn.active_link_to(ip, "asn#", "Shodan Query")) if "hostnames" in result and result["hostnames"] is not None: for hostname in result["hostnames"]: h = Hostname.get_or_create(value=hostname) links.update(h.active_link_to(ip, "A record", "Shodan Query")) if "isp" in result and result["isp"] is not None: o_isp = Company.get_or_create(name=result["isp"]) links.update(ip.active_link_to(o_isp, "hosting", "Shodan Query")) for context in ip.context: if context["source"] == "shodan_query": break else: # Remove the data part (Shodan Crawler Data, etc.) result.pop("data", None) result["source"] = "shodan_query" ip.add_context(result) return list(links)
def analyze(observable, results): links = set() json_result = VirustotalApi.fetch( observable, results.settings['virutotal_api_key']) json_string = json.dumps( json_result, sort_keys=True, indent=4, separators=(',', ': ')) results.update(raw=json_string) result = {'raw': json_string} if isinstance(observable, Ip): # Parse results for ip if json_result.get('as_owner'): result['Owner'] = json_result['as_owner'] o_isp = Company.get_or_create(name=json_result['as_owner']) links.update( observable.active_link_to( o_isp, 'hosting', 'virustotal_query')) if json_result.get('detected_urls'): result['detected_urls'] = json_result['detected_urls'] for detected_url in json_result['detected_urls']: o_url = Url.get_or_create(value=detected_url['url']) links.update( o_url.active_link_to( o_url, 'hostname', 'virustotal_query')) elif isinstance(observable, Hostname): if json_result.get('permalink'): result['permalink'] = json_result['permalink'] result['positives'] = json_result.get('positives', 0) if json_result.get('total'): result['total'] = json_result['total'] elif isinstance(observable, Hash): result['positives'] = json_result['positives'] if 'permalink' in json_result: result['permalink'] = json_result['permalink'] if 'total' in json_result: result['total'] = json_result['total'] hashes = { 'md5': json_result['md5'], 'sha1': json_result['sha1'], 'sha256': json_result['sha256'] } create_hashes = [ (k, v) for k, v in hashes.items() if v != observable.value ] for k, v in create_hashes: new_hash = Hash.get_or_create(value=v) new_hash.tag(observable.get_tags()) links.update( new_hash.active_link_to(observable, k, 'virustotal_query')) result['source'] = 'virustotal_query' observable.add_context(result) return list(links)
def analyze(hostname, results): links = set() data = whois.whois(hostname.value) if not data["domain_name"]: return list(links) should_add_context = False for context in hostname.context: if context["source"] == "whois": break else: should_add_context = True context = {"source": "whois"} context["whois_server"] = data["whois_server"] if data["dnssec"]: context["dnssec"] = data["dnssec"] if isinstance(data["creation_date"], list): context["creation_date"] = sorted(data["creation_date"])[0] else: context["creation_date"] = data["creation_date"] if isinstance(data["updated_date"], list): context["updated_date"] = sorted(data["updated_date"], reverse=True)[0] else: context["updated_date"] = data["updated_date"] if isinstance(data["expiration_date"], list): context["expiration_date"] = sorted(data["expiration_date"], reverse=True)[0] else: context["expiration_date"] = data["expiration_date"] name_servers = data["name_servers"] if isinstance(name_servers, list): for ns in name_servers: ns_obs = Hostname.get_or_create(value=ns) links.update( ns_obs.active_link_to(hostname, "NS", context["source"])) else: ns_obs = Hostname.get_or_create(value=name_servers) links.update( ns_obs.active_link_to(hostname, "NS", context["source"])) for email in data["emails"]: email_obs = Email.get_or_create(value=email) links.update( email_obs.active_link_to(hostname, "email registrar", context["source"])) if data["org"]: company_org = Company.get_or_create(name=data["org"]) links.update( company_org.active_link_to(hostname, "Org", context["source"])) if data["registrar"]: company_registrar = Company.get_or_create(name=data["registrar"]) links.update( company_registrar.active_link_to(hostname, "registrar", context["source"])) if should_add_context: hostname.add_context(context) else: hostname.save() return list(links)