def analyze(observable, results):
links = set()
lookup_results = MacAddressIoApi.get(observable.value,
results.settings)
results.update(raw=json.dumps(lookup_results, indent=2))
if lookup_results["blockDetails"]["dateCreated"]:
date_created = \
datetime.strptime(lookup_results["blockDetails"]["dateCreated"], "%Y-%m-%d")
else:
date_created = None
if lookup_results["blockDetails"]["dateUpdated"]:
date_updated = \
datetime.strptime(lookup_results["blockDetails"]["dateUpdated"], "%Y-%m-%d")
else:
date_updated = None
try:
if lookup_results["vendorDetails"]["companyName"] != "":
vendor = \
Company.get_or_create(name=lookup_results["vendorDetails"]["companyName"])
links.update(
observable.link_to(vendor, 'Vendor',
MacAddressIoApi.__MODULE_GROUP__,
date_created, date_updated))
except KeyError:
pass
MacAddressIo.add_context_to_observable(observable, lookup_results)
return list(links)
def analyze(ip, results):
links = set()
result = ShodanApi.fetch(ip, results.settings['shodan_api_key'])
results.update(raw=pformat(result))
if 'tags' in result and result['tags'] is not None:
ip.tag(result['tags'])
if 'asn' in result and result['asn'] is not None:
o_asn = Text.get_or_create(value=result['asn'])
links.update(ip.active_link_to(o_asn, 'asn#', 'Shodan Query'))
if 'hostnames' in result and result['hostnames'] is not None:
for hostname in result['hostnames']:
h = Hostname.get_or_create(value=hostname)
links.update(h.active_link_to(ip, 'A record', 'Shodan Query'))
if 'isp' in result and result['isp'] is not None:
o_isp = Company.get_or_create(name=result['isp'])
links.update(ip.active_link_to(o_isp, 'hosting', 'Shodan Query'))
for context in ip.context:
if context['source'] == 'shodan_query':
break
else:
# Remove the data part (Shodan Crawler Data, etc.)
result.pop("data", None)
result['source'] = 'shodan_query'
ip.add_context(result)
return list(links)
def analyze(ip, results):
links = set()
result = ShodanApi.fetch(ip, results.settings['shodan_api_key'])
results.update(raw=pformat(result))
if 'tags' in result and result['tags'] is not None:
ip.tag(result['tags'])
if 'asn' in result and result['asn'] is not None:
o_asn = Text.get_or_create(value=result['asn'])
links.update(ip.active_link_to(o_asn, 'asn#', 'Shodan Query'))
if 'hostnames' in result and result['hostnames'] is not None:
for hostname in result['hostnames']:
h = Hostname.get_or_create(value=hostname)
links.update(h.active_link_to(ip, 'A record', 'Shodan Query'))
if 'isp' in result and result['isp'] is not None:
o_isp = Company.get_or_create(name=result['isp'])
links.update(ip.active_link_to(o_isp, 'hosting', 'Shodan Query'))
for context in ip.context:
if context['source'] == 'shodan_query':
break
else:
# Remove the data part (Shodan Crawler Data, etc.)
result.pop("data", None)
result['source'] = 'shodan_query'
ip.add_context(result)
return list(links)
def analyze(observable, results):
links = set()
parts = extract(observable.value)
if parts.subdomain == '':
data = DomainToolsApi.get("/{}/whois/history".format(observable.value), results.settings)
results.update(raw=json.dumps(data, indent=2))
for record in data['response']['history']:
created = datetime.strptime(record['whois']['registration']['created'], "%Y-%m-%d")
expires = datetime.strptime(record['whois']['registration']['expires'], "%Y-%m-%d")
registrar = Company.get_or_create(name=record['whois']['registration']['registrar'])
registrant = Text.get_or_create(value=record['whois']['registrant'])
links.update(observable.link_to(registrar, 'Registrar', 'DomainTools', created, expires))
links.update(observable.link_to(registrant, 'Registrant', 'DomainTools', created, expires))
parsed = parse_raw_whois([record['whois']['record']], normalized=True)
email = get_value_at(parsed, 'contacts.registrant.email')
if email:
email = Email.get_or_create(value=email)
links.update(observable.link_to(email, 'Registrant Email', 'DomainTools', created, expires))
return list(links)
def analyze(ip, results):
links = set()
r = IPWhois(ip.value)
result = r.lookup_whois()
results.update(raw=pformat(result))
# Let's focus on the most specific information
# Which should be in the smallest subnet
n = 0
smallest_subnet = None
for network in result['nets']:
cidr_bits = int(network['cidr'].split('/')[1].split(',')[0])
if cidr_bits > n:
n = cidr_bits
smallest_subnet = network
if smallest_subnet:
# Create the company
company = Company.get_or_create(
name=smallest_subnet['description'].split("\n")[0])
links.update(ip.active_link_to(company, 'hosting',
'Network Whois'))
# Link it to every email address referenced
if smallest_subnet['emails']:
for email_address in smallest_subnet['emails']:
email = Email.get_or_create(value=email_address)
links.update(company.link_to(email, None, 'Network Whois'))
# Copy the subnet info into the main dict
for key in smallest_subnet:
if smallest_subnet[key]:
result["net_{}".format(key)] = smallest_subnet[key]
# Add the network whois to the context if not already present
for context in ip.context:
if context['source'] == 'network_whois':
break
else:
# Remove the nets info (the main one was copied)
result.pop("nets", None)
result.pop("raw", None)
result.pop("raw_referral", None)
result.pop("referral", None)
result.pop("query", None)
result['source'] = 'network_whois'
ip.add_context(result)
return list(links)
def analyze(ip, results):
links = set()
r = IPWhois(ip.value)
result = r.lookup_whois()
results.update(raw=pformat(result))
# Let's focus on the most specific information
# Which should be in the smallest subnet
n = 0
smallest_subnet = None
for network in result['nets']:
cidr_bits = int(network['cidr'].split('/')[1].split(',')[0])
if cidr_bits > n:
n = cidr_bits
smallest_subnet = network
if smallest_subnet:
# Create the company
company = Company.get_or_create(
name=smallest_subnet['description'].split("\n")[0])
links.update(ip.active_link_to(company, 'hosting', 'Network Whois'))
# Link it to every email address referenced
if smallest_subnet['emails']:
for email_address in smallest_subnet['emails']:
email = Email.get_or_create(value=email_address)
links.update(company.link_to(email, None, 'Network Whois'))
# Copy the subnet info into the main dict
for key in smallest_subnet:
if smallest_subnet[key]:
result["net_{}".format(key)] = smallest_subnet[key]
# Add the network whois to the context if not already present
for context in ip.context:
if context['source'] == 'network_whois':
break
else:
# Remove the nets info (the main one was copied)
result.pop("nets", None)
result.pop("raw", None)
result.pop("raw_referral", None)
result.pop("referral", None)
result.pop("query", None)
result['source'] = 'network_whois'
ip.add_context(result)
return list(links)
def analyze(observable, results):
links = set()
json_result = VirustotalApi.fetch(observable, results.settings['virutotal_api_key'])
json_string = json.dumps(json_result, sort_keys=True, indent=4, separators=(',', ': '))
results.update(raw=json_string)
result = {'raw': json_string}
if isinstance(observable, Ip):
# Parse results for ip
if json_result.get('as_owner'):
result['Owner'] = json_result['as_owner']
o_isp = Company.get_or_create(name=json_result['as_owner'])
links.update(observable.active_link_to(o_isp, 'hosting', 'virustotal_query'))
if json_result.get('detected_urls'):
result['detected_urls'] = json_result['detected_urls']
for detected_url in json_result['detected_urls']:
o_url = Url.get_or_create(value=detected_url['url'])
links.update(o_url.active_link_to(o_url, 'hostname', 'virustotal_query'))
elif isinstance(observable, Hostname):
if json_result.get('permalink'):
result['permalink'] = json_result['permalink']
result['positives'] = json_result.get('positives', 0)
if json_result.get('total'):
result['total'] = json_result['total']
elif isinstance(observable, Hash):
result['positives'] = json_result['positives']
if 'permalink' in json_result:
result['permalink'] = json_result['permalink']
if 'total' in json_result:
result['total'] = json_result['total']
hashes ={ 'md5': json_result['md5'], 'sha1': json_result['sha1'], 'sha256': json_result['sha256']}
create_hashes = [(k, v) for k,v in hashes.items() if v != observable.value]
for k, v in create_hashes:
new_hash = Hash.get_or_create(value=v)
new_hash.tag(observable.get_tags())
links.update(new_hash.active_link_to(observable, k, 'virustotal_query'))
result['source'] = 'virustotal_query'
observable.add_context(result)
return list(links)
def analyze(observable, results):
links = set()
parts = tldextract_parser(observable.value)
if parts.subdomain == "":
data = DomainToolsApi.get(
"/{}/whois/history".format(observable.value), results.settings
)
results.update(raw=json.dumps(data, indent=2))
for record in data["response"]["history"]:
created = datetime.strptime(
record["whois"]["registration"]["created"], "%Y-%m-%d"
)
expires = datetime.strptime(
record["whois"]["registration"]["expires"], "%Y-%m-%d"
)
registrar = Company.get_or_create(
name=record["whois"]["registration"]["registrar"]
)
registrant = Text.get_or_create(value=record["whois"]["registrant"])
links.update(
observable.link_to(
registrar, "Registrar", "DomainTools", created, expires
)
)
links.update(
observable.link_to(
registrant, "Registrant", "DomainTools", created, expires
)
)
parsed = parse_raw_whois([record["whois"]["record"]], normalized=True)
email = get_value_at(parsed, "contacts.registrant.email")
if email:
email = Email.get_or_create(value=email)
links.update(
observable.link_to(
email, "Registrant Email", "DomainTools", created, expires
)
)
return list(links)
def analyze(observable, results):
links = set()
json_result = VirustotalApi.fetch(
observable, results.settings['virutotal_api_key'])
json_string = json.dumps(json_result,
sort_keys=True,
indent=4,
separators=(',', ': '))
results.update(raw=json_string)
result = {'raw': json_string}
if isinstance(observable, Ip):
# Parse results for ip
if json_result.get('as_owner'):
result['Owner'] = json_result['as_owner']
o_isp = Company.get_or_create(name=json_result['as_owner'])
links.update(
observable.active_link_to(o_isp, 'hosting',
'virustotal_query'))
if json_result.get('detected_urls'):
result['detected_urls'] = json_result['detected_urls']
for detected_url in json_result['detected_urls']:
o_url = Url.get_or_create(value=detected_url['url'])
links.update(
o_url.active_link_to(o_url, 'hostname',
'virustotal_query'))
elif isinstance(observable, Hostname):
if json_result.get('permalink'):
result['permalink'] = json_result['permalink']
result['positives'] = json_result.get('positives', 0)
if json_result.get('total'):
result['total'] = json_result['total']
result['source'] = 'virustotal_query'
observable.add_context(result)
return list(links)
def analyze(ip):
links = []
results = IPWhois(ip.value)
results = results.lookup_rdap()
for entity in results['objects']:
entity = results['objects'][entity]
if entity['contact']['kind'] != 'individual':
# Create the company
company = Company.get_or_create(name=entity['contact']['name'], rdap=entity)
link = Link.connect(ip, company)
link.add_history('hosting')
links.append(link)
# Link it to every email address referenced
for email_info in entity['contact']['email']:
email = Email.get_or_create(value=email_info['value'])
link = Link.connect(company, email)
links.append(link)
return links
def analyze(ip, results):
links = set()
result = ShodanApi.fetch(ip, results.settings['shodan_api_key'])
json_string = json.dumps(result,
sort_keys=True,
indent=4,
separators=(',', ': '))
results.update(raw=json_string)
if 'tags' in result and result['tags'] is not None:
ip.tag(result['tags'])
if 'asn' in result and result['asn'] is not None:
o_asn = AutonomousSystem.get_or_create(
value=result['asn'].replace("AS", ""))
links.update(ip.active_link_to(o_asn, 'asn#', 'Shodan Query'))
if 'hostnames' in result and result['hostnames'] is not None:
for hostname in result['hostnames']:
h = Hostname.get_or_create(value=hostname)
links.update(h.active_link_to(ip, 'A record', 'Shodan Query'))
if 'isp' in result and result['isp'] is not None:
o_isp = Company.get_or_create(name=result['isp'])
links.update(ip.active_link_to(o_isp, 'hosting', 'Shodan Query'))
for context in ip.context:
if context['source'] == 'shodan_query':
break
else:
# Remove the data part (Shodan Crawler Data, etc.)
result.pop("data", None)
result['source'] = 'shodan_query'
ip.add_context(result)
return list(links)
def analyze(ip, results):
links = set()
result = ShodanApi.fetch(ip, results.settings["shodan_api_key"])
json_string = json.dumps(
result, sort_keys=True, indent=4, separators=(",", ": ")
)
results.update(raw=json_string)
if "tags" in result and result["tags"] is not None:
ip.tag(result["tags"])
if "asn" in result and result["asn"] is not None:
o_asn = AutonomousSystem.get_or_create(
value=result["asn"].replace("AS", "")
)
links.update(o_asn.active_link_to(ip, "asn#", "Shodan Query"))
if "hostnames" in result and result["hostnames"] is not None:
for hostname in result["hostnames"]:
h = Hostname.get_or_create(value=hostname)
links.update(h.active_link_to(ip, "A record", "Shodan Query"))
if "isp" in result and result["isp"] is not None:
o_isp = Company.get_or_create(name=result["isp"])
links.update(ip.active_link_to(o_isp, "hosting", "Shodan Query"))
for context in ip.context:
if context["source"] == "shodan_query":
break
else:
# Remove the data part (Shodan Crawler Data, etc.)
result.pop("data", None)
result["source"] = "shodan_query"
ip.add_context(result)
return list(links)
def analyze(observable, results):
links = set()
json_result = VirustotalApi.fetch(
observable, results.settings['virutotal_api_key'])
json_string = json.dumps(
json_result, sort_keys=True, indent=4, separators=(',', ': '))
results.update(raw=json_string)
result = {'raw': json_string}
if isinstance(observable, Ip):
# Parse results for ip
if json_result.get('as_owner'):
result['Owner'] = json_result['as_owner']
o_isp = Company.get_or_create(name=json_result['as_owner'])
links.update(
observable.active_link_to(
o_isp, 'hosting', 'virustotal_query'))
if json_result.get('detected_urls'):
result['detected_urls'] = json_result['detected_urls']
for detected_url in json_result['detected_urls']:
o_url = Url.get_or_create(value=detected_url['url'])
links.update(
o_url.active_link_to(
o_url, 'hostname', 'virustotal_query'))
elif isinstance(observable, Hostname):
if json_result.get('permalink'):
result['permalink'] = json_result['permalink']
result['positives'] = json_result.get('positives', 0)
if json_result.get('total'):
result['total'] = json_result['total']
elif isinstance(observable, Hash):
result['positives'] = json_result['positives']
if 'permalink' in json_result:
result['permalink'] = json_result['permalink']
if 'total' in json_result:
result['total'] = json_result['total']
hashes = {
'md5': json_result['md5'],
'sha1': json_result['sha1'],
'sha256': json_result['sha256']
}
create_hashes = [
(k, v) for k, v in hashes.items() if v != observable.value
]
for k, v in create_hashes:
new_hash = Hash.get_or_create(value=v)
new_hash.tag(observable.get_tags())
links.update(
new_hash.active_link_to(observable, k, 'virustotal_query'))
result['source'] = 'virustotal_query'
observable.add_context(result)
return list(links)
def analyze(hostname, results):
links = set()
data = whois.whois(hostname.value)
if not data["domain_name"]:
return list(links)
should_add_context = False
for context in hostname.context:
if context["source"] == "whois":
break
else:
should_add_context = True
context = {"source": "whois"}
context["whois_server"] = data["whois_server"]
if data["dnssec"]:
context["dnssec"] = data["dnssec"]
if isinstance(data["creation_date"], list):
context["creation_date"] = sorted(data["creation_date"])[0]
else:
context["creation_date"] = data["creation_date"]
if isinstance(data["updated_date"], list):
context["updated_date"] = sorted(data["updated_date"],
reverse=True)[0]
else:
context["updated_date"] = data["updated_date"]
if isinstance(data["expiration_date"], list):
context["expiration_date"] = sorted(data["expiration_date"],
reverse=True)[0]
else:
context["expiration_date"] = data["expiration_date"]
name_servers = data["name_servers"]
if isinstance(name_servers, list):
for ns in name_servers:
ns_obs = Hostname.get_or_create(value=ns)
links.update(
ns_obs.active_link_to(hostname, "NS", context["source"]))
else:
ns_obs = Hostname.get_or_create(value=name_servers)
links.update(
ns_obs.active_link_to(hostname, "NS", context["source"]))
for email in data["emails"]:
email_obs = Email.get_or_create(value=email)
links.update(
email_obs.active_link_to(hostname, "email registrar",
context["source"]))
if data["org"]:
company_org = Company.get_or_create(name=data["org"])
links.update(
company_org.active_link_to(hostname, "Org", context["source"]))
if data["registrar"]:
company_registrar = Company.get_or_create(name=data["registrar"])
links.update(
company_registrar.active_link_to(hostname, "registrar",
context["source"]))
if should_add_context:
hostname.add_context(context)
else:
hostname.save()
return list(links)
