def _generate_hit_from_search_result(self, search_result):
"""
Query urlscan io Result API for the given URL search result - hit.
:param search_result: dictionary
"""
result_id = search_result.get('_id', None)
result_url = "{0}{1}".format(self.urlscan_io_result_api_url, result_id)
result_response = requests.get(result_url, headers=self.HEADERS)
if result_response.status_code == 200:
result_content = result_response.json()
stats = result_content.get('stats', None)
if stats:
malicious_flag = stats.get('malicious', None)
if malicious_flag == 1:
# Some malicious scans show as failed, do not include those
if self._verify_for_scan_failed_flag(result_content):
return None
task = result_content.get('task', None)
page = result_content.get('page', None)
png_url = task.get('screenshotURL', None) if task else None
scan_time = task.get('time', None) if task else None
report_url = task.get('reportURL', None) if task else None
uniq_countries_int = stats.get('uniqCountries', None)
city_country_list = self._prepare_city_contry(
page.get('city', None), page.get(
'country', None)) if page else None
city_country = ",".join(
city_country_list) if city_country_list else None
server = page.get('server', None) if page else None
asn = page.get('asnname', None) if page else None
return Hit(
StringProp(name="Time Last Scanner", value=scan_time),
NumberProp(name="Number of Countries",
value=uniq_countries_int),
StringProp(name="City and Country",
value=city_country),
StringProp(name="Server", value=server),
StringProp(name="ASN Name", value=asn),
UriProp(name="Report Link", value=report_url),
UriProp(name="Screenshot Link", value=png_url))
else:
LOG.info(
"No Result information found on URL: {0}".format(result_url))
LOG.debug(result_response.text)
def _lookup_net_uri(self, event, *args, **kwargs):
"""Return hits for URL artifacts"""
hits = []
# event.artifact is a ThreatServiceArtifactDTO
artifact_type = event.artifact['type']
artifact_value = event.artifact['value']
# Return zero or more hits. Here's one example.
hits.append(
Hit(NumberProp(name="id", value=123),
StringProp(name="Type", value=artifact_type),
UriProp(name="Link", value=artifact_value),
IpProp(name="IP Address", value="127.0.0.1"),
LatLngProp(name="Location", lat=42.366, lng=-71.081)))
hits.append(
Hit(
NumberProp(name="id", value=456),
StringProp(name="Type", value=artifact_type),
UriProp(name="Link", value=artifact_value),
IpProp(name="IP Address", value="0.0.0.0"),
))
yield hits
def lookup_artifact(self, event, *args, **kwargs):
"""
Use YETI to search for artifact value
"""
if not isinstance(event, ThreatServiceLookupEvent):
return
hits = []
LOG.info("Querying YETI")
artifact = event.artifact
artifact_value = artifact['value']
LOG.info(artifact)
LOG.info("Looking up ({art_type}): {art_value}".format(
art_type=artifact['type'], art_value=artifact_value))
try:
# init new indicators object
indicators = self.yeti_client.observable_search(
regex=False, value=artifact_value)
LOG.debug(indicators)
except ValueError as e:
LOG.error(traceback.format_exc())
raise e
if not indicators or len(indicators) < 1:
return hits
tags = ""
for tag in indicators[0]["tags"]:
if tags != "":
tags += ", "
tags += tag["name"]
description = indicators[0]["description"] if indicators[0][
"description"] else "None"
try:
hits.append(
Hit(StringProp(name="Type", value=indicators[0]["type"]),
StringProp(name="Value", value=indicators[0]["value"]),
StringProp(name="Tags", value=tags),
StringProp(name="Created", value=indicators[0]["created"]),
UriProp(name="URL", value=indicators[0]["human_url"]),
StringProp(name="Description", value=description)))
return hits
except Exception as e:
LOG.error(traceback.format_exc())
raise e
def _query_hibp_api(self, artifact_value):
hits = []
retry = True
while (retry):
try:
# Return zero or more hits. Here's one example.
url = "{0}/{1}".format(self.HAVE_I_BEEN_PWNED_URL,
artifact_value)
response = requests.get(
url, headers={'User-Agent': 'Resilient HIBP CTS'})
if response.status_code == 200:
content = json.loads(response.text)
breaches = content["Breaches"]
if breaches is None:
breaches = []
pastes = content["Pastes"]
if pastes is None:
pastes = []
hits.append(
Hit(
NumberProp(name="Breached Sites",
value=len(breaches)),
NumberProp(name="Pastes", value=len(pastes)),
UriProp(name="View data from Have I Been Pwned",
value=url)))
retry = False
# 404 is returned when an email was not found
elif response.status_code == 404:
LOG.info("No hit information found on email address: {0}".
format(artifact_value))
retry = False
elif response.status_code == 429:
# Rate limit was hit, wait 2 seconds and try again
time.sleep(2)
else:
LOG.warn("Have I Been pwned returned expected status code")
retry = False
raise ThreatLookupIncompleteException()
except BaseException as e:
LOG.exception(e.message)
return hits
def _lookup_net_uri(self, event, *args, **kwargs):
LOG.info("Looking up with Google Safe Browsing API")
value = event.artifact['value']
LOG.info("Looking up URL: " + str(value))
sb = SafeBrowsingAPI(self.apikey)
resp = sb.lookup_urls(value)
hits = []
for match in resp.get("matches", []):
linkurl = match["threat"]["url"]
link = LINK_URL.format(match["threat"]["url"])
hits.append(Hit(
StringProp(name="Threat Type", value=match["threatType"]),
UriProp(name="Report Link", value=link),
StringProp(name="Platform Type", value=match["platformType"]),
StringProp(name="URL Name", value=linkurl)
))
return hits
def _lookup_artifact(self, event, *args, **kwargs):
"""Lookup an artifact"""
# This is a generic handler - we only care about lookup events but might be sent others
if not isinstance(event, ThreatServiceLookupEvent):
return
# event.artifact is a ThreatServiceArtifactDTO
artifact_type = event.artifact['type']
artifact_value = event.artifact['value']
# Check that the event matches an artifact type that we want to search in MISP
if artifact_type not in MISP_TYPES:
# Nothing to do
LOG.info(u"MISP lookup not implemented for %s", artifact_type)
return
# Doc says: MISP search for IP addresses using CIDR, uses '|' (pipe) instead of '/' (slashes) in the value.
# But this appears not to be the case, we just search for the unchanged value.
# if artifact_type == "net.cidr":
# artifact_value = str(artifact_value).replace('/', '|')
LOG.info("MISP Lookup: " + str(artifact_value))
misp_api = PyMISP(self.misp_url, self.misp_key, self.misp_verifycert,
'json')
# MISP search_index: produces a list of events, and their key attributes, based on search criteria.
# But does not filter by attribute type (only by the value that matched),
# and the value matches include substrings, so we can't filter the results effectively.
# matches = misp_api.search_index(published=1,
# tag=self.misp_tag,
# org=self.misp_org,
# attribute=str(artifact_value))
# MISP search: produce events or attribute values, but only for a single type of attribute (or all types).
# Attribute value matches partial strings (LIKE %value%), but we usually want exact-match searching.
# Our search strategy is this:
# - search for each type, returning only attributes;
# - filter the attribute results for case-insensitive exact matches only,
# - collect the list of events that the attributes belong to,
# - finally retrieve the event metadata for our results.
#
# (This strategy will likely change with future versions of MISP!)
misp_types = MISP_TYPES[artifact_type]
search_value = str(artifact_value).lower()
event_ids = set()
def matches(an_attribute):
"""Does the attribute value match? MISP does substring searches but we want exact."""
if artifact_type == "net.cidr":
# Match any CIDR, we assume the server did the logical search
return True
if an_attribute["value"].lower() == search_value:
# Match the whole attribute value, lowercase.
return True
return False
for misp_type in misp_types:
# Search for this one attribute-type
result = misp_api.search(controller='attributes',
values=[search_value],
type_attribute=misp_type,
tags=self.misp_tag,
org=self.misp_org,
withAttachments=0)
LOG.debug("Search for %s", misp_type)
LOG.debug(json.dumps(result))
if result:
response = result.get("response", {})
if isinstance(response, dict):
attributes = response.get("Attribute", [])
for attribute in attributes:
if matches(attribute):
event_ids.add(attribute["event_id"])
hits = []
if not event_ids:
# Nothing to do
return hits
# Finally let's get summary for each event.
# (Don't use the search api for this, it will match partial event ids).
for event_id in event_ids:
result = misp_api.get_event(event_id)
if "Event" in result:
event = result["Event"]
event_id = event["id"]
link = self.misp_link_url + "/events/view/" + str(event_id)
info = event.get("info")
datestr = event.get("date")
hit = Hit(
StringProp(name="Info", value=info),
StringProp(name="Date", value=datestr),
UriProp(name="MISP Link", value=link),
)
# Add all the tags as separate properties
for tag in event.get("Tag"):
tag_name, tag_value = tag["name"].split(":", 1)
hit.append(
StringProp(name="{}:".format(tag_name),
value=tag_value))
hits.append(hit)
return hits
def _generate_hit(self, artifact_value, tags_hits_list):
"""
Query RiskIQ PassiveTotal API for the given 'net.name' (domain name artifact), 'net.uri' (URL) or 'net.ip'
(IP address) and generate a Hit.
:param artifact_value
:param tags_hits_list
"""
# Passive DNS Results - Hits
# We grab the totalRecords number and show the First Seen date to Last Seen date interval
pdns_results_response = self._passivetotal_get_response(
self.passivetotal_passive_dns_api_url, artifact_value)
pdns_hit_number, pdns_first_seen, pdns_last_seen = None, None, None
if pdns_results_response.status_code == 200:
pdns_results = pdns_results_response.json()
pdns_hit_number = pdns_results.get("totalRecords", None)
pdns_first_seen = pdns_results.get("firstSeen", None)
pdns_last_seen = pdns_results.get("lastSeen", None)
LOG.info(pdns_hit_number)
LOG.info(pdns_first_seen)
LOG.info(pdns_last_seen)
else:
LOG.info(
"No Passive DNS information found for artifact value: {0}".
format(artifact_value))
LOG.debug(pdns_results_response.text)
# URL Classification - suspicious, malicious etc
classification_results_response = self._passivetotal_get_response(
self.passivetotal_actions_class_api_url, artifact_value)
classification_hit = None
if classification_results_response.status_code == 200:
classification_results = classification_results_response.json()
classification_hit = classification_results.get(
"classification", None)
LOG.info(classification_hit)
else:
LOG.info(
"No URL classification found for artifact value: {0}".format(
artifact_value))
LOG.debug(classification_results_response.text)
# Count of subdomains
subdomain_results_response = self._passivetotal_get_response(
self.passivetotal_enrich_subdom_api_url, artifact_value)
subdomain_hits_number, first_ten_subdomains = None, None
if subdomain_results_response.status_code == 200:
subdomain_results = subdomain_results_response.json()
subdomain_hits = subdomain_results.get("subdomains", None)
subdomain_hits_number = len(
subdomain_hits) if subdomain_hits else None
first_ten_subdomains = ', '.join(
subdomain_hits[:10]) if subdomain_hits else None
LOG.info(subdomain_hits_number)
LOG.info(first_ten_subdomains)
else:
LOG.info("No subdomain information found for artifact value: {0}".
format(artifact_value))
LOG.debug(subdomain_results_response.text)
# Convert tags hits list to str
tags_hits = ", ".join(tags_hits_list) if tags_hits_list else None
# Construct url back to to PassiveThreat
report_url = self.passivetotal_community_url + artifact_value
return Hit(
NumberProp(name="Number of Passive DNS Records",
value=pdns_hit_number),
StringProp(name="First Seen", value=pdns_first_seen),
StringProp(name="Last Seen", value=pdns_last_seen),
NumberProp(name="Subdomains - All", value=subdomain_hits_number),
StringProp(name="Subdomains - First ten Hostnames",
value=first_ten_subdomains),
StringProp(name="Tags", value=tags_hits),
StringProp(name="Classification", value=classification_hit),
UriProp(name="Report Link", value=report_url))