def _query_shadow_server(self, artifact_type, artifact_value):
hits = []
try:
url = "{0}?{1}={2}".format(
self.options.get("shadow_server_url",
"http://bin-test.shadowserver.org/api"),
self.allowed_artifacts.get(artifact_type), artifact_value)
LOG.debug("Getting info from {0}".format(url))
response = requests.get(url)
if response.status_code == 200:
LOG.debug(response.text)
# return hash {...} for found result or just hash
if not response.text.strip() == artifact_value:
resp_json = json.loads(
response.text.replace(artifact_value, "", 1))
hit = Hit()
for attribute, value in resp_json.items():
if attribute not in self.data_to_ignore:
hit.append(StringProp(name=attribute, value=value))
# Return zero or more hits. Here's one example.
hits.append(hit)
else:
LOG.warn("Got response status {0} from Shadow Server".format(
response.status_code))
except BaseException as e:
LOG.exception(str(e))
return hits
def _query_abuseipdb(self, artifact_value):
hits = []
ignore_white_listed = True
if self.options.get("ignore_white_listed", "True").lower() != "true":
ignore_white_listed = False
try:
url = "{0}/{1}/json?key={2}".format(
self.options.get("abuseipdb_url",
"https://www.abuseipdb.com/check"),
artifact_value, self.options.get("abuseipdb_key"))
adapter = requests_toolbelt.SSLAdapter("SSLv23")
session = requests.Session()
session.mount('https://', adapter)
LOG.debug("Getting info from {0}".format(url))
response = session.get(url)
if response.status_code == 200:
resp_json = json.loads(response.text)
category_list = set([])
number_of_reports = len(resp_json)
country = resp_json[0]["country"]
most_recent_report = resp_json[0]["created"]
"""First we clean list of duplicated categories
We also check for whitelisted reports.
If the option to ignore is set to true, we ignore ALL reports if we found at least one white listed """
for report in resp_json:
if report["isWhitelisted"] and ignore_white_listed:
LOG.info(
"Ignoring white listed reports for {0}".format(
artifact_value))
return hits
for category in report["category"]:
if category != 0:
category_list.add(category)
"""Then we build the string with the values"""
categories_string = ""
for cat in category_list:
if categories_string != "":
categories_string += ", "
categories_string += self.abuseipdb_categories[cat]
# Return zero or more hits. Here's one example.
hits.append(
Hit(
NumberProp(name="Number of reports",
value=number_of_reports),
StringProp(name="Country", value=country),
StringProp(name="Most Recent Report",
value=most_recent_report),
StringProp(name="Categories",
value=categories_string)))
else:
LOG.warn("Got response status {0} from AbuseIPDB".format(
response.status_code))
except BaseException as e:
LOG.exception(e.message)
return hits
def _query_abuseipdb(self, rc, artifact_value):
"""
perform the query to abuseipdb
:param rc: resilient-lib object
:param artifact_value:
:return: hits object
"""
hits = []
try:
headers = HEADER_TEMPLATE.copy()
headers['Key'] = self.options.get("abuseipdb_key")
url = self.options.get("abuseipdb_url")
params = {
'ipAddress': artifact_value,
'isWhitelisted': self.options.get("ignore_white_listed", "True"),
'verbose': True
}
response = rc.execute_call_v2("get", url, params=params, headers=headers)
LOG.debug(response.json())
resp_data = response.json()['data']
number_of_reports = resp_data['totalReports']
country = resp_data['countryName']
most_recent_report = resp_data['lastReportedAt']
confidence_score = resp_data.get("abuseConfidenceScore", 0)
# get clean list of de-duped categories
categories_names = ""
if resp_data.get('reports'):
categories_list = []
for report in resp_data['reports']:
categories_list.extend(report["categories"])
categories_set = set(categories_list) # dedup list
categories_names = u', '.join((self.abuseipdb_categories.get(item, 'unknown') for item in categories_set))
# only return data if there's anything useful
if number_of_reports or confidence_score:
# Return zero or more hits. Here's one example.
hits.append(
Hit(
NumberProp(name="Confidence Score", value=confidence_score),
NumberProp(name="Number of Reports", value=number_of_reports),
StringProp(name="Country", value=country),
StringProp(name="Most Recent Report", value=most_recent_report),
StringProp(name="Categories", value=categories_names)
)
)
except Exception as err:
LOG.exception(str(err))
return hits
def _generate_hit_from_search_result(self, search_result):
"""
Query urlscan io Result API for the given URL search result - hit.
:param search_result: dictionary
"""
result_id = search_result.get('_id', None)
result_url = "{0}{1}".format(self.urlscan_io_result_api_url, result_id)
result_response = requests.get(result_url, headers=self.HEADERS)
if result_response.status_code == 200:
result_content = result_response.json()
stats = result_content.get('stats', None)
if stats:
malicious_flag = stats.get('malicious', None)
if malicious_flag == 1:
# Some malicious scans show as failed, do not include those
if self._verify_for_scan_failed_flag(result_content):
return None
task = result_content.get('task', None)
page = result_content.get('page', None)
png_url = task.get('screenshotURL', None) if task else None
scan_time = task.get('time', None) if task else None
report_url = task.get('reportURL', None) if task else None
uniq_countries_int = stats.get('uniqCountries', None)
city_country_list = self._prepare_city_contry(
page.get('city', None), page.get(
'country', None)) if page else None
city_country = ",".join(
city_country_list) if city_country_list else None
server = page.get('server', None) if page else None
asn = page.get('asnname', None) if page else None
return Hit(
StringProp(name="Time Last Scanner", value=scan_time),
NumberProp(name="Number of Countries",
value=uniq_countries_int),
StringProp(name="City and Country",
value=city_country),
StringProp(name="Server", value=server),
StringProp(name="ASN Name", value=asn),
UriProp(name="Report Link", value=report_url),
UriProp(name="Screenshot Link", value=png_url))
else:
LOG.info(
"No Result information found on URL: {0}".format(result_url))
LOG.debug(result_response.text)
def _lookup_net_uri(self, event, *args, **kwargs):
"""Return hits for URL artifacts"""
hits = []
# event.artifact is a ThreatServiceArtifactDTO
artifact_type = event.artifact['type']
artifact_value = event.artifact['value']
# Return zero or more hits. Here's one example.
hits.append(
Hit(NumberProp(name="id", value=123),
StringProp(name="Type", value=artifact_type),
UriProp(name="Link", value=artifact_value),
IpProp(name="IP Address", value="127.0.0.1"),
LatLngProp(name="Location", lat=42.366, lng=-71.081)))
hits.append(
Hit(
NumberProp(name="id", value=456),
StringProp(name="Type", value=artifact_type),
UriProp(name="Link", value=artifact_value),
IpProp(name="IP Address", value="0.0.0.0"),
))
yield hits
def lookup_artifact(self, event, *args, **kwargs):
"""
Use YETI to search for artifact value
"""
if not isinstance(event, ThreatServiceLookupEvent):
return
hits = []
LOG.info("Querying YETI")
artifact = event.artifact
artifact_value = artifact['value']
LOG.info(artifact)
LOG.info("Looking up ({art_type}): {art_value}".format(
art_type=artifact['type'], art_value=artifact_value))
try:
# init new indicators object
indicators = self.yeti_client.observable_search(
regex=False, value=artifact_value)
LOG.debug(indicators)
except ValueError as e:
LOG.error(traceback.format_exc())
raise e
if not indicators or len(indicators) < 1:
return hits
tags = ""
for tag in indicators[0]["tags"]:
if tags != "":
tags += ", "
tags += tag["name"]
description = indicators[0]["description"] if indicators[0][
"description"] else "None"
try:
hits.append(
Hit(StringProp(name="Type", value=indicators[0]["type"]),
StringProp(name="Value", value=indicators[0]["value"]),
StringProp(name="Tags", value=tags),
StringProp(name="Created", value=indicators[0]["created"]),
UriProp(name="URL", value=indicators[0]["human_url"]),
StringProp(name="Description", value=description)))
return hits
except Exception as e:
LOG.error(traceback.format_exc())
raise e
def _query_hibp_api(self, artifact_value):
hits = []
retry = True
while (retry):
try:
# Return zero or more hits. Here's one example.
breach_url = "{0}/breachedaccount/{1}".format(
self.HAVE_I_BEEN_PWNED_URL, artifact_value)
breaches_response = requests.get(
breach_url, headers={'User-Agent': 'Resilient HIBP CTS'})
paste_url = "{0}/pasteaccount/{1}".format(
self.HAVE_I_BEEN_PWNED_URL, artifact_value)
pastes_response = requests.get(
paste_url, headers={'User-Agent': 'Resilient HIBP CTS'})
if breaches_response.status_code == 200 and pastes_response.status_code == 200:
b_content = breaches_response.json()
if b_content is None:
b_content = []
p_content = pastes_response.json()
if p_content is None:
p_content = []
hits.append(
Hit(
NumberProp(name="Breached Sites",
value=len(b_content)),
NumberProp(name="Pastes", value=len(p_content))))
retry = False
# 404 is returned when an email was not found
elif breaches_response.status_code == 404 or pastes_response.status_code == 404:
LOG.info("No hit information found on email address: {0}".
format(artifact_value))
retry = False
elif breaches_response.status_code == 429 or pastes_response.status_code == 429:
# Rate limit was hit, wait 2 seconds and try again
time.sleep(2)
else:
LOG.warn("Have I Been pwned returned expected status code")
retry = False
raise ThreatLookupIncompleteException()
except BaseException as e:
LOG.exception(e.message)
return hits
def _lookup_net_uri(self, event, *args, **kwargs):
LOG.info("Looking up with Google Safe Browsing API")
value = event.artifact['value']
LOG.info("Looking up URL: " + str(value))
sb = SafeBrowsingAPI(self.apikey)
resp = sb.lookup_urls(value)
hits = []
for match in resp.get("matches", []):
linkurl = match["threat"]["url"]
link = LINK_URL.format(match["threat"]["url"])
hits.append(Hit(
StringProp(name="Threat Type", value=match["threatType"]),
UriProp(name="Report Link", value=link),
StringProp(name="Platform Type", value=match["platformType"]),
StringProp(name="URL Name", value=linkurl)
))
return hits
def _query_mcafee_tie(self, reputations_dict):
hit = Hit()
# Check Enterprise File Provider
hit = self._get_enterprise_info(reputations_dict, hit)
# Check GTI File Provider
hit = self._get_gti_info(reputations_dict, hit)
# Check ATD File Provider
hit = self._get_atd_info(reputations_dict, hit)
# Check MWG File Provider
hit = self._get_mwg_info(reputations_dict, hit)
# Verifies a trust level was set before returning a hit
for prop in hit["props"]:
if fnmatch(prop["name"], "*Trust Level"):
return hit
return []
def _lookup_artifact(self, event, *args, **kwargs):
"""Lookup an artifact"""
# This is a generic handler - we only care about lookup events but might be sent others
if not isinstance(event, ThreatServiceLookupEvent):
return
# event.artifact is a ThreatServiceArtifactDTO
artifact_type = event.artifact['type']
artifact_value = event.artifact['value']
# Check that the event matches an artifact type that we want to search in MISP
if artifact_type not in MISP_TYPES:
# Nothing to do
LOG.info(u"MISP lookup not implemented for %s", artifact_type)
return
# Doc says: MISP search for IP addresses using CIDR, uses '|' (pipe) instead of '/' (slashes) in the value.
# But this appears not to be the case, we just search for the unchanged value.
# if artifact_type == "net.cidr":
# artifact_value = str(artifact_value).replace('/', '|')
LOG.info("MISP Lookup: " + str(artifact_value))
misp_api = PyMISP(self.misp_url, self.misp_key, self.misp_verifycert,
'json')
# MISP search_index: produces a list of events, and their key attributes, based on search criteria.
# But does not filter by attribute type (only by the value that matched),
# and the value matches include substrings, so we can't filter the results effectively.
# matches = misp_api.search_index(published=1,
# tag=self.misp_tag,
# org=self.misp_org,
# attribute=str(artifact_value))
# MISP search: produce events or attribute values, but only for a single type of attribute (or all types).
# Attribute value matches partial strings (LIKE %value%), but we usually want exact-match searching.
# Our search strategy is this:
# - search for each type, returning only attributes;
# - filter the attribute results for case-insensitive exact matches only,
# - collect the list of events that the attributes belong to,
# - finally retrieve the event metadata for our results.
#
# (This strategy will likely change with future versions of MISP!)
misp_types = MISP_TYPES[artifact_type]
search_value = str(artifact_value).lower()
event_ids = set()
def matches(an_attribute):
"""Does the attribute value match? MISP does substring searches but we want exact."""
if artifact_type == "net.cidr":
# Match any CIDR, we assume the server did the logical search
return True
if an_attribute["value"].lower() == search_value:
# Match the whole attribute value, lowercase.
return True
return False
for misp_type in misp_types:
# Search for this one attribute-type
result = misp_api.search(controller='attributes',
values=[search_value],
type_attribute=misp_type,
tags=self.misp_tag,
org=self.misp_org,
withAttachments=0)
LOG.debug("Search for %s", misp_type)
LOG.debug(json.dumps(result))
if result:
response = result.get("response", {})
if isinstance(response, dict):
attributes = response.get("Attribute", [])
for attribute in attributes:
if matches(attribute):
event_ids.add(attribute["event_id"])
hits = []
if not event_ids:
# Nothing to do
return hits
# Finally let's get summary for each event.
# (Don't use the search api for this, it will match partial event ids).
for event_id in event_ids:
result = misp_api.get_event(event_id)
if "Event" in result:
event = result["Event"]
event_id = event["id"]
link = self.misp_link_url + "/events/view/" + str(event_id)
info = event.get("info")
datestr = event.get("date")
hit = Hit(
StringProp(name="Info", value=info),
StringProp(name="Date", value=datestr),
UriProp(name="MISP Link", value=link),
)
# Add all the tags as separate properties
for tag in event.get("Tag"):
tag_name, tag_value = tag["name"].split(":", 1)
hit.append(
StringProp(name="{}:".format(tag_name),
value=tag_value))
hits.append(hit)
return hits
def _generate_hit(self, artifact_value, tags_hits_list):
"""
Query RiskIQ PassiveTotal API for the given 'net.name' (domain name artifact), 'net.uri' (URL) or 'net.ip'
(IP address) and generate a Hit.
:param artifact_value
:param tags_hits_list
"""
# Passive DNS Results - Hits
# We grab the totalRecords number and show the First Seen date to Last Seen date interval
pdns_results_response = self._passivetotal_get_response(
self.passivetotal_passive_dns_api_url, artifact_value)
pdns_hit_number, pdns_first_seen, pdns_last_seen = None, None, None
if pdns_results_response.status_code == 200:
pdns_results = pdns_results_response.json()
pdns_hit_number = pdns_results.get("totalRecords", None)
pdns_first_seen = pdns_results.get("firstSeen", None)
pdns_last_seen = pdns_results.get("lastSeen", None)
LOG.info(pdns_hit_number)
LOG.info(pdns_first_seen)
LOG.info(pdns_last_seen)
else:
LOG.info(
"No Passive DNS information found for artifact value: {0}".
format(artifact_value))
LOG.debug(pdns_results_response.text)
# URL Classification - suspicious, malicious etc
classification_results_response = self._passivetotal_get_response(
self.passivetotal_actions_class_api_url, artifact_value)
classification_hit = None
if classification_results_response.status_code == 200:
classification_results = classification_results_response.json()
classification_hit = classification_results.get(
"classification", None)
LOG.info(classification_hit)
else:
LOG.info(
"No URL classification found for artifact value: {0}".format(
artifact_value))
LOG.debug(classification_results_response.text)
# Count of subdomains
subdomain_results_response = self._passivetotal_get_response(
self.passivetotal_enrich_subdom_api_url, artifact_value)
subdomain_hits_number, first_ten_subdomains = None, None
if subdomain_results_response.status_code == 200:
subdomain_results = subdomain_results_response.json()
subdomain_hits = subdomain_results.get("subdomains", None)
subdomain_hits_number = len(
subdomain_hits) if subdomain_hits else None
first_ten_subdomains = ', '.join(
subdomain_hits[:10]) if subdomain_hits else None
LOG.info(subdomain_hits_number)
LOG.info(first_ten_subdomains)
else:
LOG.info("No subdomain information found for artifact value: {0}".
format(artifact_value))
LOG.debug(subdomain_results_response.text)
# Convert tags hits list to str
tags_hits = ", ".join(tags_hits_list) if tags_hits_list else None
# Construct url back to to PassiveThreat
report_url = self.passivetotal_community_url + artifact_value
return Hit(
NumberProp(name="Number of Passive DNS Records",
value=pdns_hit_number),
StringProp(name="First Seen", value=pdns_first_seen),
StringProp(name="Last Seen", value=pdns_last_seen),
NumberProp(name="Subdomains - All", value=subdomain_hits_number),
StringProp(name="Subdomains - First ten Hostnames",
value=first_ten_subdomains),
StringProp(name="Tags", value=tags_hits),
StringProp(name="Classification", value=classification_hit),
UriProp(name="Report Link", value=report_url))