def _query_abuseipdb(self, artifact_value):
hits = []
ignore_white_listed = True
if self.options.get("ignore_white_listed", "True").lower() != "true":
ignore_white_listed = False
try:
url = "{0}/{1}/json?key={2}".format(
self.options.get("abuseipdb_url",
"https://www.abuseipdb.com/check"),
artifact_value, self.options.get("abuseipdb_key"))
adapter = requests_toolbelt.SSLAdapter("SSLv23")
session = requests.Session()
session.mount('https://', adapter)
LOG.debug("Getting info from {0}".format(url))
response = session.get(url)
if response.status_code == 200:
resp_json = json.loads(response.text)
category_list = set([])
number_of_reports = len(resp_json)
country = resp_json[0]["country"]
most_recent_report = resp_json[0]["created"]
"""First we clean list of duplicated categories
We also check for whitelisted reports.
If the option to ignore is set to true, we ignore ALL reports if we found at least one white listed """
for report in resp_json:
if report["isWhitelisted"] and ignore_white_listed:
LOG.info(
"Ignoring white listed reports for {0}".format(
artifact_value))
return hits
for category in report["category"]:
if category != 0:
category_list.add(category)
"""Then we build the string with the values"""
categories_string = ""
for cat in category_list:
if categories_string != "":
categories_string += ", "
categories_string += self.abuseipdb_categories[cat]
# Return zero or more hits. Here's one example.
hits.append(
Hit(
NumberProp(name="Number of reports",
value=number_of_reports),
StringProp(name="Country", value=country),
StringProp(name="Most Recent Report",
value=most_recent_report),
StringProp(name="Categories",
value=categories_string)))
else:
LOG.warn("Got response status {0} from AbuseIPDB".format(
response.status_code))
except BaseException as e:
LOG.exception(e.message)
return hits
def _query_abuseipdb(self, rc, artifact_value):
"""
perform the query to abuseipdb
:param rc: resilient-lib object
:param artifact_value:
:return: hits object
"""
hits = []
try:
headers = HEADER_TEMPLATE.copy()
headers['Key'] = self.options.get("abuseipdb_key")
url = self.options.get("abuseipdb_url")
params = {
'ipAddress': artifact_value,
'isWhitelisted': self.options.get("ignore_white_listed", "True"),
'verbose': True
}
response = rc.execute_call_v2("get", url, params=params, headers=headers)
LOG.debug(response.json())
resp_data = response.json()['data']
number_of_reports = resp_data['totalReports']
country = resp_data['countryName']
most_recent_report = resp_data['lastReportedAt']
confidence_score = resp_data.get("abuseConfidenceScore", 0)
# get clean list of de-duped categories
categories_names = ""
if resp_data.get('reports'):
categories_list = []
for report in resp_data['reports']:
categories_list.extend(report["categories"])
categories_set = set(categories_list) # dedup list
categories_names = u', '.join((self.abuseipdb_categories.get(item, 'unknown') for item in categories_set))
# only return data if there's anything useful
if number_of_reports or confidence_score:
# Return zero or more hits. Here's one example.
hits.append(
Hit(
NumberProp(name="Confidence Score", value=confidence_score),
NumberProp(name="Number of Reports", value=number_of_reports),
StringProp(name="Country", value=country),
StringProp(name="Most Recent Report", value=most_recent_report),
StringProp(name="Categories", value=categories_names)
)
)
except Exception as err:
LOG.exception(str(err))
return hits
def _generate_hit_from_search_result(self, search_result):
"""
Query urlscan io Result API for the given URL search result - hit.
:param search_result: dictionary
"""
result_id = search_result.get('_id', None)
result_url = "{0}{1}".format(self.urlscan_io_result_api_url, result_id)
result_response = requests.get(result_url, headers=self.HEADERS)
if result_response.status_code == 200:
result_content = result_response.json()
stats = result_content.get('stats', None)
if stats:
malicious_flag = stats.get('malicious', None)
if malicious_flag == 1:
# Some malicious scans show as failed, do not include those
if self._verify_for_scan_failed_flag(result_content):
return None
task = result_content.get('task', None)
page = result_content.get('page', None)
png_url = task.get('screenshotURL', None) if task else None
scan_time = task.get('time', None) if task else None
report_url = task.get('reportURL', None) if task else None
uniq_countries_int = stats.get('uniqCountries', None)
city_country_list = self._prepare_city_contry(
page.get('city', None), page.get(
'country', None)) if page else None
city_country = ",".join(
city_country_list) if city_country_list else None
server = page.get('server', None) if page else None
asn = page.get('asnname', None) if page else None
return Hit(
StringProp(name="Time Last Scanner", value=scan_time),
NumberProp(name="Number of Countries",
value=uniq_countries_int),
StringProp(name="City and Country",
value=city_country),
StringProp(name="Server", value=server),
StringProp(name="ASN Name", value=asn),
UriProp(name="Report Link", value=report_url),
UriProp(name="Screenshot Link", value=png_url))
else:
LOG.info(
"No Result information found on URL: {0}".format(result_url))
LOG.debug(result_response.text)
def lookup_artifact(self, event, *args, **kwargs):
"""
Use YETI to search for artifact value
"""
if not isinstance(event, ThreatServiceLookupEvent):
return
hits = []
LOG.info("Querying YETI")
artifact = event.artifact
artifact_value = artifact['value']
LOG.info(artifact)
LOG.info("Looking up ({art_type}): {art_value}".format(
art_type=artifact['type'], art_value=artifact_value))
try:
# init new indicators object
indicators = self.yeti_client.observable_search(
regex=False, value=artifact_value)
LOG.debug(indicators)
except ValueError as e:
LOG.error(traceback.format_exc())
raise e
if not indicators or len(indicators) < 1:
return hits
tags = ""
for tag in indicators[0]["tags"]:
if tags != "":
tags += ", "
tags += tag["name"]
description = indicators[0]["description"] if indicators[0][
"description"] else "None"
try:
hits.append(
Hit(StringProp(name="Type", value=indicators[0]["type"]),
StringProp(name="Value", value=indicators[0]["value"]),
StringProp(name="Tags", value=tags),
StringProp(name="Created", value=indicators[0]["created"]),
UriProp(name="URL", value=indicators[0]["human_url"]),
StringProp(name="Description", value=description)))
return hits
except Exception as e:
LOG.error(traceback.format_exc())
raise e
def _query_shadow_server(self, artifact_type, artifact_value):
hits = []
try:
url = "{0}?{1}={2}".format(
self.options.get("shadow_server_url",
"http://bin-test.shadowserver.org/api"),
self.allowed_artifacts.get(artifact_type), artifact_value)
LOG.debug("Getting info from {0}".format(url))
response = requests.get(url)
if response.status_code == 200:
LOG.debug(response.text)
# return hash {...} for found result or just hash
if not response.text.strip() == artifact_value:
resp_json = json.loads(
response.text.replace(artifact_value, "", 1))
hit = Hit()
for attribute, value in resp_json.items():
if attribute not in self.data_to_ignore:
hit.append(StringProp(name=attribute, value=value))
# Return zero or more hits. Here's one example.
hits.append(hit)
else:
LOG.warn("Got response status {0} from Shadow Server".format(
response.status_code))
except BaseException as e:
LOG.exception(str(e))
return hits
def _lookup_net_uri(self, event, *args, **kwargs):
LOG.info("Looking up with Google Safe Browsing API")
value = event.artifact['value']
LOG.info("Looking up URL: " + str(value))
sb = SafeBrowsingAPI(self.apikey)
resp = sb.lookup_urls(value)
hits = []
for match in resp.get("matches", []):
linkurl = match["threat"]["url"]
link = LINK_URL.format(match["threat"]["url"])
hits.append(Hit(
StringProp(name="Threat Type", value=match["threatType"]),
UriProp(name="Report Link", value=link),
StringProp(name="Platform Type", value=match["platformType"]),
StringProp(name="URL Name", value=linkurl)
))
return hits
def _get_mwg_info(self, reputations_dict, hit):
# Information for file provider
if FileProvider.MWG in reputations_dict:
mwg_rep = reputations_dict[FileProvider.MWG]
trust_level = self._get_trust_level(mwg_rep[ReputationProp.TRUST_LEVEL])
if trust_level:
# Not a hit until trust level has been verified to less than or equal to MIGHT BE MALICIOUS
hit.append(StringProp(name="MWG Trust Level", value=trust_level))
return hit
def _get_atd_info(self, reputations_dict, hit):
# Information for Advanced Threat Defense file provider
if FileProvider.ATD in reputations_dict:
atd_rep = reputations_dict[FileProvider.ATD]
trust_level = self._get_trust_level(atd_rep[ReputationProp.TRUST_LEVEL])
if trust_level:
# Not a hit until trust level has been verified to less than or equal to MIGHT BE MALICIOUS
hit.append(StringProp(name="ATD Trust Level", value=trust_level))
return hit
def _lookup_net_uri(self, event, *args, **kwargs):
"""Return hits for URL artifacts"""
hits = []
# event.artifact is a ThreatServiceArtifactDTO
artifact_type = event.artifact['type']
artifact_value = event.artifact['value']
# Return zero or more hits. Here's one example.
hits.append(
Hit(NumberProp(name="id", value=123),
StringProp(name="Type", value=artifact_type),
UriProp(name="Link", value=artifact_value),
IpProp(name="IP Address", value="127.0.0.1"),
LatLngProp(name="Location", lat=42.366, lng=-71.081)))
hits.append(
Hit(
NumberProp(name="id", value=456),
StringProp(name="Type", value=artifact_type),
UriProp(name="Link", value=artifact_value),
IpProp(name="IP Address", value="0.0.0.0"),
))
yield hits
def _get_gti_info(self, reputations_dict, hit):
# Information for GTI file provider
if FileProvider.GTI in reputations_dict:
gti_rep = reputations_dict[FileProvider.GTI]
trust_level = self._get_trust_level(gti_rep[ReputationProp.TRUST_LEVEL])
if trust_level:
# Not a hit until trust level has been verified to less than or equal to MIGHT BE MALICIOUS
hit.append(StringProp(name="GTI Trust Level", value=trust_level))
# Retrieve the GTI reputation attributes
gti_rep_attribs = gti_rep[ReputationProp.ATTRIBUTES]
# Get prevalence (if it exists)
if FileGtiAttrib.PREVALENCE in gti_rep_attribs:
hit.append(StringProp(name="GTI Prevalence", value=gti_rep_attribs[FileGtiAttrib.PREVALENCE]))
# Get First Contact Date (if it exists)
if FileGtiAttrib.FIRST_CONTACT in gti_rep_attribs:
hit.append(StringProp(name="GTI First Contact", value=EpochMixin.to_localtime_string(
gti_rep_attribs[FileGtiAttrib.FIRST_CONTACT])))
return hit
def _get_enterprise_info(self, reputations_dict, hit):
# Information for Enterprise file provider
if FileProvider.ENTERPRISE in reputations_dict:
ent_rep = reputations_dict[FileProvider.ENTERPRISE]
trust_level = self._get_trust_level(ent_rep[ReputationProp.TRUST_LEVEL])
if trust_level:
# Not a hit until trust level has been verified to less than or equal to MIGHT BE MALICIOUS
hit.append(StringProp(name="Enterprise Trust Level", value=trust_level))
# Retrieve the enterprise reputation attributes
ent_rep_attribs = ent_rep[ReputationProp.ATTRIBUTES]
# Get Average Local Rep
if FileEnterpriseAttrib.AVG_LOCAL_REP in ent_rep_attribs:
local_rep = self._get_trust_level(int(ent_rep_attribs[FileEnterpriseAttrib.AVG_LOCAL_REP]))
if local_rep:
hit.append(StringProp(name="Enterprise Avg Local Trust Level", value=local_rep))
# Get prevalence (if it exists)
if FileEnterpriseAttrib.PREVALENCE in ent_rep_attribs:
hit.append(StringProp(name="Enterprise Prevalence",
value=ent_rep_attribs[FileEnterpriseAttrib.PREVALENCE]))
# Get Enterprise Size (if it exists)
if FileEnterpriseAttrib.ENTERPRISE_SIZE in ent_rep_attribs:
hit.append(StringProp(name="Enterprise Size",
value=ent_rep_attribs[FileEnterpriseAttrib.ENTERPRISE_SIZE]))
# Get First Contact Date (if it exists)
if FileEnterpriseAttrib.FIRST_CONTACT in ent_rep_attribs:
hit.append(StringProp(name="Enterprise First Contact",
value=FileEnterpriseAttrib.to_localtime_string(
ent_rep_attribs[FileEnterpriseAttrib.FIRST_CONTACT])))
return hit
def _lookup_artifact(self, event, *args, **kwargs):
"""Lookup an artifact"""
# This is a generic handler - we only care about lookup events but might be sent others
if not isinstance(event, ThreatServiceLookupEvent):
return
# event.artifact is a ThreatServiceArtifactDTO
artifact_type = event.artifact['type']
artifact_value = event.artifact['value']
# Check that the event matches an artifact type that we want to search in MISP
if artifact_type not in MISP_TYPES:
# Nothing to do
LOG.info(u"MISP lookup not implemented for %s", artifact_type)
return
# Doc says: MISP search for IP addresses using CIDR, uses '|' (pipe) instead of '/' (slashes) in the value.
# But this appears not to be the case, we just search for the unchanged value.
# if artifact_type == "net.cidr":
# artifact_value = str(artifact_value).replace('/', '|')
LOG.info("MISP Lookup: " + str(artifact_value))
misp_api = PyMISP(self.misp_url, self.misp_key, self.misp_verifycert,
'json')
# MISP search_index: produces a list of events, and their key attributes, based on search criteria.
# But does not filter by attribute type (only by the value that matched),
# and the value matches include substrings, so we can't filter the results effectively.
# matches = misp_api.search_index(published=1,
# tag=self.misp_tag,
# org=self.misp_org,
# attribute=str(artifact_value))
# MISP search: produce events or attribute values, but only for a single type of attribute (or all types).
# Attribute value matches partial strings (LIKE %value%), but we usually want exact-match searching.
# Our search strategy is this:
# - search for each type, returning only attributes;
# - filter the attribute results for case-insensitive exact matches only,
# - collect the list of events that the attributes belong to,
# - finally retrieve the event metadata for our results.
#
# (This strategy will likely change with future versions of MISP!)
misp_types = MISP_TYPES[artifact_type]
search_value = str(artifact_value).lower()
event_ids = set()
def matches(an_attribute):
"""Does the attribute value match? MISP does substring searches but we want exact."""
if artifact_type == "net.cidr":
# Match any CIDR, we assume the server did the logical search
return True
if an_attribute["value"].lower() == search_value:
# Match the whole attribute value, lowercase.
return True
return False
for misp_type in misp_types:
# Search for this one attribute-type
result = misp_api.search(controller='attributes',
values=[search_value],
type_attribute=misp_type,
tags=self.misp_tag,
org=self.misp_org,
withAttachments=0)
LOG.debug("Search for %s", misp_type)
LOG.debug(json.dumps(result))
if result:
response = result.get("response", {})
if isinstance(response, dict):
attributes = response.get("Attribute", [])
for attribute in attributes:
if matches(attribute):
event_ids.add(attribute["event_id"])
hits = []
if not event_ids:
# Nothing to do
return hits
# Finally let's get summary for each event.
# (Don't use the search api for this, it will match partial event ids).
for event_id in event_ids:
result = misp_api.get_event(event_id)
if "Event" in result:
event = result["Event"]
event_id = event["id"]
link = self.misp_link_url + "/events/view/" + str(event_id)
info = event.get("info")
datestr = event.get("date")
hit = Hit(
StringProp(name="Info", value=info),
StringProp(name="Date", value=datestr),
UriProp(name="MISP Link", value=link),
)
# Add all the tags as separate properties
for tag in event.get("Tag"):
tag_name, tag_value = tag["name"].split(":", 1)
hit.append(
StringProp(name="{}:".format(tag_name),
value=tag_value))
hits.append(hit)
return hits
def _generate_hit(self, artifact_value, tags_hits_list):
"""
Query RiskIQ PassiveTotal API for the given 'net.name' (domain name artifact), 'net.uri' (URL) or 'net.ip'
(IP address) and generate a Hit.
:param artifact_value
:param tags_hits_list
"""
# Passive DNS Results - Hits
# We grab the totalRecords number and show the First Seen date to Last Seen date interval
pdns_results_response = self._passivetotal_get_response(
self.passivetotal_passive_dns_api_url, artifact_value)
pdns_hit_number, pdns_first_seen, pdns_last_seen = None, None, None
if pdns_results_response.status_code == 200:
pdns_results = pdns_results_response.json()
pdns_hit_number = pdns_results.get("totalRecords", None)
pdns_first_seen = pdns_results.get("firstSeen", None)
pdns_last_seen = pdns_results.get("lastSeen", None)
LOG.info(pdns_hit_number)
LOG.info(pdns_first_seen)
LOG.info(pdns_last_seen)
else:
LOG.info(
"No Passive DNS information found for artifact value: {0}".
format(artifact_value))
LOG.debug(pdns_results_response.text)
# URL Classification - suspicious, malicious etc
classification_results_response = self._passivetotal_get_response(
self.passivetotal_actions_class_api_url, artifact_value)
classification_hit = None
if classification_results_response.status_code == 200:
classification_results = classification_results_response.json()
classification_hit = classification_results.get(
"classification", None)
LOG.info(classification_hit)
else:
LOG.info(
"No URL classification found for artifact value: {0}".format(
artifact_value))
LOG.debug(classification_results_response.text)
# Count of subdomains
subdomain_results_response = self._passivetotal_get_response(
self.passivetotal_enrich_subdom_api_url, artifact_value)
subdomain_hits_number, first_ten_subdomains = None, None
if subdomain_results_response.status_code == 200:
subdomain_results = subdomain_results_response.json()
subdomain_hits = subdomain_results.get("subdomains", None)
subdomain_hits_number = len(
subdomain_hits) if subdomain_hits else None
first_ten_subdomains = ', '.join(
subdomain_hits[:10]) if subdomain_hits else None
LOG.info(subdomain_hits_number)
LOG.info(first_ten_subdomains)
else:
LOG.info("No subdomain information found for artifact value: {0}".
format(artifact_value))
LOG.debug(subdomain_results_response.text)
# Convert tags hits list to str
tags_hits = ", ".join(tags_hits_list) if tags_hits_list else None
# Construct url back to to PassiveThreat
report_url = self.passivetotal_community_url + artifact_value
return Hit(
NumberProp(name="Number of Passive DNS Records",
value=pdns_hit_number),
StringProp(name="First Seen", value=pdns_first_seen),
StringProp(name="Last Seen", value=pdns_last_seen),
NumberProp(name="Subdomains - All", value=subdomain_hits_number),
StringProp(name="Subdomains - First ten Hostnames",
value=first_ten_subdomains),
StringProp(name="Tags", value=tags_hits),
StringProp(name="Classification", value=classification_hit),
UriProp(name="Report Link", value=report_url))
