class SearcherExample(BaseComponent):
"""
Example of a custom threat searcher component.
You can use this as a template when implementing custom searcher(s) in your own package.
"""
# Register this as an async searcher for the URL /<root>/example
channel = searcher_channel("example")
# Handle lookups for artifacts of type 'net.uri' (see doc for full list)
@handler("net.uri")
def _lookup_net_uri(self, event, *args, **kwargs):
"""Return hits for URL artifacts"""
hits = []
# event.artifact is a ThreatServiceArtifactDTO
artifact_type = event.artifact['type']
artifact_value = event.artifact['value']
# Return zero or more hits. Here's one example.
hits.append(
Hit(NumberProp(name="id", value=123),
StringProp(name="Type", value=artifact_type),
UriProp(name="Link", value=artifact_value),
IpProp(name="IP Address", value="127.0.0.1"),
LatLngProp(name="Location", lat=42.366, lng=-71.081)))
hits.append(
Hit(
NumberProp(name="id", value=456),
StringProp(name="Type", value=artifact_type),
UriProp(name="Link", value=artifact_value),
IpProp(name="IP Address", value="0.0.0.0"),
))
yield hits
class GoogleSafeBrowsingThreatSearcher(BaseComponent):
"""
Custom threat lookup for Google Safe Browsing API
"""
channel = searcher_channel("gsb")
def __init__(self, opts):
super(GoogleSafeBrowsingThreatSearcher, self).__init__(opts)
self.apikey = opts.get(CONFIG_SECTION, {}).get("google_api_key")
if self.apikey is None:
exc = "Configuration value `google_api_key=XXX` is missing in [{}] section".format(CONFIG_SECTION)
raise Exception(exc)
@handler("net.uri")
def _lookup_net_uri(self, event, *args, **kwargs):
LOG.info("Looking up with Google Safe Browsing API")
value = event.artifact['value']
LOG.info("Looking up URL: " + str(value))
sb = SafeBrowsingAPI(self.apikey)
resp = sb.lookup_urls(value)
hits = []
for match in resp.get("matches", []):
linkurl = match["threat"]["url"]
link = LINK_URL.format(match["threat"]["url"])
hits.append(Hit(
StringProp(name="Threat Type", value=match["threatType"]),
UriProp(name="Report Link", value=link),
StringProp(name="Platform Type", value=match["platformType"]),
StringProp(name="URL Name", value=linkurl)
))
return hits
def __init__(self,
request_id=None,
name="unknown",
artifact=None,
channel=searcher_channel()):
super(ThreatServiceLookupEvent, self).__init__(name=name)
self.channels = (channel, )
self.request_id = request_id
self.cts_channel = channel
self.artifact = artifact
self.name = name
def _handle_get_request(self, event, *args, **kwargs):
"""
Responds to GET /cts/<anything>/<request-id>
The URL below /cts/ is specific to this threat service. For example,
/cts/one and /cts/two are considered two separate threat sources.
Response is a ResponseDTO containing the response, or 'please retry'
"""
LOG.info(event.args[0])
response = event.args[1]
request_id = None
if not args:
return {"id": request_id, "hits": []}
# The ID of the lookup request
request_id = args[-1]
# The channel that searchers are listening for events
cts_channel = searcher_channel(*args[:-1])
response_object = {"id": request_id, "hits": []}
cache_key = (cts_channel, request_id)
request_data = self.cache.get(cache_key)
if not request_data:
# There's no record of this request in our cache, return empty hits
response.status = 200
return response_object
response_object["hits"] = request_data["hits"]
if not request_data["complete"]:
# The searchers haven't finished yet, return partial hits if available
response.status = 303
response_object["retry_secs"] = self.later_retry_secs
# Update the counter, so we can detect "stale" failures
request_data["count"] = request_data.get("count", 0) + 1
if request_data["count"] > self.max_retries:
LOG.info("Exceeded max retries for {}".format(cache_key))
try:
self.cache.pop(cache_key)
except KeyError:
pass
response.status = 200
return response_object
return response_object
# Remove the result from cache
# self.cache.pop(cache_key)
return response_object
class UrlScanIoSearcher(BaseComponent):
"""
A custom threat service 'searcher' for urlscan.io
Test using 'curl':
curl -v -k --header "Content-Type: application/json" --data-binary '{"type":"net.uri","value":"http://example.org"}' 'http://127.0.0.1:9000/cts/usio'
curl -v 'http://127.0.0.1:9000/cts/example/f9acc1b7-6184-5746-873e-e385e6214261'
Test example of a potentially malicious url in urlscan.io search database is "http://detailsindia.in".
Test example of a non-malicious url in urlscan.io search database is "www.bai.org"
"""
CONFIG_SECTION = "urlscanio"
HEADERS = {'Content-Type': 'application/json'}
SEARCH_API_SIZE_QUERY_PARAM = "&size="
SEARCH_API_QUERY_TERM_PAGE_URL_PARAM = "?q=page.url:"
def __init__(self, opts):
super(UrlScanIoSearcher, self).__init__(opts)
self.options = opts.get(self.CONFIG_SECTION, {})
# Register this as an async searcher for the URL /<root>/example
channel = searcher_channel("usio")
@handler("net.uri")
def _lookup_net_uri(self, event, *args, **kwargs):
"""
Handle lookups for artifacts of type 'net uri' (URL artifacts)
"""
# Read configuration settings:
self.urlscan_io_search_api_url = self._get_value_from_options(
"urlscan_io_search_api_url")
self.urlscan_io_result_api_url = self._get_value_from_options(
"urlscan_io_result_api_url")
self.urlscan_io_search_size = self.options.get(
"urlscan_io_search_size", None)
# event.artifact is a ThreatServiceArtifactDTO
artifact_type = event.artifact['type']
artifact_value = event.artifact['value']
LOG.info(
"Looking up with urlscan.io Search API started for Artifact Type {0} - Artifact Value {1}"
.format(artifact_type, artifact_value))
hits = self._query_urlscan_io_api(artifact_value)
yield hits
def _get_value_from_options(self, app_config_setting_key):
"""
Get value from options dict or raise ValueError for the mandatory config setting.
:param app_config_setting_name key
"""
if app_config_setting_key in self.options:
return self.options[app_config_setting_key]
else:
error_msg = "Mandatory config setting '{}' not set.".format(
app_config_setting_key)
LOG.error(error_msg)
raise ValueError(error_msg)
def _query_urlscan_io_api(self, artifact_value):
"""
Query urlscan io Search API for the given URL.
:param artifact_value: URL
"""
hits = []
try:
optional_size_param = "{0}{1}".format(self.SEARCH_API_SIZE_QUERY_PARAM, self.urlscan_io_search_size) \
if self.urlscan_io_search_size else ""
url = "{0}{1}\"{2}\"{3}".format(
self.urlscan_io_search_api_url,
self.SEARCH_API_QUERY_TERM_PAGE_URL_PARAM, artifact_value,
optional_size_param)
search_response = requests.get(url, headers=self.HEADERS)
if search_response.status_code == 200:
content = search_response.json()
total_hits = content.get('total', None)
if total_hits is None or total_hits == 0:
LOG.info("No Results for the URL.")
LOG.debug(search_response.text)
return hits
LOG.info("Getting the report for the URL.")
search_results = content.get('results', None)
if not search_results:
LOG.info("No Results for the URL.")
LOG.debug(search_response.text)
return hits
for result in search_results:
if not result:
continue
result_hit = self._generate_hit_from_search_result(result)
if result_hit: # Do not include None value
hits.append(result_hit)
if not hits:
LOG.info("URL {0} isn't marked as malicious.".format(
artifact_value))
else:
LOG.info("No hit information found on URL: {0}".format(
artifact_value))
LOG.debug(search_response.text)
except BaseException as ex:
LOG.exception(ex.message)
return hits
def _generate_hit_from_search_result(self, search_result):
"""
Query urlscan io Result API for the given URL search result - hit.
:param search_result: dictionary
"""
result_id = search_result.get('_id', None)
result_url = "{0}{1}".format(self.urlscan_io_result_api_url, result_id)
result_response = requests.get(result_url, headers=self.HEADERS)
if result_response.status_code == 200:
result_content = result_response.json()
stats = result_content.get('stats', None)
if stats:
malicious_flag = stats.get('malicious', None)
if malicious_flag == 1:
# Some malicious scans show as failed, do not include those
if self._verify_for_scan_failed_flag(result_content):
return None
task = result_content.get('task', None)
page = result_content.get('page', None)
png_url = task.get('screenshotURL', None) if task else None
scan_time = task.get('time', None) if task else None
report_url = task.get('reportURL', None) if task else None
uniq_countries_int = stats.get('uniqCountries', None)
city_country_list = self._prepare_city_contry(
page.get('city', None), page.get(
'country', None)) if page else None
city_country = ",".join(
city_country_list) if city_country_list else None
server = page.get('server', None) if page else None
asn = page.get('asnname', None) if page else None
return Hit(
StringProp(name="Time Last Scanner", value=scan_time),
NumberProp(name="Number of Countries",
value=uniq_countries_int),
StringProp(name="City and Country",
value=city_country),
StringProp(name="Server", value=server),
StringProp(name="ASN Name", value=asn),
UriProp(name="Report Link", value=report_url),
UriProp(name="Screenshot Link", value=png_url))
else:
LOG.info(
"No Result information found on URL: {0}".format(result_url))
LOG.debug(result_response.text)
@staticmethod
def _verify_for_scan_failed_flag(result_content):
""" Verify if scan failed """
result_data = result_content.get('data', None)
if not result_data:
return True
result_data_requests_list = result_data.get('requests', None)
if not result_data_requests_list:
return True
# get first element from the list
requests_first_el = result_data_requests_list[0]
if not requests_first_el:
return True
response = requests_first_el.get('response', None)
if not response or 'failed' in response:
return True
return False
@staticmethod
def _prepare_city_contry(*argv):
"""
Prepare a list of non None value or blank "Falsy" parameters.
:param *argv - city, country
:return: list
"""
city_country_list = [el for el in argv if el]
return city_country_list
class MISPThreatSearcher(BaseComponent):
"""
Custom threat lookup for MISP
"""
channel = searcher_channel("misp")
def __init__(self, opts):
super(MISPThreatSearcher, self).__init__(opts)
# misp_url is the base URL for the MISP server (e.g. 'https://misp.example.com:883/')
self.misp_url = opts.get(CONFIG_SECTION, {}).get("misp_url")
if not self.misp_url:
exc = "Configuration value `misp_url=XXX` is missing in [{}] section".format(
CONFIG_SECTION)
raise Exception(exc)
# misp_link_url is the base URL for hyperlinks - default to same as 'misp_url'
self.misp_link_url = opts.get(CONFIG_SECTION,
{}).get("misp_link_url", self.misp_url)
# Authentication key for API access, e.g.: 3PhYSFDeC8xpqjC0ZrFZDwazoSRDUQ1j4IlKbu0G
# (get this from MISP: Event Actions -> Automation)
self.misp_key = opts.get(CONFIG_SECTION, {}).get("misp_key")
if not self.misp_key:
exc = "Configuration value `misp_key=XXX` is missing in [{}] section".format(
CONFIG_SECTION)
raise Exception(exc)
# verify the MISP server HTTPS certificate?
self.misp_verifycert = opts.get(CONFIG_SECTION,
{}).get("misp_verifycert", True)
# Optionally, filter on:
# tags=one,two
self.misp_tag = opts.get(CONFIG_SECTION, {}).get("misp_tag")
if self.misp_tag:
self.misp_tag = self.misp_tag.split(",")
# org=one,two
self.misp_org = opts.get(CONFIG_SECTION, {}).get("misp_org")
if self.misp_org:
self.misp_org = self.misp_org.split(",")
@handler()
def _lookup_artifact(self, event, *args, **kwargs):
"""Lookup an artifact"""
# This is a generic handler - we only care about lookup events but might be sent others
if not isinstance(event, ThreatServiceLookupEvent):
return
# event.artifact is a ThreatServiceArtifactDTO
artifact_type = event.artifact['type']
artifact_value = event.artifact['value']
# Check that the event matches an artifact type that we want to search in MISP
if artifact_type not in MISP_TYPES:
# Nothing to do
LOG.info(u"MISP lookup not implemented for %s", artifact_type)
return
# Doc says: MISP search for IP addresses using CIDR, uses '|' (pipe) instead of '/' (slashes) in the value.
# But this appears not to be the case, we just search for the unchanged value.
# if artifact_type == "net.cidr":
# artifact_value = str(artifact_value).replace('/', '|')
LOG.info("MISP Lookup: " + str(artifact_value))
misp_api = PyMISP(self.misp_url, self.misp_key, self.misp_verifycert,
'json')
# MISP search_index: produces a list of events, and their key attributes, based on search criteria.
# But does not filter by attribute type (only by the value that matched),
# and the value matches include substrings, so we can't filter the results effectively.
# matches = misp_api.search_index(published=1,
# tag=self.misp_tag,
# org=self.misp_org,
# attribute=str(artifact_value))
# MISP search: produce events or attribute values, but only for a single type of attribute (or all types).
# Attribute value matches partial strings (LIKE %value%), but we usually want exact-match searching.
# Our search strategy is this:
# - search for each type, returning only attributes;
# - filter the attribute results for case-insensitive exact matches only,
# - collect the list of events that the attributes belong to,
# - finally retrieve the event metadata for our results.
#
# (This strategy will likely change with future versions of MISP!)
misp_types = MISP_TYPES[artifact_type]
search_value = str(artifact_value).lower()
event_ids = set()
def matches(an_attribute):
"""Does the attribute value match? MISP does substring searches but we want exact."""
if artifact_type == "net.cidr":
# Match any CIDR, we assume the server did the logical search
return True
if an_attribute["value"].lower() == search_value:
# Match the whole attribute value, lowercase.
return True
return False
for misp_type in misp_types:
# Search for this one attribute-type
result = misp_api.search(controller='attributes',
values=[search_value],
type_attribute=misp_type,
tags=self.misp_tag,
org=self.misp_org,
withAttachments=0)
LOG.debug("Search for %s", misp_type)
LOG.debug(json.dumps(result))
if result:
response = result.get("response", {})
if isinstance(response, dict):
attributes = response.get("Attribute", [])
for attribute in attributes:
if matches(attribute):
event_ids.add(attribute["event_id"])
hits = []
if not event_ids:
# Nothing to do
return hits
# Finally let's get summary for each event.
# (Don't use the search api for this, it will match partial event ids).
for event_id in event_ids:
result = misp_api.get_event(event_id)
if "Event" in result:
event = result["Event"]
event_id = event["id"]
link = self.misp_link_url + "/events/view/" + str(event_id)
info = event.get("info")
datestr = event.get("date")
hit = Hit(
StringProp(name="Info", value=info),
StringProp(name="Date", value=datestr),
UriProp(name="MISP Link", value=link),
)
# Add all the tags as separate properties
for tag in event.get("Tag"):
tag_name, tag_value = tag["name"].split(":", 1)
hit.append(
StringProp(name="{}:".format(tag_name),
value=tag_value))
hits.append(hit)
return hits
class McAfeeTieSearcher(BaseComponent):
"""
McAfee TIE custom threat searcher component
Test using 'curl':
curl -v -X OPTIONS 'http://127.0.0.1:9000/cts/mcafee_tie_searcher'
curl -v -k --header "Content-Type: application/json" --data-binary '{"type":"hash.md5",
"value":"1CF5B6CC0E6B742F6DEF8BF96D847A25"}' 'http://127.0.0.1:9000/cts/mcafee_tie_searcher'
curl -v -k --header "Content-Type: application/json" --data-binary '{"type":"hash.sha1",
"value":"19A82049C4336E8A5A30426FEC3F560358FAB6ED"}' 'http://127.0.0.1:9000/cts/mcafee_tie_searcher'
curl -v -k --header "Content-Type: application/json" --data-binary '{"type":"hash.sha256",
"value":"A8B03AD33BC6D7A7F376B943F763EEDD1CDAF125D012F9018F2F56678AE67EA4"}'
'http://127.0.0.1:9000/cts/mcafee_tie_searcher'
"""
# Register this as an async searcher for the URL /<root>/mcafee_tie_searcher
channel = searcher_channel("mcafee_tie_searcher")
config_file = "dxlclient_config"
def __init__(self, opts):
super(McAfeeTieSearcher, self).__init__(opts)
try:
config = opts.get("mcafee").get(self.config_file)
if config is None:
LOG.error(self.config_file + " is not set. You must set this path to run this threat service")
raise ValueError(self.config_file + " is not set. You must set this path to run this threat service")
# Create configuration from file for DxlClient
self.config = DxlClientConfig.create_dxl_config_from_file(config)
except AttributeError:
LOG.error("There is no [mcafee] section in the config file,"
"please set that by running resilient-circuits config -u")
raise AttributeError("[mcafee] section is not set in the config file")
# Create client
self.client = DxlClient(self.config)
self.client.connect()
# Handle lookup for artifacts of type md5, sha1, and sha256 hashes
@handler("hash.md5", "hash.sha1", "hash.sha256")
def _lookup_hash(self, event, *args, **kwargs):
artifact_type = event.artifact["type"]
artifact_value = event.artifact["value"]
LOG.debug("_lookup_hash started for Artifact Type {0} - Artifact Value {1}".format(
artifact_type, artifact_value))
tie_client = TieClient(self.client)
if artifact_type == "hash.md5":
resilient_hash = {HashType.MD5: artifact_value}
elif artifact_type == "hash.sha1":
resilient_hash = {HashType.SHA1: artifact_value}
elif artifact_type == "hash.sha256":
resilient_hash = {HashType.SHA256: artifact_value}
else:
raise ValueError("Something went wrong setting the hash value")
reputations_dict = \
tie_client.get_file_reputation(
resilient_hash
)
hits = self._query_mcafee_tie(reputations_dict)
yield hits
def _query_mcafee_tie(self, reputations_dict):
hit = Hit()
# Check Enterprise File Provider
hit = self._get_enterprise_info(reputations_dict, hit)
# Check GTI File Provider
hit = self._get_gti_info(reputations_dict, hit)
# Check ATD File Provider
hit = self._get_atd_info(reputations_dict, hit)
# Check MWG File Provider
hit = self._get_mwg_info(reputations_dict, hit)
# Verifies a trust level was set before returning a hit
for prop in hit["props"]:
if fnmatch(prop["name"], "*Trust Level"):
return hit
return []
def _get_enterprise_info(self, reputations_dict, hit):
# Information for Enterprise file provider
if FileProvider.ENTERPRISE in reputations_dict:
ent_rep = reputations_dict[FileProvider.ENTERPRISE]
trust_level = self._get_trust_level(ent_rep[ReputationProp.TRUST_LEVEL])
if trust_level:
# Not a hit until trust level has been verified to less than or equal to MIGHT BE MALICIOUS
hit.append(StringProp(name="Enterprise Trust Level", value=trust_level))
# Retrieve the enterprise reputation attributes
ent_rep_attribs = ent_rep[ReputationProp.ATTRIBUTES]
# Get Average Local Rep
if FileEnterpriseAttrib.AVG_LOCAL_REP in ent_rep_attribs:
local_rep = self._get_trust_level(int(ent_rep_attribs[FileEnterpriseAttrib.AVG_LOCAL_REP]))
if local_rep:
hit.append(StringProp(name="Enterprise Avg Local Trust Level", value=local_rep))
# Get prevalence (if it exists)
if FileEnterpriseAttrib.PREVALENCE in ent_rep_attribs:
hit.append(StringProp(name="Enterprise Prevalence",
value=ent_rep_attribs[FileEnterpriseAttrib.PREVALENCE]))
# Get Enterprise Size (if it exists)
if FileEnterpriseAttrib.ENTERPRISE_SIZE in ent_rep_attribs:
hit.append(StringProp(name="Enterprise Size",
value=ent_rep_attribs[FileEnterpriseAttrib.ENTERPRISE_SIZE]))
# Get First Contact Date (if it exists)
if FileEnterpriseAttrib.FIRST_CONTACT in ent_rep_attribs:
hit.append(StringProp(name="Enterprise First Contact",
value=FileEnterpriseAttrib.to_localtime_string(
ent_rep_attribs[FileEnterpriseAttrib.FIRST_CONTACT])))
return hit
def _get_gti_info(self, reputations_dict, hit):
# Information for GTI file provider
if FileProvider.GTI in reputations_dict:
gti_rep = reputations_dict[FileProvider.GTI]
trust_level = self._get_trust_level(gti_rep[ReputationProp.TRUST_LEVEL])
if trust_level:
# Not a hit until trust level has been verified to less than or equal to MIGHT BE MALICIOUS
hit.append(StringProp(name="GTI Trust Level", value=trust_level))
# Retrieve the GTI reputation attributes
gti_rep_attribs = gti_rep[ReputationProp.ATTRIBUTES]
# Get prevalence (if it exists)
if FileGtiAttrib.PREVALENCE in gti_rep_attribs:
hit.append(StringProp(name="GTI Prevalence", value=gti_rep_attribs[FileGtiAttrib.PREVALENCE]))
# Get First Contact Date (if it exists)
if FileGtiAttrib.FIRST_CONTACT in gti_rep_attribs:
hit.append(StringProp(name="GTI First Contact", value=EpochMixin.to_localtime_string(
gti_rep_attribs[FileGtiAttrib.FIRST_CONTACT])))
return hit
def _get_atd_info(self, reputations_dict, hit):
# Information for Advanced Threat Defense file provider
if FileProvider.ATD in reputations_dict:
atd_rep = reputations_dict[FileProvider.ATD]
trust_level = self._get_trust_level(atd_rep[ReputationProp.TRUST_LEVEL])
if trust_level:
# Not a hit until trust level has been verified to less than or equal to MIGHT BE MALICIOUS
hit.append(StringProp(name="ATD Trust Level", value=trust_level))
return hit
def _get_mwg_info(self, reputations_dict, hit):
# Information for file provider
if FileProvider.MWG in reputations_dict:
mwg_rep = reputations_dict[FileProvider.MWG]
trust_level = self._get_trust_level(mwg_rep[ReputationProp.TRUST_LEVEL])
if trust_level:
# Not a hit until trust level has been verified to less than or equal to MIGHT BE MALICIOUS
hit.append(StringProp(name="MWG Trust Level", value=trust_level))
return hit
@staticmethod
def _get_trust_level(trust_level_number):
trust_level = ""
if TrustLevel.MIGHT_BE_MALICIOUS is trust_level_number:
trust_level = "Might be Malicious"
elif TrustLevel.MOST_LIKELY_MALICIOUS is trust_level_number:
trust_level = "Most Likely Malicious"
elif TrustLevel.KNOWN_MALICIOUS is trust_level_number:
trust_level = "Known Malicious"
return trust_level
class HaveIBeenPwnedSearcher(BaseComponent):
"""
Have I been Pwned custom threat searcher component
Test using 'curl':
curl -v -X OPTIONS 'http://127.0.0.1:9000/cts/have_i_been_pwned_threat_service'
curl -v -k --header "Content-Type: application/json" --data-binary '{"type":"email.header",
"value":"*****@*****.**"}' 'http://127.0.0.1:9000/cts/have_i_been_pwned_threat_service'
curl -v -k --header "Content-Type: application/json" --data-binary '{"type":"email.header.to",
"value":"*****@*****.**"}' 'http://127.0.0.1:9000/cts/have_i_been_pwned_threat_service'
"""
HAVE_I_BEEN_PWNED_URL = "https://haveibeenpwned.com/api/v2"
def __init__(self, opts):
super(HaveIBeenPwnedSearcher, self).__init__(opts)
LOG.debug(opts)
# Register this as an async searcher for the URL /<root>/example
channel = searcher_channel("have_i_been_pwned_threat_service")
# Handle lookups for artifacts of type 'email sender' and 'email receiver
@handler("email.header", "email.header.to")
def _lookup_email_sender(self, event, *args, **kwargs):
# event.artifact is a ThreatServiceArtifactDTO
artifact_type = event.artifact['type']
artifact_value = event.artifact['value']
LOG.debug("_lookup_email_sender started for Artifact Type {0} - Artifact Value {1}".format(
artifact_type, artifact_value))
hits = self._query_hibp_api(artifact_value)
yield hits
def _query_hibp_api(self, artifact_value):
hits = []
retry = True
while(retry):
try:
# Return zero or more hits. Here's one example.
breach_url = "{0}/breachedaccount/{1}".format(self.HAVE_I_BEEN_PWNED_URL, artifact_value)
breaches_response = requests.get(breach_url, headers={'User-Agent': 'Resilient HIBP CTS'})
paste_url = "{0}/pasteaccount/{1}".format(self.HAVE_I_BEEN_PWNED_URL, artifact_value)
pastes_response = requests.get(paste_url, headers={'User-Agent': 'Resilient HIBP CTS'})
if breaches_response.status_code == 200 and pastes_response.status_code == 200:
b_content = breaches_response.json()
if b_content is None:
b_content = []
p_content = pastes_response.json()
if p_content is None:
p_content = []
hits.append(
Hit(
NumberProp(name="Breached Sites", value=len(b_content)),
NumberProp(name="Pastes", value=len(p_content))
)
)
retry = False
# 404 is returned when an email was not found
elif breaches_response.status_code == 404 or pastes_response.status_code == 404:
LOG.info("No hit information found on email address: {0}".format(artifact_value))
retry = False
elif breaches_response.status_code == 429 or pastes_response.status_code == 429:
# Rate limit was hit, wait 2 seconds and try again
time.sleep(2)
else:
LOG.warn("Have I Been pwned returned expected status code")
retry = False
raise ThreatLookupIncompleteException()
except BaseException as e:
LOG.exception(e.message)
return hits
def _handle_post_request(self, event, *args, **kwargs):
"""
Responds to POST /cts/<anything>
The URL below /cts/ is specific to this threat service. For example,
/cts/one and /cts/two can be registered as two separate threat sources.
The string 'one' or 'two' becomes the channel that searcher events are dispatched on.
Request is a ThreatServiceArtifactDTO containing the artifact to be scanned
Response is a ResponseDTO containing the response, or 'please retry' (HTTP status 303).
"""
request = event.args[0]
response = event.args[1]
# The channels that searchers are listening for events
cts_channel = searcher_channel(*args)
value = request.body.getvalue()
if not value:
err = "Empty request"
LOG.warn(err)
return {"id": str(uuid4()), "hits": []}
# Resilient sends artifacts in two formats: multi-part MIME, or plain JSON.
# server may send either, even for cases where there is no file content,
# so check content-type and decode appropriately.
try:
if request.headers and "form-data" in request.headers.get(
"Content-Type", ""):
multipart_data = decoder.MultipartDecoder(
value, request.headers["Content-Type"])
body = json.loads(multipart_data.parts[0].text)
LOG.debug(body)
else:
body = json.loads(value.decode("utf-8"))
LOG.debug(body)
except (ValueError, NonMultipartContentTypeException) as e:
err = "Can't handle request: {}".format(e)
LOG.warn(err)
LOG.debug(value)
return {"id": str(uuid4()), "hits": []}
if not isinstance(body, dict):
# Valid JSON but not a valid request.
err = "Invalid request: {}".format(json.dumps(body))
LOG.warn(err)
return {"id": str(uuid4()), "hits": []}
# Generate a request ID, derived from the artifact being requested.
request_id = str(uuid5(self.namespace, json.dumps(body)))
artifact_type = body.get("type", "unknown")
artifact_value = body.get("value")
response_object = {"id": request_id, "hits": []}
cache_key = (cts_channel, request_id)
if artifact_type == "net.name" and artifact_value == "localhost":
# Hard-coded response to 'net.name' of 'localhost'
# because this is used in 'resutil threatservicetest'
# and we want to return an immediate (not async) response
return response_object
# If we already have a completed query for this key, return it immmediately
request_data = self.cache.get(cache_key)
if request_data and request_data.get("complete"):
response_object["hits"] = request_data.get("hits", [])
return response_object
response.status = 303
response_object["retry_secs"] = self.first_retry_secs
# Add the request to the cache, then notify searchers that there's a new request
self.cache.setdefault(cache_key, {
"id": request_id,
"artifact": body,
"hits": [],
"complete": False
})
evt = ThreatServiceLookupEvent(request_id=request_id,
name=artifact_type,
artifact=body,
channel=cts_channel)
self.async_helper.fire(evt, HELPER_CHANNEL)
return response_object
class AbuseIPDBThreatFeedSearcher(ResilientComponent):
"""
custom threat searcher component for abuseipdb
"""
def __init__(self, opts):
super(AbuseIPDBThreatFeedSearcher, self).__init__(opts)
self.opts = opts
self.options = opts.get("abuseipdb_cts", {})
LOG.debug(opts)
# check https://www.abuseipdb.com/categories for more information
self.abuseipdb_categories = {
3: "Fraud Orders",
4: "DDoS Attack",
5: "FTP Brute-Force",
6: "Ping of Death",
7: "Phishing",
8: "Fraud VoIP",
9: "Open Proxy",
10: "Web Spam",
11: "Email Spam",
12: "Blog Spam",
13: "VPN IP",
14: "Port Scan",
15: "Hacking",
16: "SQL Injection",
17: "Spoofing",
18: "Brute-Force",
19: "Bad Web Bot",
20: "Exploited Host",
21: "Web App Attack",
22: "SSH",
23: "IoT Targeted",
}
# Register this as an async searcher for the URL /<root>/example
channel = searcher_channel("abuseipdb_threat_feed")
@handler("net.ip")
def _lookup_net_ip(self, event, *args, **kwargs):
"""Lookup an artifact"""
# This is a generic handler - we only care about lookup events but might be sent others
if not isinstance(event, ThreatServiceLookupEvent):
return
# event.artifact is a ThreatServiceArtifactDTO
artifact_type = event.artifact['type']
artifact_value = event.artifact['value']
validate_fields(("abuseipdb_key", "abuseipdb_url"), self.options)
LOG.info(
"AbuseIPDB lookup started for Artifact Type {0} - Artifact Value {1}"
.format(artifact_type, artifact_value))
rc = RequestsCommon(self.opts, self.options)
hits = self._query_abuseipdb(rc, artifact_value)
yield hits
def _query_abuseipdb(self, rc, artifact_value):
"""
perform the query to abuseipdb
:param rc: resilient-lib object
:param artifact_value:
:return: hits object
"""
hits = []
try:
headers = HEADER_TEMPLATE.copy()
headers['Key'] = self.options.get("abuseipdb_key")
url = self.options.get("abuseipdb_url")
params = {
'ipAddress': artifact_value,
'isWhitelisted': self.options.get("ignore_white_listed",
"True"),
'verbose': True
}
response = rc.execute_call_v2("get",
url,
params=params,
headers=headers)
LOG.debug(response.json())
resp_data = response.json()['data']
number_of_reports = resp_data['totalReports']
country = resp_data['countryName']
most_recent_report = resp_data['lastReportedAt']
confidence_score = resp_data.get("abuseConfidenceScore", 0)
# get clean list of de-duped categories
categories_names = ""
if resp_data.get('reports'):
categories_list = []
for report in resp_data['reports']:
categories_list.extend(report["categories"])
categories_set = set(categories_list) # dedup list
categories_names = u', '.join(
(self.abuseipdb_categories.get(item, 'unknown')
for item in categories_set))
# only return data if there's anything useful
if number_of_reports or confidence_score:
# Return zero or more hits. Here's one example.
hits.append(
Hit(
NumberProp(name="Confidence Score",
value=confidence_score),
NumberProp(name="Number of Reports",
value=number_of_reports),
StringProp(name="Country", value=country),
StringProp(name="Most Recent Report",
value=most_recent_report),
StringProp(name="Categories", value=categories_names)))
except Exception as err:
LOG.exception(str(err))
return hits
class ShadowServerThreatFeedSearcher(ResilientComponent):
"""
Example of a custom threat searcher component
"""
def __init__(self, opts):
super(ShadowServerThreatFeedSearcher, self).__init__(opts)
self.options = opts.get("shadow_server_url", {})
self.allowed_artifacts = {"hash.md5": "md5", "hash.sha1": "sha1"}
self.data_to_ignore = ["md5", "sha1", "sha256", "sha512"]
# Register this as an async searcher for the URL /<root>/example
channel = searcher_channel("shadow_server_threat_feed")
@handler()
def _lookup_hash_shadow_server(self, event, *args, **kwargs):
"""Lookup an artifact"""
# This is a generic handler - we only care about lookup events but might be sent others
if not isinstance(event, ThreatServiceLookupEvent):
return
# event.artifact is a ThreatServiceArtifactDTO
artifact_type = event.artifact['type']
artifact_value = event.artifact['value']
# Check that the event matches an artifact type that we want to search in Shadow Server
if artifact_type not in self.allowed_artifacts:
# Nothing to do
LOG.info("Shadow Server lookup not implemented for {0}".format(
artifact_type))
return
LOG.info(
"Shadow Server lookup started for Artifact Type {0} - Artifact Value {1}"
.format(artifact_type, artifact_value))
hits = self._query_shadow_server(artifact_type, artifact_value)
yield hits
def _query_shadow_server(self, artifact_type, artifact_value):
hits = []
try:
url = "{0}?{1}={2}".format(
self.options.get("shadow_server_url",
"http://bin-test.shadowserver.org/api"),
self.allowed_artifacts.get(artifact_type), artifact_value)
LOG.debug("Getting info from {0}".format(url))
response = requests.get(url)
if response.status_code == 200:
LOG.debug(response.text)
# return hash {...} for found result or just hash
if not response.text.strip() == artifact_value:
resp_json = json.loads(
response.text.replace(artifact_value, "", 1))
hit = Hit()
for attribute, value in resp_json.items():
if attribute not in self.data_to_ignore:
hit.append(StringProp(name=attribute, value=value))
# Return zero or more hits. Here's one example.
hits.append(hit)
else:
LOG.warn("Got response status {0} from Shadow Server".format(
response.status_code))
except BaseException as e:
LOG.exception(str(e))
return hits
class YetiThreatFeedSearcher(BaseComponent):
"""
YETI custom threat searcher component
Test using 'curl':
curl -v -X OPTIONS 'http://127.0.0.1:9000/cts/yeti_threat_service'
curl -v -k --header "Content-Type: application/json" --data-binary '{"type":"net.ip",
"value":"11.11.11.11"}' 'http://127.0.0.1:9000/cts/yeti_threat_service'
"""
channel = searcher_channel("yeti_threat_service")
def __init__(self, opts):
super(YetiThreatFeedSearcher, self).__init__(opts)
self.options = opts.get(CONFIG_SECTION)
url = self.options.get("urlbase")
username = self.options.get("username")
password = self.options.get("password")
api_key = self.options.get("api_key")
self.yeti_client = pyeti.YetiApi(url, (username, password), api_key)
@handler()
def lookup_artifact(self, event, *args, **kwargs):
"""
Use YETI to search for artifact value
"""
if not isinstance(event, ThreatServiceLookupEvent):
return
hits = []
LOG.info("Querying YETI")
artifact = event.artifact
artifact_value = artifact['value']
LOG.info(artifact)
LOG.info("Looking up ({art_type}): {art_value}".format(
art_type=artifact['type'], art_value=artifact_value))
try:
# init new indicators object
indicators = self.yeti_client.observable_search(
regex=False, value=artifact_value)
LOG.debug(indicators)
except ValueError as e:
LOG.error(traceback.format_exc())
raise e
if not indicators or len(indicators) < 1:
return hits
tags = ""
for tag in indicators[0]["tags"]:
if tags != "":
tags += ", "
tags += tag["name"]
description = indicators[0]["description"] if indicators[0][
"description"] else "None"
try:
hits.append(
Hit(StringProp(name="Type", value=indicators[0]["type"]),
StringProp(name="Value", value=indicators[0]["value"]),
StringProp(name="Tags", value=tags),
StringProp(name="Created", value=indicators[0]["created"]),
UriProp(name="URL", value=indicators[0]["human_url"]),
StringProp(name="Description", value=description)))
return hits
except Exception as e:
LOG.error(traceback.format_exc())
raise e
class PassiveTotalSearcher(BaseComponent):
"""
A custom threat service 'searcher' for Passive Total
Test using 'curl':
curl -v -k --header "Content-Type: application/json" --data-binary '{"type":"net.uri","value":"http://example.org"}' 'http://127.0.0.1:9000/cts/pst'
curl -v 'http://127.0.0.1:9000/cts/example/f9acc1b7-6184-5746-873e-e385e6214261'
"""
CONFIG_SECTION = "passivetotal"
def __init__(self, opts):
super(PassiveTotalSearcher, self).__init__(opts)
self.options = opts.get(self.CONFIG_SECTION, {})
# Register this as an async searcher for the URL /<root>/example
channel = searcher_channel("pst")
@handler("net.name", "net.uri", "net.ip")
def _lookup(self, event, *args, **kwargs):
"""
Handle lookups for artifacts of type 'net.name' (domain name artifact), 'net.uri' (URL) or 'net.ip' (IP address)
"""
# Read configuration settings:
self.passivetotal_api_key = self._get_value_from_options(
"passivetotal_api_key")
self.passivetotal_username = self._get_value_from_options(
"passivetotal_username")
self.passivetotal_base_url = self._get_value_from_options(
"passivetotal_base_url")
self.passivetotal_account_api_url = self._get_value_from_options(
"passivetotal_account_api_url")
self.passivetotal_actions_tags_api_url = self._get_value_from_options(
"passivetotal_actions_tags_api_url")
self.passivetotal_passive_dns_api_url = self._get_value_from_options(
"passivetotal_passive_dns_api_url")
self.passivetotal_actions_class_api_url = self._get_value_from_options(
"passivetotal_actions_class_api_url")
self.passivetotal_enrich_subdom_api_url = self._get_value_from_options(
"passivetotal_enrich_subdom_api_url")
self.passivetotal_community_url = self._get_value_from_options(
"passivetotal_community_url")
self.passivetotal_tags = self._get_value_from_options(
"passivetotal_tags")
# event.artifact is a ThreatServiceArtifactDTO
artifact_type = event.artifact['type']
artifact_value = event.artifact['value']
LOG.info("_lookup started for Artifact Type {0} - Artifact Value {1}".
format(artifact_type, artifact_value))
hits = self._query_passivetotal_api(artifact_value)
yield hits
def _get_value_from_options(self, app_config_setting_key):
"""
Get value from options dict or raise ValueError for the mandatory config setting.
:param app_config_setting_key key
"""
if app_config_setting_key in self.options:
return self.options[app_config_setting_key]
else:
error_msg = "Mandatory config setting '{}' not set.".format(
app_config_setting_key)
LOG.error(error_msg)
raise ValueError(error_msg)
def _query_passivetotal_api(self, artifact_value):
"""
Validate user account if API call quota has exceeded, verify if tags are match, query RiskIQ PassiveTotal API
for the given 'net.name' (domain name artifact), 'net.uri' (URL) or 'net.ip' (IP address) and generate Hits.
:param artifact_value
"""
hits = []
if self._validate_user_account_exceeded():
return hits
validate_tags_match, tags_hits = self._validate_tag_match(
artifact_value)
if not validate_tags_match:
# failure condition if the site doesn't match your definition
LOG.info(
"The site isn't currently listed as compromised according to your definition."
)
return hits
LOG.info("Positive Threat Intel for %s", artifact_value)
# Create the hits array to sent back to Resilient
hits.append(self._generate_hit(artifact_value, tags_hits))
return hits
def _validate_user_account_exceeded(self):
"""
Validate if user's API call quota has exceeded.
Return True if user may not proceed.
"""
account_metadata_response = self._passivetotal_get_response(
self.passivetotal_account_api_url, '')
if account_metadata_response.status_code == 200:
account = account_metadata_response.json()
else:
LOG.info("No Account information found for username: {0}".format(
self.passivetotal_username))
LOG.debug(account_metadata_response.text)
return True
account_quota_exceeded = account.get("searchApiQuotaExceeded", None)
if account_quota_exceeded:
LOG.info("Your PassiveTotal Account has no API queries left.")
LOG.debug(account_metadata_response)
return True
return False
def _validate_tag_match(self, artifact_value):
"""
Validate if returned PassiveTotal tag hits for given artifact_value include user's flagged tags from app.config file.
Return True if tags intersect, there is a match. PT returns tags users has flagged.
:param artifact_value
"""
tags_response = self._passivetotal_get_response(
self.passivetotal_actions_tags_api_url, artifact_value)
if tags_response.status_code == 200:
tags = tags_response.json()
else:
LOG.info("No Tag information found for artifact value: {0}".format(
artifact_value))
LOG.debug(tags_response.text)
return False, None
tags_hits = tags.get('tags', None)
LOG.info("Comparing tags that result with a hit " + str(tags_hits) +
" with flagged tags in app.config file " +
self.passivetotal_tags)
# Tests the site has tags you have flagged
passive_tag_set = set(item.lower().strip()
for item in self.passivetotal_tags.split(","))
tags_hit_set = set(item.lower() for item in tags_hits)
return bool(passive_tag_set.intersection(tags_hit_set)), tags_hits
def _passivetotal_get_response(self, path, query):
"""
Get response from the given API for the given query.
:param path PT API url
:param query the domain or IP being queried
"""
url = self.passivetotal_base_url + path
data = {'query': query}
auth = (self.passivetotal_username, self.passivetotal_api_key)
response = requests.get(url, auth=auth, json=data)
return response
def _generate_hit(self, artifact_value, tags_hits_list):
"""
Query RiskIQ PassiveTotal API for the given 'net.name' (domain name artifact), 'net.uri' (URL) or 'net.ip'
(IP address) and generate a Hit.
:param artifact_value
:param tags_hits_list
"""
# Passive DNS Results - Hits
# We grab the totalRecords number and show the First Seen date to Last Seen date interval
pdns_results_response = self._passivetotal_get_response(
self.passivetotal_passive_dns_api_url, artifact_value)
pdns_hit_number, pdns_first_seen, pdns_last_seen = None, None, None
if pdns_results_response.status_code == 200:
pdns_results = pdns_results_response.json()
pdns_hit_number = pdns_results.get("totalRecords", None)
pdns_first_seen = pdns_results.get("firstSeen", None)
pdns_last_seen = pdns_results.get("lastSeen", None)
LOG.info(pdns_hit_number)
LOG.info(pdns_first_seen)
LOG.info(pdns_last_seen)
else:
LOG.info(
"No Passive DNS information found for artifact value: {0}".
format(artifact_value))
LOG.debug(pdns_results_response.text)
# URL Classification - suspicious, malicious etc
classification_results_response = self._passivetotal_get_response(
self.passivetotal_actions_class_api_url, artifact_value)
classification_hit = None
if classification_results_response.status_code == 200:
classification_results = classification_results_response.json()
classification_hit = classification_results.get(
"classification", None)
LOG.info(classification_hit)
else:
LOG.info(
"No URL classification found for artifact value: {0}".format(
artifact_value))
LOG.debug(classification_results_response.text)
# Count of subdomains
subdomain_results_response = self._passivetotal_get_response(
self.passivetotal_enrich_subdom_api_url, artifact_value)
subdomain_hits_number, first_ten_subdomains = None, None
if subdomain_results_response.status_code == 200:
subdomain_results = subdomain_results_response.json()
subdomain_hits = subdomain_results.get("subdomains", None)
subdomain_hits_number = len(
subdomain_hits) if subdomain_hits else None
first_ten_subdomains = ', '.join(
subdomain_hits[:10]) if subdomain_hits else None
LOG.info(subdomain_hits_number)
LOG.info(first_ten_subdomains)
else:
LOG.info("No subdomain information found for artifact value: {0}".
format(artifact_value))
LOG.debug(subdomain_results_response.text)
# Convert tags hits list to str
tags_hits = ", ".join(tags_hits_list) if tags_hits_list else None
# Construct url back to to PassiveThreat
report_url = self.passivetotal_community_url + artifact_value
return Hit(
NumberProp(name="Number of Passive DNS Records",
value=pdns_hit_number),
StringProp(name="First Seen", value=pdns_first_seen),
StringProp(name="Last Seen", value=pdns_last_seen),
NumberProp(name="Subdomains - All", value=subdomain_hits_number),
StringProp(name="Subdomains - First ten Hostnames",
value=first_ten_subdomains),
StringProp(name="Tags", value=tags_hits),
StringProp(name="Classification", value=classification_hit),
UriProp(name="Report Link", value=report_url))
class AbuseIPDBThreatFeedSearcher(ResilientComponent):
"""
Example of a custom threat searcher component
"""
def __init__(self, opts):
super(AbuseIPDBThreatFeedSearcher, self).__init__(opts)
self.options = opts.get("abuseipdb_cts", {})
LOG.debug(opts)
"""check https://www.abuseipdb.com/categories for more information"""
self.abuseipdb_categories = {
3: "Fraud Orders",
4: "DDoS Attack",
5: "FTP Brute-Force",
6: "Ping of Death",
7: "Phishing",
8: "Fraud VoIP",
9: "Open Proxy",
10: "Web Spam",
11: "Email Spam",
12: "Blog Spam",
13: "VPN IP",
14: "Port Scan",
15: "Hacking",
16: "SQL Injection",
17: "Spoofing",
18: "Brute-Force",
19: "Bad Web Bot",
20: "Exploited Host",
21: "Web App Attack",
22: "SSH",
23: "IoT Targeted",
}
# Register this as an async searcher for the URL /<root>/example
channel = searcher_channel("abuseipdb_threat_feed")
@handler("net.ip")
def _lookup_net_ip(self, event, *args, **kwargs):
"""Lookup an artifact"""
# This is a generic handler - we only care about lookup events but might be sent others
if not isinstance(event, ThreatServiceLookupEvent):
return
# event.artifact is a ThreatServiceArtifactDTO
artifact_type = event.artifact['type']
artifact_value = event.artifact['value']
if not self.options.get("abuseipdb_key"):
LOG.error(
"AbuseIPDB api key not set. You must set an api key to run this CTS."
)
return
LOG.info(
"AbuseIPDB lookup started for Artifact Type {0} - Artifact Value {1}"
.format(artifact_type, artifact_value))
hits = self._query_abuseipdb(artifact_value)
yield hits
def _query_abuseipdb(self, artifact_value):
hits = []
ignore_white_listed = True
if self.options.get("ignore_white_listed", "True").lower() != "true":
ignore_white_listed = False
try:
url = "{0}/{1}/json?key={2}".format(
self.options.get("abuseipdb_url",
"https://www.abuseipdb.com/check"),
artifact_value, self.options.get("abuseipdb_key"))
adapter = requests_toolbelt.SSLAdapter("SSLv23")
session = requests.Session()
session.mount('https://', adapter)
LOG.debug("Getting info from {0}".format(url))
response = session.get(url)
if response.status_code == 200:
resp_json = json.loads(response.text)
category_list = set([])
number_of_reports = len(resp_json)
country = resp_json[0]["country"]
most_recent_report = resp_json[0]["created"]
"""First we clean list of duplicated categories
We also check for whitelisted reports.
If the option to ignore is set to true, we ignore ALL reports if we found at least one white listed """
for report in resp_json:
if report["isWhitelisted"] and ignore_white_listed:
LOG.info(
"Ignoring white listed reports for {0}".format(
artifact_value))
return hits
for category in report["category"]:
if category != 0:
category_list.add(category)
"""Then we build the string with the values"""
categories_string = ""
for cat in category_list:
if categories_string != "":
categories_string += ", "
categories_string += self.abuseipdb_categories[cat]
# Return zero or more hits. Here's one example.
hits.append(
Hit(
NumberProp(name="Number of reports",
value=number_of_reports),
StringProp(name="Country", value=country),
StringProp(name="Most Recent Report",
value=most_recent_report),
StringProp(name="Categories",
value=categories_string)))
else:
LOG.warn("Got response status {0} from AbuseIPDB".format(
response.status_code))
except BaseException as e:
LOG.exception(e.message)
return hits
