def test_resp_types(self):
IPIFY = TestFunctionRequests.URL_TEST_DATA_RESULTS
rc = RequestsCommon(None, None)
# J S O N
json_result = rc.execute_call("get", "{}?format=json".format(IPIFY),
None)
try:
json.dumps(json_result)
except (TypeError, OverflowError):
self.fail("result is not json")
# T E X T
text_result = rc.execute_call("get",
"{}?format=text".format(IPIFY),
None,
resp_type='text')
self.assertIsNotNone(text_result)
# B Y T E S
bytes_result = rc.execute_call("get",
"{}?format=text".format(IPIFY),
None,
resp_type='bytes')
self.assertIsNotNone(bytes_result)
self.assertTrue(isinstance(bytes_result, bytes))
def test_statuscode(self):
URL = TestFunctionRequests.URL_TEST_HTTP_STATUS_CODES
rc = RequestsCommon(None, None)
resp = rc.execute_call("get",
"/".join((URL, "200")),
None,
resp_type='text')
with self.assertRaises(IntegrationError):
resp = rc.execute_call("get",
"/".join((URL, "300")),
None,
resp_type='text')
def selftest_function(opts):
"""
Placeholder for selftest function. An example use would be to test package api connectivity.
Suggested return values are be unimplemented, success, or failure.
"""
# Getting the Config file parameters.
options = opts.get("fn_spamhaus_query", {})
wqs_url = options.get('spamhaus_wqs_url') + "{}/{}"
dqs_key = options.get('spamhaus_dqs_key')
# Header data
header_data = {'Authorization': 'Bearer {}'.format(dqs_key)}
requestcommon_object = RequestsCommon(function_opts=options)
proxies_data = requestcommon_object.get_proxies()
try:
response_json = requestcommon_object.execute_call(
'GET',
wqs_url.format('AUTHBL', '127.0.0.2'),
headers=header_data,
proxies=proxies_data,
callback=spamhaus_call_error)
return {"state": "Success"}
except Exception as err_msg:
log.info(err_msg)
return {"state": "Failed"}
def test_basicauth(self):
URL = "/".join(
(TestFunctionRequests.URL_TEST_HTTP_VERBS, "basic-auth"))
basicauth = ("postman", "password")
rc = RequestsCommon(None, None)
resp = rc.execute_call("get", URL, None, basicauth=basicauth)
self.assertTrue(resp.get("authenticated"))
def sitereview(self, url, value):
payload = {"url": value, "captcha":""}
rc = RequestsCommon(self.opts, self.options)
result = rc.execute_call('post', url, payload=payload, headers=HEADERS, resp_type='text')
dict_to_str = json.dumps(xmltodict.parse(result))
return json.loads(dict_to_str)
def test_timeout(self):
URL = "/".join((TestFunctionRequests.URL_TEST_HTTP_STATUS_CODES,
"200?sleep=30000"))
rc = RequestsCommon(None, None)
with self.assertRaises(IntegrationError):
resp = rc.execute_call("get",
URL,
None,
resp_type='text',
timeout=2)
def test_headers(self):
# G E T with headers
headers = {
"Content-type": "application/json; charset=UTF-8",
"my-sample-header": "my header"
}
URL = "/".join((TestFunctionRequests.URL_TEST_HTTP_VERBS, "headers"))
rc = RequestsCommon()
json_result = rc.execute_call("get", URL, None, headers=headers)
self.assertEqual(json_result['headers'].get("my-sample-header"),
"my header")
def test_proxy(self):
rc = RequestsCommon()
proxy_url = TestFunctionRequests.URL_TEST_PROXY
proxy_result = rc.execute_call("get", proxy_url, None)
proxies = {
'http':
proxy_result['curl']
if proxy_result['protocol'] == 'http' else None,
'https':
proxy_result['curl']
if proxy_result['protocol'] == 'https' else None
}
URL = "?".join(
(TestFunctionRequests.URL_TEST_DATA_RESULTS, "format=json"))
# J S O N
json_result = rc.execute_call("get", URL, None, proxies=proxies)
self.assertTrue(json_result.get("ip"))
integrations = {
"integrations": {
'http_proxy':
proxy_result['curl']
if proxy_result['protocol'] == 'http' else None,
'https_proxy':
proxy_result['curl']
if proxy_result['protocol'] == 'https' else None
}
}
rc = RequestsCommon(opts=integrations)
json_result = rc.execute_call("get", URL, None)
self.assertTrue(json_result.get("ip"))
def test_statuscode_callback(self):
URL = "/".join(
(TestFunctionRequests.URL_TEST_HTTP_STATUS_CODES, "300"))
def callback(resp):
if resp.status_code != 300:
raise ValueError(resp.status_code)
rc = RequestsCommon(None, None)
resp = rc.execute_call("get",
URL,
None,
resp_type='text',
callback=callback)
def test_verbs(self):
URL = TestFunctionRequests.URL_TEST_HTTP_VERBS
headers = {"Content-type": "application/json; charset=UTF-8"}
payload = {'title': 'foo', 'body': 'bar', 'userId': 1}
rc = RequestsCommon(None, None)
# P O S T
# test json argument without headers
resp = rc.execute_call("post",
"/".join((URL, "post")),
payload,
log=TestFunctionRequests.LOG)
print(resp)
self.assertEqual(resp['json'].get("body"), "bar")
# test json argument with headers
resp = rc.execute_call("post",
"/".join((URL, "post")),
payload,
headers=headers,
log=TestFunctionRequests.LOG)
print(resp)
self.assertEqual(resp['json'].get("body"), "bar")
# test data argument
headers_data = {"Content-type": "application/x-www-form-urlencoded"}
resp = rc.execute_call("post",
"/".join((URL, "post")),
payload,
headers=headers_data,
log=TestFunctionRequests.LOG)
print(resp)
self.assertEqual(resp['json'].get("body"), "bar")
# G E T
resp = rc.execute_call("get",
"/".join((URL, "get")),
payload,
log=TestFunctionRequests.LOG)
self.assertTrue(resp['args'].get("userId"))
self.assertEqual(resp['args'].get("userId"), '1')
# P U T
resp = rc.execute_call("put",
"/".join((URL, "put")),
payload,
headers=headers,
log=TestFunctionRequests.LOG)
TestFunctionRequests.LOG.info(resp)
self.assertTrue(resp['args'].get("title"))
self.assertEqual(resp['args'].get("title"), 'foo')
# P A T C H
patch = {'title': 'patch'}
resp = rc.execute_call("patch",
"/".join((URL, "patch")),
patch,
headers=headers,
log=TestFunctionRequests.LOG)
print("resp {}".format(resp))
self.assertTrue(resp['args'].get("title"))
self.assertEqual(resp['args'].get("title"), 'patch')
# D E L E T E
DEL_URL = "/".join((URL, "delete"))
resp = rc.execute_call("delete",
DEL_URL,
None,
log=TestFunctionRequests.LOG)
self.assertEqual(resp.get("url"), DEL_URL)
# bad verb
with self.assertRaises(IntegrationError):
resp = rc.execute_call("bad",
URL,
None,
log=TestFunctionRequests.LOG)
def selftest_function(opts):
"""
Placeholder for selftest function. An example use would be to test package api connectivity.
Suggested return values are be unimplemented, success, or failure.
"""
options = opts.get("fn_rsa_netwitness", {})
nw_packet_server_url = options.get("nw_packet_server_url")
nw_packet_server_user = options.get("nw_packet_server_user")
nw_packet_server_password = options.get("nw_packet_server_password")
nw_packet_server_verify = str_to_bool(
options.get("nw_packet_server_verify"))
nw_log_server_url = options.get("nw_log_server_url")
nw_log_server_user = options.get("nw_log_server_user")
nw_log_server_password = options.get("nw_log_server_password")
nw_log_server_verify = str_to_bool(options.get("nw_log_server_verify"))
try:
rc = RequestsCommon(options, opts)
# Test PCAP server connection
headers = get_headers(nw_packet_server_user, nw_packet_server_password)
request_url = "{}/sdk/packets?sessions={}&render=pcap".format(
nw_packet_server_url, "100")
rc.execute_call("GET",
request_url,
verify_flag=nw_packet_server_verify,
headers=headers,
resp_type='bytes')
# Test Logger server connection
time1 = int(time.time()) * 1000
time2 = int(time.time()) * 1000
headers = get_headers(nw_log_server_user, nw_log_server_password)
request_url = "{}/sdk/packets?time1={}&time2={}&render={}".format(
nw_log_server_url, time1, time2, "logs")
rc.execute_call("GET",
request_url,
verify_flag=nw_log_server_verify,
headers=headers,
resp_type="text")
return {"state": "success"}
except Exception as err:
err_reason_msg = """Could not connect to NetWitness.
error: {0}
---------
Current Configs in app.config file::
---------
nw_packet_server_url: {1}
nw_packet_server_user: {2}
nw_packet_server_verify: {3}
nw_log_server_url: {4}
nw_log_server_user: {5}
nw_log_server_verify: {6}\n""".format(
err, nw_packet_server_url, nw_packet_server_user,
nw_packet_server_verify, nw_log_server_url, nw_log_server_user,
nw_log_server_verify)
log.error(err_reason_msg)
return {"state": "failed"}
def _fn_cs_falcon_search_function(self, event, *args, **kwargs):
"""Function that queries your CrowdStrike Falcon Hosts for a list of Devices using a Filter and/or Query. If Devices are found they are returned as a Python List"""
err_msg = None
log = logging.getLogger(__name__)
try:
# Instansiate helper (which gets appconfigs)
cs_helper = CrowdStrikeHelper(self.function_opts)
# Get the function inputs:
fn_inputs = {
"cs_filter_string":
cs_helper.get_function_input(kwargs, "cs_filter_string",
True), # text (optional)
"cs_query":
cs_helper.get_function_input(kwargs, "cs_query",
True) # text (optional)
}
# Create new Function ResultPayload with appconfigs and function inputs
payload = ResultPayload(CrowdStrikeHelper.app_config_section,
**fn_inputs)
# Get crowdstrike filter and query
cs_filter_string = fn_inputs.get("cs_filter_string")
cs_query = fn_inputs.get("cs_query")
# At least on of them has to be defined
if cs_filter_string is None and cs_query is None:
raise IntegrationError(
"Function Input cs_filter_string or cs_query must be defined"
)
yield StatusMessage("> Function Inputs OK")
# Instansiate new RequestCommon object to facilitate CrowdStrike REST API calls
rqc = RequestsCommon(self.opts, self.function_opts)
# Fist we look ip device ids
# Set the request URL and payload
get_device_ids_url = "{0}{1}".format(
cs_helper.bauth_base_url, "/devices/queries/devices/v1")
get_device_ids_payload = {}
if cs_filter_string is not None:
get_device_ids_payload["filter"] = cs_filter_string
if cs_query is not None:
get_device_ids_payload["q"] = cs_query
yield StatusMessage(
u'> Searching CrowdStrike for devices. Filter: "{0}" Query: {1}'
.format(cs_helper.str_to_unicode(cs_filter_string), cs_query))
# Make GET request for device_ids
get_device_ids_response = rqc.execute_call(
verb="GET",
url=get_device_ids_url,
payload=get_device_ids_payload,
basicauth=(cs_helper.bauth_api_uuid, cs_helper.bauth_api_key),
headers=cs_helper.json_header)
device_ids = get_device_ids_response.get("resources", [])
if len(device_ids) > 0:
# Then we get device_details for each device_id
yield StatusMessage("> Devices found. Getting device details")
get_device_details_url = "{0}{1}".format(
cs_helper.bauth_base_url, "/devices/entities/devices/v1")
get_device_details_payload = {"ids": device_ids}
get_device_details_response = rqc.execute_call(
verb="GET",
url=get_device_details_url,
payload=get_device_details_payload,
basicauth=(cs_helper.bauth_api_uuid,
cs_helper.bauth_api_key),
headers=cs_helper.json_header)
device_details = get_device_details_response.get(
"resources", [])
if len(device_details) > 0:
yield StatusMessage(
"> Device details received. Finishing...")
# For each device, convert their string timestamps to utc_time in ms
for device in device_details:
device[
"agent_local_time"] = cs_helper.timestamp_to_ms_epoch(
device.get("agent_local_time"),
timestamp_format="%Y-%m-%dT%H:%M:%S.%fZ")
device["first_seen"] = cs_helper.timestamp_to_ms_epoch(
device.get("first_seen"))
device[
"modified_timestamp"] = cs_helper.timestamp_to_ms_epoch(
device.get("modified_timestamp"))
device["last_seen"] = cs_helper.timestamp_to_ms_epoch(
device.get("last_seen"))
payload = payload.done(True, device_details)
else:
err_msg = u'> Could not get device details from CrowdStrike. Filter: "{0}" Query: {1}'.format(
cs_helper.str_to_unicode(cs_filter_string), cs_query)
yield StatusMessage(err_msg)
payload = payload.done(False, None, reason=err_msg)
else:
err_msg = u'> No devices found in CrowdStrike. Filter: "{0}" Query: {1}'.format(
cs_helper.str_to_unicode(cs_filter_string), cs_query)
yield StatusMessage(err_msg)
payload = payload.done(False, None, reason=err_msg)
results = payload
log.debug("RESULTS: %s", results)
log.info("Complete")
# Produce a FunctionResult with the results
yield FunctionResult(results)
except Exception:
yield FunctionError()
def _fn_spamhaus_query_submit_artifact_function(self, event, *args,
**kwargs):
"""Function: Function to check IP Address & Domain Names against Spamhaus Database to see whether IP Address or Domain Names appears in Spamhaus block list records or not."""
try:
# Get the function parameters:
spamhaus_query_string = kwargs.get("spamhaus_query_string") # text
spamhaus_search_resource = kwargs.get(
"spamhaus_search_resource") # text
# Get the app.config parameters:
spamhaus_wqs_url = self.options.get("spamhaus_wqs_url") + "{}/{}"
spamhaus_dqs_key = self.options.get("spamhaus_dqs_key")
log = logging.getLogger(__name__)
log.info("spamhaus_query_string: %s", spamhaus_query_string)
log.info("spamhaus_search_resource: %s", spamhaus_search_resource)
log.info("spamhaus_wqs_url: %s", spamhaus_wqs_url)
log.info("spamhaus_dqs_key: %s", spamhaus_dqs_key)
yield StatusMessage(
u"Checking Artifact: {} against Spamhaus block list resource {} database."
.format(spamhaus_query_string, spamhaus_search_resource))
# Checking API Key
if not spamhaus_dqs_key:
raise ApikeyError(
"API key must be defined in App.config file Spamhaus section."
)
# Initialising the Result payload object
result_object = ResultPayload("fn_spamhaus_query", **kwargs)
# Initialising Request Common for REST Api Call
request_common_obj = RequestsCommon(opts=self.options)
# Get proxy Configuration
proxies = request_common_obj.get_proxies()
# Construct call header with api key
header_data = {
'Authorization': 'Bearer {}'.format(spamhaus_dqs_key)
}
# Make Get Call to Spamhaus website
response_object = request_common_obj.execute_call(
verb='GET',
url=spamhaus_wqs_url.format(spamhaus_search_resource,
spamhaus_query_string),
headers=header_data,
proxies=proxies,
callback=spamhaus_call_error)
# Get Received data in JSON format.
response_json = response_object.json()
if not response_json:
raise SpamhausRequestCallError(
"No Response Returned from Api call")
if response_object.status_code == 404:
response_json[
'is_in_blocklist'] = False # a bool flag to for block list status
elif response_object.status_code == 200:
response_json['is_in_blocklist'] = True
resp_code_list = response_json.get('resp')
# Checking STATIC_INFO_RESPONSE for more information on returned info code.
for code in resp_code_list:
code_information = STATIC_INFO_RESPONSE.get(code)
# If information not found in `STATIC_INFO_RESPONSE`, Then trying with Spamhaus info API Call
if not code_information:
code_reponse_obj = request_common_obj.execute_call(
verb='GET',
url=spamhaus_wqs_url.format('info', code),
headers=header_data,
proxies=proxies,
callback=spamhaus_call_error)
if code_reponse_obj.status_code == 404:
response_json[code] = None
elif code_reponse_obj.status_code == 200:
response_json[code] = code_reponse_obj.json()
else:
response_json[code] = code_information
yield StatusMessage(
u"Completed Checking artifact against Spamhaus block list.")
# populating the result output set
results = result_object.done(success=True, content=response_json)
# Produce a FunctionResult with the results
yield FunctionResult(results)
except Exception as err_msg:
yield FunctionError(err_msg)
def _isitphishing_html_document_function(self, event, *args, **kwargs):
"""Function: isitphishing_html_document
This function takes an incident id as a required parameter and
task_id, attachment_id, and artifact_id as optional input which
specifies an HTML document to be base64 encoded and sent to the
Vade Secure API endpoint:
https://ws.isitphishing.org/api/v2/document
for analysis to detemine if the document contains phishing.
The "results" dictionary contains the result of the API query in
"contents" and the "inputs" parameters to the function.
"""
try:
rp = ResultPayload(CONFIG_DATA_SECTION, **kwargs)
# Get the function parameters:
incident_id = kwargs.get("incident_id") # number
task_id = kwargs.get("task_id") # number
attachment_id = kwargs.get("attachment_id") # number
artifact_id = kwargs.get("artifact_id") # number
log = logging.getLogger(__name__)
log.info("incident_id: %s", incident_id)
log.info("task_id: %s", task_id)
log.info("attachment_id: %s", attachment_id)
log.info("artifact_id: %s", artifact_id)
# Form the URL for API request.
API_URL = u"{0}/document".format(self.options["isitphishing_api_url"])
# Get the license key to access the API endpoint.
auth_token = get_license_key(self.options["isitphishing_name"], self.options["isitphishing_license"])
# Build the header and the data payload.
headers = {
"Authorization": u'Bearer {}'.format(auth_token),
"Content-type": "application/json",
"Accept": "application/json"
}
# Build the document payload which is a base64-encoded string.
client = self.rest_client()
# Get the attachment data
data = get_file_attachment(client, incident_id, artifact_id, task_id, attachment_id)
filename = get_file_attachment_name(client, incident_id, artifact_id, task_id, attachment_id)
# Base64 encode the document string and build payload.
base64encoded_doc = base64.b64encode(data).decode("ascii")
payload = {"document": base64encoded_doc}
yield StatusMessage("Query isitPhishing endpoint for status of document.")
# Make API URL request
rc = RequestsCommon(self.opts, self.options)
results_analysis = rc.execute_call("post", API_URL, payload, log=log, headers=headers)
results = rp.done(True, results_analysis)
# add back in the filename
results["inputs"]["filename"] = filename
# Produce a FunctionResult with the results
yield FunctionResult(results)
except Exception as err:
yield FunctionError()
def _isitphishing_url_function(self, event, *args, **kwargs):
"""Function: isitphishing_url
This function takes URL string as a required parameter. It
queries the Vade Secure isiphishing API to analyze the
URL to determine if it is PHISHING or SPAM.
The results contain the result of the query in 'contents' and the
URL input parameter to the function.
"""
try:
rp = ResultPayload(CONFIG_DATA_SECTION, **kwargs)
# Get the function parameters:
isitphishing_url = kwargs.get("isitphishing_url") # text
log = logging.getLogger(__name__)
log.info("isitphishing_url: %s", isitphishing_url)
# Form the URL for API request.
API_URL = u"{0}/url".format(self.options["isitphishing_api_url"])
# Get the license key to access the API endpoint.
auth_token = get_license_key(self.options["isitphishing_name"],
self.options["isitphishing_license"])
# Build the header and the data payload.
headers = {
"Authorization": u'Bearer {}'.format(auth_token),
"Content-type": "application/json",
}
payload = {
"url": isitphishing_url,
"force": False,
"smart": True,
"timeout": 8000
}
yield StatusMessage(
"Query isitPhishing.org endpoint for status of URL {0}.".
format(isitphishing_url))
# Make URL request
rc = RequestsCommon(self.opts, self.options)
results_analysis = rc.execute_call("post",
API_URL,
payload,
log=log,
headers=headers)
results = rp.done(True, results_analysis)
# Send back the results and the input parameter.
results = {
"content": results_analysis,
"inputs": {
"URL": isitphishing_url
}
}
# Produce a FunctionResult with the results
yield FunctionResult(results)
except Exception as err:
yield FunctionError(err)
