def test_dictionary_merge(self, input_dict):
""" Test that the dictionary merge function works as expected.
The function should return a dictionary
If 1 or all inputs or None or empty it should not affect the return type"""
helper = ICDXHelper({})
result = helper.merge_dicts(input_dict)
assert (isinstance(result, dict))
def test_file_artifact_parse_success(self, search_result):
""" Test that the file artifact function works as expected
The function should return a dictionary
With a known good input the return should be Truth-y (has elements)
"""
helper = ICDXHelper({})
result = helper.parse_for_file_artifacts(search_result)
assert (isinstance(result, dict))
assert (bool(result))
def test_dictionary_merge_failure(self, input_dict):
""" Test that the dictionary merge function handles empty inputs.
The function should return a dictionary
If 1 or all inputs or None or empty it should not affect the return type
But if all inputs are None or empty the return should be False-y i.e empty dict"""
helper = ICDXHelper({})
result = helper.merge_dicts(input_dict)
assert (isinstance(result, dict))
assert (bool(result) == False)
def test_multi_artifact_parse_success(self, search_result):
""" Test that the file artifact function works as expected
The function should return a dictionary
With a known good input the return should be Truth-y (has elements)
"""
helper = ICDXHelper({})
result = helper.search_for_artifact_data(search_result)
assert (isinstance(result, dict))
assert (result is not None)
assert '10.1.1.1' in result['IP Address']
if 'email object' in search_result['name']:
assert '127.0.0.1' in result['IP Address']
def __init__(self, opts):
super(ICDXComponentObserver, self).__init__(opts)
self.options = opts.get("fn_icdx", {})
add_methods_to_global()
helper = ICDXHelper(self.options)
if "pytest" in sys.modules:
"""
Reaching here indicates that the component is invoked from within a testing session.
In this case, don't start the forwarder
"""
log.info(
"Running within a test environment. Not starting consumer")
else:
if helper.str_to_bool(
self.options.get("icdx_forwarder_toggle", None)):
self.setup_observer()
def test_successful_instantiation(self):
"""Should not raise an exception when AMQPAsyncConsumer is instantiated with correct params"""
obj = AMQPAsyncConsumer(host="192.168.1.254",
port=1,
virtual_host="dx",
username="******",
amqp_password="******",
rest_client=SimpleClient(),
helper=ICDXHelper(options={}))
def test_full_unsuccessful_instantiation(self):
"""Should raise an exception when AMQPAsyncConsumer is instantiated with malformed params"""
with pytest.raises(Exception) as e_info:
obj = AMQPAsyncConsumer(host="192.168.1.254",
port=1,
virtual_host="dx",
username="******",
amqp_password="******",
rest_client=SimpleClient(),
helper=ICDXHelper())
def test_restclient_validation(self):
"""Should raise an exception when AMQPAsyncConsumer is instantiated without params"""
obj = AMQPAsyncConsumer(host="192.168.1.254",
port=1,
virtual_host="dx",
username="******",
amqp_password="******",
rest_client=SimpleClient(),
helper=ICDXHelper(options={}))
assert isinstance(getattr(obj, "_client"), SimpleClient)
obj_with_no_client = AMQPAsyncConsumer(host="192.168.1.254",
port=1,
virtual_host="dx",
username="******",
amqp_password="******",
rest_client=123,
helper=ICDXHelper(options={}))
assert isinstance(getattr(obj_with_no_client, "_client"), type(None))
def setup_observer(self):
"""
Attempts to create an instance of the Consumer class and begin consuming forwarded messages.
If the connection is closed unexpectedly, Consumer attempts to reopen it.
AMQP sends hearbeats to keep a connection alive, if 2 consecutive heartbeats are missed
the Consumer will completly stop.
"""
helper = ICDXHelper(self.options)
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
# Init the Consumer, passing in an instance of RestClient and ICDXHelper as injected deps
observer = AMQPAsyncConsumer(
host=helper.get_config_option("icdx_amqp_host"),
port=helper.get_config_option("icdx_amqp_port", True),
virtual_host=helper.get_config_option("icdx_amqp_vhost"),
username=helper.get_config_option("icdx_amqp_username"),
amqp_password=helper.get_config_option("icdx_amqp_password"),
rest_client=self.rest_client(),
helper=ICDXHelper(self.options))
"""
Start our consumer, will attempt a connection to ICDX
When a connection is attempted a chain of functions are called
Each of these functions has a callback which in turn will trigger the next part of the setup
Interfacing with Pika asynchronously is done by passing in callback methods you would like to have invoked
when a certain event completes.
The on_message function is triggered to potentially create an incident every time a new forwarded event
comes off the message queue
The below pattern is similar to how threads work wherein all logic is encapsulated -- we only call run()
All functions are event-based which requires that the output of one function be the input to another.
"""
try:
# Create a thread which targets the observers run function
# N.B note the lack of parentheses on observer.run
res_daemon_thread = threading.Thread(target=observer.run)
# Make the thread a daemon (background)
res_daemon_thread.daemon = True
# Start daemon thread in bg so rest of resilient-circuits is not blocked
res_daemon_thread.start()
except Exception as e:
log.error(
"Encountered an issue when starting the thread: {}".format(
str(e)))
def _icdx_get_archive_list_function(self, event, *args, **kwargs):
"""Function: The Get Archive List API is used to return a list of archives in the ICDx system. The response is an unsorted list of archive metadata objects which can then be searched by a user."""
log = logging.getLogger(__name__)
try:
rc = ResultPayload(ICDX_SECTION, **kwargs)
helper = ICDXHelper(self.options)
yield StatusMessage(
"Attempting to gather config and setup the AMQP Client")
try:
# Initialise the AmqpFacade, pass in config values
amqp_client = AmqpFacade(
host=helper.get_config_option("icdx_amqp_host"),
port=helper.get_config_option("icdx_amqp_port", True),
virtual_host=helper.get_config_option("icdx_amqp_vhost"),
username=helper.get_config_option("icdx_amqp_username"),
amqp_password=helper.get_config_option(
"icdx_amqp_password"))
yield StatusMessage(
"Config options gathered and AMQP client setup.")
except Exception:
raise FunctionError(
"Encountered error while initialising the AMQP Client")
# Prepare request payload
request = {"id": GET_ARCHIVE_LIST_CODE}
# Make the call to ICDx and get a handle on any results
archives, status = amqp_client.call(json.dumps(request))
yield StatusMessage(
"ICDX call complete with status: {}".format(status))
# If status code in the message header is 200 we have results, 204 is empty response.
results = rc.done(
success=False if status != 200 else True,
content={
"archives":
(json.loads(archives) if archives is not None else None)
})
results.update({
"archives":
(json.loads(archives) if archives is not None else None)
})
# Produce a FunctionResult with the results
yield FunctionResult(results)
log.info("Complete")
except Exception:
yield FunctionError()
def test_default_port_usage(self):
"""Should raise an exception when AMQPAsyncConsumer is instantiated without params"""
obj = AMQPAsyncConsumer(host="192.168.1.254",
port=1,
virtual_host="dx",
username="******",
amqp_password="******",
rest_client=SimpleClient(),
helper=ICDXHelper(options={}))
assert isinstance(getattr(obj, "_port"), int)
assert getattr(obj, "_port") == 1
obj_with_no_client = AMQPAsyncConsumer(host="192.168.1.254",
port="notaPort",
virtual_host="dx",
username="******",
amqp_password="******",
rest_client=SimpleClient(),
helper=123)
assert isinstance(getattr(obj_with_no_client, "_port"), int)
assert getattr(obj_with_no_client, "_port") == 5672
def _icdx_get_event_function(self, event, *args, **kwargs):
"""Function: Takes in an input of a UUID for an event and attempts to get the details of this event from the ICDx platform."""
try:
rc = ResultPayload(ICDX_SECTION, **kwargs)
# Get the function parameters:
icdx_uuid = kwargs.get("icdx_uuid") # text
log = logging.getLogger(__name__)
log.info("icdx_uuid: %s", icdx_uuid)
if icdx_uuid is None:
raise FunctionError(
"Encountered error: icdx_uuid may not be None")
helper = ICDXHelper(self.options)
yield StatusMessage(
"Attempting to gather config and setup the AMQP Client")
try:
# Initialise the AmqpFacade, pass in config values
amqp_client = AmqpFacade(
host=helper.get_config_option("icdx_amqp_host"),
port=helper.get_config_option("icdx_amqp_port", True),
virtual_host=helper.get_config_option("icdx_amqp_vhost"),
username=helper.get_config_option("icdx_amqp_username"),
amqp_password=helper.get_config_option(
"icdx_amqp_password"))
yield StatusMessage(
"Config options gathered and AMQP client setup.")
except Exception:
raise FunctionError(
"Encountered error while initialising the AMQP Client")
yield StatusMessage(
"Sending request to ICDx with a UUID of {0}".format(icdx_uuid))
# Prepare request payload
request = {
"id": GET_EVENT_API_CODE,
"uuid": icdx_uuid,
}
# Make the call to ICDx and get a handle on any results
search_result_payload, status = amqp_client.call(
json.dumps(request))
yield StatusMessage(
"ICDx call complete with status: {}".format(status))
# Define variables related to artifact search to avoid reference before assignment
artifact_dictionary = None
artifact_keys_as_list = None
artifact_values_as_list = None
if 'connection'.encode('utf-8') in search_result_payload:
yield StatusMessage(
"Network Connection attribute found on Event. Parsing for artifact data"
)
if 'file'.encode('utf-8') in search_result_payload:
yield StatusMessage(
"File attribute found on Event. Parsing for artifact data")
if 'email'.encode('utf-8') in search_result_payload:
yield StatusMessage(
"Email attribute found on Event. Parsing for artifact data"
)
if status == 200:
# Parse the search result for any artifacts
artifact_dictionary = helper.search_for_artifact_data(
json.loads(search_result_payload))
"""Lastly get the keys and the values as separate lists
This is needed in v30 or lower.
Post-processing script shows how to use the data in both v30 and v31
"""
artifact_keys_as_list = artifact_dictionary.keys()
artifact_values_as_list = artifact_dictionary.values()
elif status == 400:
yield StatusMessage(
"Received a 400 (Bad Request). Check the formatting of your ICDx Query"
)
elif status == 401:
yield StatusMessage(
"Received a 401 (Auth). Check the credentials in app.config"
)
# Return the results
event = None if status != 200 else json.loads(
search_result_payload.decode('utf-8')),
artifacts = artifact_dictionary,
artifact_keys_as_list = None if isinstance(
artifact_keys_as_list,
type(None)) else list(artifact_keys_as_list)
artifact_values_as_list = None if isinstance(
artifact_values_as_list,
type(None)) else list(artifact_values_as_list)
results = rc.done(success=False if status != 200 else True,
content={
"event":
event,
"artifacts":
artifact_dictionary,
"artifact_keys_as_list":
artifact_keys_as_list,
"artifact_values_as_list":
artifact_values_as_list
})
results.update({
"event": event,
"artifacts": artifact_dictionary,
"artifact_keys_as_list": artifact_keys_as_list,
"artifact_values_as_list": artifact_values_as_list
})
# Produce a FunctionResult with the results
yield FunctionResult(results)
except Exception:
yield FunctionError()
def test_file_artifact_parse_failure(self, search_result):
helper = ICDXHelper({})
result = helper.parse_for_file_artifacts(search_result)
assert (not result)
assert (bool(result) is False)
def _icdx_find_events_function(self, event, *args, **kwargs):
"""Function: Takes a number of parameters in a search request and attempts to gather events from the ICDx Platform. Returns a response containing a list of events or a response with a 204 status code when no results are found."""
try:
# Get the function parameters:
icdx_search_request = self.get_textarea_param(
kwargs.get("icdx_search_request")) # textarea
log = logging.getLogger(__name__)
log.info("icdx_search_request: %s", icdx_search_request)
if icdx_search_request is None:
raise FunctionError(
"Encountered error: icdx_search_request may not be None")
helper = ICDXHelper(self.options)
yield StatusMessage(
"Attempting to gather config and setup the AMQP Client")
try:
# Initialise the AmqpFacade, pass in config values
amqp_client = AmqpFacade(
host=helper.get_config_option("icdx_amqp_host"),
port=helper.get_config_option("icdx_amqp_port", True),
virtual_host=helper.get_config_option("icdx_amqp_vhost"),
username=helper.get_config_option("icdx_amqp_username"),
amqp_password=helper.get_config_option(
"icdx_amqp_password"))
yield StatusMessage(
"Config options gathered and AMQP client setup.")
except Exception as e:
yield StatusMessage(
"Encountered error while initialising the AMQP Client; Reason (if any): {0}"
.format(str(e)))
raise FunctionError()
# Prepare request payload
try:
search_payload = json.loads(icdx_search_request)
# Set the hard limit for displaying in the UI
"""
if hard limit is not set, it defaults to 100
if hard limit is set and exceeds 100 that is the new limit
if hard limit is set and does not exceed 100, 100 is the limit
Max limit appears to be 500
"""
search_payload['hard_limit'] = max([
helper.safe_cast(val=i, to_type=int) for i in
[helper.get_config_option("icdx_search_limit", True), 100]
if i is not None
])
# Payload is now a dict -- read 'limit' and replace with app.config value if exceeds it
if search_payload['limit'] > search_payload['hard_limit']:
search_payload['limit'] = search_payload['hard_limit']
# All okay, request has valid JSON and an ID
yield StatusMessage(
"Request payload validated. Sending request with a where attribute of {0} and a filter attribute of {1}"
.format(
search_payload.get('where', None)
or '"No Where Condition"',
search_payload.get('filter', None)
or '"No Filter Condition"'))
except Exception:
raise FunctionError(
"Problem parsing the JSON for search request")
# Make the call to ICDx and get a handle on any results
search_result, status = amqp_client.call(
json.dumps(search_payload))
yield StatusMessage(
"ICDX call complete with status: {}".format(status))
result_set, num_of_results = None, 0
if not search_result:
log.info("Received an empty result set for search query.")
else:
log.info("Received response. Parsing for information")
# Get results dict and number of results
if status == 200:
res = json.loads(search_result)
result_set, num_of_results = res["result"], len(
res["result"])
elif status == 400:
yield StatusMessage(
"Received a 400 (Bad Request). Check the formatting of your ICDx Query"
)
results = {
"inputs": {
"icdx_search_request": search_payload
},
"success": (False if status != 200 else True),
"result_set": result_set,
"num_of_results": num_of_results,
"execution_time": int(time.time() * 1000)
}
yield StatusMessage(
"Finishing. Received results: {}. Number of results: {}".
format(results["success"], num_of_results))
# Produce a FunctionResult with the results
yield FunctionResult(results)
except Exception:
yield FunctionError()
