def test_create_values_for_returned_dict(self):
from EDL import create_values_for_returned_dict, EDL_VALUES_KEY, RequestArguments
with open('EDL_test/TestHelperFunctions/demisto_url_iocs.json',
'r') as iocs_json_f:
iocs_json = json.loads(iocs_json_f.read())
# strips port numbers
request_args = RequestArguments(query='',
drop_invalids=True,
url_port_stripping=True)
returned_dict, num_of_indicators = create_values_for_returned_dict(
iocs_json, request_args)
returned_output = returned_dict.get(EDL_VALUES_KEY)
assert returned_output == "1.2.3.4/wget\nwww.demisto.com/cool"
assert num_of_indicators == 2
# should ignore indicators with port numbers
request_args = RequestArguments(query='',
drop_invalids=True,
url_port_stripping=False)
returned_dict, num_of_indicators = create_values_for_returned_dict(
iocs_json, request_args)
returned_output = returned_dict.get(EDL_VALUES_KEY)
assert returned_output == 'www.demisto.com/cool'
assert num_of_indicators == 1
# should not ignore indicators with '*' in them
request_args = RequestArguments(query='',
drop_invalids=False,
url_port_stripping=False)
returned_dict, num_of_indicators = create_values_for_returned_dict(
iocs_json, request_args)
returned_output = returned_dict.get(EDL_VALUES_KEY)
assert returned_output == 'www.demisto.com/cool\nwww.demisto.com/*'
assert num_of_indicators == 2
def test_create_proxysg_out_format(self):
"""
Given:
- RequestArguments
- Indicator info
When:
- request proxysg outbound format
Then:
- assert files_by_category as 3 keys
- assert the result
"""
from EDL import create_proxysg_out_format, RequestArguments, create_proxysg_all_category_out_format
files_by_category = {}
with open('test_data/demisto_url_iocs.json', 'r') as iocs_json_f:
iocs_json = json.loads(iocs_json_f.read())
request_args = RequestArguments(query='', drop_invalids=True, url_port_stripping=True,
url_protocol_stripping=True)
for ioc in iocs_json:
files_by_category = create_proxysg_out_format(ioc, files_by_category, request_args)
assert len(files_by_category) == 3
result_file = tempfile.TemporaryFile(mode='w+t')
result_file = create_proxysg_all_category_out_format(result_file, files_by_category)
result_file.seek(0)
assert result_file.read() == 'define category category1\n1.2.3.4/wget\nend\ndefine category category2\n' \
'www.demisto.com/cool\nend\ndefine category bc_category\nwww.demisto.com/*cool\n' \
'end'
def test_create_mwg_out_format(self):
"""
Given:
- RequestArguments
- Indicator info
When:
- request mwg outbound format
Then:
- assert the result
"""
from EDL import create_mwg_out_format, RequestArguments
with open('test_data/demisto_url_iocs.json', 'r') as iocs_json_f:
iocs_json = json.loads(iocs_json_f.read())
request_args = RequestArguments(query='', drop_invalids=True, url_port_stripping=True,
url_protocol_stripping=True)
returned_output = ''
not_first_call = False
for ioc in iocs_json:
returned_output += (
create_mwg_out_format(ioc, request_args, not_first_call))
not_first_call = True
assert returned_output == 'type=string\n"1.2.3.4/wget" "AutoFocus Feed"\n"www.demisto.com/cool" ' \
'"AutoFocus V2,VirusTotal,Alien Vault OTX TAXII Feed"\n"www.demisto.com/*cool" ' \
'"AutoFocus V2,VirusTotal,Alien Vault OTX TAXII Feed"'
def test_create_json_out_format(self):
"""
Given:
- RequestArguments
- Indicator info
When:
- request json outbound format
Then:
- assert the result
"""
from EDL import create_json_out_format, RequestArguments
returned_output = []
with open('test_data/demisto_url_iocs.json', 'r') as iocs_json_f:
iocs_json = json.loads(iocs_json_f.read())
# strips port numbers
request_args = RequestArguments(query='', drop_invalids=True, url_port_stripping=True)
for ioc in iocs_json:
returned_output.append(create_json_out_format(['value', 'indicator_type'], ioc, request_args, True))
assert returned_output == [', {"value": "1.2.3.4/wget", "indicator_type": "URL"}',
', {"value": "https://www.demisto.com/cool", "indicator_type": "URL"}',
', {"value": "https://www.demisto.com/*cool", "indicator_type": "URL"}']
returned_output = ''
not_first_call = False
for ioc in iocs_json:
returned_output += (create_json_out_format([], ioc, request_args, not_first_call))
not_first_call = True
returned_output += ']'
assert json.loads(returned_output) == iocs_json
def test_create_csv_out_format(self):
"""
Given:
- RequestArguments
- Indicator info With CustomFields
When:
- request csv outbound format
Then:
- assert the result
"""
from EDL import create_csv_out_format, RequestArguments
with open('test_data/demisto_url_iocs.json', 'r') as iocs_json_f:
iocs_json = json.loads(iocs_json_f.read())
request_args = RequestArguments(query='',
drop_invalids=True,
url_port_stripping=True,
url_protocol_stripping=True)
returned_output = ''
not_first_call = False
for ioc in iocs_json:
returned_output += (create_csv_out_format(
not_first_call, ['value', 'indicator_type', 'test'], ioc,
request_args))
not_first_call = True
assert returned_output == 'name,type\n"1.2.3.4/wget","URL","test"\n"www.demisto.com/cool","URL","test"\n' \
'"www.demisto.com/*cool","URL","None"'
def test_get_edl_ioc_values_1(self, mocker):
"""Test on_demand"""
from EDL import get_edl_ioc_values, RequestArguments
with open('EDL_test/TestHelperFunctions/iocs_cache_values_text.json',
'r') as iocs_text_values_f:
iocs_text_dict = json.loads(iocs_text_values_f.read())
integration_context = {"last_output": iocs_text_dict}
request_args = RequestArguments(query='', limit=50, offset=0)
ioc_list = get_edl_ioc_values(on_demand=True,
request_args=request_args,
edl_cache=integration_context)
for ioc_row in ioc_list:
assert ioc_row in iocs_text_dict
class TestRequestArguments:
from EDL import RequestArguments
context_json = {
RequestArguments.CTX_QUERY_KEY: "query",
RequestArguments.CTX_LIMIT_KEY: 10,
RequestArguments.CTX_OFFSET_KEY: 1,
RequestArguments.CTX_INVALIDS_KEY: True,
RequestArguments.CTX_PORT_STRIP_KEY: True,
RequestArguments.CTX_COLLAPSE_IPS_KEY: "collapse",
RequestArguments.CTX_EMPTY_EDL_COMMENT_KEY: True
}
request_args = RequestArguments(
query=context_json[RequestArguments.CTX_QUERY_KEY],
limit=context_json[RequestArguments.CTX_LIMIT_KEY],
offset=context_json[RequestArguments.CTX_OFFSET_KEY],
url_port_stripping=context_json[RequestArguments.CTX_PORT_STRIP_KEY],
drop_invalids=context_json[RequestArguments.CTX_PORT_STRIP_KEY],
collapse_ips=context_json[RequestArguments.CTX_COLLAPSE_IPS_KEY])
def test_to_context_json(self):
"""
Test to_context_json transforms the class to the expected context_json
Given:
- request_args
When:
- calling to_context_json()
Then:
- creates a dict in the expected format
"""
assert self.request_args.to_context_json() == self.context_json
def test_from_context_json(self):
"""
Test from_context_json creates an instance of the class with all the values
Given:
- context json with data
When:
- calling from_context_json()
Then:
- creates an instance of RequestArguments with the proper values
"""
actual_request_args_dict = self.RequestArguments.from_context_json(self.context_json).__dict__
expected_request_args_dict = self.request_args.__dict__
for key, val in actual_request_args_dict.items():
assert expected_request_args_dict[key] == val
def test_format_indicators__filters(self):
from EDL import format_indicators, RequestArguments
iocs = [
{'value': '2603:1006:1400::/40', 'indicator_type': 'IPv6'},
{'value': '2002:ac8:b8d:0:0:0:0:0', 'indicator_type': 'IPv6'},
{'value': 'demisto.com:369/rest/of/path', 'indicator_type': 'URL'},
{'value': 'panw.com/path', 'indicator_type': 'URL'},
{'value': '*.domain.com', 'indicator_type': 'URL'},
]
request_args = RequestArguments(query='', drop_invalids=True, url_port_stripping=True)
returned_output = format_indicators(iocs, request_args)
assert '2603:1006:1400::/40' in returned_output
assert '2002:ac8:b8d:0:0:0:0:0' in returned_output
assert 'demisto.com/rest/of/path' in returned_output # port stripping
assert 'panw.com/path' in returned_output
assert '*.domain.com' in returned_output
assert 'domain.com' in returned_output # PAN-OS URLs
assert len(returned_output) == 6
def test_create_values_for_returned_dict__filters(self):
from EDL import create_values_for_returned_dict, EDL_VALUES_KEY, RequestArguments
iocs = [
{
'value': '2603:1006:1400::/40',
'indicator_type': 'IPv6'
},
{
'value': '2002:ac8:b8d:0:0:0:0:0',
'indicator_type': 'IPv6'
},
{
'value': 'demisto.com:369/rest/of/path',
'indicator_type': 'URL'
},
{
'value': 'panw.com/path',
'indicator_type': 'URL'
},
{
'value': '*.domain.com',
'indicator_type': 'URL'
},
]
request_args = RequestArguments(query='',
drop_invalids=True,
url_port_stripping=True)
returned_dict, num_of_indicators = create_values_for_returned_dict(
iocs, request_args)
returned_output = returned_dict.get(EDL_VALUES_KEY, '').split('\n')
assert '2603:1006:1400::/40' in returned_output
assert '2002:ac8:b8d:0:0:0:0:0' in returned_output
assert 'demisto.com/rest/of/path' in returned_output # port stripping
assert 'panw.com/path' in returned_output
assert '*.domain.com' in returned_output
assert 'domain.com' in returned_output # PAN-OS URLs
assert num_of_indicators == 6
def test_format_indicators(self):
from EDL import format_indicators, RequestArguments, COLLAPSE_TO_RANGES
with open('EDL_test/TestHelperFunctions/demisto_url_iocs.json', 'r') as iocs_json_f:
iocs_json = json.loads(iocs_json_f.read())
# strips port numbers
request_args = RequestArguments(query='', drop_invalids=True, url_port_stripping=True)
returned_output = format_indicators(iocs_json, request_args)
assert returned_output == {'1.2.3.4/wget', 'www.demisto.com/cool'}
# should ignore indicators with port numbers
request_args = RequestArguments(query='', drop_invalids=True, url_port_stripping=False)
returned_output = format_indicators(iocs_json, request_args)
assert returned_output == {'www.demisto.com/cool'}
# should not ignore indicators with '*' in them
request_args = RequestArguments(query='', drop_invalids=False, url_port_stripping=False)
returned_output = format_indicators(iocs_json, request_args)
assert returned_output == {'www.demisto.com/cool', 'www.demisto.com/*'}
with open('EDL_test/TestHelperFunctions/simplified_demisto_ip_iocs.json', 'r') as iocs_json_f:
iocs_json = json.loads(iocs_json_f.read())
# collapse IPs to CIDRs
request_args = RequestArguments(query='', collapse_ips=True)
returned_output = format_indicators(iocs_json, request_args)
assert returned_output == {'1.1.1.2/31', '1.1.1.1'}
# collapse IPs to ranges
request_args = RequestArguments(query='', collapse_ips=COLLAPSE_TO_RANGES)
returned_output = format_indicators(iocs_json, request_args)
assert returned_output == {'1.1.1.1-1.1.1.3'}
with open('EDL_test/TestHelperFunctions/simplified_demisto_ip_v6_iocs.json', 'r') as iocs_json_f:
iocs_json = json.loads(iocs_json_f.read())
# collapse IPv6
request_args = RequestArguments(query='', collapse_ips=True)
returned_output = format_indicators(iocs_json, request_args)
assert returned_output == {'1:1:1:1:1:1:1:2/127', '1:1:1:1:1:1:1:1'}
class TestRequestArguments:
from EDL import RequestArguments
context_json = {
RequestArguments.CTX_QUERY_KEY: "query",
RequestArguments.CTX_LIMIT_KEY: 10,
RequestArguments.CTX_OFFSET_KEY: 1,
RequestArguments.CTX_INVALIDS_KEY: True,
RequestArguments.CTX_PORT_STRIP_KEY: True,
RequestArguments.CTX_COLLAPSE_IPS_KEY: "collapse",
RequestArguments.CTX_EMPTY_EDL_COMMENT_KEY: True,
RequestArguments.CTX_OUT_FORMAT: 'text',
RequestArguments.CTX_MWG_TYPE: 'string',
RequestArguments.CTX_CATEGORY_DEFAULT: 'bc_category',
RequestArguments.CTX_CATEGORY_ATTRIBUTE: [],
RequestArguments.CTX_FIELDS_TO_PRESENT: 'name,type',
RequestArguments.CTX_CSV_TEXT: False,
RequestArguments.CTX_PROTOCOL_STRIP_KEY: False,
RequestArguments.CTX_URL_TRUNCATE_KEY: False,
RequestArguments.CTX_NO_TLD: True,
RequestArguments.CTX_MAXIMUM_CIDR: 8
}
request_args = RequestArguments(
query=context_json[RequestArguments.CTX_QUERY_KEY],
limit=context_json[RequestArguments.CTX_LIMIT_KEY],
offset=context_json[RequestArguments.CTX_OFFSET_KEY],
url_port_stripping=context_json[RequestArguments.CTX_PORT_STRIP_KEY],
drop_invalids=context_json[RequestArguments.CTX_PORT_STRIP_KEY],
collapse_ips=context_json[RequestArguments.CTX_COLLAPSE_IPS_KEY],
add_comment_if_empty=context_json[RequestArguments.CTX_EMPTY_EDL_COMMENT_KEY],
out_format=context_json[RequestArguments.CTX_OUT_FORMAT],
mwg_type=context_json[RequestArguments.CTX_MWG_TYPE],
category_default=context_json[RequestArguments.CTX_CATEGORY_DEFAULT],
category_attribute='',
fields_to_present=context_json[RequestArguments.CTX_FIELDS_TO_PRESENT],
csv_text=context_json[RequestArguments.CTX_CSV_TEXT],
url_protocol_stripping=context_json[RequestArguments.CTX_PROTOCOL_STRIP_KEY],
url_truncate=context_json[RequestArguments.CTX_URL_TRUNCATE_KEY],
no_wildcard_tld=context_json[RequestArguments.CTX_NO_TLD],
maximum_cidr_size=context_json[RequestArguments.CTX_MAXIMUM_CIDR],
)
def test_to_context_json(self):
"""
Test to_context_json transforms the class to the expected context_json
Given:
- request_args
When:
- calling to_context_json()
Then:
- creates a dict in the expected format
"""
assert self.request_args.to_context_json() == self.context_json
def test_from_context_json(self):
"""
Test from_context_json creates an instance of the class with all the values
Given:
- context json with data
When:
- calling from_context_json()
Then:
- creates an instance of RequestArguments with the proper values
"""
actual_request_args_dict = self.RequestArguments.from_context_json(self.context_json).__dict__
expected_request_args_dict = self.request_args.__dict__
for key, val in actual_request_args_dict.items():
assert expected_request_args_dict[key] == val