def test_build_iterator_with_version_6_2_0(mocker):
"""
Given
- server version 6.2.0
When
- Running build_iterator method.
Then
- Ensure that the no_update value is True
- Request is called without headers "If-None-Match" and "If-Modified-Since"
"""
feed_name = 'mock_feed_name'
mocker.patch.object(demisto, 'debug')
mocker.patch('CommonServerPython.get_demisto_version',
return_value={"version": "6.2.0"})
with requests_mock.Mocker() as m:
m.get('https://api.github.com/meta', status_code=304)
client = Client(url='https://api.github.com/meta', headers={})
result, no_update = client.build_iterator(
feed={'url': 'https://api.github.com/meta'}, feed_name=feed_name)
assert not result
assert no_update
assert 'If-None-Match' not in client.headers
assert 'If-Modified-Since' not in client.headers
def test_build_iterator_not_modified_header(mocker):
"""
Given
- Last run has etag and last_modified in it
- response with status code 304(Not Modified)
When
- Running build_iterator method.
Then
- Ensure that the no_update value is True
- Request is called with the headers "If-None-Match" and "If-Modified-Since"
"""
feed_name = 'mock_feed_name'
mocker.patch.object(demisto, 'debug')
mocker.patch.object(demisto, 'getLastRun', return_value={feed_name: {'etag': '0', 'last_modified': 'now'}})
with requests_mock.Mocker() as m:
m.get('https://api.github.com/meta', status_code=304)
client = Client(
url='https://api.github.com/meta'
)
result, no_update = client.build_iterator(feed={'url': 'https://api.github.com/meta'}, feed_name=feed_name)
assert not result
assert no_update
assert demisto.debug.call_args[0][0] == 'No new indicators fetched, ' \
'createIndicators will be executed with noUpdate=True.'
assert 'If-None-Match' in client.headers
assert 'If-Modified-Since' in client.headers
def test_json_feed_with_config():
with open('test_data/amazon_ip_ranges.json') as ip_ranges_json:
ip_ranges = json.load(ip_ranges_json)
feed_name_to_config = {
'AMAZON': {
'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json',
'extractor': "prefixes[?service=='AMAZON']",
'indicator': 'ip_prefix',
'indicator_type': FeedIndicatorType.CIDR,
'fields': ['region', 'service']
}
}
with requests_mock.Mocker() as m:
m.get('https://ip-ranges.amazonaws.com/ip-ranges.json', json=ip_ranges)
client = Client(url='https://ip-ranges.amazonaws.com/ip-ranges.json',
credentials={
'username': '******',
'password': '******'
},
feed_name_to_config=feed_name_to_config,
insecure=True)
indicators, _ = fetch_indicators_command(client=client,
indicator_type='CIDR',
feedTags=['test'],
auto_detect=False)
assert len(
jmespath.search(expression="[].rawJSON.service",
data=indicators)) == 1117
def test_post_of_indicators_with_no_json_object():
feed_name_to_config = {
'Github': {
'url': 'https://api.github.com/meta',
'extractor': "hooks",
'indicator': None
}
}
with requests_mock.Mocker() as m:
matcher = m.post('https://api.github.com/meta',
json=json.loads(FLAT_LIST_OF_INDICATORS),
request_headers={
'content-type':
'application/x-www-form-urlencoded'
})
client = Client(url='https://api.github.com/meta',
feed_name_to_config=feed_name_to_config,
insecure=True,
data='test=1')
indicators, _ = fetch_indicators_command(client=client,
indicator_type=None,
feedTags=['test'],
auto_detect=True)
assert matcher.last_request.text == 'test=1'
assert len(indicators) == 3
assert indicators[0].get('value') == '1.1.1.1'
assert indicators[0].get('type') == 'IP'
assert indicators[1].get('rawJSON') == {'indicator': '2.2.2.2'}
def test_list_of_indicators_with_no_json_object():
feed_name_to_config = {
'Github': {
'url': 'https://api.github.com/meta',
'extractor': "hooks",
'indicator': None
}
}
with requests_mock.Mocker() as m:
m.get('https://api.github.com/meta',
json=json.loads(FLAT_LIST_OF_INDICATORS))
client = Client(url='https://api.github.com/meta',
feed_name_to_config=feed_name_to_config,
insecure=True)
indicators, _ = fetch_indicators_command(client=client,
indicator_type=None,
feedTags=['test'],
auto_detect=True)
assert len(indicators) == 3
assert indicators[0].get('value') == '1.1.1.1'
assert indicators[0].get('type') == 'IP'
assert indicators[1].get('rawJSON') == {'indicator': '2.2.2.2'}
def test_build_iterator_change_extractor():
"""
Given:
- feed configuration with wrong feed extractor
When:
- jmespath search for the given expression to be queried
Then:
- The json does not contain the given expression and TypeError raised
"""
url_page1 = 'https://api.intelgraph.idefense.com/rest/threatindicator/v0/domain?page_size=200&page=1'
status_code = 200
client = Client(**PARAMS)
json_data = {
"ccc": [{
"display_text": "3.3.3.3"
}],
"total_size": 1,
"page": 1,
"page_size": 200,
"more": False
}
with requests_mock.Mocker() as m:
m.get(url_page1, status_code=status_code, json=json_data)
with pytest.raises(TypeError) as e:
custom_build_iterator(client,
PARAMS['feed_name_to_config']['Domain'], 0)
if not e:
assert False
def test_parse_headers():
headers = """Authorization: Bearer X
User-Agent:test
Stam : Ba
"""
res = Client.parse_headers(headers)
assert res['Authorization'] == 'Bearer X'
assert res['User-Agent'] == 'test'
assert res['Stam'] == 'Ba'
assert len(res) == 3
def test_build_iterator_paging():
"""
Given:
- feed configuration with no filters to fetch with
When:
- fetch indicators using jsonFeedApiModule with an api that his response method is pagination
Then:
- assert that results returned from 2 different pages
"""
url_page1 = 'https://api.intelgraph.idefense.com/rest/threatindicator/v0/ip?page_size=200&page=1'
status_code = 200
json_data_page1 = {
"results": [{
"display_text": "2.2.2.2"
}, {
"display_text": "1.1.1.1"
}],
"total_size": 4,
"page": 1,
"page_size": 200,
"more": True
}
url_page2 = 'https://api.intelgraph.idefense.com/rest/threatindicator/v0/ip?page_size=200&page=2'
json_data_page2 = {
"results": [{
"display_text": "3.3.3.3"
}, {
"display_text": "4.4.4.4"
}],
"total_size": 4,
"page": 2,
"page_size": 200,
"more": False
}
with requests_mock.Mocker() as m:
m.get(url_page1, status_code=status_code, json=json_data_page1)
m.get(url_page2, status_code=status_code, json=json_data_page2)
client = Client(**PARAMS)
results = custom_build_iterator(client,
PARAMS['feed_name_to_config']['IP'], 0)
assert len(results) == 4
def test_json_feed_no_config():
with open('test_data/amazon_ip_ranges.json') as ip_ranges_json:
ip_ranges = json.load(ip_ranges_json)
with requests_mock.Mocker() as m:
m.get('https://ip-ranges.amazonaws.com/ip-ranges.json', json=ip_ranges)
client = Client(url='https://ip-ranges.amazonaws.com/ip-ranges.json',
credentials={
'username': '******',
'password': '******'
},
extractor="prefixes[?service=='AMAZON']",
indicator='ip_prefix',
fields=['region', 'service'],
insecure=True)
indicators = fetch_indicators_command(client=client,
indicator_type='CIDR')
assert len(
jmespath.search(expression="[].rawJSON.service",
data=indicators)) == 1117