def test_health_call_success(route, client, rsa_api_response,
gsb_api_request_get, valid_jwt):
app = client.application
gsb_api_request_get.return_value = rsa_api_response(
payload=EXPECTED_RESPONSE_OF_JWKS_ENDPOINT)
response = client.post(route, headers=headers(valid_jwt()))
expected_url = app.config['GSB_API_URL'].format(endpoint='threatLists',
key=get_key())
expected_headers = {
'User-Agent': app.config['CTR_USER_AGENT'],
}
calls = [
call('https://visibility.amp.cisco.com/.well-known/jwks'),
call().raise_for_status(),
call(expected_url, headers=expected_headers),
call().ok.__bool__(),
call('https://visibility.amp.cisco.com/.well-known/jwks'),
call().raise_for_status()
]
gsb_api_request_get.assert_has_calls(calls)
expected_payload = {'data': {'status': 'ok'}}
assert response.status_code == HTTPStatus.OK
assert response.get_json() == expected_payload
def observe_observables():
input_observables = get_observables()
g.bundle = Bundle()
key = get_key()
if key is None:
raise AuthenticationRequiredError
url = current_app.config['AVOTX_URL']
headers = {'User-Agent': current_app.config['CTR_USER_AGENT']}
client = Client(key, url, headers=headers)
limit = current_app.config['CTR_ENTITIES_LIMIT']
prepared_observables = []
for input_observable in input_observables:
prepared_observable = Observable.instance_for(**input_observable)
if prepared_observable is not None:
prepared_observables.append(prepared_observable)
def make_bundle(observable):
return observable.observe(client, limit=limit)
with ThreadPoolExecutor(
max_workers=get_workers(prepared_observables)) as executor:
bundles = executor.map(make_bundle, prepared_observables)
for bundle in bundles:
g.bundle |= bundle
data = g.bundle.json()
return jsonify_data(data)
def test_health_call_success(
route, client, hibp_api_request, rsa_api_response, valid_jwt
):
app = client.application
hibp_api_request.side_effect = (
rsa_api_response(EXPECTED_RESPONSE_OF_JWKS_ENDPOINT),
hibp_api_response(HTTPStatus.OK),
rsa_api_response(EXPECTED_RESPONSE_OF_JWKS_ENDPOINT)
)
response = client.post(route, headers=headers(valid_jwt()))
email = app.config['HIBP_TEST_EMAIL']
expected_url = app.config['HIBP_API_URL'].format(
email=quote(email, safe=''),
truncate='true',
)
expected_headers = {
'User-Agent': app.config['CTR_USER_AGENT'],
'hibp-api-key': get_key()
}
calls = [call('https://visibility.amp.cisco.com/.well-known/jwks'),
call(expected_url, headers=expected_headers),
call('https://visibility.amp.cisco.com/.well-known/jwks')]
hibp_api_request.assert_has_calls(calls)
expected_payload = {'data': {'status': 'ok'}}
assert response.status_code == HTTPStatus.OK
assert response.get_json() == expected_payload
def observe_observables():
observables = get_observables()
g.bundle = Bundle()
key = get_key()
if key is None:
raise AuthenticationRequiredError
url = current_app.config['AVOTX_URL']
headers = {'User-Agent': current_app.config['CTR_USER_AGENT']}
client = Client(key, url, headers=headers)
limit = current_app.config['CTR_ENTITIES_LIMIT']
for observable in observables:
observable = Observable.instance_for(**observable)
if observable is None:
continue
bundle = observable.observe(client, limit=limit)
g.bundle |= bundle
data = g.bundle.json()
return jsonify_data(data)
def observe_observables():
key = get_key()
observables = get_observables()
client = SecurityTrailsClient(current_app.config['API_URL'],
key,
current_app.config['USER_AGENT'],
current_app.config['NUMBER_OF_PAGES'],
current_app.config['GET_ALL_PAGES'])
g.sightings = []
try:
for observable in observables:
mapping = Mapping.for_(observable)
if mapping:
client_data = client.get_data(observable)
for record in client_data:
refer_link = client.refer_link(
current_app.config['UI_URL'], observable
)
sighting = mapping.extract_sighting(record, refer_link)
if sighting:
g.sightings.append(sighting)
except KeyError:
g.errors = [{
'type': 'fatal',
'code': 'key error',
'message': 'The data structure of SecurityTrails '
'has changed. The module is broken.'
}]
return jsonify_result()
def test_health_call_with_external_error_from_gsb_failure(
route, client, gsb_api_request_get, valid_jwt, rsa_api_response):
for code, message, status in [
(
HTTPStatus.BAD_REQUEST,
'API key not valid. Please pass a valid API key.',
'INVALID_ARGUMENT',
),
(
HTTPStatus.TOO_MANY_REQUESTS,
"Quota exceeded for quota group 'LookupAPIGroup' "
"and limit 'Lookup API requests per 100 seconds' "
"of service 'safebrowsing.googleapis.com' "
"for consumer 'project_number:314159265358'.",
'RESOURCE_EXHAUSTED',
),
]:
app = client.application
gsb_api_request_get.side_effect = (
rsa_api_response(payload=EXPECTED_RESPONSE_OF_JWKS_ENDPOINT),
gsb_api_response_error(code, message, status),
rsa_api_response(payload=EXPECTED_RESPONSE_OF_JWKS_ENDPOINT))
response = client.post(route, headers=headers(valid_jwt()))
expected_url = app.config['GSB_API_URL'].format(endpoint='threatLists',
key=get_key())
expected_headers = {
'User-Agent': app.config['CTR_USER_AGENT'],
}
calls = [
call('https://visibility.amp.cisco.com/.well-known/jwks'),
call(expected_url, headers=expected_headers),
call('https://visibility.amp.cisco.com/.well-known/jwks')
]
gsb_api_request_get.assert_has_calls(calls)
gsb_api_request_get.reset_mock()
code = status.lower().replace('_', ' ')
expected_payload = {
'errors': [{
'code': code,
'message': message,
'type': 'fatal',
}]
}
assert response.status_code == HTTPStatus.OK
assert response.get_json() == expected_payload
def health():
key = get_key()
# Use some supported entity just to check that the GTI API key is valid.
observable = current_app.config['GTI_TEST_ENTITY']
_, error = get_events(key, observable)
if error:
return jsonify_errors(error)
else:
return jsonify_data({'status': 'ok'})
def health():
key = get_key()
# Use some breached email just to check that the HIBP API key is valid.
email = current_app.config['HIBP_TEST_EMAIL']
_, error = fetch_breaches(key, email, truncate=True)
if error:
return jsonify_errors(error)
else:
return jsonify_data({'status': 'ok'})
def health():
key = get_key()
client = SecurityTrailsClient(
current_app.config['API_URL'],
key,
current_app.config['USER_AGENT'],
all_pages=current_app.config['GET_ALL_PAGES'])
_ = client.ping()
return jsonify_data({'status': 'ok'})
def health():
key = get_key()
client = FarsightClient(current_app.config['API_URL'], key,
current_app.config['USER_AGENT'])
_ = client.lookup({
'value': 'www.farsightsecurity.com',
'type': 'domain'
},
limit=1)
return jsonify_data({'status': 'ok'})
def health():
key = get_key()
if key is None:
raise AuthenticationRequiredError
url = current_app.config['AVOTX_URL']
headers = {'User-Agent': current_app.config['CTR_USER_AGENT']}
client = Client(key, url, headers=headers)
_ = client.query('/api/v1/user/me')
return jsonify_data({'status': 'ok'})
def observe_observables():
observables, error = get_observables()
if error:
return jsonify_errors(error)
observable_types = current_app.config['GTI_OBSERVABLE_TYPES']
observables = [
observable
for observable in observables
if observable['type'] in observable_types
]
key = get_key()
bundle = Bundle()
for observable in observables:
events, error = get_events_for_observable(key, observable)
if error:
# Make sure not to lose any data processed so far.
return jsonify_errors(error, data=bundle.json())
indicator_by_rule_uuid = {}
for event in events:
sighting = Sighting.map(event)
bundle.add(sighting)
if 'detection' in event:
rule = event['detection']['rule']
indicator = indicator_by_rule_uuid.get(rule['uuid'])
if indicator is None:
indicator = Indicator.map(rule)
indicator_by_rule_uuid[rule['uuid']] = indicator
bundle.add(indicator)
relationship = Relationship.map(sighting, indicator)
bundle.add(relationship)
data = bundle.json()
return jsonify_data(data)
def observe_observables():
observables, error = get_observables()
if error:
return jsonify_errors(error)
emails = [
observable['value'] for observable in observables
if observable['type'] == 'email'
]
key = get_key()
bundle = Bundle()
limit = current_app.config['CTR_ENTITIES_LIMIT']
for email in emails:
breaches, error = fetch_breaches(key, email)
if error:
return jsonify_errors(error, data=bundle.json())
breaches.sort(key=itemgetter('BreachDate'), reverse=True)
breaches = breaches[:limit]
source_uri = current_app.config['HIBP_UI_URL'].format(
email=quote(email, safe=''))
for breach in breaches:
indicator = Indicator.map(breach)
sighting = Sighting.map(breach, email, source_uri)
relationship = Relationship.map(indicator, sighting)
bundle.add(indicator)
bundle.add(sighting)
bundle.add(relationship)
data = bundle.json()
return jsonify_data(data)
def observe_observables():
key = get_key()
observables = get_observables()
client = FarsightClient(current_app.config['API_URL'],
key,
current_app.config['USER_AGENT'])
g.sightings = []
limit = current_app.config['CTR_ENTITIES_LIMIT']
aggr = current_app.config['AGGREGATE']
time_delta = (current_app.config['NUMBER_OF_DAYS_FOR_FARSIGHT_TIME_FILTER']
if aggr else None)
url_template = current_app.config['UI_SEARCH_URL']
try:
for x in observables:
mapping = Mapping.for_(x)
if mapping:
lookup_data = client.lookup(x, time_delta)
if lookup_data:
refer_link = url_template.format(query=x['value'])
g.sightings.extend(
mapping.extract_sightings(
lookup_data, refer_link, limit, aggr
)
)
except KeyError:
g.errors = [{
'type': 'fatal',
'code': 'key error',
'message': 'The data structure of Farsight DNSDB '
'has changed. The module is broken.'
}]
return jsonify_result()
def test_enrich_call_success(any_route,
client,
valid_json,
gsb_api_request_post,
gsb_api_request_get,
valid_jwt,
expected_payload,
rsa_api_response):
app = client.application
response = None
if any_route.startswith(('/deliberate', '/observe')):
gsb_api_request_post.return_value = gsb_api_response_ok()
gsb_api_request_get.return_value = rsa_api_response(
payload=EXPECTED_RESPONSE_OF_JWKS_ENDPOINT
)
response = client.post(any_route,
json=valid_json,
headers=headers(valid_jwt()))
expected_url = app.config['GSB_API_URL'].format(
endpoint='threatMatches:find',
key=get_key()
)
expected_headers = {
'User-Agent': app.config['CTR_USER_AGENT'],
}
gsb_api_request_post.assert_called_once_with(
expected_url,
json={
'client': {
'clientId': app.config['GSB_API_CLIENT_ID'],
'clientVersion': app.config['GSB_API_CLIENT_VERSION'],
},
'threatInfo': {
'threatTypes': list(
app.config['GSB_API_THREAT_TYPES'].keys()
),
'platformTypes': app.config['GSB_API_PLATFORM_TYPES'],
'threatEntryTypes': (
app.config['GSB_API_THREAT_ENTRY_TYPES']
),
'threatEntries': [
{'url': 'cisco.com'},
{'url': 'https://www.google.com/'},
],
},
},
headers=expected_headers,
)
if any_route.startswith('/refer'):
response = client.post(any_route, json=valid_json)
assert response is not None, f'Unknown route: {any_route}.'
assert response.status_code == HTTPStatus.OK
assert response.get_json() == expected_payload
def test_health_call_with_external_error_from_hibp_failure(route,
client,
hibp_api_request,
rsa_api_response,
valid_jwt):
for status_code, error_code, error_message, is_authentic in [
(
HTTPStatus.UNAUTHORIZED,
'access denied',
'Authorization failed: Unauthorized error from 3rd party',
False,
),
(
HTTPStatus.TOO_MANY_REQUESTS,
'too many requests',
'Rate limit is exceeded. Try again in 3 seconds.',
True,
),
(
HTTPStatus.SERVICE_UNAVAILABLE,
'service unavailable',
'Service temporarily unavailable. Please try again later.',
False,
),
(
HTTPStatus.INTERNAL_SERVER_ERROR,
'oops',
'Something went wrong.',
False,
),
]:
app = client.application
hibp_api_request.side_effect = (
rsa_api_response(EXPECTED_RESPONSE_OF_JWKS_ENDPOINT),
hibp_api_response(status_code),
rsa_api_response(EXPECTED_RESPONSE_OF_JWKS_ENDPOINT)
)
response = client.post(route, headers=headers(valid_jwt()))
email = app.config['HIBP_TEST_EMAIL']
expected_url = app.config['HIBP_API_URL'].format(
email=quote(email, safe=''),
truncate='true',
)
expected_headers = {
'User-Agent': app.config['CTR_USER_AGENT'],
'hibp-api-key': get_key()
}
calls = [call('https://visibility.amp.cisco.com/.well-known/jwks'),
call(expected_url, headers=expected_headers),
call('https://visibility.amp.cisco.com/.well-known/jwks')]
hibp_api_request.assert_has_calls(calls)
hibp_api_request.reset_mock()
expected_payload = {
'errors': [
{
'code': error_code,
'message': error_message,
'type': 'fatal',
}
]
}
assert response.status_code == HTTPStatus.OK
assert response.get_json() == expected_payload
def test_enrich_call_with_external_error_from_gsb_failure(gsb_api_route,
client,
valid_json,
gsb_api_request_post,
gsb_api_request_get,
valid_jwt,
rsa_api_response):
for code, message, status in [
(
HTTPStatus.BAD_REQUEST,
'API key not valid. Please pass a valid API key.',
'INVALID_ARGUMENT',
),
(
HTTPStatus.TOO_MANY_REQUESTS,
"Quota exceeded for quota group 'LookupAPIGroup' "
"and limit 'Lookup API requests per day' "
"of service 'safebrowsing.googleapis.com' "
"for consumer 'project_number:314159265358'.",
'RESOURCE_EXHAUSTED',
),
]:
app = client.application
gsb_api_request_post.return_value = gsb_api_response_error(code,
message,
status)
gsb_api_request_get.return_value = rsa_api_response(
payload=EXPECTED_RESPONSE_OF_JWKS_ENDPOINT)
response = client.post(gsb_api_route,
json=valid_json,
headers=headers(valid_jwt()))
expected_url = app.config['GSB_API_URL'].format(
endpoint='threatMatches:find',
key=get_key()
)
expected_headers = {
'User-Agent': app.config['CTR_USER_AGENT'],
}
gsb_api_request_post.assert_called_once_with(
expected_url,
json={
'client': {
'clientId': app.config['GSB_API_CLIENT_ID'],
'clientVersion': app.config['GSB_API_CLIENT_VERSION'],
},
'threatInfo': {
'threatTypes': list(
app.config['GSB_API_THREAT_TYPES'].keys()
),
'platformTypes': app.config['GSB_API_PLATFORM_TYPES'],
'threatEntryTypes': (
app.config['GSB_API_THREAT_ENTRY_TYPES']
),
'threatEntries': [
{'url': 'cisco.com'},
{'url': 'https://www.google.com/'},
],
},
},
headers=expected_headers,
)
gsb_api_request_post.reset_mock()
code = status.lower().replace('_', ' ')
expected_payload = {
'errors': [
{
'code': code,
'message': message,
'type': 'fatal',
}
]
}
assert response.status_code == HTTPStatus.OK
assert response.get_json() == expected_payload