def deliberate_observables():
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
start_time = datetime.utcnow()
token = get_jwt()
g.verdicts = []
for observable in observables:
output = get_abuse_ipdb_outputs(observable, token)
if output:
g.verdicts.append(extract_verdicts(output, start_time))
relay_output = {}
if g.verdicts:
relay_output['verdicts'] = format_docs(g.verdicts)
return jsonify_data(relay_output)
def refer_observables():
observables = filter_observables(get_observables())
data = []
if not observables:
return jsonify_data(data)
for observable in observables:
type_ = current_app.config['AUTOFOCUS_OBSERVABLE_TYPES'][
observable['type']]
data.append({
'id':
'ref-palo-alto-autofocus-search-{type}-{value}'.format(
**observable),
'title':
f'Search for this {type_}',
'description':
f'Look up this {type_} on {SOURCE_NAME}',
'url':
Entity.get_source_uri(observable['type'], observable['value']),
'categories': ['Search', SOURCE_NAME]
})
return jsonify_data(data)
def refer_observables():
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
api_key = get_jwt()
client = URLScanClient(
base_url=current_app.config['URL_SCAN_API_URL'],
api_key=api_key,
user_agent=current_app.config['USER_AGENT'],
observable_types=current_app.config['URL_SCAN_OBSERVABLE_TYPES'])
g.references = []
for observable in observables:
output = client.get_search_data(observable)
if output and output['results']:
g.references.extend(extract_references(observable))
return jsonify_data(g.references)
def observe_observables():
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
time_now = datetime.utcnow()
# get dict with actual abuseipdb categories with titles and descriptions
categories = get_categories()
token = get_jwt()
g.verdicts = []
g.judgements = []
g.indicators = []
g.sightings = []
g.relationships = []
for observable in observables:
output = get_abuse_ipdb_outputs(observable, token)
if output:
g.verdicts.append(extract_verdicts(output, time_now))
output['categories_ids'] = []
output['relations'] = {}
reports = output['data']['reports']
reports.sort(key=lambda x: x['reportedAt'], reverse=True)
if len(reports) >= current_app.config['CTR_ENTITIES_LIMIT']:
reports = reports[:current_app.config['CTR_ENTITIES_LIMIT']]
for report in reports:
g.judgements.append(
extract_judgement(report, output, categories))
g.indicators.extend(
extract_indicators(report, output, categories))
g.sightings.append(extract_sightings(report, output))
g.relationships.extend(extract_relationships(output))
relay_output = {}
if g.judgements:
relay_output['judgements'] = format_docs(g.judgements)
if g.verdicts:
relay_output['verdicts'] = format_docs(g.verdicts)
if g.sightings:
relay_output['sightings'] = format_docs(g.sightings)
if g.indicators:
relay_output['indicators'] = format_docs(g.indicators)
if g.relationships:
relay_output['relationships'] = format_docs(g.relationships)
return jsonify_data(relay_output)
def observe_observables():
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
token = get_jwt()
g.sightings = []
g.indicators = []
g.errors = []
for observable in observables:
output, spycloud_catalogs = validate_spycloud_outputs(
observable, token)
if output:
breaches = output['results']
breaches.sort(
key=lambda x: x['spycloud_publish_date'], reverse=True)
unique_catalog_id_set = set()
if len(breaches) >= current_app.config['CTR_ENTITIES_LIMIT']:
breaches = breaches[:current_app.config['CTR_ENTITIES_LIMIT']]
for breach in breaches:
g.sightings.append(
extract_sightings(breach, output, spycloud_catalogs))
catalog_id = breach['source_id']
if catalog_id not in unique_catalog_id_set:
if spycloud_catalogs[catalog_id]:
g.indicators.append(
extract_indicators(spycloud_catalogs[catalog_id]))
unique_catalog_id_set.add(catalog_id)
else:
error_message = current_app.config[
'CATALOG_ERROR_TEMPLATE'].format(
catalog_id=catalog_id)
g.errors.append(get_catalog_error(error_message))
relay_output = {}
if g.sightings:
relay_output['sightings'] = format_docs(g.sightings)
if g.indicators:
relay_output['indicators'] = format_docs(g.indicators)
return jsonify_data(relay_output, g.errors)
def respond_observables():
relay_input = get_json(ObservableSchema(many=True))
observables = get_scan_observables(relay_input)
if not observables:
return jsonify_data([])
g.actions = []
for observable in observables:
g.actions.append(extract_action(observable))
return jsonify_data(g.actions)
def observe_observables():
_ = get_jwt()
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
g.bundle = Bundle()
for observable in observables:
g.bundle.merge(observe(observable))
return jsonify_data(g.bundle.json())
def refer_observables():
relay_input = get_observables()
observables = filter_observables(relay_input)
if not observables:
return jsonify_data([])
_ = get_credentials()
relay_output = []
for observable in observables:
relay_output.append(get_search_pivots(observable))
return jsonify_data(relay_output)
def respond_observables():
relay_input = get_observables()
observables = filter_observables(relay_input)
api_client = ReferenceDataClient(get_credentials(), current_app.config)
params = {
'filter': 'element_type = "IP"',
'fields': 'number_of_elements, name'
}
result = api_client.get_reference_sets(params)
actions = []
securex_set_absent = True
for observable in observables:
for set_ in result:
if set_['name'] == current_app.config['SECUREX_SET_NAME']:
securex_set_absent = False
if set_['number_of_elements'] > 0:
data = api_client.get_reference_set_data(set_['name'])
if observable in str(data):
actions.append(remove_from(observable, set_['name']))
continue
else:
actions.append(add_to(observable, set_['name']))
if securex_set_absent:
actions.append(add_and_create(observable))
securex_set_absent = True
actions.sort(key=lambda item: item['title'])
return jsonify_data(actions)
def health():
key = get_jwt().get('key', '')
client = C1fAppClient(key)
_ = client.get_c1fapp_response('test.com')
return jsonify_data({'status': 'ok'})
def refer_observables():
observables = get_observables()
url_template = current_app.config['UI_SEARCH_URL']
observable_types_map = current_app.config['FARSIGHT_OBSERVABLES']
data = []
for observable in observables:
type_ = observable_types_map.get(observable['type'])
if type_:
data.append(
{
'id': (
'ref-farsight-dnsdb-search-{type}-{value}'.format(
**observable
)
),
'title': f'Search for this {type_}',
'description': f'Lookup this {type_} on Farsight DNSDB',
'url': url_template.format(query=observable['value']),
'categories': ['Search', 'Farsight DNSDB'],
}
)
return jsonify_data(data)
def observe_observables():
observables = get_observables()
g.bundle = Bundle()
key = get_key()
if key is None:
raise AuthenticationRequiredError
url = current_app.config['AVOTX_URL']
headers = {'User-Agent': current_app.config['CTR_USER_AGENT']}
client = Client(key, url, headers=headers)
limit = current_app.config['CTR_ENTITIES_LIMIT']
for observable in observables:
observable = Observable.instance_for(**observable)
if observable is None:
continue
bundle = observable.observe(client, limit=limit)
g.bundle |= bundle
data = g.bundle.json()
return jsonify_data(data)
def health():
credentials = get_jwt()
client = AkamaiClient(credentials, current_app.config['USER_AGENT'])
_ = client.network_lists(include_elements=False)
return jsonify_data({'status': 'ok'})
def refer_observables():
observables, error = get_observables()
if error:
return jsonify_errors(error)
observable_types = current_app.config['GTI_OBSERVABLE_TYPES']
def type_of(observable):
return observable_types[observable['type']]
url_template = current_app.config['GTI_UI_SEARCH_URL']
data = [
{
'id': 'ref-gti-search-{type}-{value}'.format(**observable),
'title': f'Search for this {type_of(observable)}',
'description': (
f'Lookup this {type_of(observable)} '
'on Gigamon ThreatINSIGHT'
),
'url': url_template.format(query=observable['value']),
'categories': ['Search', 'Gigamon ThreatINSIGHT'],
}
for observable in observables
if observable['type'] in observable_types
]
return jsonify_data(data)
def observe_observables():
input_observables = get_observables()
g.bundle = Bundle()
key = get_key()
if key is None:
raise AuthenticationRequiredError
url = current_app.config['AVOTX_URL']
headers = {'User-Agent': current_app.config['CTR_USER_AGENT']}
client = Client(key, url, headers=headers)
limit = current_app.config['CTR_ENTITIES_LIMIT']
prepared_observables = []
for input_observable in input_observables:
prepared_observable = Observable.instance_for(**input_observable)
if prepared_observable is not None:
prepared_observables.append(prepared_observable)
def make_bundle(observable):
return observable.observe(client, limit=limit)
with ThreadPoolExecutor(
max_workers=get_workers(prepared_observables)) as executor:
bundles = executor.map(make_bundle, prepared_observables)
for bundle in bundles:
g.bundle |= bundle
data = g.bundle.json()
return jsonify_data(data)
def refer_observables():
observables = get_observables()
ui_url = current_app.config['UI_URL']
data = []
for observable in observables:
type_ = ST_OBSERVABLE_TYPES.get(observable['type'])
if type_:
data.append(
{
'id': (
'ref-securitytrails-search-{type}-{value}'.format(
**observable)
),
'title': f'Search for this {type_}',
'description': (
f'Lookup this {type_} on SecurityTrails'
),
'url': SecurityTrailsClient.refer_link(ui_url, observable),
'categories': ['Search', 'SecurityTrails'],
}
)
return jsonify_data(data)
def health():
ApiClient.health_test_observable = current_app.config['HEALTH_OBSERVABLE']
client = ApiClient(api_key=get_api_key(),
base_url=current_app.config['AUTOFOCUS_API_URL'],
user_agent=current_app.config['USER_AGENT'])
client.get_tic_indicator_data(current_app.config['HEALTH_OBSERVABLE'])
return jsonify_data({'status': 'ok'})
def health():
credentials = get_credentials()
client = XForceClient(current_app.config['API_URL'], credentials,
current_app.config['USER_AGENT'])
_ = client.usage()
return jsonify_data({'status': 'ok'})
def health():
credentials = get_jwt()
url = 'https://api.securitycenter.windows.com/api/exposureScore'
client = Client(credentials)
client.open_session()
client.call_api(url)
client.close_session()
return jsonify_data({'status': 'ok'})
def health():
api_key = get_jwt()
client = URLScanClient(
base_url=current_app.config['URL_SCAN_API_URL'],
api_key=api_key,
user_agent=current_app.config['USER_AGENT'],
observable_types=current_app.config['URL_SCAN_OBSERVABLE_TYPES'])
client.get_search_data(
current_app.config['URL_SCAN_HEALTH_CHECK_OBSERVABLE'])
return jsonify_data({'status': 'ok'})
def refer_observables():
observables = get_observables()
data = []
for observable in observables:
value = observable['value']
type_ = observable['type'].lower()
if type_ == 'ip':
data.append(get_search_pivot(value))
return jsonify_data(data)
def observe_observables():
entities = get_entities()
if not entities:
return jsonify_data()
g.verdicts = []
g.judgements = []
for entity in entities:
if entity:
judgement = entity.get_judgement()
verdict = entity.get_verdict()
verdict['judgement_id'] = judgement['id']
g.judgements.append(judgement)
g.verdicts.append(verdict)
return jsonify_data()
def health():
key = get_key()
# Use some breached email just to check that the HIBP API key is valid.
email = current_app.config['HIBP_TEST_EMAIL']
_, error = fetch_breaches(key, email, truncate=True)
if error:
return jsonify_errors(error)
else:
return jsonify_data({'status': 'ok'})
def deliberate_observables():
api_key = get_jwt() # Let's get the third party API Key
data = {
} # Let's create a data directory to be sent back to Threat Response
g.verdicts = [
] # Let's create a list into which we will store valid verdicts data results for every observables
g.judgements = [
] # Let's create a list into which we will store valid judgements data results for every observables
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
observables = build_input_api(observables)
for observable in observables:
o_value = observable['value']
o_type = observable['type'].lower()
# print single observable for which to send a reputation query to the third party
print(green(o_value, bold=True))
disposition = call_api(o_value, api_key)
# query the third party for the observable
print(cyan(disposition, bold=True))
# translate the third party returned value to Threat Response Expected value
disposition_tuple = get_disposition(disposition)
print(cyan(disposition_tuple, bold=True))
# disposition_tuple is not empty then continue
if not disposition_tuple:
continue
# disposition_tuple then get the current date and calculate end date as an end of life date for judgment and verdicts
# We need these information as mandatory information to return to Threat Response
start_time = datetime.utcnow()
end_time = start_time + timedelta(weeks=1)
valid_time = {
'start_time': start_time.isoformat() + 'Z',
'end_time': end_time.isoformat() + 'Z',
}
# Let's append a new verdict item into the verdicts list with the minimum of information expected by the CTIM format
g.verdicts.append(
get_verdict(o_value, o_type, disposition_tuple, valid_time))
g.judgements.append(
get_judgement(o_value, o_type, disposition_tuple, valid_time))
# The g.verdicts list content all verdicts for every requested observable. Let's add this list into the data dictionnary and do some formatting stuffs
if g.verdicts:
data['verdicts'] = format_docs(g.verdicts)
if g.judgements:
data['judgements'] = format_docs(g.judgements)
# Let's get ready to send back a valid CTIM JSON result to the original Threat Response request . Let's put it into the result dictionnary
result = {'data': data}
print(
green(f"JSON result to be sent to Threat Response : \n{result}",
bold=True))
return jsonify(result)
def health():
key = get_key()
# Use some supported entity just to check that the GTI API key is valid.
observable = current_app.config['GTI_TEST_ENTITY']
_, error = get_events(key, observable)
if error:
return jsonify_errors(error)
else:
return jsonify_data({'status': 'ok'})
def observe_observables():
relay_input, error = validate_relay_input()
if error:
return jsonify_errors(error)
observables = group_observables(relay_input)
if not observables:
# Optimize a bit by not sending empty requests to the GSB API.
return jsonify_data({})
bundle = Bundle()
start_time = datetime.utcnow()
# Split the data into chunks and make multiple requests to the GSB API.
size = current_app.config['GSB_API_MAX_THREAT_ENTRIES_PER_REQUEST']
for observables in map(dict, chunks(observables.items(), size)):
gsb_input = build_gsb_input(observables)
gsb_output, error = fetch_gsb_output(gsb_input)
if error:
return jsonify_errors(error, data=bundle.json())
matches = group_matches(gsb_output)
# Extract judgements first in order to label each match with some
# "judgement_id", so that it can be extracted for each verdict later.
judgements = extract_judgements(observables, matches, start_time)
verdicts = extract_verdicts(observables, matches, start_time)
for entity in chain(judgements, verdicts):
bundle.add(entity)
relay_output = bundle.json()
return jsonify_data(relay_output)
def observe_observables():
observables, error = get_observables()
if error:
return jsonify_errors(error)
observables = group_observables(observables)
if not observables:
return jsonify_data({})
data = {}
g.verdicts = []
g.judgements = []
g.errors = []
observables = build_input_api(observables)
try:
for observable in observables:
o_value = observable['value']
o_type = observable['type'].lower()
response = call_api(o_value)
disposition_tuple = get_disposition(response['message'])
if not disposition_tuple:
continue
start_time = datetime.utcnow()
end_time = start_time + timedelta(weeks=1)
valid_time = {
'start_time': start_time.isoformat() + 'Z',
'end_time': end_time.isoformat() + 'Z',
}
g.verdicts.append(
get_verdict(o_value, o_type, disposition_tuple, valid_time))
g.judgements.append(
get_judgement(o_value, o_type, disposition_tuple, valid_time))
except KeyError:
g.errors.append(key_error())
if g.verdicts:
data['verdicts'] = format_docs(g.verdicts)
if g.judgements:
data['judgements'] = format_docs(g.judgements)
result = {'data': data}
if g.errors:
result['errors'] = g.errors
return jsonify(result)
def health():
key = get_key()
client = SecurityTrailsClient(
current_app.config['API_URL'],
key,
current_app.config['USER_AGENT'],
all_pages=current_app.config['GET_ALL_PAGES'])
_ = client.ping()
return jsonify_data({'status': 'ok'})
def respond_observables():
type_name_map = current_app.config['AKAMAI_OBSERVABLES']
observables = get_observables()
observables = [ob for ob in observables if ob['type'] in type_name_map]
credentials = get_jwt()
client = AkamaiClient(credentials, current_app.config['USER_AGENT'])
network_lists = client.network_lists()['networkLists']
network_lists = [
nl for nl in network_lists if nl.get('readOnly', False) is False
]
def action(id_, title_template, description_template, observable,
network_list):
type_name_map.get(observable['type'])
return {
'id':
id_,
'title':
title_template.format(network_list["name"]),
'description':
description_template.format(type_name_map.get(observable['type'])),
'categories': ['Akamai'],
'query-params': {
'observable_value': observable['value'],
'observable_type': observable['type'],
'network_list_id': network_list['uniqueId']
}
}
def add_to(observable, network_list):
return action(ADD_ACTION_ID, 'Add to {}', 'Add {} to Network List',
observable, network_list)
def remove_from(observable, network_list):
return action(REMOVE_ACTION_ID, 'Remove from {}',
'Remove {} from Network List', observable, network_list)
actions = []
for observable in observables:
for network_list in network_lists:
if observable['value'] in network_list.get('list', []):
actions.append(remove_from(observable, network_list))
else:
actions.append(add_to(observable, network_list))
actions.sort(key=lambda item: item['title'])
return jsonify_data(actions)
def health():
key = get_key()
if key is None:
raise AuthenticationRequiredError
url = current_app.config['AVOTX_URL']
headers = {'User-Agent': current_app.config['CTR_USER_AGENT']}
client = Client(key, url, headers=headers)
_ = client.query('/api/v1/user/me')
return jsonify_data({'status': 'ok'})