def refer_observables():
observables, error = get_observables()
if error:
return jsonify_errors(error)
observable_types = current_app.config['GTI_OBSERVABLE_TYPES']
def type_of(observable):
return observable_types[observable['type']]
url_template = current_app.config['GTI_UI_SEARCH_URL']
data = [
{
'id': 'ref-gti-search-{type}-{value}'.format(**observable),
'title': f'Search for this {type_of(observable)}',
'description': (
f'Lookup this {type_of(observable)} '
'on Gigamon ThreatINSIGHT'
),
'url': url_template.format(query=observable['value']),
'categories': ['Search', 'Gigamon ThreatINSIGHT'],
}
for observable in observables
if observable['type'] in observable_types
]
return jsonify_data(data)
def observe_observables():
observables, error = get_observables()
if error:
return jsonify_errors(error)
observable_types = current_app.config['GTI_OBSERVABLE_TYPES']
observables = [
observable
for observable in observables
if observable['type'] in observable_types
]
key = get_key()
bundle = Bundle()
for observable in observables:
events, error = get_events_for_observable(key, observable)
if error:
# Make sure not to lose any data processed so far.
return jsonify_errors(error, data=bundle.json())
indicator_by_rule_uuid = {}
for event in events:
sighting = Sighting.map(event)
bundle.add(sighting)
if 'detection' in event:
rule = event['detection']['rule']
indicator = indicator_by_rule_uuid.get(rule['uuid'])
if indicator is None:
indicator = Indicator.map(rule)
indicator_by_rule_uuid[rule['uuid']] = indicator
bundle.add(indicator)
relationship = Relationship.map(sighting, indicator)
bundle.add(relationship)
data = bundle.json()
return jsonify_data(data)
def health():
url = url_for('threatLists')
if url is None:
# Mimic the GSB API error response payload.
error = {
'code': HTTPStatus.FORBIDDEN,
'message': 'The request is missing a valid API key.',
'status': 'PERMISSION_DENIED',
}
return jsonify_errors(error)
_, error = execute(requests.get, url, headers=headers())
if error:
return jsonify_errors(error)
else:
return jsonify_data({'status': 'ok'})
def observe_observables():
observables, error = get_observables()
if error:
return jsonify_errors(error)
emails = [
observable['value'] for observable in observables
if observable['type'] == 'email'
]
key = get_key()
bundle = Bundle()
limit = current_app.config['CTR_ENTITIES_LIMIT']
for email in emails:
breaches, error = fetch_breaches(key, email)
if error:
return jsonify_errors(error, data=bundle.json())
breaches.sort(key=itemgetter('BreachDate'), reverse=True)
breaches = breaches[:limit]
source_uri = current_app.config['HIBP_UI_URL'].format(
email=quote(email, safe=''))
for breach in breaches:
indicator = Indicator.map(breach)
sighting = Sighting.map(breach, email, source_uri)
relationship = Relationship.map(indicator, sighting)
bundle.add(indicator)
bundle.add(sighting)
bundle.add(relationship)
data = bundle.json()
return jsonify_data(data)
def health():
key = get_key()
# Use some supported entity just to check that the GTI API key is valid.
observable = current_app.config['GTI_TEST_ENTITY']
_, error = get_events(key, observable)
if error:
return jsonify_errors(error)
else:
return jsonify_data({'status': 'ok'})
def health():
key = get_key()
# Use some breached email just to check that the HIBP API key is valid.
email = current_app.config['HIBP_TEST_EMAIL']
_, error = fetch_breaches(key, email, truncate=True)
if error:
return jsonify_errors(error)
else:
return jsonify_data({'status': 'ok'})
def observe_observables():
relay_input, error = validate_relay_input()
if error:
return jsonify_errors(error)
observables = group_observables(relay_input)
if not observables:
# Optimize a bit by not sending empty requests to the GSB API.
return jsonify_data({})
bundle = Bundle()
start_time = datetime.utcnow()
# Split the data into chunks and make multiple requests to the GSB API.
size = current_app.config['GSB_API_MAX_THREAT_ENTRIES_PER_REQUEST']
for observables in map(dict, chunks(observables.items(), size)):
gsb_input = build_gsb_input(observables)
gsb_output, error = fetch_gsb_output(gsb_input)
if error:
return jsonify_errors(error, data=bundle.json())
matches = group_matches(gsb_output)
# Extract judgements first in order to label each match with some
# "judgement_id", so that it can be extracted for each verdict later.
judgements = extract_judgements(observables, matches, start_time)
verdicts = extract_verdicts(observables, matches, start_time)
for entity in chain(judgements, verdicts):
bundle.add(entity)
relay_output = bundle.json()
return jsonify_data(relay_output)
def observe_observables():
observables, error = get_observables()
if error:
return jsonify_errors(error)
observables = group_observables(observables)
if not observables:
return jsonify_data({})
data = {}
g.verdicts = []
g.judgements = []
g.errors = []
observables = build_input_api(observables)
try:
for observable in observables:
o_value = observable['value']
o_type = observable['type'].lower()
response = call_api(o_value)
disposition_tuple = get_disposition(response['message'])
if not disposition_tuple:
continue
start_time = datetime.utcnow()
end_time = start_time + timedelta(weeks=1)
valid_time = {
'start_time': start_time.isoformat() + 'Z',
'end_time': end_time.isoformat() + 'Z',
}
g.verdicts.append(
get_verdict(o_value, o_type, disposition_tuple, valid_time))
g.judgements.append(
get_judgement(o_value, o_type, disposition_tuple, valid_time))
except KeyError:
g.errors.append(key_error())
if g.verdicts:
data['verdicts'] = format_docs(g.verdicts)
if g.judgements:
data['judgements'] = format_docs(g.judgements)
result = {'data': data}
if g.errors:
result['errors'] = g.errors
return jsonify(result)
def deliberate_observables():
relay_input, error = validate_relay_input()
if error:
return jsonify_errors(error)
observables = group_observables(relay_input)
if not observables:
# Optimize a bit by not sending empty requests to the GSB API.
return jsonify_data({})
bundle = Bundle()
start_time = datetime.utcnow()
# Split the data into chunks and make multiple requests to the GSB API.
size = current_app.config['GSB_API_MAX_THREAT_ENTRIES_PER_REQUEST']
for observables in map(dict, chunks(observables.items(), size)):
gsb_input = build_gsb_input(observables)
gsb_output, error = fetch_gsb_output(gsb_input)
if error:
return jsonify_errors(error, data=bundle.json())
matches = group_matches(gsb_output)
verdicts = extract_verdicts(observables, matches, start_time)
for entity in verdicts:
bundle.add(entity)
relay_output = bundle.json()
return jsonify_data(relay_output)
def health():
try:
response = requests.get(current_app.config['API_URL'],
headers=current_app.config['CTR_HEADERS'])
except requests.exceptions.SSLError as ex:
raise CybercrimeSSLError(ex)
if response.ok:
return jsonify_data({'status': 'ok'})
error = {
'code': 'invalid_health_check',
'message': 'Invalid 3rd party API connect.',
}
return jsonify_errors(error)
def refer_observables():
observables, error = get_observables()
if error:
return jsonify_errors(error)
observables = group_observables(observables)
data = []
for observable in observables:
o_type = observable['type']
o_value = observable['value']
if o_type in ('sha1', 'sha256'):
entity = 'files'
elif o_type in ('ip', 'ipv6'):
entity = 'ips'
elif o_type == 'domain':
entity = 'urls'
else:
continue
title = 'Search for this {o_type}'.format(
o_type=current_app.config["MD_ATP_OBSERVABLE_TYPES"][o_type])
description = \
'Lookup this {o_type} on Microsoft Defender for Endpoint'.format(
o_type=current_app.config['MD_ATP_OBSERVABLE_TYPES'][o_type]
)
url = '{host}/{entity}/{o_value}'.format(
host=current_app.config['SECURITY_CENTER_URL'],
entity=entity,
o_value=o_value)
data.append({
'id': f'ref-md-search-{o_type}-{o_value}',
'title': title,
'description': description,
'url': url,
'categories': ['Search', 'Microsoft Defender for Endpoint']
})
return jsonify_data(data)
def refer_observables():
observables, error = get_observables()
if error:
return jsonify_errors(error)
emails = [
observable['value'] for observable in observables
if observable['type'] == 'email'
]
data = [{
'id': f'ref-hibp-search-email-{email}',
'title': 'Search for this email',
'description': 'Check this email status with Have I Been Pwned',
'url': current_app.config['HIBP_UI_URL'].format(email=email),
'categories': ['Search', 'Have I Been Pwned'],
} for email in map(lambda email: quote(email, safe=''), emails)]
return jsonify_data(data)
def refer_observables():
relay_input, error = validate_relay_input()
if error:
return jsonify_errors(error)
observables = group_observables(relay_input)
relay_output = [{
'id':
f'ref-gsb-search-{type}-{quote(value, safe="")}',
'title': ('Search for this '
f'{current_app.config["GSB_OBSERVABLE_TYPES"][type]}'),
'description': ('Check this '
f'{current_app.config["GSB_OBSERVABLE_TYPES"][type]} '
'status with Google Safe Browsing'),
'url': (current_app.config['GSB_TRANSPARENCY_REPORT_URL'].format(
url=quote(value, safe=':'))),
'categories': ['Search', 'Google Safe Browsing'],
} for value, types in observables.items() for type in types]
return jsonify_data(relay_output)
def respond_observables():
observables, error = get_observables()
if error:
return jsonify_errors(error)
observables = group_observables(observables, 'respond')
if not observables:
return jsonify_data([])
g.actions = []
credentials = get_jwt()
client = Client(credentials)
client.open_session()
for observable in observables:
query_params = {
'observable_type': observable['type'],
'observable_value': observable['value']
}
if observable['type'] == 'ms_machine_id':
_actions = get_supported_actions(client, observable['value'])
_actions = get_reverse_actions(client, observable['value'],
_actions)
actions = []
for item in _actions:
action = {}
action['id'] = \
f'microsoft-defender-{_TR_SUPPORTED_ACTIONS[item]}'
action['title'] = item
action['description'] = item
action['categories'] = [
'Microsoft Defender for Endpoint', 'Machine Actions'
]
action['query-params'] = query_params
actions.append(action)
else:
params = "$filter=indicatorValue+eq+'{value}'&$top=1".format(
value=observable['value']).encode('utf-8')
response, error = client.call_api(
current_app.config['INDICATOR_URL'], params=params)
if response and response.get('value'):
obj = response['value'][0]
human_action = 'Alert and Block' \
if obj['action'] == 'AlertAndBlock' else obj['action']
query_params['indicator_id'] = obj['id']
actions = [
{
'id':
'microsoft-defender-remove-indicator',
'title':
'Remove indicator: {action} - {title}'.format(
action=human_action, title=obj['title']),
'description':
f'Remove indicator with {human_action} '
f'action for {observable["value"]}',
'categories': [
'Microsoft Defender for Endpoint',
'Remove Indicator'
],
'query-params':
query_params
},
]
else:
actions = [{
'id':
'microsoft-defender-add-indicator-alert',
'title':
'Add indicator: Alert',
'description':
f'Add indicator with Alert action '
f'for {observable["value"]}',
'categories':
['Microsoft Defender for Endpoint', 'Add Indicator'],
'query-params':
query_params
}, {
'id':
'microsoft-defender-'
'add-indicator-alert-and-block',
'title':
'Add indicator: Alert and Block',
'description':
f'Add indicator with '
f'Alert and Block action'
f' for {observable["value"]}',
'categories':
['Microsoft Defender for Endpoint', 'Add Indicator'],
'query-params':
query_params
}, {
'id':
'microsoft-defender-add-indicator-allowed',
'title':
'Add indicator: Allow',
'description':
f'Add indicator with Allow action '
f'for {observable["value"]}',
'categories':
['Microsoft Defender for Endpoint', 'Add Indicator'],
'query-params':
query_params
}]
g.actions.extend(actions)
client.close_session()
return jsonify_data(g.actions)
def handle_tr_formatted_error(error):
app.logger.error(error.json)
return jsonify_errors(error.json)
def handle_tr_error(exception):
app.logger.error(traceback.format_exc())
return jsonify_errors(exception.json)
def handle_relay_error(error):
app.logger.error(traceback.format_exc())
return jsonify_errors(error)
def handle_tr_formatted_error(error):
app.logger.error(traceback.format_exc())
return jsonify_errors(error.json)
def handle_relay_error(error):
return jsonify_errors(error)
def observe_observables():
observables, error = get_observables()
if error:
return jsonify_errors(error)
observables = group_observables(observables)
if not observables:
return jsonify_data({})
data = {}
g.sightings = []
credentials = get_jwt()
client = Client(credentials)
for observable in observables:
client.open_session()
response = get_alert(client, observable)
if not response or not response.get('value'):
alerts = []
else:
alerts = response['value']
alerts.sort(key=lambda x: x['alertCreationTime'], reverse=True)
count = len(alerts)
if count >= current_app.config['CTR_ENTITIES_LIMIT']:
alerts = alerts[:current_app.config['CTR_ENTITIES_LIMIT']]
events = []
else:
events = call_advanced_hunting(
client, observable['value'], observable['type'],
current_app.config['CTR_ENTITIES_LIMIT'] - count)
count = count + len(events)
events.sort(key=lambda x: x['Timestamp'], reverse=True)
mapping = Mapping(client, observable, count)
if alerts:
with ThreadPoolExecutor(max_workers=min(len(alerts),
cpu_count() or 1) *
5) as executor:
alerts = executor.map(mapping.build_sighting_from_alert,
alerts)
[g.sightings.append(alert) for alert in alerts if alert]
if events:
with ThreadPoolExecutor(max_workers=min(len(events),
cpu_count() or 1) *
5) as executor:
events = executor.map(mapping.build_sighting_from_ah, events)
[g.sightings.append(event) for event in events if event]
client.close_session()
if g.sightings:
data['sightings'] = format_docs(g.sightings)
return jsonify_data(data)
def handle_tr_formatted_error(exception):
app.logger.error(exception)
return jsonify_errors(exception.json)
def respond_trigger():
mapping_by_type = {
'sha1': 'FileSha1',
'sha256': 'FileSha256',
'ip': 'IpAddress',
'ipv6': 'IpAddress',
'domain': 'DomainName'
}
data, error = get_action_form_params()
if error:
return jsonify_errors(error)
title = 'From SecureX Threat Response'
description = 'This indicator was added via SecureX Threat Response ' \
'by the UI or API response actions'
if data['observable_type'] == 'ms_machine_id':
comment = 'Performed via SecureX Threat Response'
actions = {
'microsoft-defender-FullIsolation': {
'url':
'{base_url}/machines/{machine_id}/isolate'.format(
base_url=current_app.config['BASE_URL'],
machine_id=data['observable_value']),
'method':
'POST',
'data': {
'Comment': comment,
'IsolationType': 'Full'
}
},
'microsoft-defender-SelectiveIsolation': {
'url':
'{base_url}/machines/{machine_id}/isolate'.format(
base_url=current_app.config['BASE_URL'],
machine_id=data['observable_value']),
'method':
'POST',
'data': {
'Comment': comment,
'IsolationType': 'Selective'
}
},
'microsoft-defender-Unisolate': {
'url':
'{base_url}/machines/{machine_id}/unisolate'.format(
base_url=current_app.config['BASE_URL'],
machine_id=data['observable_value']),
'method':
'POST',
'data': {
'Comment': comment
}
},
'microsoft-defender-RestrictCodeExecution': {
'url':
'{base_url}/machines/{machine_id}'
'/restrictCodeExecution'.format(
base_url=current_app.config['BASE_URL'],
machine_id=data['observable_value']),
'method':
'POST',
'data': {
'Comment': comment
}
},
'microsoft-defender-UnrestrictCodeExecution': {
'url':
'{base_url}/machines/{machine_id}'
'/unrestrictCodeExecution'.format(
base_url=current_app.config['BASE_URL'],
machine_id=data['observable_value']),
'method':
'POST',
'data': {
'Comment': comment
}
},
'microsoft-defender-RunAntiVirusScanQuick': {
'url':
'{base_url}/machines/{machine_id}'
'/runAntiVirusScan'.format(
base_url=current_app.config['BASE_URL'],
machine_id=data['observable_value']),
'method':
'POST',
'data': {
'Comment': comment,
'ScanType': 'Quick'
}
},
'microsoft-defender-RunAntiVirusScanFull': {
'url':
'{base_url}/machines/{machine_id}'
'/runAntiVirusScan'.format(
base_url=current_app.config['BASE_URL'],
machine_id=data['observable_value']),
'method':
'POST',
'data': {
'Comment': comment,
'ScanType': 'Full'
}
},
'microsoft-defender-CollectInvestigationPackage': {
'url':
'{base_url}/machines/{machine_id}'
'/collectInvestigationPackage'.format(
base_url=current_app.config['BASE_URL'],
machine_id=data['observable_value']),
'method':
'POST',
'data': {
'Comment': comment
}
},
'microsoft-defender-InitiateInvestigation': {
'url':
'{base_url}/machines/{machine_id}'
'/startInvestigation'.format(
base_url=current_app.config['BASE_URL'],
machine_id=data['observable_value']),
'method':
'POST',
'data': {
'Comment': comment
}
}
}
else:
actions = {
'microsoft-defender-add-indicator-alert': {
'url': current_app.config['INDICATOR_URL'],
'method': 'POST',
'data': {
'indicatorValue': data['observable_value'],
'indicatorType':
mapping_by_type.get(data['observable_type']),
'action': 'Alert',
'title': title,
'description': description,
'severity': 'High'
}
},
'microsoft-defender-add-indicator-alert-and-block': {
'url': current_app.config['INDICATOR_URL'],
'method': 'POST',
'data': {
'indicatorValue': data['observable_value'],
'indicatorType':
mapping_by_type.get(data['observable_type']),
'action': 'AlertAndBlock',
'title': title,
'description': description,
'severity': 'High'
}
},
'microsoft-defender-add-indicator-allowed': {
'url': current_app.config['INDICATOR_URL'],
'method': 'POST',
'data': {
'indicatorValue': data['observable_value'],
'indicatorType':
mapping_by_type.get(data['observable_type']),
'action': 'Allowed',
'title': title,
'description': description
}
},
'microsoft-defender'
'-remove-indicator': {
'url':
current_app.config['INDICATOR_URL'] + '/' +
str(data.get('indicator_id', '')),
'method':
'DELETE',
'data': {}
}
}
result = {'data': {'status': 'success'}}
item = actions.get(data['action-id'])
if not item:
result['data']['status'] = 'failure'
result['errors'] = [
CTRBadRequestError("Unsupported action.").json,
]
return jsonify(result)
if data['observable_type'] != 'ms_machine_id' and \
not item['data']['indicatorType']:
raise UnsupportedTypeError(data['observable_type'])
if item['data']:
action = json.dumps(item['data']).encode('utf-8')
else:
action = None
credentials = get_jwt()
client = Client(credentials)
client.open_session()
response, error = client.call_api(item['url'], item['method'], data=action)
client.close_session()
if error is not None:
result['data']['status'] = 'failure'
result['errors'] = [
CTRBadRequestError(error).json,
]
return jsonify(result)
