def observe_observables():
http_client = get_chronicle_http_client(get_jwt())
chronicle_client = ChronicleClient(current_app.config['API_URL'],
http_client)
observables = get_observables()
limit = current_app.config['CTR_ENTITIES_LIMIT']
time_delta = current_app.config[
'DEFAULT_NUMBER_OF_DAYS_FOR_CHRONICLE_TIME_FILTER']
g.sightings = []
g.indicators = []
g.relationships = []
for x in observables:
mapping = Mapping.for_(x)
if mapping:
assets_data = chronicle_client.list_assets(x, time_delta, limit)
ioc_details = chronicle_client.list_ioc_details(x)
x_sightings = mapping.extract_sightings(assets_data, limit)
x_indicators = mapping.extract_indicators(ioc_details, limit)
g.sightings.extend(x_sightings)
g.indicators.extend(x_indicators)
g.relationships.extend(
mapping.create_relationships(x_sightings, x_indicators))
return jsonify_result()
def health():
key = get_jwt().get('key', '')
client = C1fAppClient(key)
_ = client.get_c1fapp_response('test.com')
return jsonify_data({'status': 'ok'})
def deliberate_observables():
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
start_time = datetime.utcnow()
token = get_jwt()
g.verdicts = []
for observable in observables:
output = get_abuse_ipdb_outputs(observable, token)
if output:
g.verdicts.append(extract_verdicts(output, start_time))
relay_output = {}
if g.verdicts:
relay_output['verdicts'] = format_docs(g.verdicts)
return jsonify_data(relay_output)
def health():
credentials = get_jwt()
client = AkamaiClient(credentials, current_app.config['USER_AGENT'])
_ = client.network_lists(include_elements=False)
return jsonify_data({'status': 'ok'})
def observe_observables():
client = Auth0SignalsClient(get_jwt())
observables = get_observables()
g.verdicts = []
g.judgements = []
g.sightings = []
g.indicators = []
g.relationships = []
for observable in observables:
if observable['type'] == 'ip':
response_data = client.get_auth0_response(observable)
if response_data:
g.verdicts.append(extract_verdict(response_data, observable))
g.judgements.extend(
extract_judgements(response_data, observable))
details = client.get_full_details(response_data)
sightings = extract_sightings(observable, details)
g.sightings.extend(sightings)
indicators = extract_indicators(details)
g.indicators.extend(indicators)
g.relationships.extend(
extract_relationships(sightings, indicators))
return jsonify_result()
def apod():
url = current_app.config['APOD_URL']
params = {'api_key': get_jwt(), **get_json(PayloadSchema())}
apod_request = Request(url, params=params)
response = apod_request.get().json()
if 'thumbnail_url' in response:
image_url = response['thumbnail_url']
else:
image_url = (response['hdurl']
if bool(params['hd']) else response['url'])
content = requests.get(image_url).content
path = get_media_path()
file_name = get_file_name(params['date'])
save_image(content, path, file_name)
return jsonify({
'status': 'Success! Image saved.',
'path': os.path.join(path, file_name),
})
def refer_observables():
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
api_key = get_jwt()
client = URLScanClient(
base_url=current_app.config['URL_SCAN_API_URL'],
api_key=api_key,
user_agent=current_app.config['USER_AGENT'],
observable_types=current_app.config['URL_SCAN_OBSERVABLE_TYPES'])
g.references = []
for observable in observables:
output = client.get_search_data(observable)
if output and output['results']:
g.references.extend(extract_references(observable))
return jsonify_data(g.references)
def observe_observables():
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
time_now = datetime.utcnow()
# get dict with actual abuseipdb categories with titles and descriptions
categories = get_categories()
token = get_jwt()
g.verdicts = []
g.judgements = []
g.indicators = []
g.sightings = []
g.relationships = []
for observable in observables:
output = get_abuse_ipdb_outputs(observable, token)
if output:
g.verdicts.append(extract_verdicts(output, time_now))
output['categories_ids'] = []
output['relations'] = {}
reports = output['data']['reports']
reports.sort(key=lambda x: x['reportedAt'], reverse=True)
if len(reports) >= current_app.config['CTR_ENTITIES_LIMIT']:
reports = reports[:current_app.config['CTR_ENTITIES_LIMIT']]
for report in reports:
g.judgements.append(
extract_judgement(report, output, categories))
g.indicators.extend(
extract_indicators(report, output, categories))
g.sightings.append(extract_sightings(report, output))
g.relationships.extend(extract_relationships(output))
relay_output = {}
if g.judgements:
relay_output['judgements'] = format_docs(g.judgements)
if g.verdicts:
relay_output['verdicts'] = format_docs(g.verdicts)
if g.sightings:
relay_output['sightings'] = format_docs(g.sightings)
if g.indicators:
relay_output['indicators'] = format_docs(g.indicators)
if g.relationships:
relay_output['relationships'] = format_docs(g.relationships)
return jsonify_data(relay_output)
def health():
credentials = get_jwt()
url = 'https://api.securitycenter.windows.com/api/exposureScore'
client = Client(credentials)
client.open_session()
client.call_api(url)
client.close_session()
return jsonify_data({'status': 'ok'})
def health():
api_key = get_jwt()
client = URLScanClient(
base_url=current_app.config['URL_SCAN_API_URL'],
api_key=api_key,
user_agent=current_app.config['USER_AGENT'],
observable_types=current_app.config['URL_SCAN_OBSERVABLE_TYPES'])
client.get_search_data(
current_app.config['URL_SCAN_HEALTH_CHECK_OBSERVABLE'])
return jsonify_data({'status': 'ok'})
def deliberate_observables():
api_key = get_jwt() # Let's get the third party API Key
data = {
} # Let's create a data directory to be sent back to Threat Response
g.verdicts = [
] # Let's create a list into which we will store valid verdicts data results for every observables
g.judgements = [
] # Let's create a list into which we will store valid judgements data results for every observables
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
observables = build_input_api(observables)
for observable in observables:
o_value = observable['value']
o_type = observable['type'].lower()
# print single observable for which to send a reputation query to the third party
print(green(o_value, bold=True))
disposition = call_api(o_value, api_key)
# query the third party for the observable
print(cyan(disposition, bold=True))
# translate the third party returned value to Threat Response Expected value
disposition_tuple = get_disposition(disposition)
print(cyan(disposition_tuple, bold=True))
# disposition_tuple is not empty then continue
if not disposition_tuple:
continue
# disposition_tuple then get the current date and calculate end date as an end of life date for judgment and verdicts
# We need these information as mandatory information to return to Threat Response
start_time = datetime.utcnow()
end_time = start_time + timedelta(weeks=1)
valid_time = {
'start_time': start_time.isoformat() + 'Z',
'end_time': end_time.isoformat() + 'Z',
}
# Let's append a new verdict item into the verdicts list with the minimum of information expected by the CTIM format
g.verdicts.append(
get_verdict(o_value, o_type, disposition_tuple, valid_time))
g.judgements.append(
get_judgement(o_value, o_type, disposition_tuple, valid_time))
# The g.verdicts list content all verdicts for every requested observable. Let's add this list into the data dictionnary and do some formatting stuffs
if g.verdicts:
data['verdicts'] = format_docs(g.verdicts)
if g.judgements:
data['judgements'] = format_docs(g.judgements)
# Let's get ready to send back a valid CTIM JSON result to the original Threat Response request . Let's put it into the result dictionnary
result = {'data': data}
print(
green(f"JSON result to be sent to Threat Response : \n{result}",
bold=True))
return jsonify(result)
def check_spycloud_health():
url = url_for('watchlist/example.org')
headers = {
**current_app.config['SPYCLOUD_BASE_HEADERS'],
'X-API-Key': get_jwt()
}
response = requests.get(url, headers=headers)
return get_response_data(response)
def deliberate_observables():
client = Auth0SignalsClient(get_jwt())
observables = get_observables()
g.verdicts = []
for observable in observables:
if observable['type'] == 'ip':
response_data = client.get_auth0_response(observable)
if response_data:
g.verdicts.append(extract_verdict(response_data, observable))
return jsonify_result()
def check_health_abuse_ipdb_api():
url = url_for('check')
headers = {'Accept': 'application/json', 'Key': get_jwt()}
params = {
'ipAddress': current_app.config.get('ABUSE_IPDB_HEALTH_CHECK_IP')
}
response = requests.get(url, headers=headers, params=params)
return get_response_data(response)
def respond_observables():
type_name_map = current_app.config['AKAMAI_OBSERVABLES']
observables = get_observables()
observables = [ob for ob in observables if ob['type'] in type_name_map]
credentials = get_jwt()
client = AkamaiClient(credentials, current_app.config['USER_AGENT'])
network_lists = client.network_lists()['networkLists']
network_lists = [
nl for nl in network_lists if nl.get('readOnly', False) is False
]
def action(id_, title_template, description_template, observable,
network_list):
type_name_map.get(observable['type'])
return {
'id':
id_,
'title':
title_template.format(network_list["name"]),
'description':
description_template.format(type_name_map.get(observable['type'])),
'categories': ['Akamai'],
'query-params': {
'observable_value': observable['value'],
'observable_type': observable['type'],
'network_list_id': network_list['uniqueId']
}
}
def add_to(observable, network_list):
return action(ADD_ACTION_ID, 'Add to {}', 'Add {} to Network List',
observable, network_list)
def remove_from(observable, network_list):
return action(REMOVE_ACTION_ID, 'Remove from {}',
'Remove {} from Network List', observable, network_list)
actions = []
for observable in observables:
for network_list in network_lists:
if observable['value'] in network_list.get('list', []):
actions.append(remove_from(observable, network_list))
else:
actions.append(add_to(observable, network_list))
actions.sort(key=lambda item: item['title'])
return jsonify_data(actions)
def observe_observables():
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
token = get_jwt()
g.sightings = []
g.indicators = []
g.errors = []
for observable in observables:
output, spycloud_catalogs = validate_spycloud_outputs(
observable, token)
if output:
breaches = output['results']
breaches.sort(
key=lambda x: x['spycloud_publish_date'], reverse=True)
unique_catalog_id_set = set()
if len(breaches) >= current_app.config['CTR_ENTITIES_LIMIT']:
breaches = breaches[:current_app.config['CTR_ENTITIES_LIMIT']]
for breach in breaches:
g.sightings.append(
extract_sightings(breach, output, spycloud_catalogs))
catalog_id = breach['source_id']
if catalog_id not in unique_catalog_id_set:
if spycloud_catalogs[catalog_id]:
g.indicators.append(
extract_indicators(spycloud_catalogs[catalog_id]))
unique_catalog_id_set.add(catalog_id)
else:
error_message = current_app.config[
'CATALOG_ERROR_TEMPLATE'].format(
catalog_id=catalog_id)
g.errors.append(get_catalog_error(error_message))
relay_output = {}
if g.sightings:
relay_output['sightings'] = format_docs(g.sightings)
if g.indicators:
relay_output['indicators'] = format_docs(g.indicators)
return jsonify_data(relay_output, g.errors)
def cme():
url = current_app.config['CME_URL']
params = {'api_key': get_jwt(),
**get_params(CMEPayloadSchema())}
cme_request = Request(url, params=params)
response = cme_request.get().json()
cme_info = form_cme_data(response)
return jsonify(cme_info)
def observe_observables():
_ = get_jwt()
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
g.bundle = Bundle()
for observable in observables:
g.bundle.merge(observe(observable))
return jsonify_data(g.bundle.json())
def respond_trigger():
relay_input = get_json(ActionFormParamsSchema())
api_key = get_jwt()
client = URLScanClient(
base_url=current_app.config['URL_SCAN_API_URL'],
api_key=api_key,
user_agent=current_app.config['USER_AGENT'],
observable_types=current_app.config['URL_SCAN_OBSERVABLE_TYPES']
)
client.make_scan(relay_input['observable_value'])
return jsonify_data({'status': 'success'})
def health():
http_client = get_chronicle_http_client(get_jwt())
chronicle_client = ChronicleClient(current_app.config['API_URL'],
http_client)
time_delta = current_app.config[
'DEFAULT_NUMBER_OF_DAYS_FOR_CHRONICLE_TIME_FILTER']
_ = chronicle_client.list_assets(
{
'type': 'domain',
'value': 'www.google.com'
}, time_delta)
return jsonify_data({'status': 'ok'})
def observe_observables():
payload = get_jwt()
client = APIVoidClient(payload)
observables = get_observables()
g.sightings = []
g.indicators = []
g.relationships = []
for observable in observables:
output = client.get_data(observable)
if output:
for engine in get_engines(output):
sighting = extract_sighting(observable, engine)
g.sightings.append(sighting)
indicator = extract_indicator(engine)
g.indicators.append(indicator)
g.relationships.append(
extract_relationship(sighting['id'], indicator['id']))
return jsonify_result()
def respond_trigger():
add_status('failure')
params = get_action_form_params()
credentials = get_jwt()
client = AkamaiClient(credentials, current_app.config['USER_AGENT'])
action_map = {
ADD_ACTION_ID: client.add_to_network_list,
REMOVE_ACTION_ID: client.remove_from_network_list
}
action = action_map.get(params['action-id'])
if not action:
raise InvalidArgumentError("Unsupported action.")
action(params['network_list_id'], params['observable_value'])
add_status('success')
return jsonify_result()
def observe_observables():
jwt = get_jwt()
host = jwt['hostname']
api_token = jwt['ApiToken']
observables = group_observables(get_observables())
if not observables:
return jsonify_data({})
sentinelone_outputs = get_sentinelone_outputs(host, api_token, observables)
if not sentinelone_outputs:
return jsonify_data({})
indicators = []
sightings = []
relationships = []
for output in sentinelone_outputs:
data = output.get('data', [])
observable = output.get('observable')
for entry in data:
output_indicators = entry.get('indicators')
sightings.append(extract_sightings(entry, observable, host))
if output_indicators:
indicators.extend(extract_indicators(output_indicators))
relationships.extend(extract_relationships(entry))
relay_output = {}
if sightings:
relay_output['sightings'] = format_docs(sightings)
if indicators:
relay_output['indicators'] = format_docs(indicators)
if relationships:
relay_output['relationships'] = format_docs(relationships)
return jsonify_data(relay_output)
def refer_observables():
jwt = get_jwt()
host = jwt['hostname']
observables = get_observables()
relay_output = []
observable_to_dv_query_mapping = {
"md5": 'FileMD5 = "{0}"',
"sha1": 'FileSHA1 = "{0}"',
"sha256": 'FileSHA256 = "{0}"',
"ip": 'DstIP = "{0}" OR SrcIP = "{0}"',
"ipv6": 'DstIP = "{0}" OR SrcIP = "{0}"',
"url": 'networkUrl = "{0}"',
"hostname": 'agentName = "{0}"'
}
for obs in observables:
refer_object = {
"id": "ref-sentinelone-search-{0}-{1}",
"title": "Open in Deep Visibility",
"description": "Open in Deep Visibility",
"categories": ["SentinelOne", "Deep Visibility", "Search"],
"url": None
}
if obs['type'] in observable_to_dv_query_mapping:
refer_object['id'] = refer_object['id'].format(
obs['type'], urllib.parse.quote(obs['value']))
queryString = observable_to_dv_query_mapping[obs['type']].format(
obs['value'])
url_encoded_queryString = urllib.parse.quote(queryString)
url = f'https://{host}/dv?queryString={url_encoded_queryString}&timeFrame=Last30Days'
refer_object['url'] = url
relay_output.append(refer_object)
return jsonify_data(relay_output)
def observe_observables():
key = get_jwt().get('key', '')
client = C1fAppClient(key)
observables = get_observables()
g.sightings = []
g.indicators = []
g.relationships = []
limit = current_app.config['CTR_ENTITIES_LIMIT']
for observable in observables:
mapping = Mapping.for_(observable)
if mapping:
response_data = client.get_c1fapp_response(observable['value'])
response_data.sort(key=lambda x: x['reportime'], reverse=True)
response_data = response_data[:limit]
g.sightings.extend(mapping.extract_sightings(response_data))
g.indicators.extend(mapping.extract_indicators(response_data))
g.relationships.extend(mapping.extract_relationships())
return jsonify_result()
def tiles():
_ = get_jwt()
return jsonify_data([])
def tile_data():
_ = get_jwt()
_ = get_json(DashboardTileDataSchema())
return jsonify_data({})
def respond_trigger():
mapping_by_type = {
'sha1': 'FileSha1',
'sha256': 'FileSha256',
'ip': 'IpAddress',
'ipv6': 'IpAddress',
'domain': 'DomainName'
}
data, error = get_action_form_params()
if error:
return jsonify_errors(error)
title = 'From SecureX Threat Response'
description = 'This indicator was added via SecureX Threat Response ' \
'by the UI or API response actions'
if data['observable_type'] == 'ms_machine_id':
comment = 'Performed via SecureX Threat Response'
actions = {
'microsoft-defender-FullIsolation': {
'url':
'{base_url}/machines/{machine_id}/isolate'.format(
base_url=current_app.config['BASE_URL'],
machine_id=data['observable_value']),
'method':
'POST',
'data': {
'Comment': comment,
'IsolationType': 'Full'
}
},
'microsoft-defender-SelectiveIsolation': {
'url':
'{base_url}/machines/{machine_id}/isolate'.format(
base_url=current_app.config['BASE_URL'],
machine_id=data['observable_value']),
'method':
'POST',
'data': {
'Comment': comment,
'IsolationType': 'Selective'
}
},
'microsoft-defender-Unisolate': {
'url':
'{base_url}/machines/{machine_id}/unisolate'.format(
base_url=current_app.config['BASE_URL'],
machine_id=data['observable_value']),
'method':
'POST',
'data': {
'Comment': comment
}
},
'microsoft-defender-RestrictCodeExecution': {
'url':
'{base_url}/machines/{machine_id}'
'/restrictCodeExecution'.format(
base_url=current_app.config['BASE_URL'],
machine_id=data['observable_value']),
'method':
'POST',
'data': {
'Comment': comment
}
},
'microsoft-defender-UnrestrictCodeExecution': {
'url':
'{base_url}/machines/{machine_id}'
'/unrestrictCodeExecution'.format(
base_url=current_app.config['BASE_URL'],
machine_id=data['observable_value']),
'method':
'POST',
'data': {
'Comment': comment
}
},
'microsoft-defender-RunAntiVirusScanQuick': {
'url':
'{base_url}/machines/{machine_id}'
'/runAntiVirusScan'.format(
base_url=current_app.config['BASE_URL'],
machine_id=data['observable_value']),
'method':
'POST',
'data': {
'Comment': comment,
'ScanType': 'Quick'
}
},
'microsoft-defender-RunAntiVirusScanFull': {
'url':
'{base_url}/machines/{machine_id}'
'/runAntiVirusScan'.format(
base_url=current_app.config['BASE_URL'],
machine_id=data['observable_value']),
'method':
'POST',
'data': {
'Comment': comment,
'ScanType': 'Full'
}
},
'microsoft-defender-CollectInvestigationPackage': {
'url':
'{base_url}/machines/{machine_id}'
'/collectInvestigationPackage'.format(
base_url=current_app.config['BASE_URL'],
machine_id=data['observable_value']),
'method':
'POST',
'data': {
'Comment': comment
}
},
'microsoft-defender-InitiateInvestigation': {
'url':
'{base_url}/machines/{machine_id}'
'/startInvestigation'.format(
base_url=current_app.config['BASE_URL'],
machine_id=data['observable_value']),
'method':
'POST',
'data': {
'Comment': comment
}
}
}
else:
actions = {
'microsoft-defender-add-indicator-alert': {
'url': current_app.config['INDICATOR_URL'],
'method': 'POST',
'data': {
'indicatorValue': data['observable_value'],
'indicatorType':
mapping_by_type.get(data['observable_type']),
'action': 'Alert',
'title': title,
'description': description,
'severity': 'High'
}
},
'microsoft-defender-add-indicator-alert-and-block': {
'url': current_app.config['INDICATOR_URL'],
'method': 'POST',
'data': {
'indicatorValue': data['observable_value'],
'indicatorType':
mapping_by_type.get(data['observable_type']),
'action': 'AlertAndBlock',
'title': title,
'description': description,
'severity': 'High'
}
},
'microsoft-defender-add-indicator-allowed': {
'url': current_app.config['INDICATOR_URL'],
'method': 'POST',
'data': {
'indicatorValue': data['observable_value'],
'indicatorType':
mapping_by_type.get(data['observable_type']),
'action': 'Allowed',
'title': title,
'description': description
}
},
'microsoft-defender'
'-remove-indicator': {
'url':
current_app.config['INDICATOR_URL'] + '/' +
str(data.get('indicator_id', '')),
'method':
'DELETE',
'data': {}
}
}
result = {'data': {'status': 'success'}}
item = actions.get(data['action-id'])
if not item:
result['data']['status'] = 'failure'
result['errors'] = [
CTRBadRequestError("Unsupported action.").json,
]
return jsonify(result)
if data['observable_type'] != 'ms_machine_id' and \
not item['data']['indicatorType']:
raise UnsupportedTypeError(data['observable_type'])
if item['data']:
action = json.dumps(item['data']).encode('utf-8')
else:
action = None
credentials = get_jwt()
client = Client(credentials)
client.open_session()
response, error = client.call_api(item['url'], item['method'], data=action)
client.close_session()
if error is not None:
result['data']['status'] = 'failure'
result['errors'] = [
CTRBadRequestError(error).json,
]
return jsonify(result)
def respond_observables():
observables, error = get_observables()
if error:
return jsonify_errors(error)
observables = group_observables(observables, 'respond')
if not observables:
return jsonify_data([])
g.actions = []
credentials = get_jwt()
client = Client(credentials)
client.open_session()
for observable in observables:
query_params = {
'observable_type': observable['type'],
'observable_value': observable['value']
}
if observable['type'] == 'ms_machine_id':
_actions = get_supported_actions(client, observable['value'])
_actions = get_reverse_actions(client, observable['value'],
_actions)
actions = []
for item in _actions:
action = {}
action['id'] = \
f'microsoft-defender-{_TR_SUPPORTED_ACTIONS[item]}'
action['title'] = item
action['description'] = item
action['categories'] = [
'Microsoft Defender for Endpoint', 'Machine Actions'
]
action['query-params'] = query_params
actions.append(action)
else:
params = "$filter=indicatorValue+eq+'{value}'&$top=1".format(
value=observable['value']).encode('utf-8')
response, error = client.call_api(
current_app.config['INDICATOR_URL'], params=params)
if response and response.get('value'):
obj = response['value'][0]
human_action = 'Alert and Block' \
if obj['action'] == 'AlertAndBlock' else obj['action']
query_params['indicator_id'] = obj['id']
actions = [
{
'id':
'microsoft-defender-remove-indicator',
'title':
'Remove indicator: {action} - {title}'.format(
action=human_action, title=obj['title']),
'description':
f'Remove indicator with {human_action} '
f'action for {observable["value"]}',
'categories': [
'Microsoft Defender for Endpoint',
'Remove Indicator'
],
'query-params':
query_params
},
]
else:
actions = [{
'id':
'microsoft-defender-add-indicator-alert',
'title':
'Add indicator: Alert',
'description':
f'Add indicator with Alert action '
f'for {observable["value"]}',
'categories':
['Microsoft Defender for Endpoint', 'Add Indicator'],
'query-params':
query_params
}, {
'id':
'microsoft-defender-'
'add-indicator-alert-and-block',
'title':
'Add indicator: Alert and Block',
'description':
f'Add indicator with '
f'Alert and Block action'
f' for {observable["value"]}',
'categories':
['Microsoft Defender for Endpoint', 'Add Indicator'],
'query-params':
query_params
}, {
'id':
'microsoft-defender-add-indicator-allowed',
'title':
'Add indicator: Allow',
'description':
f'Add indicator with Allow action '
f'for {observable["value"]}',
'categories':
['Microsoft Defender for Endpoint', 'Add Indicator'],
'query-params':
query_params
}]
g.actions.extend(actions)
client.close_session()
return jsonify_data(g.actions)
def respond_trigger():
_ = get_jwt()
_ = get_action_form_params()
return jsonify_data({'status': 'success'})
