def deliberate_observables():
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
start_time = datetime.utcnow()
token = get_jwt()
g.verdicts = []
for observable in observables:
output = get_abuse_ipdb_outputs(observable, token)
if output:
g.verdicts.append(extract_verdicts(output, start_time))
relay_output = {}
if g.verdicts:
relay_output['verdicts'] = format_docs(g.verdicts)
return jsonify_data(relay_output)
def refer_observables():
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
api_key = get_jwt()
client = URLScanClient(
base_url=current_app.config['URL_SCAN_API_URL'],
api_key=api_key,
user_agent=current_app.config['USER_AGENT'],
observable_types=current_app.config['URL_SCAN_OBSERVABLE_TYPES'])
g.references = []
for observable in observables:
output = client.get_search_data(observable)
if output and output['results']:
g.references.extend(extract_references(observable))
return jsonify_data(g.references)
def observe_observables():
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
time_now = datetime.utcnow()
# get dict with actual abuseipdb categories with titles and descriptions
categories = get_categories()
token = get_jwt()
g.verdicts = []
g.judgements = []
g.indicators = []
g.sightings = []
g.relationships = []
for observable in observables:
output = get_abuse_ipdb_outputs(observable, token)
if output:
g.verdicts.append(extract_verdicts(output, time_now))
output['categories_ids'] = []
output['relations'] = {}
reports = output['data']['reports']
reports.sort(key=lambda x: x['reportedAt'], reverse=True)
if len(reports) >= current_app.config['CTR_ENTITIES_LIMIT']:
reports = reports[:current_app.config['CTR_ENTITIES_LIMIT']]
for report in reports:
g.judgements.append(
extract_judgement(report, output, categories))
g.indicators.extend(
extract_indicators(report, output, categories))
g.sightings.append(extract_sightings(report, output))
g.relationships.extend(extract_relationships(output))
relay_output = {}
if g.judgements:
relay_output['judgements'] = format_docs(g.judgements)
if g.verdicts:
relay_output['verdicts'] = format_docs(g.verdicts)
if g.sightings:
relay_output['sightings'] = format_docs(g.sightings)
if g.indicators:
relay_output['indicators'] = format_docs(g.indicators)
if g.relationships:
relay_output['relationships'] = format_docs(g.relationships)
return jsonify_data(relay_output)
def deliberate_observables():
api_key = get_jwt() # Let's get the third party API Key
data = {
} # Let's create a data directory to be sent back to Threat Response
g.verdicts = [
] # Let's create a list into which we will store valid verdicts data results for every observables
g.judgements = [
] # Let's create a list into which we will store valid judgements data results for every observables
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
observables = build_input_api(observables)
for observable in observables:
o_value = observable['value']
o_type = observable['type'].lower()
# print single observable for which to send a reputation query to the third party
print(green(o_value, bold=True))
disposition = call_api(o_value, api_key)
# query the third party for the observable
print(cyan(disposition, bold=True))
# translate the third party returned value to Threat Response Expected value
disposition_tuple = get_disposition(disposition)
print(cyan(disposition_tuple, bold=True))
# disposition_tuple is not empty then continue
if not disposition_tuple:
continue
# disposition_tuple then get the current date and calculate end date as an end of life date for judgment and verdicts
# We need these information as mandatory information to return to Threat Response
start_time = datetime.utcnow()
end_time = start_time + timedelta(weeks=1)
valid_time = {
'start_time': start_time.isoformat() + 'Z',
'end_time': end_time.isoformat() + 'Z',
}
# Let's append a new verdict item into the verdicts list with the minimum of information expected by the CTIM format
g.verdicts.append(
get_verdict(o_value, o_type, disposition_tuple, valid_time))
g.judgements.append(
get_judgement(o_value, o_type, disposition_tuple, valid_time))
# The g.verdicts list content all verdicts for every requested observable. Let's add this list into the data dictionnary and do some formatting stuffs
if g.verdicts:
data['verdicts'] = format_docs(g.verdicts)
if g.judgements:
data['judgements'] = format_docs(g.judgements)
# Let's get ready to send back a valid CTIM JSON result to the original Threat Response request . Let's put it into the result dictionnary
result = {'data': data}
print(
green(f"JSON result to be sent to Threat Response : \n{result}",
bold=True))
return jsonify(result)
def observe_observables():
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
token = get_jwt()
g.sightings = []
g.indicators = []
g.errors = []
for observable in observables:
output, spycloud_catalogs = validate_spycloud_outputs(
observable, token)
if output:
breaches = output['results']
breaches.sort(
key=lambda x: x['spycloud_publish_date'], reverse=True)
unique_catalog_id_set = set()
if len(breaches) >= current_app.config['CTR_ENTITIES_LIMIT']:
breaches = breaches[:current_app.config['CTR_ENTITIES_LIMIT']]
for breach in breaches:
g.sightings.append(
extract_sightings(breach, output, spycloud_catalogs))
catalog_id = breach['source_id']
if catalog_id not in unique_catalog_id_set:
if spycloud_catalogs[catalog_id]:
g.indicators.append(
extract_indicators(spycloud_catalogs[catalog_id]))
unique_catalog_id_set.add(catalog_id)
else:
error_message = current_app.config[
'CATALOG_ERROR_TEMPLATE'].format(
catalog_id=catalog_id)
g.errors.append(get_catalog_error(error_message))
relay_output = {}
if g.sightings:
relay_output['sightings'] = format_docs(g.sightings)
if g.indicators:
relay_output['indicators'] = format_docs(g.indicators)
return jsonify_data(relay_output, g.errors)
def observe_observables():
_ = get_jwt()
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
g.bundle = Bundle()
for observable in observables:
g.bundle.merge(observe(observable))
return jsonify_data(g.bundle.json())
def respond_observables():
relay_input = get_json(ObservableSchema(many=True))
observables = get_scan_observables(relay_input)
if not observables:
return jsonify_data([])
g.actions = []
for observable in observables:
g.actions.append(extract_action(observable))
return jsonify_data(g.actions)
from functools import partial
from uuid import uuid4, uuid5
from datetime import datetime
from flask import Blueprint, g, current_app
from api.schemas import ObservableSchema
from api.utils import get_json, get_jwt, jsonify_result
from api.client import APIVoidClient
enrich_api = Blueprint('enrich', __name__)
get_observables = partial(get_json, schema=ObservableSchema(many=True))
def get_confidence(engine):
if engine.get('confidence'):
return engine.get('confidence').capitalize()
else:
return 'High'
def get_transient_id(entity_type, base_value=None):
uuid = (uuid5(current_app.config['NAMESPACE_BASE'], base_value)
if base_value else uuid4())
return f'transient:{entity_type}-{uuid}'
def extract_indicator(engine):
return {
'confidence': get_confidence(engine),
from datetime import datetime, timedelta
from http import HTTPStatus
from itertools import chain
from urllib.parse import quote
from uuid import uuid4
import requests
from flask import Blueprint, request, current_app
from api.bundle import Bundle
from api.schemas import ObservableSchema
from api.utils import url_for, execute, headers, jsonify_errors, jsonify_data
enrich_api = Blueprint('enrich', __name__)
observables_schema = ObservableSchema(many=True)
def validate_relay_input():
relay_input = request.get_json(force=True, silent=True, cache=False)
error = observables_schema.validate(relay_input) or None
if error:
relay_input = None
# Mimic the GSB API error response payload.
error = {
'code': HTTPStatus.BAD_REQUEST,
'message': f'Invalid JSON payload received. {json.dumps(error)}.',
'details': error,
'status': 'INVALID_ARGUMENT',
}
def observe_observables():
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
api_key = get_jwt()
client = URLScanClient(
base_url=current_app.config['URL_SCAN_API_URL'],
api_key=api_key,
user_agent=current_app.config['USER_AGENT'],
observable_types=current_app.config['URL_SCAN_OBSERVABLE_TYPES'])
g.sightings = []
g.judgements = []
g.indicators = []
g.relationships = []
for observable in observables:
output = client.get_search_data(observable)
if output:
output['relationships'] = {}
search_results = output['results']
search_results.sort(key=lambda x: x['task']['time'], reverse=True)
if len(search_results) > current_app.config['CTR_ENTITIES_LIMIT']:
search_results = \
search_results[:current_app.config['CTR_ENTITIES_LIMIT']]
workers_number = min((os.cpu_count() or 1) * 5,
len(search_results) or 1)
with ThreadPoolExecutor(max_workers=workers_number) as executor:
result_outputs = \
executor.map(client.get_result_data, search_results)
for search_result in search_results:
g.sightings.append(extract_sighting(output, search_result))
result_output = next(result_outputs)
if result_output and \
result_output['verdicts']['overall']['malicious']:
g.judgements.append(
extract_judgement(output, result_output))
for category in \
result_output['verdicts']['overall']['categories']:
if not output['relationships'].get(category):
g.indicators.append(
extract_indicator(result_output, category))
output['relationships'][category] = {
'sighting_ids': [g.sightings[-1]['id']],
'indicator_id': g.indicators[-1]['id']
}
else:
output['relationships'][category][
'sighting_ids'].append(g.sightings[-1]['id'])
g.relationships.extend(
extract_relationships(output['relationships']))
relay_output = {}
if g.sightings:
relay_output['sightings'] = format_docs(g.sightings)
if g.judgements:
relay_output['judgements'] = format_docs(g.judgements)
if g.indicators:
relay_output['indicators'] = format_docs(g.indicators)
if g.relationships:
relay_output['relationships'] = format_docs(g.relationships)
return jsonify_data(relay_output)
