def observe_observables():
observables, error = get_observables()
if error:
return jsonify_errors(error)
observable_types = current_app.config['GTI_OBSERVABLE_TYPES']
observables = [
observable
for observable in observables
if observable['type'] in observable_types
]
key = get_key()
bundle = Bundle()
for observable in observables:
events, error = get_events_for_observable(key, observable)
if error:
# Make sure not to lose any data processed so far.
return jsonify_errors(error, data=bundle.json())
indicator_by_rule_uuid = {}
for event in events:
sighting = Sighting.map(event)
bundle.add(sighting)
if 'detection' in event:
rule = event['detection']['rule']
indicator = indicator_by_rule_uuid.get(rule['uuid'])
if indicator is None:
indicator = Indicator.map(rule)
indicator_by_rule_uuid[rule['uuid']] = indicator
bundle.add(indicator)
relationship = Relationship.map(sighting, indicator)
bundle.add(relationship)
data = bundle.json()
return jsonify_data(data)
def observe_observables():
observables, error = get_observables()
if error:
return jsonify_errors(error)
emails = [
observable['value'] for observable in observables
if observable['type'] == 'email'
]
key = get_key()
bundle = Bundle()
limit = current_app.config['CTR_ENTITIES_LIMIT']
for email in emails:
breaches, error = fetch_breaches(key, email)
if error:
return jsonify_errors(error, data=bundle.json())
breaches.sort(key=itemgetter('BreachDate'), reverse=True)
breaches = breaches[:limit]
source_uri = current_app.config['HIBP_UI_URL'].format(
email=quote(email, safe=''))
for breach in breaches:
indicator = Indicator.map(breach)
sighting = Sighting.map(breach, email, source_uri)
relationship = Relationship.map(indicator, sighting)
bundle.add(indicator)
bundle.add(sighting)
bundle.add(relationship)
data = bundle.json()
return jsonify_data(data)
def observe_observables():
relay_input, error = validate_relay_input()
if error:
return jsonify_errors(error)
observables = group_observables(relay_input)
if not observables:
# Optimize a bit by not sending empty requests to the GSB API.
return jsonify_data({})
bundle = Bundle()
start_time = datetime.utcnow()
# Split the data into chunks and make multiple requests to the GSB API.
size = current_app.config['GSB_API_MAX_THREAT_ENTRIES_PER_REQUEST']
for observables in map(dict, chunks(observables.items(), size)):
gsb_input = build_gsb_input(observables)
gsb_output, error = fetch_gsb_output(gsb_input)
if error:
return jsonify_errors(error, data=bundle.json())
matches = group_matches(gsb_output)
# Extract judgements first in order to label each match with some
# "judgement_id", so that it can be extracted for each verdict later.
judgements = extract_judgements(observables, matches, start_time)
verdicts = extract_verdicts(observables, matches, start_time)
for entity in chain(judgements, verdicts):
bundle.add(entity)
relay_output = bundle.json()
return jsonify_data(relay_output)
def deliberate_observables():
relay_input, error = validate_relay_input()
if error:
return jsonify_errors(error)
observables = group_observables(relay_input)
if not observables:
# Optimize a bit by not sending empty requests to the GSB API.
return jsonify_data({})
bundle = Bundle()
start_time = datetime.utcnow()
# Split the data into chunks and make multiple requests to the GSB API.
size = current_app.config['GSB_API_MAX_THREAT_ENTRIES_PER_REQUEST']
for observables in map(dict, chunks(observables.items(), size)):
gsb_input = build_gsb_input(observables)
gsb_output, error = fetch_gsb_output(gsb_input)
if error:
return jsonify_errors(error, data=bundle.json())
matches = group_matches(gsb_output)
verdicts = extract_verdicts(observables, matches, start_time)
for entity in verdicts:
bundle.add(entity)
relay_output = bundle.json()
return jsonify_data(relay_output)