def process_report_data(self, report_data, number_of_days_verdict_valid,
number_of_days_judgement_valid,
number_of_days_indicator_valid, limit):
bundle = Bundle()
bundle.add(
self.extract_verdict(report_data, number_of_days_verdict_valid))
bundle.add(
self._judgement(report_data['result']['score'],
number_of_days_judgement_valid))
categories = report_data.get('result', {}).get('cats', {})
categories = sorted(categories.items())[:limit]
for category, flag in categories:
s = self._sighting(category)
i = self._indicator(category,
number_of_days_indicator_valid,
flag=flag)
bundle.add(s)
bundle.add(i)
bundle.add(self._relationship(s, i, 'sighting-of'))
return bundle
def process_resolutions(self, resolutions):
bundle = Bundle()
related = self._extract_related(resolutions)
if related:
related = sorted(related)
description = self._resolution_description()
sighting = self._sighting(DNS_INFORMATION_CATEGORY,
description=description)
sighting['relations'] = [self._resolved_to(r) for r in related]
bundle.add(sighting)
return bundle
def deliberate(observable):
output = get_cyberprotect_outputs(observable)
result = Bundle()
if output:
scores = output.get('scores', [])
if len(scores) >= current_app.config['CTR_ENTITIES_LIMIT']:
scores = scores[:current_app.config['CTR_ENTITIES_LIMIT']]
for score in scores:
# need to check because [[]] can be returned in output
if score and score.get('score') is not None:
result.add(extract_verdicts(output, score))
return result
def observe_observables():
observables = get_observables()
g.bundle = Bundle()
key = get_key()
if key is None:
raise AuthenticationRequiredError
url = current_app.config['AVOTX_URL']
headers = {'User-Agent': current_app.config['CTR_USER_AGENT']}
client = Client(key, url, headers=headers)
limit = current_app.config['CTR_ENTITIES_LIMIT']
for observable in observables:
observable = Observable.instance_for(**observable)
if observable is None:
continue
bundle = observable.observe(client, limit=limit)
g.bundle |= bundle
data = g.bundle.json()
return jsonify_data(data)
def observe_observables():
input_observables = get_observables()
g.bundle = Bundle()
key = get_key()
if key is None:
raise AuthenticationRequiredError
url = current_app.config['AVOTX_URL']
headers = {'User-Agent': current_app.config['CTR_USER_AGENT']}
client = Client(key, url, headers=headers)
limit = current_app.config['CTR_ENTITIES_LIMIT']
prepared_observables = []
for input_observable in input_observables:
prepared_observable = Observable.instance_for(**input_observable)
if prepared_observable is not None:
prepared_observables.append(prepared_observable)
def make_bundle(observable):
return observable.observe(client, limit=limit)
with ThreadPoolExecutor(
max_workers=get_workers(prepared_observables)) as executor:
bundles = executor.map(make_bundle, prepared_observables)
for bundle in bundles:
g.bundle |= bundle
data = g.bundle.json()
return jsonify_data(data)
def observe_observables():
observables, error = get_observables()
if error:
return jsonify_errors(error)
observable_types = current_app.config['GTI_OBSERVABLE_TYPES']
observables = [
observable
for observable in observables
if observable['type'] in observable_types
]
key = get_key()
bundle = Bundle()
for observable in observables:
events, error = get_events_for_observable(key, observable)
if error:
# Make sure not to lose any data processed so far.
return jsonify_errors(error, data=bundle.json())
indicator_by_rule_uuid = {}
for event in events:
sighting = Sighting.map(event)
bundle.add(sighting)
if 'detection' in event:
rule = event['detection']['rule']
indicator = indicator_by_rule_uuid.get(rule['uuid'])
if indicator is None:
indicator = Indicator.map(rule)
indicator_by_rule_uuid[rule['uuid']] = indicator
bundle.add(indicator)
relationship = Relationship.map(sighting, indicator)
bundle.add(relationship)
data = bundle.json()
return jsonify_data(data)
def observe_observables():
observables, error = get_observables()
if error:
return jsonify_errors(error)
emails = [
observable['value'] for observable in observables
if observable['type'] == 'email'
]
key = get_key()
bundle = Bundle()
limit = current_app.config['CTR_ENTITIES_LIMIT']
for email in emails:
breaches, error = fetch_breaches(key, email)
if error:
return jsonify_errors(error, data=bundle.json())
breaches.sort(key=itemgetter('BreachDate'), reverse=True)
breaches = breaches[:limit]
source_uri = current_app.config['HIBP_UI_URL'].format(
email=quote(email, safe=''))
for breach in breaches:
indicator = Indicator.map(breach)
sighting = Sighting.map(breach, email, source_uri)
relationship = Relationship.map(indicator, sighting)
bundle.add(indicator)
bundle.add(sighting)
bundle.add(relationship)
data = bundle.json()
return jsonify_data(data)
def observe_observables():
credentials = get_credentials()
observables = get_observables()
client = XForceClient(current_app.config['API_URL'], credentials,
current_app.config['USER_AGENT'])
ui_url = current_app.config['UI_URL']
number_of_days_verdict_valid = int(
current_app.config['NUMBER_OF_DAYS_VERDICT_IS_VALID'])
number_of_days_judgement_valid = int(
current_app.config['NUMBER_OF_DAYS_JUDGEMENT_IS_VALID'])
number_of_days_indicator_valid = int(
current_app.config['NUMBER_OF_DAYS_INDICATOR_IS_VALID'])
limit = current_app.config['CTR_ENTITIES_LIMIT']
g.bundle = Bundle()
try:
for observable in observables:
refer_link = client.refer_link(ui_url, observable)
mapping = Mapping.for_(observable, source_uri=refer_link)
if mapping:
if limit > 0:
report = client.report(observable)
if report:
report_bundle = mapping.process_report_data(
report, number_of_days_verdict_valid,
number_of_days_judgement_valid,
number_of_days_indicator_valid, limit)
limit -= len(report_bundle.get(SIGHTING))
g.bundle.merge(report_bundle)
if limit > 0 and isinstance(mapping, DNSInformationMapping):
resolutions_bundle = mapping.process_resolutions(
client.resolve(observable))
limit -= len(resolutions_bundle.get(SIGHTING))
g.bundle.merge(resolutions_bundle)
if limit > 0:
api_linkage = client.api_linkage(observable)
if api_linkage:
api_linkage_bundle = mapping.process_api_linkage(
api_linkage, ui_url,
number_of_days_indicator_valid, limit)
g.bundle.merge(api_linkage_bundle)
except KeyError:
add_error(XForceKeyError())
return jsonify_result()
def observe_observables():
_ = get_jwt()
relay_input = get_json(ObservableSchema(many=True))
observables = group_observables(relay_input)
if not observables:
return jsonify_data({})
g.bundle = Bundle()
for observable in observables:
g.bundle.merge(observe(observable))
return jsonify_data(g.bundle.json())
def deliberate_observables():
def deliberate(observable):
mapping = Mapping.for_(observable)
client_data = client.report(observable)
if client_data:
return mapping.extract_verdict(client_data,
number_of_days_verdict_valid)
credentials = get_credentials()
observables = get_observables()
observables = [
ob for ob in observables if ob['type'] in XFORCE_OBSERVABLE_TYPES
]
client = XForceClient(current_app.config['API_URL'], credentials,
current_app.config['USER_AGENT'])
number_of_days_verdict_valid = int(
current_app.config['NUMBER_OF_DAYS_VERDICT_IS_VALID'])
g.bundle = Bundle()
try:
with ThreadPoolExecutor(
max_workers=min(len(observables) or 1, (cpu_count() or 1) *
5)) as executor:
iterator = executor.map(deliberate, observables)
g.bundle = Bundle(
*[verdict for verdict in iterator if verdict is not None])
except KeyError:
add_error(XForceKeyError())
return jsonify_result()
def process_report_data(self, report_data, number_of_days_verdict_valid,
number_of_days_judgement_valid, *args):
bundle = Bundle()
bundle.add(
self.extract_verdict(report_data, number_of_days_verdict_valid))
bundle.add(
self._judgement(
report_data.get('malware', {}).get('risk'),
number_of_days_judgement_valid))
return bundle
def observe_observables():
relay_input, error = validate_relay_input()
if error:
return jsonify_errors(error)
observables = group_observables(relay_input)
if not observables:
# Optimize a bit by not sending empty requests to the GSB API.
return jsonify_data({})
bundle = Bundle()
start_time = datetime.utcnow()
# Split the data into chunks and make multiple requests to the GSB API.
size = current_app.config['GSB_API_MAX_THREAT_ENTRIES_PER_REQUEST']
for observables in map(dict, chunks(observables.items(), size)):
gsb_input = build_gsb_input(observables)
gsb_output, error = fetch_gsb_output(gsb_input)
if error:
return jsonify_errors(error, data=bundle.json())
matches = group_matches(gsb_output)
# Extract judgements first in order to label each match with some
# "judgement_id", so that it can be extracted for each verdict later.
judgements = extract_judgements(observables, matches, start_time)
verdicts = extract_verdicts(observables, matches, start_time)
for entity in chain(judgements, verdicts):
bundle.add(entity)
relay_output = bundle.json()
return jsonify_data(relay_output)
def deliberate_observables():
relay_input, error = validate_relay_input()
if error:
return jsonify_errors(error)
observables = group_observables(relay_input)
if not observables:
# Optimize a bit by not sending empty requests to the GSB API.
return jsonify_data({})
bundle = Bundle()
start_time = datetime.utcnow()
# Split the data into chunks and make multiple requests to the GSB API.
size = current_app.config['GSB_API_MAX_THREAT_ENTRIES_PER_REQUEST']
for observables in map(dict, chunks(observables.items(), size)):
gsb_input = build_gsb_input(observables)
gsb_output, error = fetch_gsb_output(gsb_input)
if error:
return jsonify_errors(error, data=bundle.json())
matches = group_matches(gsb_output)
verdicts = extract_verdicts(observables, matches, start_time)
for entity in verdicts:
bundle.add(entity)
relay_output = bundle.json()
return jsonify_data(relay_output)
def observe(observable):
output = get_cyberprotect_outputs(observable)
result = Bundle()
if output:
scores = output.get('scores', [])
if len(scores) >= current_app.config['CTR_ENTITIES_LIMIT']:
scores = scores[:current_app.config['CTR_ENTITIES_LIMIT']]
for score in scores:
# need to check because [[]] can be returned in output
if score and score.get('score') is not None:
result.add(extract_verdicts(output, score))
details = score['details']
if len(details) >= \
current_app.config['CTR_ENTITIES_LIMIT']:
details = \
details[:current_app.config['CTR_ENTITIES_LIMIT']]
for detail in details:
if detail.get('score') is not None:
result.add(extract_judgement(output, detail))
return result
def observe(self, client: Client, limit: Optional[int] = None) -> Bundle:
"""Build a CTIM bundle for the current observable."""
bundle = Bundle()
# Implement a workaround instead of using the "/api/v1/search/pulses"
# endpoint as it works too slow and is not really optimizable...
category = {
'ip': 'IPv4',
'ipv6': 'IPv6',
}.get(
self.type(),
self.category(),
)
endpoint = (f'/api/v1/indicators/{category}/'
f"{quote(self.value, safe='@:')}/general")
data = client.query(endpoint)
if data is None:
return bundle
# Make sure to filter out redundant pulses that do not match anyway.
pulses = [
pulse for pulse in data['pulse_info']['pulses']
if data['base_indicator']['type'] in pulse['indicator_type_counts']
]
if not pulses:
return bundle
def indicator_for(pulse, page=1):
# This limit provides a decent tradeoff between the number of
# requests to be made and the size of each response coming back.
limit = 10000
endpoint = f"/api/v1/pulses/{pulse['id']}/indicators"
params = {'sort': '-created', 'limit': limit, 'page': page}
data = client.query(endpoint, params=params)
for indicator in data['results']:
if indicator['indicator'] == self.value:
return indicator
if data['next'] is None:
return None
return indicator_for(pulse, page=(page + 1))
with ThreadPoolExecutor(max_workers=get_workers(pulses)) as executor:
iterator = executor.map(indicator_for, pulses)
indicators = []
while True:
try:
indicator = next(iterator)
except RelayError:
continue
except StopIteration:
break
else:
if indicator is None:
continue
indicators.append(indicator)
indicators.sort(key=itemgetter('created'), reverse=True)
if limit is None:
limit = len(indicators)
indicators = indicators[:limit]
observable = self.json()
for indicator in indicators:
pulse = next(pulse for pulse in pulses
if pulse['id'] == indicator['pulse_key'])
# Enrich each AVOTX pulse with some additional context in order to
# simplify further mapping of that pulse into CTIM entities.
pulse['indicator'] = indicator
pulse['observable'] = observable
pulse['url'] = client.url
sighting = Sighting.map(pulse)
indicator = Indicator.map(pulse)
relationship = Relationship.map(sighting, indicator)
bundle.add(sighting)
bundle.add(indicator)
bundle.add(relationship)
return bundle
def observe(self, client: Client, limit: Optional[int] = None) -> Bundle:
# The AVOTX API does not support searching for email addresses as the
# "/api/v1/indicators/email/{email}/general" endpoint does not exist.
return Bundle()
def process_report_data(self, report_data, number_of_days_verdict_valid,
number_of_days_judgement_valid,
number_of_days_indicator_valid, limit):
bundle = Bundle()
bundle.add(
self.extract_verdict(report_data, number_of_days_verdict_valid))
categories = sorted(report_data.get('cats', {}).items())[:limit]
for category, score in categories:
sighting = self._sighting(category)
indicator = self._indicator(category,
number_of_days_indicator_valid)
judgement = self._judgement(score / 10,
number_of_days_judgement_valid)
bundle.add(sighting)
bundle.add(indicator)
bundle.add(judgement)
bundle.add(self._relationship(sighting, judgement, 'based-on'))
bundle.add(self._relationship(judgement, indicator, 'based-on'))
return bundle
def process_api_linkage(self, api_linkage_data, ui_url,
number_of_days_indicator_valid, limit):
linked_entities = api_linkage_data.get('linkedEntities', [])
external_references_map = {
entity['id']: {
'source_name':
SOURCE,
'external_id':
entity['id'],
'url':
urljoin(ui_url,
f'/collection/{entity["title"]}-{entity["id"]}')
}
for entity in linked_entities
}
external_ids = list(external_references_map.keys())
external_references = list(external_references_map.values())
def sighting(entity):
return {
**SIGHTING_DEFAULTS,
'id': transient_id(SIGHTING, entity['id']),
'observed_time': {
'start_time': entity['created'],
'end_time': entity['created'],
},
# Original values: "1owned", "2shared", "3public", "4premier"
'internal': entity['category'] == '1owned',
'observables': [self.observable],
'title': f'Contained in Collection: {entity["title"]}',
}
def indicator(entity):
return {
**INDICATOR_DEFAULTS,
'id': transient_id(INDICATOR, entity['id']),
'producer': entity['owner']['name'],
'valid_time': self._valid_time(number_of_days_indicator_valid),
'title': entity["title"],
}
linked_entities = sorted(linked_entities,
key=itemgetter('created'),
reverse=True)[:limit]
bundle = Bundle()
for entity in linked_entities:
external_reference = external_references_map[entity['id']]
common_value = {
'external_ids': external_ids,
'external_references': external_references,
'source_uri': external_reference['url'],
}
s = {**sighting(entity), **common_value}
i = {**indicator(entity), **common_value}
bundle.add(s)
bundle.add(i)
bundle.add(self._relationship(s, i, 'member-of'))
return bundle