def observe_observables():
http_client = get_chronicle_http_client(get_jwt())
chronicle_client = ChronicleClient(current_app.config['API_URL'],
http_client)
observables = get_observables()
limit = current_app.config['CTR_ENTITIES_LIMIT']
time_delta = current_app.config[
'DEFAULT_NUMBER_OF_DAYS_FOR_CHRONICLE_TIME_FILTER']
g.sightings = []
g.indicators = []
g.relationships = []
for x in observables:
mapping = Mapping.for_(x)
if mapping:
assets_data = chronicle_client.list_assets(x, time_delta, limit)
ioc_details = chronicle_client.list_ioc_details(x)
x_sightings = mapping.extract_sightings(assets_data, limit)
x_indicators = mapping.extract_indicators(ioc_details, limit)
g.sightings.extend(x_sightings)
g.indicators.extend(x_indicators)
g.relationships.extend(
mapping.create_relationships(x_sightings, x_indicators))
return jsonify_result()
def observe_observables():
key = get_key()
observables = get_observables()
client = SecurityTrailsClient(current_app.config['API_URL'],
key,
current_app.config['USER_AGENT'],
current_app.config['NUMBER_OF_PAGES'],
current_app.config['GET_ALL_PAGES'])
g.sightings = []
try:
for observable in observables:
mapping = Mapping.for_(observable)
if mapping:
client_data = client.get_data(observable)
for record in client_data:
refer_link = client.refer_link(
current_app.config['UI_URL'], observable
)
sighting = mapping.extract_sighting(record, refer_link)
if sighting:
g.sightings.append(sighting)
except KeyError:
g.errors = [{
'type': 'fatal',
'code': 'key error',
'message': 'The data structure of SecurityTrails '
'has changed. The module is broken.'
}]
return jsonify_result()
def observe_observables():
client = Auth0SignalsClient(get_jwt())
observables = get_observables()
g.verdicts = []
g.judgements = []
g.sightings = []
g.indicators = []
g.relationships = []
for observable in observables:
if observable['type'] == 'ip':
response_data = client.get_auth0_response(observable)
if response_data:
g.verdicts.append(extract_verdict(response_data, observable))
g.judgements.extend(
extract_judgements(response_data, observable))
details = client.get_full_details(response_data)
sightings = extract_sightings(observable, details)
g.sightings.extend(sightings)
indicators = extract_indicators(details)
g.indicators.extend(indicators)
g.relationships.extend(
extract_relationships(sightings, indicators))
return jsonify_result()
def deliberate_observables():
client = Auth0SignalsClient(get_jwt())
observables = get_observables()
g.verdicts = []
for observable in observables:
if observable['type'] == 'ip':
response_data = client.get_auth0_response(observable)
if response_data:
g.verdicts.append(extract_verdict(response_data, observable))
return jsonify_result()
def observe_observables():
credentials = get_credentials()
observables = get_observables()
client = XForceClient(current_app.config['API_URL'], credentials,
current_app.config['USER_AGENT'])
ui_url = current_app.config['UI_URL']
number_of_days_verdict_valid = int(
current_app.config['NUMBER_OF_DAYS_VERDICT_IS_VALID'])
number_of_days_judgement_valid = int(
current_app.config['NUMBER_OF_DAYS_JUDGEMENT_IS_VALID'])
number_of_days_indicator_valid = int(
current_app.config['NUMBER_OF_DAYS_INDICATOR_IS_VALID'])
limit = current_app.config['CTR_ENTITIES_LIMIT']
g.bundle = Bundle()
try:
for observable in observables:
refer_link = client.refer_link(ui_url, observable)
mapping = Mapping.for_(observable, source_uri=refer_link)
if mapping:
if limit > 0:
report = client.report(observable)
if report:
report_bundle = mapping.process_report_data(
report, number_of_days_verdict_valid,
number_of_days_judgement_valid,
number_of_days_indicator_valid, limit)
limit -= len(report_bundle.get(SIGHTING))
g.bundle.merge(report_bundle)
if limit > 0 and isinstance(mapping, DNSInformationMapping):
resolutions_bundle = mapping.process_resolutions(
client.resolve(observable))
limit -= len(resolutions_bundle.get(SIGHTING))
g.bundle.merge(resolutions_bundle)
if limit > 0:
api_linkage = client.api_linkage(observable)
if api_linkage:
api_linkage_bundle = mapping.process_api_linkage(
api_linkage, ui_url,
number_of_days_indicator_valid, limit)
g.bundle.merge(api_linkage_bundle)
except KeyError:
add_error(XForceKeyError())
return jsonify_result()
def observe_observables():
misp = create_misp_instance()
observables = filter_observables(get_observables())
g.verdicts = []
g.judgements = []
g.sightings = []
g.indicators = []
g.relationships = []
for observable in observables:
mapping = Mapping(observable)
# This check was added due to the fact that MISP
# returns all data in case of searching for spaces.
if not observable['value'].strip():
continue
events = misp.search(value=observable['value'],
metadata=False,
limit=current_app.config['CTR_ENTITIES_LIMIT'])
events.sort(key=lambda elem: elem['Event']['threat_level_id'])
judgements_for_observable = []
for event in events:
judgement = mapping.extract_judgement(event['Event'])
judgements_for_observable.append(judgement)
sighting = mapping.extract_sighting(event['Event'])
g.sightings.append(sighting)
indicator = mapping.extract_indicator(event['Event'])
g.indicators.append(indicator)
g.relationships.append(
mapping.extract_relationship(
sighting['id'], indicator['id'],
current_app.config['MEMBER_OF_RELATION']))
g.relationships.append(
mapping.extract_relationship(
judgement['id'], indicator['id'],
current_app.config['ELEMENT_OF_RELATION']))
if judgements_for_observable:
g.judgements.extend(judgements_for_observable)
verdict = mapping.extract_verdict(events[0]['Event'])
verdict['judgement_id'] = judgements_for_observable[0]['id']
g.verdicts.append(verdict)
return jsonify_result()
def observe_observables():
def query_qradar(obs):
query_with_observables = arial_query.build(
observable=obs,
names=names,
time=get_time_intervals(),
limit=current_app.config['CTR_ENTITIES_LIMIT'])
if not query_with_observables:
return jsonify_data({})
search_id, status = api_client.create_search(query_with_observables)
start_time = time()
while status != 'COMPLETED':
if (time() - start_time <
current_app.config['SEARCH_TIMOUT_IN_SEC']):
status = api_client.get_search(search_id)['status']
else:
raise QRadarTimeoutError
return api_client.get_search_results(search_id)['events']
relay_input = get_observables()
observables = filter_observables(relay_input)
if not observables:
return jsonify_data({})
api_client = ArielSearchClient(get_credentials(), current_app.config)
arial_query = ArielQuery()
params = {
'fields': 'columns (name)',
'filter': 'object_value_type = "Host"'
}
names = api_client.get_metadata(params)
g.sightings = []
for observable in observables:
response = query_qradar(observable)
for event in response:
mapping = Mapping()
g.sightings.append(mapping.sighting(observable, event))
return jsonify_result()
def observe_observables():
payload = get_jwt()
client = APIVoidClient(payload)
observables = get_observables()
g.sightings = []
g.indicators = []
g.relationships = []
for observable in observables:
output = client.get_data(observable)
if output:
for engine in get_engines(output):
sighting = extract_sighting(observable, engine)
g.sightings.append(sighting)
indicator = extract_indicator(engine)
g.indicators.append(indicator)
g.relationships.append(
extract_relationship(sighting['id'], indicator['id']))
return jsonify_result()
def respond_trigger():
add_status('failure')
params = get_action_form_params()
client = ReferenceDataClient(get_credentials(), current_app.config)
action_map = {
ADD_ACTION_ID: client.add_to_reference_set,
REMOVE_ACTION_ID: client.remove_from_reference_set,
ADD_AND_CRETE_ACTION_ID: client.add_and_create_reference_set
}
action = action_map.get(params['action-id'])
if not action:
raise InvalidArgumentError('Unsupported action.')
action(params['reference_set_name'], params['observable_value'])
add_status('success')
return jsonify_result()
def respond_trigger():
add_status('failure')
params = get_action_form_params()
credentials = get_jwt()
client = AkamaiClient(credentials, current_app.config['USER_AGENT'])
action_map = {
ADD_ACTION_ID: client.add_to_network_list,
REMOVE_ACTION_ID: client.remove_from_network_list
}
action = action_map.get(params['action-id'])
if not action:
raise InvalidArgumentError("Unsupported action.")
action(params['network_list_id'], params['observable_value'])
add_status('success')
return jsonify_result()
def deliberate_observables():
misp = create_misp_instance()
observables = filter_observables(get_observables())
g.verdicts = []
for observable in observables:
mapping = Mapping(observable)
events = misp.search(value=observable['value'], metadata=False)
events.sort(key=lambda elem: elem['Event']['threat_level_id'])
# We sort events in order to create a single Verdict for a set
# of events based on the fact that High threat level has
# priority over all others, then Medium threat level,
# and so on down to Undefined.
if events:
g.verdicts.append(mapping.extract_verdict(events[0]['Event']))
return jsonify_result()
def observe_observables():
key = get_key()
observables = get_observables()
client = FarsightClient(current_app.config['API_URL'],
key,
current_app.config['USER_AGENT'])
g.sightings = []
limit = current_app.config['CTR_ENTITIES_LIMIT']
aggr = current_app.config['AGGREGATE']
time_delta = (current_app.config['NUMBER_OF_DAYS_FOR_FARSIGHT_TIME_FILTER']
if aggr else None)
url_template = current_app.config['UI_SEARCH_URL']
try:
for x in observables:
mapping = Mapping.for_(x)
if mapping:
lookup_data = client.lookup(x, time_delta)
if lookup_data:
refer_link = url_template.format(query=x['value'])
g.sightings.extend(
mapping.extract_sightings(
lookup_data, refer_link, limit, aggr
)
)
except KeyError:
g.errors = [{
'type': 'fatal',
'code': 'key error',
'message': 'The data structure of Farsight DNSDB '
'has changed. The module is broken.'
}]
return jsonify_result()
def observe_observables():
key = get_jwt().get('key', '')
client = C1fAppClient(key)
observables = get_observables()
g.sightings = []
g.indicators = []
g.relationships = []
limit = current_app.config['CTR_ENTITIES_LIMIT']
for observable in observables:
mapping = Mapping.for_(observable)
if mapping:
response_data = client.get_c1fapp_response(observable['value'])
response_data.sort(key=lambda x: x['reportime'], reverse=True)
response_data = response_data[:limit]
g.sightings.extend(mapping.extract_sightings(response_data))
g.indicators.extend(mapping.extract_indicators(response_data))
g.relationships.extend(mapping.extract_relationships())
return jsonify_result()
def deliberate_observables():
def deliberate(observable):
mapping = Mapping.for_(observable)
client_data = client.report(observable)
if client_data:
return mapping.extract_verdict(client_data,
number_of_days_verdict_valid)
credentials = get_credentials()
observables = get_observables()
observables = [
ob for ob in observables if ob['type'] in XFORCE_OBSERVABLE_TYPES
]
client = XForceClient(current_app.config['API_URL'], credentials,
current_app.config['USER_AGENT'])
number_of_days_verdict_valid = int(
current_app.config['NUMBER_OF_DAYS_VERDICT_IS_VALID'])
g.bundle = Bundle()
try:
with ThreadPoolExecutor(
max_workers=min(len(observables) or 1, (cpu_count() or 1) *
5)) as executor:
iterator = executor.map(deliberate, observables)
g.bundle = Bundle(
*[verdict for verdict in iterator if verdict is not None])
except KeyError:
add_error(XForceKeyError())
return jsonify_result()
def handle_tr_formatted_error(error):
add_error(error)
app.logger.error(error.json)
return jsonify_result()
def handle_tr_formatted_error(error):
app.logger.error(error.json)
g.errors = [error.json]
return jsonify_result()
def handle_tr_formatted_error(error):
app.logger.error(traceback.format_exc())
add_error(error)
return jsonify_result()
def handle_tr_formatted_error(error):
g.errors = [error.json]
return jsonify_result()
