def process_incidents(config, src, dest, incidents):
"""handle incoming stix incidents"""
xmlns_name = config["edge"]["sites"][src]["stix"]["xmlns_name"]
status_trans = {"New": "New", "Open": "In Progress", "Closed": "Analyzed", "Rejected": "Deprecated"}
for i in incidents.keys():
json = dict()
json["event_type"] = "Threat Report"
json["title"] = incidents[i].title
json["description"] = util.rgetattr(incidents[i], ["description", "value"])
json["status"] = status_trans[incidents[i].status.value]
# inbox the incident (we need to crits id!)
config["edge_tally"]["events"]["incoming"] += 1
config["edge_tally"]["all"]["incoming"] += 1
(crits_event_id, success) = crits.crits_inbox(config, dest, "events", json, src=src)
if not success:
config["logger"].error(
log.log_messages["obj_inbox_error"].format(
src_type="edge", id_=i, dest_type="crits events api endpoint"
)
)
continue
else:
# successfully inboxed event...
config["edge_tally"]["events"]["processed"] += 1
config["edge_tally"]["all"]["processed"] += 1
if config["daemon"]["debug"]:
config["logger"].debug(
log.log_messages["obj_inbox_success"].format(
src_type="edge", id_=i, dest_type="crits events api endpoint"
)
)
# as we've now successfully processed the event, track
# the related crits/json ids (by src/dest)
if util.rgetattr(incidents[i], ["related_observables"]) and len(incidents[i].related_observables):
for j in incidents[i].related_observables:
if util.rgetattr(j, ["item", "idref"]):
# store the pending relationship in the db for
# later processing
config["db"].set_pending_crits_link(
src, dest, lhs_id=(xmlns_name + ":" + "events" + "-" + crits_event_id), rhs_id=j.item.idref
)
if util.rgetattr(incidents[i], ["related_indicators"]) and len(incidents[i].related_indicators):
for j in incidents[i].related_indicators:
if util.rgetattr(j, ["item", "idref"]):
# store the pending relationship in the db for
# later processing
config["db"].set_pending_crits_link(
src, dest, lhs_id=(xmlns_name + ":" + "events" + "-" + crits_event_id), rhs_id=j.item.idref
)
if util.rgetattr(incidents[i], ["related_incidents"]) and len(incidents[i].related_incidents):
for j in incidents[i].related_incidents:
if util.rgetattr(j, ["item", "idref"]):
# store the pending relationship in the db for
# later processing
config["db"].set_pending_crits_link(
src, dest, lhs_id=(xmlns_name + ":" + "events" + "-" + crits_event_id), rhs_id=j.item.idref
)
config["db"].set_object_id(src, dest, edge_id=i, crits_id=(xmlns_name + ":" + "events" + "-" + crits_event_id))
def process_observables(config, src, dest, observables):
'''handle incoming cybox observables and observable compositions'''
# TODO some of the hailataxii date uses the cybox ###comma###
# construct, which is currently unsupported
for o_id, o in observables.iteritems():
json = dict()
if util.rgetattr(o, ['observable_composition']) \
and not util.rgetattr(o, ['object_']):
# it's an observable composition
# store it in the db...maybe the indicator will only come
# across in a subsequent run so we can't rely on passing
# this around in memory
config['db'].store_obs_comp(src, dest,
obs_id=o_id,
obs_comp=o.observable_composition)
continue
elif util.rgetattr(o, ['object_']):
# it's a normal observable
(json, endpoint) = cybox_observable_to_json(config, o)
if not json:
config['logger'].error(
log.log_messages[
'obj_convert_error'].format(src_type='cybox',
src_obj='observable',
id_=o_id,
dest_type='crits',
dest_obj='json'))
continue
# inbox the observable to crits
config['edge_tally'][endpoint]['incoming'] += 1
config['edge_tally']['all']['incoming'] += 1
(id_, success) = \
crits.crits_inbox(config, dest, endpoint, json,
src=src, edge_id=o_id)
if not success:
config['logger'].error(
log.log_messages['obj_inbox_error'].format(
src_type='edge', id_=o, dest_type='crits ' + endpoint + ' api endpoint'))
continue
# successfully inboxed observable
config['edge_tally'][endpoint]['processed'] += 1
config['edge_tally']['all']['processed'] += 1
if config['daemon']['debug']:
config['logger'].debug(
log.log_messages['obj_inbox_success'].format(
src_type='edge', id_=o_id,
dest_type='crits ' + endpoint + ' api endpoint'))
def process_relationships(config, src, dest):
'''forge the crits relationship links between incoming observables
and indicators'''
endpoint_trans = {'emails': 'Email', 'ips': 'IP',
'samples': 'Sample', 'domains': 'Domain',
'indicators': 'Indicator', 'events': 'Event'}
pending_crits_links = config['db'].get_pending_crits_links(src, dest)
if not pending_crits_links:
config['logger'].info(
log.log_messages['no_pending_crits_relationships'])
else:
for r in pending_crits_links:
json = dict()
json['left_type'] = endpoint_trans[r['lhs_id'].split(':')[1].split('-')[0]]
json['left_id'] = r['lhs_id'].split(':')[1].split('-')[1]
# try to fetch the crits observable id corresponding to
# the edge id
rhs = config['db'].get_object_id(src, dest,
edge_id=r['rhs_id'])
if not rhs or not rhs.get('crits_id', None):
config['logger'].error(
log.log_messages['obs_comp_dereference_error'
].format(id_=r['rhs_id']))
else:
json['right_type'] = \
endpoint_trans[
rhs['crits_id'].split(':')[1].split('-')[0]]
json['right_id'] = \
rhs['crits_id'].split(':')[1].split('-')[1]
json['rel_type'] = 'Related_To'
json['rel_confidence'] = 'unknown'
config['edge_tally']['relationships']['incoming'] += 1
config['edge_tally']['all']['incoming'] += 1
(relationship_id_, success) = \
crits.crits_inbox(config, dest,
'relationships', json, src=src)
if not success:
config['logger'].error(
log.log_messages['obj_inbox_error'].format(
src_type='edge', id_=r['rhs_id'],
dest_type='crits relationships api endpoint'))
else:
# remove the pending crits relationship from the db
config['edge_tally']['relationships']['processed'] += 1
config['edge_tally']['all']['processed'] += 1
config['db'].resolve_crits_link(src, dest,
lhs_id=r['lhs_id'],
rhs_id=r['rhs_id'])
def process_observables(config, src, dest, observables):
"""handle incoming cybox observables and observable compositions"""
# TODO some of the hailataxii date uses the cybox ###comma###
# construct, which is currently unsupported
for o_id, o in observables.iteritems():
json = dict()
if util.rgetattr(o, ["observable_composition"]) and not util.rgetattr(o, ["object_"]):
# it's an observable composition
# store it in the db...maybe the indicator will only come
# across in a subsequent run so we can't rely on passing
# this around in memory
config["db"].store_obs_comp(src, dest, obs_id=o_id, obs_comp=o.observable_composition)
continue
elif util.rgetattr(o, ["object_"]):
# it's a normal observable
(json, endpoint) = cybox_observable_to_json(config, o)
if not json:
config["logger"].error(
log.log_messages["obj_convert_error"].format(
src_type="cybox", src_obj="observable", id_=o_id, dest_type="crits", dest_obj="json"
)
)
continue
# inbox the observable to crits
config["edge_tally"][endpoint]["incoming"] += 1
config["edge_tally"]["all"]["incoming"] += 1
(id_, success) = crits.crits_inbox(config, dest, endpoint, json, src=src, edge_id=o_id)
if not success:
config["logger"].error(
log.log_messages["obj_inbox_error"].format(
src_type="edge", id_=o, dest_type="crits " + endpoint + " api endpoint"
)
)
continue
# successfully inboxed observable
config["edge_tally"][endpoint]["processed"] += 1
config["edge_tally"]["all"]["processed"] += 1
if config["daemon"]["debug"]:
config["logger"].debug(
log.log_messages["obj_inbox_success"].format(
src_type="edge", id_=o_id, dest_type="crits " + endpoint + " api endpoint"
)
)
def process_relationships(config, src, dest):
"""forge the crits relationship links between incoming observables
and indicators"""
endpoint_trans = {
"emails": "Email",
"ips": "IP",
"samples": "Sample",
"domains": "Domain",
"indicators": "Indicator",
"events": "Event",
}
pending_crits_links = config["db"].get_pending_crits_links(src, dest)
if not pending_crits_links:
config["logger"].info(log.log_messages["no_pending_crits_relationships"])
else:
for r in pending_crits_links:
json = dict()
json["left_type"] = endpoint_trans[r["lhs_id"].split(":")[1].split("-")[0]]
json["left_id"] = r["lhs_id"].split(":")[1].split("-")[1]
# try to fetch the crits observable id corresponding to
# the edge id
rhs = config["db"].get_object_id(src, dest, edge_id=r["rhs_id"])
if not rhs or not rhs.get("crits_id", None):
config["logger"].error(log.log_messages["obs_comp_dereference_error"].format(id_=r["rhs_id"]))
else:
json["right_type"] = endpoint_trans[rhs["crits_id"].split(":")[1].split("-")[0]]
json["right_id"] = rhs["crits_id"].split(":")[1].split("-")[1]
json["rel_type"] = "Related_To"
json["rel_confidence"] = "unknown"
config["edge_tally"]["relationships"]["incoming"] += 1
config["edge_tally"]["all"]["incoming"] += 1
(relationship_id_, success) = crits.crits_inbox(config, dest, "relationships", json, src=src)
if not success:
config["logger"].error(
log.log_messages["obj_inbox_error"].format(
src_type="edge", id_=r["rhs_id"], dest_type="crits relationships api endpoint"
)
)
else:
# remove the pending crits relationship from the db
config["edge_tally"]["relationships"]["processed"] += 1
config["edge_tally"]["all"]["processed"] += 1
config["db"].resolve_crits_link(src, dest, lhs_id=r["lhs_id"], rhs_id=r["rhs_id"])
def process_indicators(config, src, dest, indicators):
'''handle incoming stix indicators'''
xmlns_name = config['edge']['sites'][src]['stix']['xmlns_name']
for i in indicators.keys():
json = dict()
json['type'] = 'Related_To'
json['value'] = util.rgetattr(indicators[i], ['title'],
default_='unknown')
json['indicator_confidence'] = \
util.rgetattr(indicators[i], ['confidence', 'value', 'value'],
default_='unknown')
# TODO lookup the corresponding stix prop for indicator_impact
json['indicator_impact'] = {'rating': 'unknown'}
# inbox the indicator (we need to crits id!)
config['edge_tally']['indicators']['incoming'] += 1
config['edge_tally']['all']['incoming'] += 1
(crits_indicator_id, success) = crits.crits_inbox(config, dest,
'indicators',
json, src=src)
if not success:
config['logger'].error(
log.log_messages['obj_inbox_error'].format(
src_type='edge', id_=i,
dest_type='crits indicators api endpoint'))
continue
else:
# successfully inboxed indicator...
config['edge_tally']['indicators']['processed'] += 1
config['edge_tally']['all']['processed'] += 1
if config['daemon']['debug']:
config['logger'].debug(
log.log_messages['obj_inbox_success'].format(
src_type='edge', id_=i,
dest_type='crits indicators api endpoint'))
if util.rgetattr(indicators[i], ['observables']):
for o in indicators[i].observables:
if util.rgetattr(o, ['idref']) and \
not util.rgetattr(o, ['object_']):
# TODO need to delete observable compositions from
# mongo once we've processed them
obs_comp = \
config['db'].get_obs_comp(src, dest, obs_id=o.idref)
if not obs_comp:
# [ o == embedded observable]
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'indicators' + '-' +
crits_indicator_id),
rhs_id=o.idref)
elif obs_comp:
# [o == idref observable composition]
# try to fetch the observable composition o.idref
# points to
# assumption: the observable composition was
# previously ingested. TODO what about when
# the observable composition comes in *after*
# the indicator?
observables_list = util.rgetattr(obs_comp,
['observables'])
if not observables_list:
config['logger'].error(
log.log_messages['obs_comp_dereference_error'
].format(id_=i))
continue
else:
for j in observables_list:
# store the pending relationship in
# the db for later processing
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'indicators' + '-' +
crits_indicator_id),
rhs_id=j.idref)
# TODO (need to dig up suitable sample data)
# if it's an observable composition with inline
# observables, pass them to observable composition with
# inline observables, pass them to process_observables(),
# (which will store the edge/crits id indicator pairing
# for later processing.
else:
config['logger'].error(
log.log_messages['obs_comp_dereference_error'
].format(id_=i))
continue
# as we've now successfully processed the indicator, track
# the related crits/json ids (by src/dest)
if util.rgetattr(indicators[i], ['related_indicators']) and len(indicators[i].related_indicators):
for j in indicators[i].related_indicators:
if util.rgetattr(j, ['item', 'idref']):
# store the pending relationship in the db for
# later processing
# TODO for some reason, the crits relationship api
# is rejecting _some_ (but not _all_
# indicator-to-indicator relationships. the
# indicator ids are valid and the api post looks
# correct but...sometimes this fails :-/
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'indicators' + '-' +
crits_indicator_id),
rhs_id=j.item.idref)
config['db'].set_object_id(src, dest,
edge_id=i,
crits_id=(xmlns_name + ':' + 'indicators' + '-' +
crits_indicator_id))
def process_incidents(config, src, dest, incidents):
'''handle incoming stix incidents'''
xmlns_name = config['edge']['sites'][src]['stix']['xmlns_name']
status_trans = {'New': 'New', 'Open': 'In Progress',
'Closed': 'Analyzed', 'Rejected': 'Deprecated'}
for i in incidents.keys():
json = dict()
json['event_type'] = 'Threat Report'
json['title'] = incidents[i].title
json['description'] = util.rgetattr(incidents[i], ['description', 'value'])
json['status'] = status_trans[incidents[i].status.value]
# inbox the incident (we need to crits id!)
config['edge_tally']['events']['incoming'] += 1
config['edge_tally']['all']['incoming'] += 1
(crits_event_id, success) = crits.crits_inbox(config, dest,
'events',
json, src=src)
if not success:
config['logger'].error(
log.log_messages['obj_inbox_error'].format(
src_type='edge', id_=i,
dest_type='crits events api endpoint'))
continue
else:
# successfully inboxed event...
config['edge_tally']['events']['processed'] += 1
config['edge_tally']['all']['processed'] += 1
if config['daemon']['debug']:
config['logger'].debug(
log.log_messages['obj_inbox_success'].format(
src_type='edge', id_=i,
dest_type='crits events api endpoint'))
# as we've now successfully processed the event, track
# the related crits/json ids (by src/dest)
if util.rgetattr(incidents[i], ['related_observables']) and len(incidents[i].related_observables):
for j in incidents[i].related_observables:
if util.rgetattr(j, ['item', 'idref']):
# store the pending relationship in the db for
# later processing
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'events' + '-' +
crits_event_id),
rhs_id=j.item.idref)
if util.rgetattr(incidents[i], ['related_indicators']) and len(incidents[i].related_indicators):
for j in incidents[i].related_indicators:
if util.rgetattr(j, ['item', 'idref']):
# store the pending relationship in the db for
# later processing
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'events' + '-' +
crits_event_id),
rhs_id=j.item.idref)
if util.rgetattr(incidents[i], ['related_incidents']) and len(incidents[i].related_incidents):
for j in incidents[i].related_incidents:
if util.rgetattr(j, ['item', 'idref']):
# store the pending relationship in the db for
# later processing
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'events' + '-' +
crits_event_id),
rhs_id=j.item.idref)
config['db'].set_object_id(src, dest,
edge_id=i,
crits_id=(xmlns_name + ':' + 'events' + '-' +
crits_event_id))
def process_indicators(config, src, dest, indicators):
"""handle incoming stix indicators"""
xmlns_name = config["edge"]["sites"][src]["stix"]["xmlns_name"]
for i in indicators.keys():
json = dict()
json["type"] = "Related_To"
json["value"] = util.rgetattr(indicators[i], ["title"], default_="unknown")
json["indicator_confidence"] = util.rgetattr(
indicators[i], ["confidence", "value", "value"], default_="unknown"
)
# TODO lookup the corresponding stix prop for indicator_impact
json["indicator_impact"] = {"rating": "unknown"}
# inbox the indicator (we need to crits id!)
config["edge_tally"]["indicators"]["incoming"] += 1
config["edge_tally"]["all"]["incoming"] += 1
(crits_indicator_id, success) = crits.crits_inbox(config, dest, "indicators", json, src=src)
if not success:
config["logger"].error(
log.log_messages["obj_inbox_error"].format(
src_type="edge", id_=i, dest_type="crits indicators api endpoint"
)
)
continue
else:
# successfully inboxed indicator...
config["edge_tally"]["indicators"]["processed"] += 1
config["edge_tally"]["all"]["processed"] += 1
if config["daemon"]["debug"]:
config["logger"].debug(
log.log_messages["obj_inbox_success"].format(
src_type="edge", id_=i, dest_type="crits indicators api endpoint"
)
)
if util.rgetattr(indicators[i], ["observables"]):
for o in indicators[i].observables:
if util.rgetattr(o, ["idref"]) and not util.rgetattr(o, ["object_"]):
# TODO need to delete observable compositions from
# mongo once we've processed them
obs_comp = config["db"].get_obs_comp(src, dest, obs_id=o.idref)
if not obs_comp:
# [ o == embedded observable]
config["db"].set_pending_crits_link(
src,
dest,
lhs_id=(xmlns_name + ":" + "indicators" + "-" + crits_indicator_id),
rhs_id=o.idref,
)
elif obs_comp:
# [o == idref observable composition]
# try to fetch the observable composition o.idref
# points to
# assumption: the observable composition was
# previously ingested. TODO what about when
# the observable composition comes in *after*
# the indicator?
observables_list = util.rgetattr(obs_comp, ["observables"])
if not observables_list:
config["logger"].error(log.log_messages["obs_comp_dereference_error"].format(id_=i))
continue
else:
for j in observables_list:
# store the pending relationship in
# the db for later processing
config["db"].set_pending_crits_link(
src,
dest,
lhs_id=(xmlns_name + ":" + "indicators" + "-" + crits_indicator_id),
rhs_id=j.idref,
)
# TODO (need to dig up suitable sample data)
# if it's an observable composition with inline
# observables, pass them to observable composition with
# inline observables, pass them to process_observables(),
# (which will store the edge/crits id indicator pairing
# for later processing.
else:
config["logger"].error(log.log_messages["obs_comp_dereference_error"].format(id_=i))
continue
# as we've now successfully processed the indicator, track
# the related crits/json ids (by src/dest)
if util.rgetattr(indicators[i], ["related_indicators"]) and len(indicators[i].related_indicators):
for j in indicators[i].related_indicators:
if util.rgetattr(j, ["item", "idref"]):
# store the pending relationship in the db for
# later processing
# TODO for some reason, the crits relationship api
# is rejecting _some_ (but not _all_
# indicator-to-indicator relationships. the
# indicator ids are valid and the api post looks
# correct but...sometimes this fails :-/
config["db"].set_pending_crits_link(
src,
dest,
lhs_id=(xmlns_name + ":" + "indicators" + "-" + crits_indicator_id),
rhs_id=j.item.idref,
)
config["db"].set_object_id(
src, dest, edge_id=i, crits_id=(xmlns_name + ":" + "indicators" + "-" + crits_indicator_id)
)
