def process_observables(config, src, dest, observables):
'''handle incoming cybox observables and observable compositions'''
# TODO some of the hailataxii date uses the cybox ###comma###
# construct, which is currently unsupported
for o in observables.keys():
json = dict()
if util_.rgetattr(observables[o], ['observable_composition']) \
and not util_.rgetattr(observables[o], ['object_']):
# it's an observable composition
# store it in the db...maybe the indicator will only come
# across in a subsequent run so we can't rely on passing
# this around in memory
config['db'].store_obs_comp(src, dest,
obs_id=o,
obs_comp=observables[o].observable_composition)
continue
elif util_.rgetattr(observables[o], ['object_']):
# it's a normal observable
(json, endpoint) = cybox_observable_to_json(config, observables[o])
if json:
# mark crits releasability
json.update(mark_crits_releasability(config, dest))
else:
config['logger'].error(
log_.log_messages[
'obj_convert_error'].format(src_type='cybox',
src_obj='observable',
id_=o,
dest_type='crits',
dest_obj='json'))
continue
# inbox the observable to crits
config['edge_tally'][endpoint]['incoming'] += 1
config['edge_tally']['all']['incoming'] += 1
(id_, success) = \
crits_.crits_inbox(config, dest, endpoint, json,
src=src, edge_id=o)
if not success:
config['logger'].error(
log_.log_messages['obj_inbox_error'].format(
src_type='edge', id_=o, dest_type='crits ' + endpoint + ' api endpoint'))
continue
else:
# successfully inboxed observable
config['edge_tally'][endpoint]['processed'] += 1
config['edge_tally']['all']['processed'] += 1
if config['daemon']['debug']:
config['logger'].debug(
log_.log_messages['obj_inbox_success'].format(
src_type='edge', id_=o,
dest_type='crits ' + endpoint + ' api endpoint'))
def process_relationships(config, src, dest):
'''forge the crits relationship links between incoming observables
and indicators'''
endpoint_trans = {'emails': 'Email', 'ips': 'IP',
'samples': 'Sample', 'domains': 'Domain',
'indicators': 'Indicator', 'events': 'Event'}
pending_crits_links = config['db'].get_pending_crits_links(src, dest)
if not pending_crits_links:
config['logger'].info(
log_.log_messages['no_pending_crits_relationships'])
else:
for r in pending_crits_links:
json = dict()
json['left_type'] = endpoint_trans[r['lhs_id'].split(':')[1].split('-')[0]]
json['left_id'] = r['lhs_id'].split(':')[1].split('-')[1]
# try to fetch the crits observable id corresponding to
# the edge id
rhs = config['db'].get_object_id(src, dest,
edge_id=r['rhs_id'])
if not rhs or not rhs.get('crits_id', None):
config['logger'].error(
log_.log_messages['obs_comp_dereference_error'
].format(id_=r['rhs_id']))
else:
json['right_type'] = \
endpoint_trans[
rhs['crits_id'].split(':')[1].split('-')[0]]
json['right_id'] = \
rhs['crits_id'].split(':')[1].split('-')[1]
json['rel_type'] = 'Related_To'
json['rel_confidence'] = 'unknown'
config['edge_tally']['relationships']['incoming'] += 1
config['edge_tally']['all']['incoming'] += 1
(relationship_id_, success) = \
crits_.crits_inbox(config, dest,
'relationships', json, src=src)
if not success:
config['logger'].error(
log_.log_messages['obj_inbox_error'].format(
src_type='edge', id_=r['rhs_id'],
dest_type='crits relationships api endpoint'))
else:
# remove the pending crits relationship from the db
config['edge_tally']['relationships']['processed'] += 1
config['edge_tally']['all']['processed'] += 1
config['db'].resolve_crits_link(src, dest,
lhs_id=r['lhs_id'],
rhs_id=r['rhs_id'])
def process_indicators(config, src, dest, indicators):
'''handle incoming stix indicators'''
xmlns_name = config['edge']['sites'][src]['stix']['xmlns_name']
for i in indicators.keys():
json = dict()
json['type'] = 'Related_To'
json['value'] = util_.rgetattr(indicators[i], ['title'],
default_='unknown')
json['indicator_confidence'] = \
util_.rgetattr(indicators[i], ['confidence', 'value', 'value'],
default_='unknown')
# TODO lookup the corresponding stix prop for indicator_impact
json['indicator_impact'] = {'rating': 'unknown'}
# inbox the indicator (we need to crits id!)
config['edge_tally']['indicators']['incoming'] += 1
config['edge_tally']['all']['incoming'] += 1
(crits_indicator_id, success) = crits_.crits_inbox(config, dest,
'indicators',
json, src=src)
if not success:
config['logger'].error(
log_.log_messages['obj_inbox_error'].format(
src_type='edge', id_=i,
dest_type='crits indicators api endpoint'))
continue
else:
# successfully inboxed indicator...
config['edge_tally']['indicators']['processed'] += 1
config['edge_tally']['all']['processed'] += 1
if config['daemon']['debug']:
config['logger'].debug(
log_.log_messages['obj_inbox_success'].format(
src_type='edge', id_=i,
dest_type='crits indicators api endpoint'))
if util_.rgetattr(indicators[i], ['observables']):
for o in indicators[i].observables:
if util_.rgetattr(o, ['idref']) and \
not util_.rgetattr(o, ['object_']):
# TODO need to delete observable compositions from
# mongo once we've processed them
obs_comp = \
config['db'].get_obs_comp(src, dest, obs_id=o.idref)
if not obs_comp:
# [ o == embedded observable]
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'indicators' + '-' +
crits_indicator_id),
rhs_id=o.idref)
elif obs_comp:
# [o == idref observable composition]
# try to fetch the observable composition o.idref
# points to
# assumption: the observable composition was
# previously ingested. TODO what about when
# the observable composition comes in *after*
# the indicator?
observables_list = util_.rgetattr(obs_comp,
['observables'])
if not observables_list:
config['logger'].error(
log_.log_messages['obs_comp_dereference_error'
].format(id_=i))
continue
else:
for j in observables_list:
# store the pending relationship in
# the db for later processing
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'indicators' + '-' +
crits_indicator_id),
rhs_id=j.idref)
# TODO (need to dig up suitable sample data)
# if it's an observable composition with inline
# observables, pass them to observable composition with
# inline observables, pass them to process_observables(),
# (which will store the edge/crits id indicator pairing
# for later processing.
else:
config['logger'].error(
log_.log_messages['obs_comp_dereference_error'
].format(id_=i))
continue
# as we've now successfully processed the indicator, track
# the related crits/json ids (by src/dest)
if util_.rgetattr(indicators[i], ['related_indicators']) and len(indicators[i].related_indicators):
for j in indicators[i].related_indicators:
if util_.rgetattr(j, ['item', 'idref']):
# store the pending relationship in the db for
# later processing
# TODO for some reason, the crits relationship api
# is rejecting _some_ (but not _all_
# indicator-to-indicator relationships. the
# indicator ids are valid and the api post looks
# correct but...sometimes this fails :-/
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'indicators' + '-' +
crits_indicator_id),
rhs_id=j.item.idref)
config['db'].set_object_id(src, dest,
edge_id=i,
crits_id=(xmlns_name + ':' + 'indicators' + '-' +
crits_indicator_id))
def process_incidents(config, src, dest, incidents):
'''handle incoming stix incidents'''
xmlns_name = config['edge']['sites'][src]['stix']['xmlns_name']
status_trans = {'New': 'New', 'Open': 'In Progress',
'Closed': 'Analyzed', 'Rejected': 'Deprecated'}
for i in incidents.keys():
json = dict()
json['event_type'] = 'Threat Report'
json['title'] = incidents[i].title
json['description'] = util_.rgetattr(incidents[i], ['description', 'value'])
json['status'] = status_trans[incidents[i].status.value]
# inbox the incident (we need to crits id!)
config['edge_tally']['events']['incoming'] += 1
config['edge_tally']['all']['incoming'] += 1
(crits_event_id, success) = crits_.crits_inbox(config, dest,
'events',
json, src=src)
if not success:
config['logger'].error(
log_.log_messages['obj_inbox_error'].format(
src_type='edge', id_=i,
dest_type='crits events api endpoint'))
continue
else:
# successfully inboxed event...
config['edge_tally']['events']['processed'] += 1
config['edge_tally']['all']['processed'] += 1
if config['daemon']['debug']:
config['logger'].debug(
log_.log_messages['obj_inbox_success'].format(
src_type='edge', id_=i,
dest_type='crits events api endpoint'))
# as we've now successfully processed the event, track
# the related crits/json ids (by src/dest)
if util_.rgetattr(incidents[i], ['related_observables']) and len(incidents[i].related_observables):
for j in incidents[i].related_observables:
if util_.rgetattr(j, ['item', 'idref']):
# store the pending relationship in the db for
# later processing
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'events' + '-' +
crits_event_id),
rhs_id=j.item.idref)
if util_.rgetattr(incidents[i], ['related_indicators']) and len(incidents[i].related_indicators):
for j in incidents[i].related_indicators:
if util_.rgetattr(j, ['item', 'idref']):
# store the pending relationship in the db for
# later processing
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'events' + '-' +
crits_event_id),
rhs_id=j.item.idref)
if util_.rgetattr(incidents[i], ['related_incidents']) and len(incidents[i].related_incidents):
for j in incidents[i].related_incidents:
if util_.rgetattr(j, ['item', 'idref']):
# store the pending relationship in the db for
# later processing
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'events' + '-' +
crits_event_id),
rhs_id=j.item.idref)
config['db'].set_object_id(src, dest,
edge_id=i,
crits_id=(xmlns_name + ':' + 'events' + '-' +
crits_event_id))
def process_observables(config, src, dest, observables):
'''handle incoming cybox observables and observable compositions'''
# TODO some of the hailataxii date uses the cybox ###comma###
# construct, which is currently unsupported
for o_id, o in observables.iteritems():
json = dict()
if util_.rgetattr(o, ['observable_composition']) \
and not util_.rgetattr(o, ['object_']):
# it's an observable composition
# store it in the db...maybe the indicator will only come
# across in a subsequent run so we can't rely on passing
# this around in memory
config['db'].store_obs_comp(src, dest,
obs_id=o_id,
obs_comp=o.observable_composition)
continue
elif util_.rgetattr(o, ['object_']):
# it's a normal observable
(json, endpoint) = cybox_observable_to_json(config, o)
if not json:
config['logger'].error(
log_.log_messages[
'obj_convert_error'].format(src_type='cybox',
src_obj='observable',
id_=o_id,
dest_type='crits',
dest_obj='json'))
continue
# mark crits releasability
# TODO: Maybe remove this? Not sure if it works with
# the Crits PATCH API method for setting releasability.
json.update(mark_crits_releasability(config, dest))
# inbox the observable to crits
config['edge_tally'][endpoint]['incoming'] += 1
config['edge_tally']['all']['incoming'] += 1
(id_, success) = \
crits_.crits_inbox(config, dest, endpoint, json,
src=src, edge_id=o_id)
if not success:
config['logger'].error(
log_.log_messages['obj_inbox_error'].format(
src_type='edge', id_=o, dest_type='crits ' + endpoint + ' api endpoint'))
continue
# Successfully inboxed observable
# Send Patch request to set crits releasability
patch_endpoint = '{}/{}'.format(endpoint, id_)
releasability_json = {
'action': 'add_releasability',
'name': config['crits']['sites'][dest]['api']['releasability'],
}
releasability_success = crits_.crits_patch(config, dest,
patch_endpoint, releasability_json)
if not releasability_success:
config['logger'].error(
log_.log_messages['obj_inbox_error'].format(
src_type='edge', id_=o, dest_type='crits ' + patch_endpoint + ' api endpoint'))
continue
config['edge_tally'][endpoint]['processed'] += 1
config['edge_tally']['all']['processed'] += 1
if config['daemon']['debug']:
config['logger'].debug(
log_.log_messages['obj_inbox_success'].format(
src_type='edge', id_=o_id,
dest_type='crits ' + endpoint + ' api endpoint'))
def inject_crits_sample_data(config, target=None, datatype=None):
"""inject randomly generated sample data into crits target"""
global datatypes
observable_types = list()
observable_types.extend(datatypes)
observable_types.remove("mixed")
observable_types.remove("indicator")
endpoint = None
if datatype == "ip":
endpoint = "ips"
elif datatype == "domain":
endpoint = "domains"
elif datatype == "email":
endpoint = "emails"
elif datatype == "filehash":
endpoint = "samples"
elif datatype == "indicator":
endpoint = "indicators"
if datatype in observable_types:
# single observable types
i = 0
while i < config["crits"]["datagen"]["indicator_count"]:
(id_, success) = crits_.crits_inbox(config, target, endpoint, generate_crits_json(config, datatype))
if success:
i += 1
else:
print("error inboxing crits sample data to %s - exiting!" % target)
elif datatype == "indicator":
# indicator linked to 5-25 mixed observables
endpoint_trans = {"emails": "Email", "ips": "IP", "samples": "Sample", "domains": "Domain"}
i = 0
while i < config["crits"]["datagen"]["indicator_count"]:
# generate between 5-25 observables to be linked to a
# crits indicator
observables_dict = dict()
observable_count = random.randint(5, 25)
j = 0
while j < observable_count:
observable_type_index = random.randint(0, len(observable_types) - 1)
type_ = observable_types[observable_type_index]
if type_ == "ip":
endpoint = "ips"
elif type_ == "domain":
endpoint = "domains"
elif type_ == "email":
endpoint = "emails"
elif type_ == "filehash":
endpoint = "samples"
(id_, success) = crits_.crits_inbox(config, target, endpoint, generate_crits_json(config, type_))
if success:
j += 1
observables_dict[id_] = endpoint
else:
print("error inboxing crits sample %s indicator " "observable to %s - exiting!" % (type_, target))
# now that we've got random observables in crits, inbox an
# indicator...
(id_, success) = crits_.crits_inbox(
config, target, "indicators", generate_crits_indicator_json(config, observables_dict)
)
if success:
i += 1
for k in observables_dict.keys():
json = dict()
json["left_type"] = "Indicator"
json["left_id"] = id_
json["right_type"] = endpoint_trans[observables_dict[k]]
json["right_id"] = k
json["rel_type"] = "Contains"
json["rel_confidence"] = "unknown"
(id_, success) = crits_.crits_inbox(config, target, "relationships", json)
else:
print("error inboxing relationship for crits sample indicator " "to %s - exiting!" % target)
elif datatype == "mixed":
# mixed observables
i = 0
while i < config["crits"]["datagen"]["indicator_count"]:
observable_type_index = random.randint(0, len(observable_types) - 1)
type_ = observable_types[observable_type_index]
if type_ == "ip":
endpoint = "ips"
elif type_ == "domain":
endpoint = "domains"
elif type_ == "email":
endpoint = "emails"
elif type_ == "filehash":
endpoint = "samples"
(id_, success) = crits_.crits_inbox(config, target, endpoint, generate_crits_json(config, type_))
if success:
i += 1
else:
print("error inboxing crits sample data to %s - exiting!" % target)