def process_observables(config, src, dest, observables): '''handle incoming cybox observables and observable compositions''' # TODO some of the hailataxii date uses the cybox ###comma### # construct, which is currently unsupported for o in observables.keys(): json = dict() if util_.rgetattr(observables[o], ['observable_composition']) \ and not util_.rgetattr(observables[o], ['object_']): # it's an observable composition # store it in the db...maybe the indicator will only come # across in a subsequent run so we can't rely on passing # this around in memory config['db'].store_obs_comp(src, dest, obs_id=o, obs_comp=observables[o].observable_composition) continue elif util_.rgetattr(observables[o], ['object_']): # it's a normal observable (json, endpoint) = cybox_observable_to_json(config, observables[o]) if json: # mark crits releasability json.update(mark_crits_releasability(config, dest)) else: config['logger'].error( log_.log_messages[ 'obj_convert_error'].format(src_type='cybox', src_obj='observable', id_=o, dest_type='crits', dest_obj='json')) continue # inbox the observable to crits config['edge_tally'][endpoint]['incoming'] += 1 config['edge_tally']['all']['incoming'] += 1 (id_, success) = \ crits_.crits_inbox(config, dest, endpoint, json, src=src, edge_id=o) if not success: config['logger'].error( log_.log_messages['obj_inbox_error'].format( src_type='edge', id_=o, dest_type='crits ' + endpoint + ' api endpoint')) continue else: # successfully inboxed observable config['edge_tally'][endpoint]['processed'] += 1 config['edge_tally']['all']['processed'] += 1 if config['daemon']['debug']: config['logger'].debug( log_.log_messages['obj_inbox_success'].format( src_type='edge', id_=o, dest_type='crits ' + endpoint + ' api endpoint'))
def process_relationships(config, src, dest): '''forge the crits relationship links between incoming observables and indicators''' endpoint_trans = {'emails': 'Email', 'ips': 'IP', 'samples': 'Sample', 'domains': 'Domain', 'indicators': 'Indicator', 'events': 'Event'} pending_crits_links = config['db'].get_pending_crits_links(src, dest) if not pending_crits_links: config['logger'].info( log_.log_messages['no_pending_crits_relationships']) else: for r in pending_crits_links: json = dict() json['left_type'] = endpoint_trans[r['lhs_id'].split(':')[1].split('-')[0]] json['left_id'] = r['lhs_id'].split(':')[1].split('-')[1] # try to fetch the crits observable id corresponding to # the edge id rhs = config['db'].get_object_id(src, dest, edge_id=r['rhs_id']) if not rhs or not rhs.get('crits_id', None): config['logger'].error( log_.log_messages['obs_comp_dereference_error' ].format(id_=r['rhs_id'])) else: json['right_type'] = \ endpoint_trans[ rhs['crits_id'].split(':')[1].split('-')[0]] json['right_id'] = \ rhs['crits_id'].split(':')[1].split('-')[1] json['rel_type'] = 'Related_To' json['rel_confidence'] = 'unknown' config['edge_tally']['relationships']['incoming'] += 1 config['edge_tally']['all']['incoming'] += 1 (relationship_id_, success) = \ crits_.crits_inbox(config, dest, 'relationships', json, src=src) if not success: config['logger'].error( log_.log_messages['obj_inbox_error'].format( src_type='edge', id_=r['rhs_id'], dest_type='crits relationships api endpoint')) else: # remove the pending crits relationship from the db config['edge_tally']['relationships']['processed'] += 1 config['edge_tally']['all']['processed'] += 1 config['db'].resolve_crits_link(src, dest, lhs_id=r['lhs_id'], rhs_id=r['rhs_id'])
def process_indicators(config, src, dest, indicators): '''handle incoming stix indicators''' xmlns_name = config['edge']['sites'][src]['stix']['xmlns_name'] for i in indicators.keys(): json = dict() json['type'] = 'Related_To' json['value'] = util_.rgetattr(indicators[i], ['title'], default_='unknown') json['indicator_confidence'] = \ util_.rgetattr(indicators[i], ['confidence', 'value', 'value'], default_='unknown') # TODO lookup the corresponding stix prop for indicator_impact json['indicator_impact'] = {'rating': 'unknown'} # inbox the indicator (we need to crits id!) config['edge_tally']['indicators']['incoming'] += 1 config['edge_tally']['all']['incoming'] += 1 (crits_indicator_id, success) = crits_.crits_inbox(config, dest, 'indicators', json, src=src) if not success: config['logger'].error( log_.log_messages['obj_inbox_error'].format( src_type='edge', id_=i, dest_type='crits indicators api endpoint')) continue else: # successfully inboxed indicator... config['edge_tally']['indicators']['processed'] += 1 config['edge_tally']['all']['processed'] += 1 if config['daemon']['debug']: config['logger'].debug( log_.log_messages['obj_inbox_success'].format( src_type='edge', id_=i, dest_type='crits indicators api endpoint')) if util_.rgetattr(indicators[i], ['observables']): for o in indicators[i].observables: if util_.rgetattr(o, ['idref']) and \ not util_.rgetattr(o, ['object_']): # TODO need to delete observable compositions from # mongo once we've processed them obs_comp = \ config['db'].get_obs_comp(src, dest, obs_id=o.idref) if not obs_comp: # [ o == embedded observable] config['db'].set_pending_crits_link(src, dest, lhs_id=(xmlns_name + ':' + 'indicators' + '-' + crits_indicator_id), rhs_id=o.idref) elif obs_comp: # [o == idref observable composition] # try to fetch the observable composition o.idref # points to # assumption: the observable composition was # previously ingested. TODO what about when # the observable composition comes in *after* # the indicator? observables_list = util_.rgetattr(obs_comp, ['observables']) if not observables_list: config['logger'].error( log_.log_messages['obs_comp_dereference_error' ].format(id_=i)) continue else: for j in observables_list: # store the pending relationship in # the db for later processing config['db'].set_pending_crits_link(src, dest, lhs_id=(xmlns_name + ':' + 'indicators' + '-' + crits_indicator_id), rhs_id=j.idref) # TODO (need to dig up suitable sample data) # if it's an observable composition with inline # observables, pass them to observable composition with # inline observables, pass them to process_observables(), # (which will store the edge/crits id indicator pairing # for later processing. else: config['logger'].error( log_.log_messages['obs_comp_dereference_error' ].format(id_=i)) continue # as we've now successfully processed the indicator, track # the related crits/json ids (by src/dest) if util_.rgetattr(indicators[i], ['related_indicators']) and len(indicators[i].related_indicators): for j in indicators[i].related_indicators: if util_.rgetattr(j, ['item', 'idref']): # store the pending relationship in the db for # later processing # TODO for some reason, the crits relationship api # is rejecting _some_ (but not _all_ # indicator-to-indicator relationships. the # indicator ids are valid and the api post looks # correct but...sometimes this fails :-/ config['db'].set_pending_crits_link(src, dest, lhs_id=(xmlns_name + ':' + 'indicators' + '-' + crits_indicator_id), rhs_id=j.item.idref) config['db'].set_object_id(src, dest, edge_id=i, crits_id=(xmlns_name + ':' + 'indicators' + '-' + crits_indicator_id))
def process_incidents(config, src, dest, incidents): '''handle incoming stix incidents''' xmlns_name = config['edge']['sites'][src]['stix']['xmlns_name'] status_trans = {'New': 'New', 'Open': 'In Progress', 'Closed': 'Analyzed', 'Rejected': 'Deprecated'} for i in incidents.keys(): json = dict() json['event_type'] = 'Threat Report' json['title'] = incidents[i].title json['description'] = util_.rgetattr(incidents[i], ['description', 'value']) json['status'] = status_trans[incidents[i].status.value] # inbox the incident (we need to crits id!) config['edge_tally']['events']['incoming'] += 1 config['edge_tally']['all']['incoming'] += 1 (crits_event_id, success) = crits_.crits_inbox(config, dest, 'events', json, src=src) if not success: config['logger'].error( log_.log_messages['obj_inbox_error'].format( src_type='edge', id_=i, dest_type='crits events api endpoint')) continue else: # successfully inboxed event... config['edge_tally']['events']['processed'] += 1 config['edge_tally']['all']['processed'] += 1 if config['daemon']['debug']: config['logger'].debug( log_.log_messages['obj_inbox_success'].format( src_type='edge', id_=i, dest_type='crits events api endpoint')) # as we've now successfully processed the event, track # the related crits/json ids (by src/dest) if util_.rgetattr(incidents[i], ['related_observables']) and len(incidents[i].related_observables): for j in incidents[i].related_observables: if util_.rgetattr(j, ['item', 'idref']): # store the pending relationship in the db for # later processing config['db'].set_pending_crits_link(src, dest, lhs_id=(xmlns_name + ':' + 'events' + '-' + crits_event_id), rhs_id=j.item.idref) if util_.rgetattr(incidents[i], ['related_indicators']) and len(incidents[i].related_indicators): for j in incidents[i].related_indicators: if util_.rgetattr(j, ['item', 'idref']): # store the pending relationship in the db for # later processing config['db'].set_pending_crits_link(src, dest, lhs_id=(xmlns_name + ':' + 'events' + '-' + crits_event_id), rhs_id=j.item.idref) if util_.rgetattr(incidents[i], ['related_incidents']) and len(incidents[i].related_incidents): for j in incidents[i].related_incidents: if util_.rgetattr(j, ['item', 'idref']): # store the pending relationship in the db for # later processing config['db'].set_pending_crits_link(src, dest, lhs_id=(xmlns_name + ':' + 'events' + '-' + crits_event_id), rhs_id=j.item.idref) config['db'].set_object_id(src, dest, edge_id=i, crits_id=(xmlns_name + ':' + 'events' + '-' + crits_event_id))
def process_observables(config, src, dest, observables): '''handle incoming cybox observables and observable compositions''' # TODO some of the hailataxii date uses the cybox ###comma### # construct, which is currently unsupported for o_id, o in observables.iteritems(): json = dict() if util_.rgetattr(o, ['observable_composition']) \ and not util_.rgetattr(o, ['object_']): # it's an observable composition # store it in the db...maybe the indicator will only come # across in a subsequent run so we can't rely on passing # this around in memory config['db'].store_obs_comp(src, dest, obs_id=o_id, obs_comp=o.observable_composition) continue elif util_.rgetattr(o, ['object_']): # it's a normal observable (json, endpoint) = cybox_observable_to_json(config, o) if not json: config['logger'].error( log_.log_messages[ 'obj_convert_error'].format(src_type='cybox', src_obj='observable', id_=o_id, dest_type='crits', dest_obj='json')) continue # mark crits releasability # TODO: Maybe remove this? Not sure if it works with # the Crits PATCH API method for setting releasability. json.update(mark_crits_releasability(config, dest)) # inbox the observable to crits config['edge_tally'][endpoint]['incoming'] += 1 config['edge_tally']['all']['incoming'] += 1 (id_, success) = \ crits_.crits_inbox(config, dest, endpoint, json, src=src, edge_id=o_id) if not success: config['logger'].error( log_.log_messages['obj_inbox_error'].format( src_type='edge', id_=o, dest_type='crits ' + endpoint + ' api endpoint')) continue # Successfully inboxed observable # Send Patch request to set crits releasability patch_endpoint = '{}/{}'.format(endpoint, id_) releasability_json = { 'action': 'add_releasability', 'name': config['crits']['sites'][dest]['api']['releasability'], } releasability_success = crits_.crits_patch(config, dest, patch_endpoint, releasability_json) if not releasability_success: config['logger'].error( log_.log_messages['obj_inbox_error'].format( src_type='edge', id_=o, dest_type='crits ' + patch_endpoint + ' api endpoint')) continue config['edge_tally'][endpoint]['processed'] += 1 config['edge_tally']['all']['processed'] += 1 if config['daemon']['debug']: config['logger'].debug( log_.log_messages['obj_inbox_success'].format( src_type='edge', id_=o_id, dest_type='crits ' + endpoint + ' api endpoint'))
def inject_crits_sample_data(config, target=None, datatype=None): """inject randomly generated sample data into crits target""" global datatypes observable_types = list() observable_types.extend(datatypes) observable_types.remove("mixed") observable_types.remove("indicator") endpoint = None if datatype == "ip": endpoint = "ips" elif datatype == "domain": endpoint = "domains" elif datatype == "email": endpoint = "emails" elif datatype == "filehash": endpoint = "samples" elif datatype == "indicator": endpoint = "indicators" if datatype in observable_types: # single observable types i = 0 while i < config["crits"]["datagen"]["indicator_count"]: (id_, success) = crits_.crits_inbox(config, target, endpoint, generate_crits_json(config, datatype)) if success: i += 1 else: print("error inboxing crits sample data to %s - exiting!" % target) elif datatype == "indicator": # indicator linked to 5-25 mixed observables endpoint_trans = {"emails": "Email", "ips": "IP", "samples": "Sample", "domains": "Domain"} i = 0 while i < config["crits"]["datagen"]["indicator_count"]: # generate between 5-25 observables to be linked to a # crits indicator observables_dict = dict() observable_count = random.randint(5, 25) j = 0 while j < observable_count: observable_type_index = random.randint(0, len(observable_types) - 1) type_ = observable_types[observable_type_index] if type_ == "ip": endpoint = "ips" elif type_ == "domain": endpoint = "domains" elif type_ == "email": endpoint = "emails" elif type_ == "filehash": endpoint = "samples" (id_, success) = crits_.crits_inbox(config, target, endpoint, generate_crits_json(config, type_)) if success: j += 1 observables_dict[id_] = endpoint else: print("error inboxing crits sample %s indicator " "observable to %s - exiting!" % (type_, target)) # now that we've got random observables in crits, inbox an # indicator... (id_, success) = crits_.crits_inbox( config, target, "indicators", generate_crits_indicator_json(config, observables_dict) ) if success: i += 1 for k in observables_dict.keys(): json = dict() json["left_type"] = "Indicator" json["left_id"] = id_ json["right_type"] = endpoint_trans[observables_dict[k]] json["right_id"] = k json["rel_type"] = "Contains" json["rel_confidence"] = "unknown" (id_, success) = crits_.crits_inbox(config, target, "relationships", json) else: print("error inboxing relationship for crits sample indicator " "to %s - exiting!" % target) elif datatype == "mixed": # mixed observables i = 0 while i < config["crits"]["datagen"]["indicator_count"]: observable_type_index = random.randint(0, len(observable_types) - 1) type_ = observable_types[observable_type_index] if type_ == "ip": endpoint = "ips" elif type_ == "domain": endpoint = "domains" elif type_ == "email": endpoint = "emails" elif type_ == "filehash": endpoint = "samples" (id_, success) = crits_.crits_inbox(config, target, endpoint, generate_crits_json(config, type_)) if success: i += 1 else: print("error inboxing crits sample data to %s - exiting!" % target)