def inject_edge_sample_data(config, target=None, datatype=None): """inject randomly generated sample data into edge target""" global datatypes observable_types = list() observable_types.extend(datatypes) observable_types.remove("mixed") observable_types.remove("indicator") if datatype in observable_types: i = 0 while i < config["edge"]["datagen"]["indicator_count"]: try: (observable_id, stix_) = gen_stix_observable_sample(config, target=target, datatype=datatype) success = edge_.taxii_inbox(config, target, stix_) if success: i += 1 else: print("error inboxing edge sample data to %s - exiting!" % target) exit() except: continue elif datatype == "indicator": # indicator linked to 5-25 mixed observables i = 0 while i < config["edge"]["datagen"]["indicator_count"]: observable_count = random.randint(5, 25) observables_list = list() j = 0 while j < observable_count: try: observable_type_index = random.randint(0, len(observable_types) - 1) type_ = observable_types[observable_type_index] (observable_id, stix_) = gen_stix_observable_sample(config, target=target, datatype=type_) success = edge_.taxii_inbox(config, target, stix_) if success: j += 1 observables_list.append(observable_id) else: continue except: continue try: stix_ = gen_stix_indicator_sample( config, target=target, datatype=type_, observables_list=observables_list ) success = edge_.taxii_inbox(config, target, stix_) if success: i += 1 else: continue except: continue elif datatype == "mixed": i = 0 while i < config["edge"]["datagen"]["indicator_count"]: try: observable_type_index = random.randint(0, len(observable_types) - 1) type_ = observable_types[observable_type_index] (observable_id, stix_) = gen_stix_observable_sample(config, target=target, datatype=type_) success = edge_.taxii_inbox(config, target, stix_) if success: i += 1 else: continue except: continue
def crits2edge(config, src, dest, daemon=False, now=None, last_run=None): xmlns_name = config['edge']['sites'][dest]['stix']['xmlns_name'] # check if (and when) we synced src and dest... if not now: now = util_.nowutc() if not last_run: last_run = config['db'].get_last_sync(src=src, dest=dest, direction='c2e') config['logger'].info( log_.log_messages['start_sync'].format( type_='crits', last_run=last_run, src=src, dest=dest)) endpoints = ['ips', 'domains', 'samples', 'emails', 'indicators', 'events'] # setup the tally counters config['crits_tally'] = dict() config['crits_tally']['all'] = {'incoming': 0, 'processed': 0} for endpoint in endpoints: config['crits_tally'][endpoint] = {'incoming': 0, 'processed': 0} ids = dict() for endpoint in endpoints: ids[endpoint] = fetch_crits_object_ids(config, src, endpoint, last_run) if not len(ids[endpoint]): continue else: for crits_id in ids[endpoint]: (id_, json_) = crits_poll(config, src, endpoint, crits_id,) if endpoint == 'indicators': indicator = json2indicator(config, src, dest, endpoint, json_, id_) config['crits_tally']['indicators']['incoming'] += 1 config['crits_tally']['all']['incoming'] += 1 if not indicator: config['logger'].info( log_.log_messages['obj_inbox_error'].format( src_type='crits', id_=crits_id, dest_type='edge')) continue stix_ = stix_pkg(config, src, endpoint, indicator, dest=dest) if not stix_: config['logger'].info( log_.log_messages['obj_inbox_error'].format( src_type='crits', id_=crits_id, dest_type='edge')) continue success = edge_.taxii_inbox(config, dest, stix_, src=src, crits_id=endpoint + ':' + crits_id) if not success: config['logger'].info( log_.log_messages['obj_inbox_error'].format( src_type='crits', id_=crits_id, dest_type='edge')) continue else: # track the related crits/json ids (by src/dest) config['db'].set_object_id(src, dest, edge_id=indicator.id_, crits_id=(xmlns_name + ':' + endpoint + '-' + crits_id)) config['crits_tally']['indicators']['processed'] += 1 config['crits_tally']['all']['processed'] += 1 elif endpoint == 'events': incident = json2incident(config, src, dest, endpoint, json_, id_) config['crits_tally']['events']['incoming'] += 1 config['crits_tally']['all']['incoming'] += 1 if not incident: config['logger'].info( log_.log_messages['obj_inbox_error'].format( src_type='crits', id_=crits_id, dest_type='edge')) continue stix_ = stix_pkg(config, src, endpoint, incident, dest=dest) if not stix_: config['logger'].info( log_.log_messages['obj_inbox_error'].format( src_type='crits', id_=crits_id, dest_type='edge')) continue success = edge_.taxii_inbox(config, dest, stix_, src=src, crits_id=endpoint + ':' + crits_id) if not success: config['logger'].info( log_.log_messages['obj_inbox_error'].format( src_type='crits', id_=crits_id, dest_type='edge')) continue else: # track the related crits/json ids (by src/dest) config['db'].set_object_id(src, dest, edge_id=incident.id_, crits_id=(xmlns_name + ':' + endpoint + '-' + crits_id)) config['crits_tally']['events']['processed'] += 1 config['crits_tally']['all']['processed'] += 1 else: observable = json2observable(config, src, dest, endpoint, json_, crits_id) config['crits_tally'][endpoint]['incoming'] += 1 config['crits_tally']['all']['incoming'] += 1 if not observable: config['logger'].info( log_.log_messages['obj_inbox_error'].format( src_type='crits', id_=crits_id, dest_type='edge')) continue stix_ = stix_pkg(config, src, endpoint, observable, dest=dest) if not stix_: config['logger'].info( log_.log_messages['obj_inbox_error'].format( src_type='crits', id_=crits_id, dest_type='edge')) continue success = edge_.taxii_inbox(config, dest, stix_) if not success: config['logger'].info( log_.log_messages['obj_inbox_error'].format( src_type='crits', id_=crits_id, dest_type='edge')) continue else: config['crits_tally'][endpoint]['processed'] += 1 config['crits_tally']['all']['processed'] += 1 config['db'].set_object_id(src, dest, edge_id=observable.id_, crits_id=(xmlns_name + ':' + endpoint + '-' + crits_id)) for endpoint in endpoints: if config['crits_tally'][endpoint]['incoming'] > 0: config['logger'].info(log_.log_messages['incoming_tally'].format( count=config['crits_tally'][endpoint]['incoming'], type_=endpoint, src='crits', dest='edge')) if (config['crits_tally'][endpoint]['incoming'] - config['crits_tally'][endpoint]['processed']) > 0: config['logger'].info(log_.log_messages['failed_tally'].format( count=(config['crits_tally'][endpoint]['incoming'] - config['crits_tally'][endpoint]['processed']), type_=endpoint, src='crits', dest='edge')) if config['crits_tally'][endpoint]['processed'] > 0: config['logger'].info(log_.log_messages['processed_tally'].format( count=config['crits_tally'][endpoint]['processed'], type_=endpoint, src='crits', dest='edge')) if config['crits_tally']['all']['incoming'] > 0: config['logger'].info(log_.log_messages['incoming_tally'].format( count=config['crits_tally']['all']['incoming'], type_='total', src='crits', dest='edge')) if (config['crits_tally']['all']['incoming'] - config['crits_tally']['all']['processed']) > 0: config['logger'].info(log_.log_messages['failed_tally'].format( count=(config['crits_tally']['all']['incoming'] - config['crits_tally']['all']['processed']), type_='total', src='crits', dest='edge')) if config['crits_tally']['all']['processed'] > 0: config['logger'].info(log_.log_messages['processed_tally'].format( count=config['crits_tally']['all']['processed'], type_='total', src='crits', dest='edge')) # save state to disk for next run... if config['daemon']['debug']: poll_interval = config['crits']['sites'][src]['api']['poll_interval'] config['logger'].debug( log_.log_messages['saving_state'].format( next_run=str(now + datetime.timedelta(seconds=poll_interval)))) if not daemon: config['db'].set_last_sync(src=src, dest=dest, direction='c2e', timestamp=now) return(None) else: return(util_.nowutc())