Exemplo n.º 1
0
def inject_edge_sample_data(config, target=None, datatype=None):
    """inject randomly generated sample data into edge target"""
    global datatypes
    observable_types = list()
    observable_types.extend(datatypes)
    observable_types.remove("mixed")
    observable_types.remove("indicator")
    if datatype in observable_types:
        i = 0
        while i < config["edge"]["datagen"]["indicator_count"]:
            try:
                (observable_id, stix_) = gen_stix_observable_sample(config, target=target, datatype=datatype)
                success = edge_.taxii_inbox(config, target, stix_)
                if success:
                    i += 1
                else:
                    print("error inboxing edge sample data to %s - exiting!" % target)
                    exit()
            except:
                continue
    elif datatype == "indicator":
        # indicator linked to 5-25 mixed observables
        i = 0
        while i < config["edge"]["datagen"]["indicator_count"]:
            observable_count = random.randint(5, 25)
            observables_list = list()
            j = 0
            while j < observable_count:
                try:
                    observable_type_index = random.randint(0, len(observable_types) - 1)
                    type_ = observable_types[observable_type_index]
                    (observable_id, stix_) = gen_stix_observable_sample(config, target=target, datatype=type_)
                    success = edge_.taxii_inbox(config, target, stix_)
                    if success:
                        j += 1
                        observables_list.append(observable_id)
                    else:
                        continue
                except:
                    continue
            try:
                stix_ = gen_stix_indicator_sample(
                    config, target=target, datatype=type_, observables_list=observables_list
                )
                success = edge_.taxii_inbox(config, target, stix_)
                if success:
                    i += 1
                else:
                    continue
            except:
                continue
    elif datatype == "mixed":
        i = 0
        while i < config["edge"]["datagen"]["indicator_count"]:
            try:
                observable_type_index = random.randint(0, len(observable_types) - 1)
                type_ = observable_types[observable_type_index]
                (observable_id, stix_) = gen_stix_observable_sample(config, target=target, datatype=type_)
                success = edge_.taxii_inbox(config, target, stix_)
                if success:
                    i += 1
                else:
                    continue
            except:
                continue
Exemplo n.º 2
0
def crits2edge(config, src, dest, daemon=False,
               now=None, last_run=None):
    xmlns_name = config['edge']['sites'][dest]['stix']['xmlns_name']
    # check if (and when) we synced src and dest...
    if not now:
        now = util_.nowutc()
    if not last_run:
        last_run = config['db'].get_last_sync(src=src, dest=dest,
                                              direction='c2e')
    config['logger'].info(
        log_.log_messages['start_sync'].format(
            type_='crits', last_run=last_run, src=src, dest=dest))
    endpoints = ['ips', 'domains', 'samples', 'emails', 'indicators', 'events']
    # setup the tally counters
    config['crits_tally'] = dict()
    config['crits_tally']['all'] = {'incoming': 0, 'processed': 0}
    for endpoint in endpoints:
        config['crits_tally'][endpoint] = {'incoming': 0, 'processed': 0}
    ids = dict()
    for endpoint in endpoints:
        ids[endpoint] = fetch_crits_object_ids(config, src, endpoint, last_run)
        if not len(ids[endpoint]):
            continue
        else:
            for crits_id in ids[endpoint]:
                (id_, json_) = crits_poll(config, src, endpoint, crits_id,)
                if endpoint == 'indicators':
                    indicator = json2indicator(config, src, dest,
                                              endpoint, json_, id_)
                    config['crits_tally']['indicators']['incoming'] += 1
                    config['crits_tally']['all']['incoming'] += 1
                    if not indicator:
                        config['logger'].info(
                            log_.log_messages['obj_inbox_error'].format(
                                src_type='crits', id_=crits_id, dest_type='edge'))
                        continue
                    stix_ = stix_pkg(config, src, endpoint, indicator, dest=dest)
                    if not stix_:
                        config['logger'].info(
                            log_.log_messages['obj_inbox_error'].format(
                                src_type='crits', id_=crits_id, dest_type='edge'))
                        continue
                    success = edge_.taxii_inbox(config, dest, stix_, src=src,
                                                crits_id=endpoint + ':'
                                                + crits_id)
                    if not success:
                        config['logger'].info(
                            log_.log_messages['obj_inbox_error'].format(
                                src_type='crits', id_=crits_id, dest_type='edge'))
                        continue
                    else:
                        # track the related crits/json ids (by src/dest)
                        config['db'].set_object_id(src, dest,
                                                   edge_id=indicator.id_,
                                                   crits_id=(xmlns_name + ':' + 
                                                             endpoint + '-' + crits_id))
                        config['crits_tally']['indicators']['processed'] += 1
                        config['crits_tally']['all']['processed'] += 1
                elif endpoint == 'events':
                    incident = json2incident(config, src, dest,
                                              endpoint, json_, id_)
                    config['crits_tally']['events']['incoming'] += 1
                    config['crits_tally']['all']['incoming'] += 1
                    if not incident:
                        config['logger'].info(
                            log_.log_messages['obj_inbox_error'].format(
                                src_type='crits', id_=crits_id, dest_type='edge'))
                        continue
                    stix_ = stix_pkg(config, src, endpoint, incident, dest=dest)
                    if not stix_:
                        config['logger'].info(
                            log_.log_messages['obj_inbox_error'].format(
                                src_type='crits', id_=crits_id, dest_type='edge'))
                        continue
                    success = edge_.taxii_inbox(config, dest, stix_, src=src,
                                                crits_id=endpoint + ':'
                                                + crits_id)
                    if not success:
                        config['logger'].info(
                            log_.log_messages['obj_inbox_error'].format(
                                src_type='crits', id_=crits_id, dest_type='edge'))
                        continue
                    else:
                        # track the related crits/json ids (by src/dest)
                        config['db'].set_object_id(src, dest,
                                                   edge_id=incident.id_,
                                                   crits_id=(xmlns_name + ':' + 
                                                             endpoint + '-' + crits_id))
                        config['crits_tally']['events']['processed'] += 1
                        config['crits_tally']['all']['processed'] += 1
                else:
                    observable = json2observable(config, src, dest, endpoint, json_, crits_id)
                    config['crits_tally'][endpoint]['incoming'] += 1
                    config['crits_tally']['all']['incoming'] += 1
                    if not observable:
                        config['logger'].info(
                            log_.log_messages['obj_inbox_error'].format(
                                src_type='crits', id_=crits_id, dest_type='edge'))
                        continue
                    stix_ = stix_pkg(config, src, endpoint, observable, dest=dest)
                    if not stix_:
                        config['logger'].info(
                            log_.log_messages['obj_inbox_error'].format(
                                src_type='crits', id_=crits_id, dest_type='edge'))
                        continue
                    success = edge_.taxii_inbox(config, dest, stix_)
                    if not success:
                        config['logger'].info(
                            log_.log_messages['obj_inbox_error'].format(
                                src_type='crits', id_=crits_id, dest_type='edge'))
                        continue
                    else:
                        config['crits_tally'][endpoint]['processed'] += 1
                        config['crits_tally']['all']['processed'] += 1
                        config['db'].set_object_id(src, dest,
                                                   edge_id=observable.id_,
                                                   crits_id=(xmlns_name + ':' + 
                                                             endpoint + '-' + crits_id))
    for endpoint in endpoints:
        if config['crits_tally'][endpoint]['incoming'] > 0:
            config['logger'].info(log_.log_messages['incoming_tally'].format(
                    count=config['crits_tally'][endpoint]['incoming'],
                    type_=endpoint, src='crits', dest='edge'))
        if (config['crits_tally'][endpoint]['incoming'] -
                   config['crits_tally'][endpoint]['processed']) > 0:
            config['logger'].info(log_.log_messages['failed_tally'].format(
                    count=(config['crits_tally'][endpoint]['incoming'] -
                           config['crits_tally'][endpoint]['processed']),
                    type_=endpoint, src='crits', dest='edge'))
        if config['crits_tally'][endpoint]['processed'] > 0:
            config['logger'].info(log_.log_messages['processed_tally'].format(
                    count=config['crits_tally'][endpoint]['processed'], 
                    type_=endpoint, src='crits', dest='edge'))
    if config['crits_tally']['all']['incoming'] > 0:
        config['logger'].info(log_.log_messages['incoming_tally'].format(
                count=config['crits_tally']['all']['incoming'], type_='total',
                src='crits', dest='edge'))
    if (config['crits_tally']['all']['incoming'] -
               config['crits_tally']['all']['processed']) > 0:
        config['logger'].info(log_.log_messages['failed_tally'].format(
                count=(config['crits_tally']['all']['incoming'] -
                       config['crits_tally']['all']['processed']),
                type_='total', src='crits', dest='edge'))
    if config['crits_tally']['all']['processed'] > 0:
        config['logger'].info(log_.log_messages['processed_tally'].format(
                count=config['crits_tally']['all']['processed'], type_='total',
                src='crits', dest='edge'))
    # save state to disk for next run...
    if config['daemon']['debug']:
        poll_interval = config['crits']['sites'][src]['api']['poll_interval']
        config['logger'].debug(
            log_.log_messages['saving_state'].format(
                next_run=str(now + datetime.timedelta(seconds=poll_interval))))
    if not daemon:
        config['db'].set_last_sync(src=src, dest=dest,
                                   direction='c2e', timestamp=now)
        return(None)
    else:
        return(util_.nowutc())