Exemple #1
0
 def set_object_id(self, src, dest, crits_id=None, edge_id=None):
     timestamp=util.nowutc()
     query = {'src': src, 'dest': dest,
              'crits_id': crits_id, 'edge_id': edge_id}
     doc = self.get_object_id(src, dest, crits_id=crits_id, edge_id=edge_id)
     if doc:
         # there's already a crits-edge mapping so just update
         # the timestamp
         self.collection.update(doc, {'$set': {'modified': timestamp}})
     else:
         # insert a new mapping
         query['created'] = timestamp
         query['modified'] = timestamp
         self.collection.insert(query)
Exemple #2
0
 def set_object_id(self, src, dest, crits_id=None, edge_id=None):
     timestamp = util.nowutc()
     query = {
         'src': src,
         'dest': dest,
         'crits_id': crits_id,
         'edge_id': edge_id
     }
     doc = self.get_object_id(src, dest, crits_id=crits_id, edge_id=edge_id)
     if doc:
         # there's already a crits-edge mapping so just update
         # the timestamp
         self.collection.update(doc, {'$set': {'modified': timestamp}})
     else:
         # insert a new mapping
         query['created'] = timestamp
         query['modified'] = timestamp
         self.collection.insert(query)
Exemple #3
0
def taxii_poll(config, src, dest, timestamp=None):
    """pull stix from edge via taxii"""
    client = tc.HttpClient()
    client.setUseHttps(config["edge"]["sites"][src]["taxii"]["ssl"])
    client.setAuthType(client.AUTH_BASIC)
    client.setAuthCredentials(
        {
            "username": config["edge"]["sites"][src]["taxii"]["user"],
            "password": config["edge"]["sites"][src]["taxii"]["pass"],
        }
    )
    if not timestamp:
        earliest = util.epoch_start()
    else:
        earliest = timestamp
    latest = util.nowutc()
    poll_request = tm10.PollRequest(
        message_id=tm10.generate_message_id(),
        feed_name=config["edge"]["sites"][src]["taxii"]["collection"],
        exclusive_begin_timestamp_label=earliest,
        inclusive_end_timestamp_label=latest,
        content_bindings=[t.CB_STIX_XML_11],
    )
    http_response = client.callTaxiiService2(
        config["edge"]["sites"][src]["host"],
        config["edge"]["sites"][src]["taxii"]["path"],
        t.VID_TAXII_XML_10,
        poll_request.to_xml(),
        port=config["edge"]["sites"][src]["taxii"]["port"],
    )
    taxii_message = t.get_message_from_http_response(http_response, poll_request.message_id)
    if isinstance(taxii_message, tm10.StatusMessage):
        config["logger"].error(log.log_messages["polling_error"].format(type_="taxii", error=taxii_message.message))
    elif isinstance(taxii_message, tm10.PollResponse):
        incidents = dict()
        indicators = dict()
        observables = dict()
        for content_block in taxii_message.content_blocks:
            (incidents_, indicators_, observables_) = process_taxii_content_blocks(config, content_block)
            incidents.update(incidents_)
            indicators.update(indicators_)
            observables.update(observables_)
        return (latest, incidents, indicators, observables)
Exemple #4
0
def taxii_poll(host=None, port=None, endpoint=None, collection=None, user=None, passwd=None, use_ssl=None, attempt_validation=None):
    '''poll cti via taxii'''
    client = tc.HttpClient()
    client.setUseHttps(use_ssl)
    client.setAuthType(client.AUTH_BASIC)
    client.setAuthCredentials(
        {'username': user,
         'password': passwd})
    earliest = epoch_start()
    latest = nowutc()
    poll_request = tm10.PollRequest(
       message_id=tm10.generate_message_id(),
        feed_name=collection,
        exclusive_begin_timestamp_label=earliest,
        inclusive_end_timestamp_label=latest,
        content_bindings=[t.CB_STIX_XML_11])
    http_response = client.callTaxiiService2(
        host, endpoint,
        t.VID_TAXII_XML_10, poll_request.to_xml(),
        port=port)
    taxii_message = t.get_message_from_http_response(http_response,
                                                     poll_request.message_id)
    if isinstance(taxii_message, tm10.StatusMessage):
        print('''TAXII connection error! Exiting...
%s''' % (taxii_message.message))
    elif isinstance(taxii_message, tm10.PollResponse):
        cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
                     'exploit_targets': set(), 'incidents': set(), \
                     'indicators': set(), 'threat_actors': set(), \
                     'ttps': set()}
        cooked_cybox_objs = dict()
        for content_block in taxii_message.content_blocks:
            stix_package = taxii_content_block_to_stix(content_block)
            (raw_stix_objs, raw_cybox_objs) = \
                process_stix_pkg(stix_package)
            for k in raw_stix_objs.keys():
                cooked_stix_objs[k].update(raw_stix_objs[k])
            for k in raw_cybox_objs.keys():
                if not k in cooked_cybox_objs.keys():
                    cooked_cybox_objs[k] = set()
                cooked_cybox_objs[k].update(raw_cybox_objs[k])
        return(cooked_stix_objs, cooked_cybox_objs)
Exemple #5
0
def taxii_poll(config, src, dest, timestamp=None):
    '''pull stix from edge via taxii'''
    client = tc.HttpClient()
    client.setUseHttps(config['edge']['sites'][src]['taxii']['ssl'])
    client.setAuthType(client.AUTH_BASIC)
    client.setAuthCredentials(
        {'username': config['edge']['sites'][src]['taxii']['user'],
         'password': config['edge']['sites'][src]['taxii']['pass']})
    if not timestamp:
        earliest = util.epoch_start()
    else:
        earliest = timestamp
    latest = util.nowutc()
    poll_request = tm10.PollRequest(
       message_id=tm10.generate_message_id(),
        feed_name=config['edge']['sites'][src]['taxii']['collection'],
        exclusive_begin_timestamp_label=earliest,
        inclusive_end_timestamp_label=latest,
        content_bindings=[t.CB_STIX_XML_11])
    http_response = client.callTaxiiService2(
        config['edge']['sites'][src]['host'],
        config['edge']['sites'][src]['taxii']['path'],
        t.VID_TAXII_XML_10, poll_request.to_xml(),
        port=config['edge']['sites'][src]['taxii']['port'])
    taxii_message = t.get_message_from_http_response(http_response,
                                                     poll_request.message_id)
    if isinstance(taxii_message, tm10.StatusMessage):
        config['logger'].error(log.log_messages['polling_error'].format(
            type_='taxii', error=taxii_message.message))
    elif isinstance(taxii_message, tm10.PollResponse):
        incidents = dict()
        indicators = dict()
        observables = dict()
        for content_block in taxii_message.content_blocks:
            (incidents_, indicators_, observables_) = \
                process_taxii_content_blocks(config, content_block)
            incidents.update(incidents_)
            indicators.update(indicators_)
            observables.update(observables_)
        return(latest, incidents, indicators, observables)
Exemple #6
0
def taxii_poll(host=None, port=None, endpoint=None, collection=None, user=None, passwd=None, use_ssl=None, attempt_validation=None, time_range=None, quiet=None):
    '''poll cti via taxii'''
    client = tc.HttpClient()
    client.setUseHttps(use_ssl)
    client.setAuthType(client.AUTH_BASIC)
    client.setAuthCredentials(
        {'username': user,
         'password': passwd})
    cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
                        'exploit_targets': set(), 'incidents': set(), \
                        'indicators': set(), 'threat_actors': set(), \
                        'ttps': set()}
    cooked_cybox_objs = dict()
    earliest = poll_start(time_range)
    latest = nowutc()
    poll_window = 43200 # 12 hour blocks seem reasonable
    total_windows = (latest - earliest) / poll_window
    if (latest - earliest) % poll_window:
        total_windows += 1
    if not quiet:
        widgets = ['TAXII Poll: ', Percentage(), ' ', Bar(marker=RotatingMarker()),
                   ' ', ETA()]
        progress = ProgressBar(widgets=widgets, maxval=total_windows).start()
    window_latest = latest
    window_earliest = window_latest - poll_window
    for i in range(total_windows):
        window_latest -= poll_window
        if window_earliest - poll_window < earliest:
            window_earliest = earliest
        else:
            window_earliest -= poll_window
        poll_params = tm11.PollParameters(
            allow_asynch=False,
            response_type=RT_FULL,
            content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)])
        poll_request = tm11.PollRequest(
            message_id=tm11.generate_message_id(),
            collection_name=collection,
            exclusive_begin_timestamp_label=datetime.datetime.fromtimestamp(window_earliest).replace(tzinfo=pytz.utc),
            inclusive_end_timestamp_label=datetime.datetime.fromtimestamp(window_latest).replace(tzinfo=pytz.utc),
            poll_parameters=(poll_params))
        http_response = client.callTaxiiService2(
            host, endpoint,
            t.VID_TAXII_XML_11, poll_request.to_xml(),
            port=port)
        taxii_message = t.get_message_from_http_response(http_response,
            poll_request.message_id)
        if isinstance(taxii_message, tm11.StatusMessage):
            print("TAXII connection error! %s" % (taxii_message.message))
        elif isinstance(taxii_message, tm11.PollResponse):
            for content_block in taxii_message.content_blocks:
                try:
                    stix_package = taxii_content_block_to_stix(content_block)
                    (raw_stix_objs, raw_cybox_objs) = \
                        process_stix_pkg(stix_package)
                    for k in raw_stix_objs.keys():
                        cooked_stix_objs[k].update(raw_stix_objs[k])
                    for k in raw_cybox_objs.keys():
                        if not k in cooked_cybox_objs.keys():
                            cooked_cybox_objs[k] = set()
                        cooked_cybox_objs[k].update(raw_cybox_objs[k])
                except:
                    next
        if not quiet:
            progress.update(i)
    if not quiet:
        progress.finish()
    return(cooked_stix_objs, cooked_cybox_objs)
Exemple #7
0
def crits2edge(config, src, dest, daemon=False,
               now=None, last_run=None):
    xmlns_name = config['edge']['sites'][dest]['stix']['xmlns_name']
    # check if (and when) we synced src and dest...
    if not now:
        now = util.nowutc()
    if not last_run:
        last_run = config['db'].get_last_sync(src=src, dest=dest,
                                              direction='c2e')
    config['logger'].info(
        log.log_messages['start_sync'].format(
            type_='crits', last_run=last_run, src=src, dest=dest))
    endpoints = ['ips', 'domains', 'samples', 'emails', 'indicators', 'events']
    # setup the tally counters
    config['crits_tally'] = dict()
    config['crits_tally']['all'] = {'incoming': 0, 'processed': 0}
    for endpoint in endpoints:
        config['crits_tally'][endpoint] = {'incoming': 0, 'processed': 0}
    ids = dict()
    for endpoint in endpoints:
        ids[endpoint] = fetch_crits_object_ids(config, src, endpoint, last_run)
        if not len(ids[endpoint]):
            continue
        else:
            for crits_id in ids[endpoint]:
                (id_, json_) = crits_poll(config, src, endpoint, crits_id,)
                if endpoint == 'indicators':
                    indicator = json2indicator(config, src, dest,
                                              endpoint, json_, id_)
                    config['crits_tally']['indicators']['incoming'] += 1
                    config['crits_tally']['all']['incoming'] += 1
                    if not indicator:
                        config['logger'].info(
                            log.log_messages['obj_inbox_error'].format(
                                src_type='crits', id_=crits_id, dest_type='edge'))
                        continue
                    stix_ = stix_pkg(config, src, endpoint, indicator, dest=dest)
                    if not stix_:
                        config['logger'].info(
                            log.log_messages['obj_inbox_error'].format(
                                src_type='crits', id_=crits_id, dest_type='edge'))
                        continue
                    success = edge.taxii_inbox(config, dest, stix_, src=src,
                                                crits_id=endpoint + ':'
                                                + crits_id)
                    if not success:
                        config['logger'].info(
                            log.log_messages['obj_inbox_error'].format(
                                src_type='crits', id_=crits_id, dest_type='edge'))
                        continue
                    else:
                        # track the related crits/json ids (by src/dest)
                        config['db'].set_object_id(src, dest,
                                                   edge_id=indicator.id_,
                                                   crits_id=(xmlns_name + ':' + 
                                                             endpoint + '-' + crits_id))
                        config['crits_tally']['indicators']['processed'] += 1
                        config['crits_tally']['all']['processed'] += 1
                elif endpoint == 'events':
                    incident = json2incident(config, src, dest,
                                              endpoint, json_, id_)
                    config['crits_tally']['events']['incoming'] += 1
                    config['crits_tally']['all']['incoming'] += 1
                    if not incident:
                        config['logger'].info(
                            log.log_messages['obj_inbox_error'].format(
                                src_type='crits', id_=crits_id, dest_type='edge'))
                        continue
                    stix_ = stix_pkg(config, src, endpoint, incident, dest=dest)
                    if not stix_:
                        config['logger'].info(
                            log.log_messages['obj_inbox_error'].format(
                                src_type='crits', id_=crits_id, dest_type='edge'))
                        continue
                    success = edge.taxii_inbox(config, dest, stix_, src=src,
                                                crits_id=endpoint + ':'
                                                + crits_id)
                    if not success:
                        config['logger'].info(
                            log.log_messages['obj_inbox_error'].format(
                                src_type='crits', id_=crits_id, dest_type='edge'))
                        continue
                    else:
                        # track the related crits/json ids (by src/dest)
                        config['db'].set_object_id(src, dest,
                                                   edge_id=incident.id_,
                                                   crits_id=(xmlns_name + ':' + 
                                                             endpoint + '-' + crits_id))
                        config['crits_tally']['events']['processed'] += 1
                        config['crits_tally']['all']['processed'] += 1
                else:
                    observable = json2observable(config, src, dest, endpoint, json_, crits_id)
                    config['crits_tally'][endpoint]['incoming'] += 1
                    config['crits_tally']['all']['incoming'] += 1
                    if not observable:
                        config['logger'].info(
                            log.log_messages['obj_inbox_error'].format(
                                src_type='crits', id_=crits_id, dest_type='edge'))
                        continue
                    stix_ = stix_pkg(config, src, endpoint, observable, dest=dest)
                    if not stix_:
                        config['logger'].info(
                            log.log_messages['obj_inbox_error'].format(
                                src_type='crits', id_=crits_id, dest_type='edge'))
                        continue
                    success = edge.taxii_inbox(config, dest, stix_)
                    if not success:
                        config['logger'].info(
                            log.log_messages['obj_inbox_error'].format(
                                src_type='crits', id_=crits_id, dest_type='edge'))
                        continue
                    else:
                        config['crits_tally'][endpoint]['processed'] += 1
                        config['crits_tally']['all']['processed'] += 1
                        config['db'].set_object_id(src, dest,
                                                   edge_id=observable.id_,
                                                   crits_id=(xmlns_name + ':' + 
                                                             endpoint + '-' + crits_id))
    for endpoint in endpoints:
        if config['crits_tally'][endpoint]['incoming'] > 0:
            config['logger'].info(log.log_messages['incoming_tally'].format(
                    count=config['crits_tally'][endpoint]['incoming'],
                    type_=endpoint, src='crits', dest='edge'))
        if (config['crits_tally'][endpoint]['incoming'] -
                   config['crits_tally'][endpoint]['processed']) > 0:
            config['logger'].info(log.log_messages['failed_tally'].format(
                    count=(config['crits_tally'][endpoint]['incoming'] -
                           config['crits_tally'][endpoint]['processed']),
                    type_=endpoint, src='crits', dest='edge'))
        if config['crits_tally'][endpoint]['processed'] > 0:
            config['logger'].info(log.log_messages['processed_tally'].format(
                    count=config['crits_tally'][endpoint]['processed'], 
                    type_=endpoint, src='crits', dest='edge'))
    if config['crits_tally']['all']['incoming'] > 0:
        config['logger'].info(log.log_messages['incoming_tally'].format(
                count=config['crits_tally']['all']['incoming'], type_='total',
                src='crits', dest='edge'))
    if (config['crits_tally']['all']['incoming'] -
               config['crits_tally']['all']['processed']) > 0:
        config['logger'].info(log.log_messages['failed_tally'].format(
                count=(config['crits_tally']['all']['incoming'] -
                       config['crits_tally']['all']['processed']),
                type_='total', src='crits', dest='edge'))
    if config['crits_tally']['all']['processed'] > 0:
        config['logger'].info(log.log_messages['processed_tally'].format(
                count=config['crits_tally']['all']['processed'], type_='total',
                src='crits', dest='edge'))
    # save state to disk for next run...
    if config['daemon']['debug']:
        poll_interval = config['crits']['sites'][src]['api']['poll_interval']
        config['logger'].debug(
            log.log_messages['saving_state'].format(
                next_run=str(now + datetime.timedelta(seconds=poll_interval))))
    if not daemon:
        config['db'].set_last_sync(src=src, dest=dest,
                                   direction='c2e', timestamp=now)
        return(None)
    else:
        return(util.nowutc())
Exemple #8
0
def crits2edge(config, src, dest, daemon=False, now=None, last_run=None):
    xmlns_name = config['edge']['sites'][dest]['stix']['xmlns_name']
    # check if (and when) we synced src and dest...
    if not now:
        now = util.nowutc()
    if not last_run:
        last_run = config['db'].get_last_sync(src=src,
                                              dest=dest,
                                              direction='c2e')
    config['logger'].info(log.log_messages['start_sync'].format(
        type_='crits', last_run=last_run, src=src, dest=dest))
    endpoints = ['ips', 'domains', 'samples', 'emails', 'indicators', 'events']
    # setup the tally counters
    config['crits_tally'] = dict()
    config['crits_tally']['all'] = {'incoming': 0, 'processed': 0}
    for endpoint in endpoints:
        config['crits_tally'][endpoint] = {'incoming': 0, 'processed': 0}
    ids = dict()
    for endpoint in endpoints:
        ids[endpoint] = fetch_crits_object_ids(config, src, endpoint, last_run)
        if not len(ids[endpoint]):
            continue
        else:
            for crits_id in ids[endpoint]:
                (id_, json_) = crits_poll(
                    config,
                    src,
                    endpoint,
                    crits_id,
                )
                if endpoint == 'indicators':
                    indicator = json2indicator(config, src, dest, endpoint,
                                               json_, id_)
                    config['crits_tally']['indicators']['incoming'] += 1
                    config['crits_tally']['all']['incoming'] += 1
                    if not indicator:
                        config['logger'].info(
                            log.log_messages['obj_inbox_error'].format(
                                src_type='crits',
                                id_=crits_id,
                                dest_type='edge'))
                        continue
                    stix_ = stix_pkg(config,
                                     src,
                                     endpoint,
                                     indicator,
                                     dest=dest)
                    if not stix_:
                        config['logger'].info(
                            log.log_messages['obj_inbox_error'].format(
                                src_type='crits',
                                id_=crits_id,
                                dest_type='edge'))
                        continue
                    success = edge.taxii_inbox(config,
                                               dest,
                                               stix_,
                                               src=src,
                                               crits_id=endpoint + ':' +
                                               crits_id)
                    if not success:
                        config['logger'].info(
                            log.log_messages['obj_inbox_error'].format(
                                src_type='crits',
                                id_=crits_id,
                                dest_type='edge'))
                        continue
                    else:
                        # track the related crits/json ids (by src/dest)
                        config['db'].set_object_id(
                            src,
                            dest,
                            edge_id=indicator.id_,
                            crits_id=(xmlns_name + ':' + endpoint + '-' +
                                      crits_id))
                        config['crits_tally']['indicators']['processed'] += 1
                        config['crits_tally']['all']['processed'] += 1
                elif endpoint == 'events':
                    incident = json2incident(config, src, dest, endpoint,
                                             json_, id_)
                    config['crits_tally']['events']['incoming'] += 1
                    config['crits_tally']['all']['incoming'] += 1
                    if not incident:
                        config['logger'].info(
                            log.log_messages['obj_inbox_error'].format(
                                src_type='crits',
                                id_=crits_id,
                                dest_type='edge'))
                        continue
                    stix_ = stix_pkg(config,
                                     src,
                                     endpoint,
                                     incident,
                                     dest=dest)
                    if not stix_:
                        config['logger'].info(
                            log.log_messages['obj_inbox_error'].format(
                                src_type='crits',
                                id_=crits_id,
                                dest_type='edge'))
                        continue
                    success = edge.taxii_inbox(config,
                                               dest,
                                               stix_,
                                               src=src,
                                               crits_id=endpoint + ':' +
                                               crits_id)
                    if not success:
                        config['logger'].info(
                            log.log_messages['obj_inbox_error'].format(
                                src_type='crits',
                                id_=crits_id,
                                dest_type='edge'))
                        continue
                    else:
                        # track the related crits/json ids (by src/dest)
                        config['db'].set_object_id(
                            src,
                            dest,
                            edge_id=incident.id_,
                            crits_id=(xmlns_name + ':' + endpoint + '-' +
                                      crits_id))
                        config['crits_tally']['events']['processed'] += 1
                        config['crits_tally']['all']['processed'] += 1
                else:
                    observable = json2observable(config, src, dest, endpoint,
                                                 json_, crits_id)
                    config['crits_tally'][endpoint]['incoming'] += 1
                    config['crits_tally']['all']['incoming'] += 1
                    if not observable:
                        config['logger'].info(
                            log.log_messages['obj_inbox_error'].format(
                                src_type='crits',
                                id_=crits_id,
                                dest_type='edge'))
                        continue
                    stix_ = stix_pkg(config,
                                     src,
                                     endpoint,
                                     observable,
                                     dest=dest)
                    if not stix_:
                        config['logger'].info(
                            log.log_messages['obj_inbox_error'].format(
                                src_type='crits',
                                id_=crits_id,
                                dest_type='edge'))
                        continue
                    success = edge.taxii_inbox(config, dest, stix_)
                    if not success:
                        config['logger'].info(
                            log.log_messages['obj_inbox_error'].format(
                                src_type='crits',
                                id_=crits_id,
                                dest_type='edge'))
                        continue
                    else:
                        config['crits_tally'][endpoint]['processed'] += 1
                        config['crits_tally']['all']['processed'] += 1
                        config['db'].set_object_id(
                            src,
                            dest,
                            edge_id=observable.id_,
                            crits_id=(xmlns_name + ':' + endpoint + '-' +
                                      crits_id))
    for endpoint in endpoints:
        if config['crits_tally'][endpoint]['incoming'] > 0:
            config['logger'].info(log.log_messages['incoming_tally'].format(
                count=config['crits_tally'][endpoint]['incoming'],
                type_=endpoint,
                src='crits',
                dest='edge'))
        if (config['crits_tally'][endpoint]['incoming'] -
                config['crits_tally'][endpoint]['processed']) > 0:
            config['logger'].info(log.log_messages['failed_tally'].format(
                count=(config['crits_tally'][endpoint]['incoming'] -
                       config['crits_tally'][endpoint]['processed']),
                type_=endpoint,
                src='crits',
                dest='edge'))
        if config['crits_tally'][endpoint]['processed'] > 0:
            config['logger'].info(log.log_messages['processed_tally'].format(
                count=config['crits_tally'][endpoint]['processed'],
                type_=endpoint,
                src='crits',
                dest='edge'))
    if config['crits_tally']['all']['incoming'] > 0:
        config['logger'].info(log.log_messages['incoming_tally'].format(
            count=config['crits_tally']['all']['incoming'],
            type_='total',
            src='crits',
            dest='edge'))
    if (config['crits_tally']['all']['incoming'] -
            config['crits_tally']['all']['processed']) > 0:
        config['logger'].info(log.log_messages['failed_tally'].format(
            count=(config['crits_tally']['all']['incoming'] -
                   config['crits_tally']['all']['processed']),
            type_='total',
            src='crits',
            dest='edge'))
    if config['crits_tally']['all']['processed'] > 0:
        config['logger'].info(log.log_messages['processed_tally'].format(
            count=config['crits_tally']['all']['processed'],
            type_='total',
            src='crits',
            dest='edge'))
    # save state to disk for next run...
    if config['daemon']['debug']:
        poll_interval = config['crits']['sites'][src]['api']['poll_interval']
        config['logger'].debug(log.log_messages['saving_state'].format(
            next_run=str(now + datetime.timedelta(seconds=poll_interval))))
    if not daemon:
        config['db'].set_last_sync(src=src,
                                   dest=dest,
                                   direction='c2e',
                                   timestamp=now)
        return (None)
    else:
        return (util.nowutc())
Exemple #9
0
def taxii_poll(host=None, port=None, endpoint=None, collection=None, user=None, passwd=None, ssl_cert=None, use_ssl=None, attempt_validation=None, time_range=None, quiet=None):
    '''poll cti via taxii'''
    client = tc.HttpClient()
    client.set_use_https(use_ssl)
    if ssl_cert:
        client.setAuthType(client.AUTH_CERT_BASIC)
        client.setAuthCredentials(
            {'username': user,
             'password': passwd,
             'key_file': ssl_cert,
             'cert_file': ssl_cert})
    else:
        client.setAuthType(client.AUTH_BASIC)
        client.setAuthCredentials(
            {'username': user,
             'password': passwd})
    cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
                        'exploit_targets': set(), 'incidents': set(), \
                        'indicators': set(), 'threat_actors': set(), \
                        'ttps': set()}
    cooked_cybox_objs = {'AccountObjectType': set(),
                      'AddressObjectType': set(),
                      'APIObjectType': set(),
                      'ArchiveFileObjectType': set(),
                      'ARPCacheObjectType': set(),
                      'ArtifactObjectType': set(),
                      'ASObjectType': set(),
                      'CodeObjectType': set(),
                      'CustomObjectType': set(),
                      'DeviceObjectType': set(),
                      'DiskObjectType': set(),
                      'DiskPartitionObjectType': set(),
                      'DNSCacheObjectType': set(),
                      'DNSQueryObjectType': set(),
                      'DNSRecordObjectType': set(),
                      'DomainNameObjectType': set(),
                      'EmailMessageObjectType': set(),
                      'FileObjectType': set(),
                      'GUIDialogboxObjectType': set(),
                      'GUIObjectType': set(),
                      'GUIWindowObjectType': set(),
                      'HostnameObjectType': set(),
                      'HTTPSessionObjectType': set(),
                      'ImageFileObjectType': set(),
                      'LibraryObjectType': set(),
                      'LinkObjectType': set(),
                      'LinuxPackageObjectType': set(),
                      'MemoryObjectType': set(),
                      'MutexObjectType': set(),
                      'NetworkConnectionObjectType': set(),
                      'NetworkFlowObjectType': set(),
                      'NetworkPacketObjectType': set(),
                      'NetworkRouteEntryObjectType': set(),
                      'NetRouteObjectType': set(),
                      'NetworkSocketObjectType': set(),
                      'NetworkSubnetObjectType': set(),
                      'PDFFileObjectType': set(),
                      'PipeObjectType': set(),
                      'PortObjectType': set(),
                      'ProcessObjectType': set(),
                      'ProductObjectType': set(),
                      'SemaphoreObjectType': set(),
                      'SMSMessageObjectType': set(),
                      'SocketAddressObjectType': set(),
                      'SystemObjectType': set(),
                      'UnixFileObjectType': set(),
                      'UnixNetworkRouteEntryObjectType': set(),
                      'UnixPipeObjectType': set(),
                      'UnixProcessObjectType': set(),
                      'UnixUserAccountObjectType': set(),
                      'UnixVolumeObjectType': set(),
                      'URIObjectType': set(),
                      'URLHistoryObjectType': set(),
                      'UserAccountObjectType': set(),
                      'UserSessionObjectType': set(),
                      'VolumeObjectType': set(),
                      'WhoisObjectType': set(),
                      'WindowsComputerAccountObjectType': set(),
                      'WindowsCriticalSectionObjectType': set(),
                      'WindowsDriverObjectType': set(),
                      'WindowsEventLogObjectType': set(),
                      'WindowsEventObjectType': set(),
                      'WindowsExecutableFileObjectType': set(),
                      'WindowsFilemappingObjectType': set(),
                      'WindowsFileObjectType': set(),
                      'WindowsHandleObjectType': set(),
                      'WindowsHookObjectType': set(),
                      'WindowsKernelHookObjectType': set(),
                      'WindowsKernelObjectType': set(),
                      'WindowsMailslotObjectType': set(),
                      'WindowsMemoryPageRegionObjectType': set(),
                      'WindowsMutexObjectType': set(),
                      'WindowsNetworkRouteEntryObjectType': set(),
                      'WindowsNetworkShareObjectType': set(),
                      'WindowsPipeObjectType': set(),
                      'WindowsPrefetchObjectType': set(),
                      'WindowsProcessObjectType': set(),
                      'WindowsRegistryKeyObjectType': set(),
                      'WindowsSemaphoreObjectType': set(),
                      'WindowsServiceObjectType': set(),
                      'WindowsSystemObjectType': set(),
                      'WindowsSystemRestoreObjectType': set(),
                      'WindowsTaskObjectType': set(),
                      'WindowsThreadObjectType': set(),
                      'WindowsUserAccountObjectType': set(),
                      'WindowsVolumeObjectType': set(),
                      'WindowsWaitableTimerObjectType': set(),
                      'X509CertificateObjectType': set(),
    }
    earliest = poll_start(time_range)
    latest = nowutc()
    poll_window = 43200 # 12 hour blocks seem reasonable
    total_windows = (latest - earliest) / poll_window
    if (latest - earliest) % poll_window:
        total_windows += 1
    if not quiet:
        widgets = ['TAXII Poll: ', Percentage(), ' ', Bar(marker=RotatingMarker()),
                   ' ', ETA()]
        progress = ProgressBar(widgets=widgets, maxval=total_windows).start()
    window_latest = latest
    window_earliest = window_latest - poll_window
    for i in range(total_windows):
        window_latest -= poll_window
        if window_earliest - poll_window < earliest:
            window_earliest = earliest
        else:
            window_earliest -= poll_window
        if window_latest <= window_earliest:
            if not quiet:
                progress.update(i)
            break
        poll_params = tm11.PollParameters(
            allow_asynch=False,
            response_type=RT_FULL,
            content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)])
        poll_request = tm11.PollRequest(
            message_id=tm11.generate_message_id(),
            collection_name=collection,
            exclusive_begin_timestamp_label=datetime.datetime.fromtimestamp(window_earliest).replace(tzinfo=pytz.utc),
            inclusive_end_timestamp_label=datetime.datetime.fromtimestamp(window_latest).replace(tzinfo=pytz.utc),
            poll_parameters=(poll_params))
        try:
            http_response = client.callTaxiiService2(
                host, endpoint,
                t.VID_TAXII_XML_11, poll_request.to_xml(),
                port=port)
            taxii_message = t.get_message_from_http_response(http_response,
                poll_request.message_id)
            if isinstance(taxii_message, tm11.StatusMessage):
                print("TAXII connection error! %s" % (taxii_message.message))
            elif isinstance(taxii_message, tm11.PollResponse):
                for content_block in taxii_message.content_blocks:
                    try:
                        stix_package = taxii_content_block_to_stix(content_block)
                        (raw_stix_objs, raw_cybox_objs) = \
                            process_stix_pkg(stix_package)
                        for k in raw_stix_objs.keys():
                            cooked_stix_objs[k].update(raw_stix_objs[k])
                        for k in raw_cybox_objs.keys():
                            if not k in cooked_cybox_objs.keys():
                                cooked_cybox_objs[k] = set()
                            cooked_cybox_objs[k].update(raw_cybox_objs[k])
                    except:
                        next
        except:
            next
        if not quiet:
            progress.update(i)
    if not quiet:
        progress.finish()
    return(cooked_stix_objs, cooked_cybox_objs)
Exemple #10
0
def taxii_poll(host=None,
               port=None,
               endpoint=None,
               collection=None,
               user=None,
               passwd=None,
               use_ssl=None,
               attempt_validation=None,
               time_range=None,
               quiet=None):
    '''poll cti via taxii'''
    client = tc.HttpClient()
    client.setUseHttps(use_ssl)
    client.setAuthType(client.AUTH_BASIC)
    client.setAuthCredentials({'username': user, 'password': passwd})
    cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
                        'exploit_targets': set(), 'incidents': set(), \
                        'indicators': set(), 'threat_actors': set(), \
                        'ttps': set()}
    cooked_cybox_objs = dict()
    earliest = poll_start(time_range)
    latest = nowutc()
    poll_window = 43200  # 12 hour blocks seem reasonable
    total_windows = (latest - earliest) / poll_window
    if (latest - earliest) % poll_window:
        total_windows += 1
    if not quiet:
        widgets = [
            'TAXII Poll: ',
            Percentage(), ' ',
            Bar(marker=RotatingMarker()), ' ',
            ETA()
        ]
        progress = ProgressBar(widgets=widgets, maxval=total_windows).start()
    window_latest = latest
    window_earliest = window_latest - poll_window
    for i in range(total_windows):
        window_latest -= poll_window
        if window_earliest - poll_window < earliest:
            window_earliest = earliest
        else:
            window_earliest -= poll_window
        poll_params = tm11.PollParameters(
            allow_asynch=False,
            response_type=RT_FULL,
            content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)])
        poll_request = tm11.PollRequest(
            message_id=tm11.generate_message_id(),
            collection_name=collection,
            exclusive_begin_timestamp_label=datetime.datetime.fromtimestamp(
                window_earliest).replace(tzinfo=pytz.utc),
            inclusive_end_timestamp_label=datetime.datetime.fromtimestamp(
                window_latest).replace(tzinfo=pytz.utc),
            poll_parameters=(poll_params))
        http_response = client.callTaxiiService2(host,
                                                 endpoint,
                                                 t.VID_TAXII_XML_11,
                                                 poll_request.to_xml(),
                                                 port=port)
        taxii_message = t.get_message_from_http_response(
            http_response, poll_request.message_id)
        if isinstance(taxii_message, tm11.StatusMessage):
            print("TAXII connection error! %s" % (taxii_message.message))
        elif isinstance(taxii_message, tm11.PollResponse):
            for content_block in taxii_message.content_blocks:
                try:
                    stix_package = taxii_content_block_to_stix(content_block)
                    (raw_stix_objs, raw_cybox_objs) = \
                        process_stix_pkg(stix_package)
                    for k in raw_stix_objs.keys():
                        cooked_stix_objs[k].update(raw_stix_objs[k])
                    for k in raw_cybox_objs.keys():
                        if not k in cooked_cybox_objs.keys():
                            cooked_cybox_objs[k] = set()
                        cooked_cybox_objs[k].update(raw_cybox_objs[k])
                except:
                    next
        if not quiet:
            progress.update(i)
    if not quiet:
        progress.finish()
    return (cooked_stix_objs, cooked_cybox_objs)
Exemple #11
0
def edge2crits(config, src, dest, daemon=False, now=None,
               last_run=None):
    '''sync an edge instance with crits'''
    # check if (and when) we synced src and dest...
    if not now:
        now = util.nowutc()
    if not last_run:
        # didn't get last_run as an arg so check the db...
        last_run = config['db'].get_last_sync(src=src, dest=dest,
                                              direction='e2c')
    config['logger'].info(log.log_messages['start_sync'].format(
        type_='edge', last_run=str(last_run), src=src, dest=dest))
    # setup the tally counters
    config['edge_tally'] = dict()
    endpoints = ['ips', 'domains', 'samples', 'emails', 'indicators', 'relationships', 'events']
    config['edge_tally']['all'] = {'incoming': 0, 'processed': 0}
    for endpoint in endpoints:
        config['edge_tally'][endpoint] = {'incoming': 0, 'processed': 0}
    # poll for new edge data...
    (latest, incidents, indicators, observables) = \
        taxii_poll(config, src, dest, last_run)
    process_observables(config, src, dest, observables)
    process_incidents(config, src, dest, incidents)
    process_indicators(config, src, dest, indicators)
    process_relationships(config, src, dest)
    for endpoint in endpoints:
        if config['edge_tally'][endpoint]['incoming'] > 0:
            config['logger'].info(log.log_messages['incoming_tally'].format(
                    count=config['edge_tally'][endpoint]['incoming'],
                    type_=endpoint, src='edge', dest='crits'))
        if (config['edge_tally'][endpoint]['incoming'] -
                   config['edge_tally'][endpoint]['processed']) > 0:
            config['logger'].info(log.log_messages['failed_tally'].format(
                    count=(config['edge_tally'][endpoint]['incoming'] -
                           config['edge_tally'][endpoint]['processed']),
                    type_=endpoint, src='edge', dest='crits'))
        if config['edge_tally'][endpoint]['processed'] > 0:
            config['logger'].info(log.log_messages['processed_tally'].format(
                    count=config['edge_tally'][endpoint]['processed'], 
                    type_=endpoint, src='edge', dest='crits'))
    if config['edge_tally']['all']['incoming'] > 0:
        config['logger'].info(log.log_messages['incoming_tally'].format(
                count=config['edge_tally']['all']['incoming'], type_='total',
                src='edge', dest='crits'))
    if (config['edge_tally']['all']['incoming'] -
               config['edge_tally']['all']['processed']) > 0:
        config['logger'].info(log.log_messages['failed_tally'].format(
                count=(config['edge_tally']['all']['incoming'] -
                       config['edge_tally']['all']['processed']),
                type_='total', src='edge', dest='crits'))
    if config['edge_tally']['all']['processed'] > 0:
        config['logger'].info(log.log_messages['processed_tally'].format(
                count=config['edge_tally']['all']['processed'], type_='total',
                src='edge', dest='crits'))
    # save state to disk for next run...
    if config['daemon']['debug']:
        poll_interval = \
            config['edge']['sites'][src]['taxii']['poll_interval']
        next_run = str(now + datetime.timedelta(seconds=poll_interval))
        config['logger'].debug(log.log_messages['saving_state'].format(
            next_run=next_run))
    if not daemon:
        config['db'].set_last_sync(src=src, dest=dest,
                                   direction='e2c', timestamp=now)
        return(None)
    else:
        return(util.nowutc())
Exemple #12
0
def edge2crits(config, src, dest, daemon=False, now=None, last_run=None):
    """sync an edge instance with crits"""
    # check if (and when) we synced src and dest...
    if not now:
        now = util.nowutc()
    if not last_run:
        # didn't get last_run as an arg so check the db...
        last_run = config["db"].get_last_sync(src=src, dest=dest, direction="e2c")
    config["logger"].info(
        log.log_messages["start_sync"].format(type_="edge", last_run=str(last_run), src=src, dest=dest)
    )
    # setup the tally counters
    config["edge_tally"] = dict()
    endpoints = ["ips", "domains", "samples", "emails", "indicators", "relationships", "events"]
    config["edge_tally"]["all"] = {"incoming": 0, "processed": 0}
    for endpoint in endpoints:
        config["edge_tally"][endpoint] = {"incoming": 0, "processed": 0}
    # poll for new edge data...
    (latest, incidents, indicators, observables) = taxii_poll(config, src, dest, last_run)
    process_observables(config, src, dest, observables)
    process_incidents(config, src, dest, incidents)
    process_indicators(config, src, dest, indicators)
    process_relationships(config, src, dest)
    for endpoint in endpoints:
        if config["edge_tally"][endpoint]["incoming"] > 0:
            config["logger"].info(
                log.log_messages["incoming_tally"].format(
                    count=config["edge_tally"][endpoint]["incoming"], type_=endpoint, src="edge", dest="crits"
                )
            )
        if (config["edge_tally"][endpoint]["incoming"] - config["edge_tally"][endpoint]["processed"]) > 0:
            config["logger"].info(
                log.log_messages["failed_tally"].format(
                    count=(config["edge_tally"][endpoint]["incoming"] - config["edge_tally"][endpoint]["processed"]),
                    type_=endpoint,
                    src="edge",
                    dest="crits",
                )
            )
        if config["edge_tally"][endpoint]["processed"] > 0:
            config["logger"].info(
                log.log_messages["processed_tally"].format(
                    count=config["edge_tally"][endpoint]["processed"], type_=endpoint, src="edge", dest="crits"
                )
            )
    if config["edge_tally"]["all"]["incoming"] > 0:
        config["logger"].info(
            log.log_messages["incoming_tally"].format(
                count=config["edge_tally"]["all"]["incoming"], type_="total", src="edge", dest="crits"
            )
        )
    if (config["edge_tally"]["all"]["incoming"] - config["edge_tally"]["all"]["processed"]) > 0:
        config["logger"].info(
            log.log_messages["failed_tally"].format(
                count=(config["edge_tally"]["all"]["incoming"] - config["edge_tally"]["all"]["processed"]),
                type_="total",
                src="edge",
                dest="crits",
            )
        )
    if config["edge_tally"]["all"]["processed"] > 0:
        config["logger"].info(
            log.log_messages["processed_tally"].format(
                count=config["edge_tally"]["all"]["processed"], type_="total", src="edge", dest="crits"
            )
        )
    # save state to disk for next run...
    if config["daemon"]["debug"]:
        poll_interval = config["edge"]["sites"][src]["taxii"]["poll_interval"]
        next_run = str(now + datetime.timedelta(seconds=poll_interval))
        config["logger"].debug(log.log_messages["saving_state"].format(next_run=next_run))
    if not daemon:
        config["db"].set_last_sync(src=src, dest=dest, direction="e2c", timestamp=now)
        return None
    else:
        return util.nowutc()
Exemple #13
0
def taxii_poll(host=None,
               port=None,
               endpoint=None,
               collection=None,
               user=None,
               passwd=None,
               ssl_cert=None,
               use_ssl=None,
               attempt_validation=None,
               time_range=None,
               quiet=None):
    '''poll cti via taxii'''
    client = tc.HttpClient()
    client.set_use_https(use_ssl)
    if ssl_cert:
        client.setAuthType(client.AUTH_CERT_BASIC)
        client.setAuthCredentials({
            'username': user,
            'password': passwd,
            'key_file': ssl_cert,
            'cert_file': ssl_cert
        })
    else:
        client.setAuthType(client.AUTH_BASIC)
        client.setAuthCredentials({'username': user, 'password': passwd})
    cooked_stix_objs = {'campaigns': set(), 'courses_of_action': set(), \
                        'exploit_targets': set(), 'incidents': set(), \
                        'indicators': set(), 'threat_actors': set(), \
                        'ttps': set()}
    cooked_cybox_objs = {
        'AccountObjectType': set(),
        'AddressObjectType': set(),
        'APIObjectType': set(),
        'ArchiveFileObjectType': set(),
        'ARPCacheObjectType': set(),
        'ArtifactObjectType': set(),
        'ASObjectType': set(),
        'CodeObjectType': set(),
        'CustomObjectType': set(),
        'DeviceObjectType': set(),
        'DiskObjectType': set(),
        'DiskPartitionObjectType': set(),
        'DNSCacheObjectType': set(),
        'DNSQueryObjectType': set(),
        'DNSRecordObjectType': set(),
        'DomainNameObjectType': set(),
        'EmailMessageObjectType': set(),
        'FileObjectType': set(),
        'GUIDialogboxObjectType': set(),
        'GUIObjectType': set(),
        'GUIWindowObjectType': set(),
        'HostnameObjectType': set(),
        'HTTPSessionObjectType': set(),
        'ImageFileObjectType': set(),
        'LibraryObjectType': set(),
        'LinkObjectType': set(),
        'LinuxPackageObjectType': set(),
        'MemoryObjectType': set(),
        'MutexObjectType': set(),
        'NetworkConnectionObjectType': set(),
        'NetworkFlowObjectType': set(),
        'NetworkPacketObjectType': set(),
        'NetworkRouteEntryObjectType': set(),
        'NetRouteObjectType': set(),
        'NetworkSocketObjectType': set(),
        'NetworkSubnetObjectType': set(),
        'PDFFileObjectType': set(),
        'PipeObjectType': set(),
        'PortObjectType': set(),
        'ProcessObjectType': set(),
        'ProductObjectType': set(),
        'SemaphoreObjectType': set(),
        'SMSMessageObjectType': set(),
        'SocketAddressObjectType': set(),
        'SystemObjectType': set(),
        'UnixFileObjectType': set(),
        'UnixNetworkRouteEntryObjectType': set(),
        'UnixPipeObjectType': set(),
        'UnixProcessObjectType': set(),
        'UnixUserAccountObjectType': set(),
        'UnixVolumeObjectType': set(),
        'URIObjectType': set(),
        'URLHistoryObjectType': set(),
        'UserAccountObjectType': set(),
        'UserSessionObjectType': set(),
        'VolumeObjectType': set(),
        'WhoisObjectType': set(),
        'WindowsComputerAccountObjectType': set(),
        'WindowsCriticalSectionObjectType': set(),
        'WindowsDriverObjectType': set(),
        'WindowsEventLogObjectType': set(),
        'WindowsEventObjectType': set(),
        'WindowsExecutableFileObjectType': set(),
        'WindowsFilemappingObjectType': set(),
        'WindowsFileObjectType': set(),
        'WindowsHandleObjectType': set(),
        'WindowsHookObjectType': set(),
        'WindowsKernelHookObjectType': set(),
        'WindowsKernelObjectType': set(),
        'WindowsMailslotObjectType': set(),
        'WindowsMemoryPageRegionObjectType': set(),
        'WindowsMutexObjectType': set(),
        'WindowsNetworkRouteEntryObjectType': set(),
        'WindowsNetworkShareObjectType': set(),
        'WindowsPipeObjectType': set(),
        'WindowsPrefetchObjectType': set(),
        'WindowsProcessObjectType': set(),
        'WindowsRegistryKeyObjectType': set(),
        'WindowsSemaphoreObjectType': set(),
        'WindowsServiceObjectType': set(),
        'WindowsSystemObjectType': set(),
        'WindowsSystemRestoreObjectType': set(),
        'WindowsTaskObjectType': set(),
        'WindowsThreadObjectType': set(),
        'WindowsUserAccountObjectType': set(),
        'WindowsVolumeObjectType': set(),
        'WindowsWaitableTimerObjectType': set(),
        'X509CertificateObjectType': set(),
    }
    earliest = poll_start(time_range)
    latest = nowutc()
    poll_window = 43200  # 12 hour blocks seem reasonable
    total_windows = (latest - earliest) / poll_window
    if (latest - earliest) % poll_window:
        total_windows += 1
    if not quiet:
        widgets = [
            'TAXII Poll: ',
            Percentage(), ' ',
            Bar(marker=RotatingMarker()), ' ',
            ETA()
        ]
        progress = ProgressBar(widgets=widgets, maxval=total_windows).start()
    window_latest = latest
    window_earliest = window_latest - poll_window
    for i in range(total_windows):
        window_latest -= poll_window
        if window_earliest - poll_window < earliest:
            window_earliest = earliest
        else:
            window_earliest -= poll_window
        if window_latest <= window_earliest:
            if not quiet:
                progress.update(i)
            break
        poll_params = tm11.PollParameters(
            allow_asynch=False,
            response_type=RT_FULL,
            content_bindings=[tm11.ContentBinding(binding_id=CB_STIX_XML_11)])
        poll_request = tm11.PollRequest(
            message_id=tm11.generate_message_id(),
            collection_name=collection,
            exclusive_begin_timestamp_label=datetime.datetime.fromtimestamp(
                window_earliest).replace(tzinfo=pytz.utc),
            inclusive_end_timestamp_label=datetime.datetime.fromtimestamp(
                window_latest).replace(tzinfo=pytz.utc),
            poll_parameters=(poll_params))
        try:
            http_response = client.callTaxiiService2(host,
                                                     endpoint,
                                                     t.VID_TAXII_XML_11,
                                                     poll_request.to_xml(),
                                                     port=port)
            taxii_message = t.get_message_from_http_response(
                http_response, poll_request.message_id)
            if isinstance(taxii_message, tm11.StatusMessage):
                print("TAXII connection error! %s" % (taxii_message.message))
            elif isinstance(taxii_message, tm11.PollResponse):
                for content_block in taxii_message.content_blocks:
                    try:
                        stix_package = taxii_content_block_to_stix(
                            content_block)
                        (raw_stix_objs, raw_cybox_objs) = \
                            process_stix_pkg(stix_package)
                        for k in raw_stix_objs.keys():
                            cooked_stix_objs[k].update(raw_stix_objs[k])
                        for k in raw_cybox_objs.keys():
                            if not k in cooked_cybox_objs.keys():
                                cooked_cybox_objs[k] = set()
                            cooked_cybox_objs[k].update(raw_cybox_objs[k])
                    except:
                        next
        except:
            next
        if not quiet:
            progress.update(i)
    if not quiet:
        progress.finish()
    return (cooked_stix_objs, cooked_cybox_objs)