예제 #1
0
def cybox_address_to_json(config, observable):
    '''translate a cybox address object to crits json'''
    crits_types = {'cidr': 'Address - cidr',
                   'ipv4-addr': 'Address - ipv4-addr',
                   'ipv4-net': 'Address - ipv4-net',
                   'ipv4-netmask': 'Address - ipv4-net-mask',
                   'ipv6-addr': 'Address - ipv6-addr',
                   'ipv6-net': 'Address - ipv6-net',
                   'ipv6-netmask': 'Address - ipv6-net-mask'}
    condition = util_.rgetattr(observable.object_.properties, ['condition'])
    if condition in ['Equals', None]:
        # currently not handling other observable conditions as
        # it's not clear that crits even supports these...
        ip_category = util_.rgetattr(observable.object_.properties,
                                     ['category'])
        ip_value = util_.rgetattr(observable.object_.properties,
                                  ['address_value', 'value'])
        if ip_value and ip_category:
            if ip_category not in crits_types.keys():
                config['logger'].error(
                    log_.log_messages['unsupported_object_error'].format(
                        type_='edge', obj_type=(str(type(observable.object_.properties)) 
                                                + ', %s' % ip_category), id_=observable.id_))
                return(None)
            json = {'ip': ip_value, 'ip_type': crits_types[ip_category]}
            json['stix_id'] = observable.id_
            return(json)
예제 #2
0
def cybox_domain_to_json(config, observable):
    '''translate a cybox domain object to crits json'''
    crits_types = {'FQDN': 'A'}
    # crits doesn't appear to support tlds...
    domain_category = util_.rgetattr(observable.object_.properties, ['type_'])
    domain_value = util_.rgetattr(observable.object_.properties,
                                  ['value', 'value'])
    if domain_category and domain_value:
        json = {'domain': domain_value, 'type': crits_types[domain_category]}
        json['stix_id'] = observable.id_
        return(json)
예제 #3
0
def process_observables(config, src, dest, observables):
    '''handle incoming cybox observables and observable compositions'''
    # TODO some of the hailataxii date uses the cybox ###comma###
    #      construct, which is currently unsupported
    for o in observables.keys():
        json = dict()
        if util_.rgetattr(observables[o], ['observable_composition']) \
           and not util_.rgetattr(observables[o], ['object_']):
            # it's an observable composition
            # store it in the db...maybe the indicator will only come
            # across in a subsequent run so we can't rely on passing
            # this around in memory
            config['db'].store_obs_comp(src, dest,
                                        obs_id=o,
                                        obs_comp=observables[o].observable_composition)
            continue
        elif util_.rgetattr(observables[o], ['object_']):
            # it's a normal observable
            (json, endpoint) = cybox_observable_to_json(config, observables[o])
            if json:
                # mark crits releasability
                json.update(mark_crits_releasability(config, dest))
            else:
                config['logger'].error(
                    log_.log_messages[
                        'obj_convert_error'].format(src_type='cybox',
                                                    src_obj='observable',
                                                    id_=o,
                                                    dest_type='crits',
                                                    dest_obj='json'))
                continue
            # inbox the observable to crits
            config['edge_tally'][endpoint]['incoming'] += 1
            config['edge_tally']['all']['incoming'] += 1
            (id_, success) = \
                crits_.crits_inbox(config, dest, endpoint, json,
                                   src=src, edge_id=o)
            if not success:
                config['logger'].error(
                    log_.log_messages['obj_inbox_error'].format(
                        src_type='edge', id_=o, dest_type='crits ' + endpoint + ' api endpoint'))
                continue
            else:
                # successfully inboxed observable
                config['edge_tally'][endpoint]['processed'] += 1
                config['edge_tally']['all']['processed'] += 1
                if config['daemon']['debug']:
                    config['logger'].debug(
                        log_.log_messages['obj_inbox_success'].format(
                            src_type='edge', id_=o,
                            dest_type='crits ' + endpoint + ' api endpoint'))
예제 #4
0
def cybox_observable_to_json(config, observable):
    '''translate a cybox observable to crits json'''
    props = util_.rgetattr(observable.object_, ['properties'])
    if props and isinstance(props, Address):
        endpoint = 'ips'
        json = cybox_address_to_json(config, observable)
    elif props and isinstance(props, DomainName):
        endpoint = 'domains'
        json = cybox_domain_to_json(config, observable)
    elif props and isinstance(props, URI):
        endpoint = 'domains'
        json = cybox_uri_to_json(config, observable)
    elif props and isinstance(props, File):
        endpoint = 'samples'
        json = cybox_file_to_json(config, observable)
    elif props and isinstance(props, EmailMessage):
        endpoint = 'emails'
        json = cybox_email_to_json(config, observable)
    if json and endpoint:
        # TODO: this would all be a helluva lot easier if the crits
        #       api supported manually setting an _id
        #
        # json['_id'] = observable.id_.split('-')[1]
        return(json, endpoint)
    else:
        config['logger'].error(
            log_.log_messages['unsupported_object_error'].format(
                type_='edge', obj_type=type(props), id_=observable.id_))
        return(None, None)
예제 #5
0
def cybox_uri_to_json(config, observable):
    '''translate a cybox uri object to crits json'''
    crits_types = {'Domain Name': 'A'}
    # urls currently not supported...
    domain_category = util_.rgetattr(observable.object_.properties,
                                     ['type_'])
    domain_value = util_.rgetattr(observable.object_.properties,
                                  ['value', 'value'])
    if domain_category and domain_value:
        if domain_category not in crits_types.keys():
            config['logger'].error(
                log_.log_messages['unsupported_object_error'].format(
                    type_='edge', obj_type=(str(type(observable.object_.properties)) 
                                            + ', %s' % domain_category), id_=observable.id_))
            return(None)
        json = {'domain': domain_value, 'type': crits_types[domain_category]}
        json['stix_id'] = observable.id_
        return(json)
예제 #6
0
def cybox_file_to_json(config, observable):
    '''translate a cybox file object to crits json'''
    crits_types = {'MD5': 'md5',
                   'SHA1': 'sha1',
                   'SHA224': 'sha224',
                   'SHA256': 'sha256',
                   'SHA384': 'sha384',
                   'SHA512': 'sha512',
                   'SSDEEP': 'ssdeep'}
    json = {'upload_type': 'metadata'}
    hashes = util_.rgetattr(observable.object_.properties, ['hashes'])
    if hashes:
        for hash in hashes:
            hash_type = util_.rgetattr(hash, ['type_', 'value'])
            hash_value = util_.rgetattr(hash, ['simple_hash_value', 'value'])
            if hash_type and hash_value:
                json[crits_types[hash_type]] = hash_value
    file_name = util_.rgetattr(observable.object_.properties,
                               ['file_name', 'value'])
    if file_name:
        json['filename'] = file_name
    file_format = util_.rgetattr(observable.object_.properties,
                                 ['file_format', 'value'])
    if file_format:
        json['filetype'] = file_format
    file_size = util_.rgetattr(observable.object_.properties,
                               ['size_in_bytes', 'value'])
    if file_size:
        json['size'] = file_size
    json['stix_id'] = observable.id_
    return(json)
예제 #7
0
def process_indicators(config, src, dest, indicators):
    '''handle incoming stix indicators'''
    xmlns_name = config['edge']['sites'][src]['stix']['xmlns_name']
    for i in indicators.keys():
        json = dict()
        json['type'] = 'Related_To'
        json['value'] = util_.rgetattr(indicators[i], ['title'],
                                       default_='unknown')
        json['indicator_confidence'] = \
            util_.rgetattr(indicators[i], ['confidence', 'value', 'value'],
                           default_='unknown')
        # TODO lookup the corresponding stix prop for indicator_impact
        json['indicator_impact'] = {'rating': 'unknown'}
        # inbox the indicator (we need to crits id!)
        config['edge_tally']['indicators']['incoming'] += 1
        config['edge_tally']['all']['incoming'] += 1
        (crits_indicator_id, success) = crits_.crits_inbox(config, dest,
                                                           'indicators',
                                                           json, src=src)
        if not success:
            config['logger'].error(
                log_.log_messages['obj_inbox_error'].format(
                    src_type='edge', id_=i, 
                    dest_type='crits indicators api endpoint'))
            continue
        else:
            # successfully inboxed indicator...
            config['edge_tally']['indicators']['processed'] += 1
            config['edge_tally']['all']['processed'] += 1
            if config['daemon']['debug']:
                config['logger'].debug(
                    log_.log_messages['obj_inbox_success'].format(
                        src_type='edge', id_=i,
                        dest_type='crits indicators api endpoint'))
        if util_.rgetattr(indicators[i], ['observables']):
            for o in indicators[i].observables:
                if util_.rgetattr(o, ['idref']) and \
                   not util_.rgetattr(o, ['object_']):
                    # TODO need to delete observable compositions from
                    #      mongo once we've processed them
                    obs_comp = \
                        config['db'].get_obs_comp(src, dest, obs_id=o.idref)
                    if not obs_comp:
                        # [ o == embedded observable]
                        config['db'].set_pending_crits_link(src, dest,
                                                            lhs_id=(xmlns_name + ':' + 
                                                                      'indicators' + '-' + 
                                                                      crits_indicator_id),
                                                            rhs_id=o.idref)
                    elif obs_comp:
                        # [o == idref observable composition]
                        # try to fetch the observable composition o.idref
                        # points to
                        # assumption: the observable composition was
                        # previously ingested. TODO what about when
                        # the observable composition comes in *after*
                        # the indicator?
                        observables_list = util_.rgetattr(obs_comp,
                                                          ['observables'])
                        if not observables_list:
                            config['logger'].error(
                                log_.log_messages['obs_comp_dereference_error'
                                              ].format(id_=i))
                            continue
                        else:
                            for j in observables_list:
                                # store the pending relationship in
                                # the db for later processing
                                config['db'].set_pending_crits_link(src, dest,
                                                                    lhs_id=(xmlns_name + ':' + 
                                                                              'indicators' + '-' +
                                                                              crits_indicator_id),
                                                                    rhs_id=j.idref)
                    # TODO (need to dig up suitable sample data)
                    # if it's an observable composition with inline
                    # observables, pass them to observable composition with
                    # inline observables, pass them to process_observables(),
                    # (which will store the edge/crits id indicator pairing
                    # for later processing.
                    else:
                        config['logger'].error(
                            log_.log_messages['obs_comp_dereference_error'
                                          ].format(id_=i))
                        continue
        # as we've now successfully processed the indicator, track
        # the related crits/json ids (by src/dest)
        if util_.rgetattr(indicators[i], ['related_indicators']) and len(indicators[i].related_indicators):
            for j in indicators[i].related_indicators:
                if util_.rgetattr(j, ['item', 'idref']):
                    # store the pending relationship in the db for
                    # later processing 

                    # TODO for some reason, the crits relationship api
                    # is rejecting _some_ (but not _all_
                    # indicator-to-indicator relationships. the
                    # indicator ids are valid and the api post looks
                    # correct but...sometimes this fails :-/
                    config['db'].set_pending_crits_link(src, dest,
                                                        lhs_id=(xmlns_name + ':' + 
                                                                  'indicators' + '-' +
                                                                  crits_indicator_id),
                                                        rhs_id=j.item.idref)
                                        
        config['db'].set_object_id(src, dest,
                                   edge_id=i,
                                   crits_id=(xmlns_name + ':' + 'indicators' + '-' +
                                             crits_indicator_id))
예제 #8
0
def process_incidents(config, src, dest, incidents):
    '''handle incoming stix incidents'''
    xmlns_name = config['edge']['sites'][src]['stix']['xmlns_name']
    status_trans = {'New': 'New', 'Open': 'In Progress',
                    'Closed': 'Analyzed', 'Rejected': 'Deprecated'}
    for i in incidents.keys():
        json = dict()
        json['event_type'] = 'Threat Report'
        json['title'] = incidents[i].title
        json['description'] = util_.rgetattr(incidents[i], ['description', 'value'])
        json['status'] = status_trans[incidents[i].status.value]
        # inbox the incident (we need to crits id!)
        config['edge_tally']['events']['incoming'] += 1
        config['edge_tally']['all']['incoming'] += 1
        (crits_event_id, success) = crits_.crits_inbox(config, dest,
                                                       'events',
                                                       json, src=src)
        if not success:
            config['logger'].error(
                log_.log_messages['obj_inbox_error'].format(
                    src_type='edge', id_=i, 
                    dest_type='crits events api endpoint'))
            continue
        else:
            # successfully inboxed event...
            config['edge_tally']['events']['processed'] += 1
            config['edge_tally']['all']['processed'] += 1
            if config['daemon']['debug']:
                config['logger'].debug(
                    log_.log_messages['obj_inbox_success'].format(
                        src_type='edge', id_=i,
                        dest_type='crits events api endpoint'))
        # as we've now successfully processed the event, track
        # the related crits/json ids (by src/dest)
        if util_.rgetattr(incidents[i], ['related_observables']) and len(incidents[i].related_observables):
            for j in incidents[i].related_observables:
                if util_.rgetattr(j, ['item', 'idref']):
                    # store the pending relationship in the db for
                    # later processing 
                    config['db'].set_pending_crits_link(src, dest,
                                                        lhs_id=(xmlns_name + ':' + 
                                                                  'events' + '-' +
                                                                  crits_event_id),
                                                        rhs_id=j.item.idref)
        if util_.rgetattr(incidents[i], ['related_indicators']) and len(incidents[i].related_indicators):
            for j in incidents[i].related_indicators:
                if util_.rgetattr(j, ['item', 'idref']):
                    # store the pending relationship in the db for
                    # later processing 
                    config['db'].set_pending_crits_link(src, dest,
                                                        lhs_id=(xmlns_name + ':' + 
                                                                  'events' + '-' +
                                                                  crits_event_id),
                                                        rhs_id=j.item.idref)
        if util_.rgetattr(incidents[i], ['related_incidents']) and len(incidents[i].related_incidents):
            for j in incidents[i].related_incidents:
                if util_.rgetattr(j, ['item', 'idref']):
                    # store the pending relationship in the db for
                    # later processing 
                    config['db'].set_pending_crits_link(src, dest,
                                                        lhs_id=(xmlns_name + ':' + 
                                                                  'events' + '-' +
                                                                  crits_event_id),
                                                        rhs_id=j.item.idref)
                                        
        config['db'].set_object_id(src, dest,
                                   edge_id=i,
                                   crits_id=(xmlns_name + ':' + 'events' + '-' +
                                             crits_event_id))
예제 #9
0
def cybox_email_to_json(config, observable):
    '''translate a cybox email object to crits json'''
    crits_types = {'subject': 'subject', 'to': 'to', 'cc': 'cc',
                   'from_': 'from_address', 'sender': 'sender', 'date': 'date',
                   'message_id': 'message_id', 'reply_to': 'reply_to',
                   'boundary': 'boundary', 'x_mailer': 'x_mailer',
                   'x_originating_ip': 'x_originating_ip'}
    json = {'upload_type': 'fields'}
    subject = util_.rgetattr(observable.object_.properties,
                             ['header', 'subject', 'value'])
    if subject:
        json['subject'] = subject
    to = util_.rgetattr(observable.object_.properties, ['header', 'to'])
    if to:
        json['to'] = []
        for i in to:
            addr = util_.rgetattr(i, ['address_value', 'values'])
            if addr:
                json['to'].append(addr)
    cc = util_.rgetattr(observable.object_.properties, ['header', 'cc'])
    if cc:
        json['cc'] = []
        for i in cc:
            addr = util_.rgetattr(i, ['address_value', 'values'])
            if addr:
                json['cc'].append(addr)
    from_ = util_.rgetattr(observable.object_.properties,
                           ['header', 'from_', 'address_value', 'value'])
    if from_:
        json['from_address'] = [from_]
    sender = util_.rgetattr(observable.object_.properties,
                            ['header', 'sender', 'address_value', 'value'])
    if sender:
        json['sender'] = sender
    date = util_.rgetattr(observable.object_.properties,
                          ['header', 'date', 'value'])
    if date:
        json['date'] = date
    message_id = util_.rgetattr(observable.object_.properties,
                                ['header', 'message_id', 'value'])
    if message_id:
        json['message_id'] = message_id
    reply_to = util_.rgetattr(observable.object_.properties,
                              ['header', 'reply_to', 'address_value', 'value'])
    if reply_to:
        json['reply_to'] = reply_to
    boundary = util_.rgetattr(observable.object_.properties,
                              ['header', 'boundary', 'value'])
    if boundary:
        json['boundary'] = boundary
    x_mailer = util_.rgetattr(observable.object_.properties,
                              ['header', 'x_mailer', 'value'])
    if x_mailer:
        json['x_mailer'] = x_mailer
    x_originating_ip = util_.rgetattr(observable.object_.properties,
                                      ['header', 'x_originating_ip', 'value'])
    if x_originating_ip:
        json['x_originating_ip'] = x_originating_ip
    json['stix_id'] = observable.id_
    return(json)
예제 #10
0
def process_observables(config, src, dest, observables):
    '''handle incoming cybox observables and observable compositions'''
    # TODO some of the hailataxii date uses the cybox ###comma###
    #      construct, which is currently unsupported
    for o_id, o in observables.iteritems():
        json = dict()
        if util_.rgetattr(o, ['observable_composition']) \
           and not util_.rgetattr(o, ['object_']):
            # it's an observable composition
            # store it in the db...maybe the indicator will only come
            # across in a subsequent run so we can't rely on passing
            # this around in memory
            config['db'].store_obs_comp(src, dest,
                                        obs_id=o_id,
                                        obs_comp=o.observable_composition)
            continue
        elif util_.rgetattr(o, ['object_']):
            # it's a normal observable
            (json, endpoint) = cybox_observable_to_json(config, o)
            if not json:
                config['logger'].error(
                    log_.log_messages[
                        'obj_convert_error'].format(src_type='cybox',
                                                    src_obj='observable',
                                                    id_=o_id,
                                                    dest_type='crits',
                                                    dest_obj='json'))
                continue

            # mark crits releasability
            # TODO: Maybe remove this? Not sure if it works with
            # the Crits PATCH API method for setting releasability. 
            json.update(mark_crits_releasability(config, dest))

            # inbox the observable to crits
            config['edge_tally'][endpoint]['incoming'] += 1
            config['edge_tally']['all']['incoming'] += 1
            (id_, success) = \
                crits_.crits_inbox(config, dest, endpoint, json,
                                   src=src, edge_id=o_id)
            if not success:
                config['logger'].error(
                    log_.log_messages['obj_inbox_error'].format(
                        src_type='edge', id_=o, dest_type='crits ' + endpoint + ' api endpoint'))
                continue

            # Successfully inboxed observable

            # Send Patch request to set crits releasability
            patch_endpoint = '{}/{}'.format(endpoint, id_)
            releasability_json = {
                'action': 'add_releasability',
                'name': config['crits']['sites'][dest]['api']['releasability'],
            }
            releasability_success = crits_.crits_patch(config, dest, 
                patch_endpoint, releasability_json)

            if not releasability_success:
                config['logger'].error(
                    log_.log_messages['obj_inbox_error'].format(
                        src_type='edge', id_=o, dest_type='crits ' + patch_endpoint + ' api endpoint'))
                continue


            config['edge_tally'][endpoint]['processed'] += 1
            config['edge_tally']['all']['processed'] += 1
            if config['daemon']['debug']:
                config['logger'].debug(
                    log_.log_messages['obj_inbox_success'].format(
                        src_type='edge', id_=o_id,
                        dest_type='crits ' + endpoint + ' api endpoint'))