Ejemplo n.º 1
0
def cybox_address_to_json(config, observable):
    """translate a cybox address object to crits json"""
    crits_types = {
        "cidr": "Address - cidr",
        "ipv4-addr": "Address - ipv4-addr",
        "ipv4-net": "Address - ipv4-net",
        "ipv4-netmask": "Address - ipv4-net-mask",
        "ipv6-addr": "Address - ipv6-addr",
        "ipv6-net": "Address - ipv6-net",
        "ipv6-netmask": "Address - ipv6-net-mask",
    }
    condition = util.rgetattr(observable.object_.properties, ["condition"])
    if condition in ["Equals", None]:
        # currently not handling other observable conditions as
        # it's not clear that crits even supports these...
        ip_category = util.rgetattr(observable.object_.properties, ["category"])
        ip_value = util.rgetattr(observable.object_.properties, ["address_value", "value"])
        if ip_value and ip_category:
            if ip_category not in crits_types.keys():
                config["logger"].error(
                    log.log_messages["unsupported_object_error"].format(
                        type_="edge",
                        obj_type=(str(type(observable.object_.properties)) + ", %s" % ip_category),
                        id_=observable.id_,
                    )
                )
                return None
            json = {"ip": ip_value, "ip_type": crits_types[ip_category]}
            json["stix_id"] = observable.id_
            return json
Ejemplo n.º 2
0
def cybox_address_to_json(config, observable):
    '''translate a cybox address object to crits json'''
    crits_types = {'cidr': 'Address - cidr',
                   'ipv4-addr': 'Address - ipv4-addr',
                   'ipv4-net': 'Address - ipv4-net',
                   'ipv4-netmask': 'Address - ipv4-net-mask',
                   'ipv6-addr': 'Address - ipv6-addr',
                   'ipv6-net': 'Address - ipv6-net',
                   'ipv6-netmask': 'Address - ipv6-net-mask'}
    condition = util.rgetattr(observable.object_.properties, ['condition'])
    if condition in ['Equals', None]:
        # currently not handling other observable conditions as
        # it's not clear that crits even supports these...
        ip_category = util.rgetattr(observable.object_.properties,
                                     ['category'])
        ip_value = util.rgetattr(observable.object_.properties,
                                  ['address_value', 'value'])
        if ip_value and ip_category:
            if ip_category not in crits_types.keys():
                config['logger'].error(
                    log.log_messages['unsupported_object_error'].format(
                        type_='edge', obj_type=(str(type(observable.object_.properties)) 
                                                + ', %s' % ip_category), id_=observable.id_))
                return(None)
            json = {'ip': ip_value, 'ip_type': crits_types[ip_category]}
            json['stix_id'] = observable.id_
            return(json)
Ejemplo n.º 3
0
def cybox_domain_to_json(config, observable):
    """translate a cybox domain object to crits json"""
    crits_types = {"FQDN": "A"}
    # crits doesn't appear to support tlds...
    domain_category = util.rgetattr(observable.object_.properties, ["type_"])
    domain_value = util.rgetattr(observable.object_.properties, ["value", "value"])
    if domain_category and domain_value:
        json = {"domain": domain_value, "type": crits_types[domain_category]}
        json["stix_id"] = observable.id_
        return json
Ejemplo n.º 4
0
def cybox_domain_to_json(config, observable):
    '''translate a cybox domain object to crits json'''
    crits_types = {'FQDN': 'A'}
    # crits doesn't appear to support tlds...
    domain_category = util.rgetattr(observable.object_.properties, ['type_'])
    domain_value = util.rgetattr(observable.object_.properties,
                                  ['value', 'value'])
    if domain_category and domain_value:
        json = {'domain': domain_value, 'type': crits_types[domain_category]}
        json['stix_id'] = observable.id_
        return(json)
Ejemplo n.º 5
0
def process_observables(config, src, dest, observables):
    '''handle incoming cybox observables and observable compositions'''
    # TODO some of the hailataxii date uses the cybox ###comma###
    #      construct, which is currently unsupported
    for o_id, o in observables.iteritems():
        json = dict()
        if util.rgetattr(o, ['observable_composition']) \
           and not util.rgetattr(o, ['object_']):
            # it's an observable composition
            # store it in the db...maybe the indicator will only come
            # across in a subsequent run so we can't rely on passing
            # this around in memory
            config['db'].store_obs_comp(src, dest,
                                        obs_id=o_id,
                                        obs_comp=o.observable_composition)
            continue
        elif util.rgetattr(o, ['object_']):
            # it's a normal observable
            (json, endpoint) = cybox_observable_to_json(config, o)
            if not json:
                config['logger'].error(
                    log.log_messages[
                        'obj_convert_error'].format(src_type='cybox',
                                                    src_obj='observable',
                                                    id_=o_id,
                                                    dest_type='crits',
                                                    dest_obj='json'))
                continue

            # inbox the observable to crits
            config['edge_tally'][endpoint]['incoming'] += 1
            config['edge_tally']['all']['incoming'] += 1
            (id_, success) = \
                crits.crits_inbox(config, dest, endpoint, json,
                                   src=src, edge_id=o_id)
            if not success:
                config['logger'].error(
                    log.log_messages['obj_inbox_error'].format(
                        src_type='edge', id_=o, dest_type='crits ' + endpoint + ' api endpoint'))
                continue

            # successfully inboxed observable
            config['edge_tally'][endpoint]['processed'] += 1
            config['edge_tally']['all']['processed'] += 1
            if config['daemon']['debug']:
                config['logger'].debug(
                    log.log_messages['obj_inbox_success'].format(
                        src_type='edge', id_=o_id,
                        dest_type='crits ' + endpoint + ' api endpoint'))
Ejemplo n.º 6
0
def cybox_observable_to_json(config, observable):
    '''translate a cybox observable to crits json'''
    props = util.rgetattr(observable.object_, ['properties'])
    endpoint = None
    json = None
    if props and isinstance(props, Address):
        endpoint = 'ips'
        json = cybox_address_to_json(config, observable)
    elif props and isinstance(props, DomainName):
        endpoint = 'domains'
        json = cybox_domain_to_json(config, observable)
    elif props and isinstance(props, URI):
        endpoint = 'domains'
        json = cybox_uri_to_json(config, observable)
    elif props and isinstance(props, File):
        endpoint = 'samples'
        json = cybox_file_to_json(config, observable)
    elif props and isinstance(props, EmailMessage):
        endpoint = 'emails'
        json = cybox_email_to_json(config, observable)
    if endpoint and json:
        # TODO: this would all be a helluva lot easier if the crits
        #       api supported manually setting an _id
        #
        # json['_id'] = observable.id_.split('-')[1]
        return(json, endpoint)
    else:
        config['logger'].error(
            log.log_messages['unsupported_object_error'].format(
                type_='edge', obj_type=type(props), id_=observable.id_))
        return(None, None)
Ejemplo n.º 7
0
def cybox_observable_to_json(config, observable):
    """translate a cybox observable to crits json"""
    props = util.rgetattr(observable.object_, ["properties"])
    endpoint = None
    json = None
    if props and isinstance(props, Address):
        endpoint = "ips"
        json = cybox_address_to_json(config, observable)
    elif props and isinstance(props, DomainName):
        endpoint = "domains"
        json = cybox_domain_to_json(config, observable)
    elif props and isinstance(props, URI):
        endpoint = "domains"
        json = cybox_uri_to_json(config, observable)
    elif props and isinstance(props, File):
        endpoint = "samples"
        json = cybox_file_to_json(config, observable)
    elif props and isinstance(props, EmailMessage):
        endpoint = "emails"
        json = cybox_email_to_json(config, observable)
    if endpoint and json:
        # TODO: this would all be a helluva lot easier if the crits
        #       api supported manually setting an _id
        #
        # json['_id'] = observable.id_.split('-')[1]
        return (json, endpoint)
    else:
        config["logger"].error(
            log.log_messages["unsupported_object_error"].format(type_="edge", obj_type=type(props), id_=observable.id_)
        )
        return (None, None)
Ejemplo n.º 8
0
def process_observables(config, src, dest, observables):
    """handle incoming cybox observables and observable compositions"""
    # TODO some of the hailataxii date uses the cybox ###comma###
    #      construct, which is currently unsupported
    for o_id, o in observables.iteritems():
        json = dict()
        if util.rgetattr(o, ["observable_composition"]) and not util.rgetattr(o, ["object_"]):
            # it's an observable composition
            # store it in the db...maybe the indicator will only come
            # across in a subsequent run so we can't rely on passing
            # this around in memory
            config["db"].store_obs_comp(src, dest, obs_id=o_id, obs_comp=o.observable_composition)
            continue
        elif util.rgetattr(o, ["object_"]):
            # it's a normal observable
            (json, endpoint) = cybox_observable_to_json(config, o)
            if not json:
                config["logger"].error(
                    log.log_messages["obj_convert_error"].format(
                        src_type="cybox", src_obj="observable", id_=o_id, dest_type="crits", dest_obj="json"
                    )
                )
                continue

            # inbox the observable to crits
            config["edge_tally"][endpoint]["incoming"] += 1
            config["edge_tally"]["all"]["incoming"] += 1
            (id_, success) = crits.crits_inbox(config, dest, endpoint, json, src=src, edge_id=o_id)
            if not success:
                config["logger"].error(
                    log.log_messages["obj_inbox_error"].format(
                        src_type="edge", id_=o, dest_type="crits " + endpoint + " api endpoint"
                    )
                )
                continue

            # successfully inboxed observable
            config["edge_tally"][endpoint]["processed"] += 1
            config["edge_tally"]["all"]["processed"] += 1
            if config["daemon"]["debug"]:
                config["logger"].debug(
                    log.log_messages["obj_inbox_success"].format(
                        src_type="edge", id_=o_id, dest_type="crits " + endpoint + " api endpoint"
                    )
                )
Ejemplo n.º 9
0
def cybox_uri_to_json(config, observable):
    '''translate a cybox uri object to crits json'''
    crits_types = {'Domain Name': 'A'}
    # urls currently not supported...
    domain_category = util.rgetattr(observable.object_.properties,
                                     ['type_'])
    domain_value = util.rgetattr(observable.object_.properties,
                                  ['value', 'value'])
    if domain_category and domain_value:
        if domain_category not in crits_types.keys():
            config['logger'].error(
                log.log_messages['unsupported_object_error'].format(
                    type_='edge', obj_type=(str(type(observable.object_.properties)) 
                                            + ', %s' % domain_category), id_=observable.id_))
            return(None)
        json = {'domain': domain_value, 'type': crits_types[domain_category]}
        json['stix_id'] = observable.id_
        return(json)
Ejemplo n.º 10
0
def process_incidents(config, src, dest, incidents):
    """handle incoming stix incidents"""
    xmlns_name = config["edge"]["sites"][src]["stix"]["xmlns_name"]
    status_trans = {"New": "New", "Open": "In Progress", "Closed": "Analyzed", "Rejected": "Deprecated"}
    for i in incidents.keys():
        json = dict()
        json["event_type"] = "Threat Report"
        json["title"] = incidents[i].title
        json["description"] = util.rgetattr(incidents[i], ["description", "value"])
        json["status"] = status_trans[incidents[i].status.value]
        # inbox the incident (we need to crits id!)
        config["edge_tally"]["events"]["incoming"] += 1
        config["edge_tally"]["all"]["incoming"] += 1
        (crits_event_id, success) = crits.crits_inbox(config, dest, "events", json, src=src)
        if not success:
            config["logger"].error(
                log.log_messages["obj_inbox_error"].format(
                    src_type="edge", id_=i, dest_type="crits events api endpoint"
                )
            )
            continue
        else:
            # successfully inboxed event...
            config["edge_tally"]["events"]["processed"] += 1
            config["edge_tally"]["all"]["processed"] += 1
            if config["daemon"]["debug"]:
                config["logger"].debug(
                    log.log_messages["obj_inbox_success"].format(
                        src_type="edge", id_=i, dest_type="crits events api endpoint"
                    )
                )
        # as we've now successfully processed the event, track
        # the related crits/json ids (by src/dest)
        if util.rgetattr(incidents[i], ["related_observables"]) and len(incidents[i].related_observables):
            for j in incidents[i].related_observables:
                if util.rgetattr(j, ["item", "idref"]):
                    # store the pending relationship in the db for
                    # later processing
                    config["db"].set_pending_crits_link(
                        src, dest, lhs_id=(xmlns_name + ":" + "events" + "-" + crits_event_id), rhs_id=j.item.idref
                    )
        if util.rgetattr(incidents[i], ["related_indicators"]) and len(incidents[i].related_indicators):
            for j in incidents[i].related_indicators:
                if util.rgetattr(j, ["item", "idref"]):
                    # store the pending relationship in the db for
                    # later processing
                    config["db"].set_pending_crits_link(
                        src, dest, lhs_id=(xmlns_name + ":" + "events" + "-" + crits_event_id), rhs_id=j.item.idref
                    )
        if util.rgetattr(incidents[i], ["related_incidents"]) and len(incidents[i].related_incidents):
            for j in incidents[i].related_incidents:
                if util.rgetattr(j, ["item", "idref"]):
                    # store the pending relationship in the db for
                    # later processing
                    config["db"].set_pending_crits_link(
                        src, dest, lhs_id=(xmlns_name + ":" + "events" + "-" + crits_event_id), rhs_id=j.item.idref
                    )

        config["db"].set_object_id(src, dest, edge_id=i, crits_id=(xmlns_name + ":" + "events" + "-" + crits_event_id))
Ejemplo n.º 11
0
def cybox_uri_to_json(config, observable):
    """translate a cybox uri object to crits json"""
    crits_types = {"Domain Name": "A"}
    # urls currently not supported...
    domain_category = util.rgetattr(observable.object_.properties, ["type_"])
    domain_value = util.rgetattr(observable.object_.properties, ["value", "value"])
    if domain_category and domain_value:
        if domain_category not in crits_types.keys():
            config["logger"].error(
                log.log_messages["unsupported_object_error"].format(
                    type_="edge",
                    obj_type=(str(type(observable.object_.properties)) + ", %s" % domain_category),
                    id_=observable.id_,
                )
            )
            return None
        json = {"domain": domain_value, "type": crits_types[domain_category]}
        json["stix_id"] = observable.id_
        return json
Ejemplo n.º 12
0
def cybox_file_to_json(config, observable):
    '''translate a cybox file object to crits json'''
    crits_types = {'MD5': 'md5',
                   'SHA1': 'sha1',
                   'SHA224': 'sha224',
                   'SHA256': 'sha256',
                   'SHA384': 'sha384',
                   'SHA512': 'sha512',
                   'SSDEEP': 'ssdeep'}
    json = {'upload_type': 'metadata'}
    hashes = util.rgetattr(observable.object_.properties, ['hashes'])
    if hashes:
        for hash in hashes:
            hash_type = util.rgetattr(hash, ['type_', 'value'])
            hash_value = util.rgetattr(hash, ['simple_hash_value', 'value'])
            if hash_type and hash_value:
                json[crits_types[hash_type]] = hash_value
    file_name = util.rgetattr(observable.object_.properties,
                               ['file_name', 'value'])
    if file_name:
        json['filename'] = file_name
    file_format = util.rgetattr(observable.object_.properties,
                                 ['file_format', 'value'])
    if file_format:
        json['filetype'] = file_format
    file_size = util.rgetattr(observable.object_.properties,
                               ['size_in_bytes', 'value'])
    if file_size:
        json['size'] = file_size
    json['stix_id'] = observable.id_
    return(json)
Ejemplo n.º 13
0
def cybox_file_to_json(config, observable):
    """translate a cybox file object to crits json"""
    crits_types = {
        "MD5": "md5",
        "SHA1": "sha1",
        "SHA224": "sha224",
        "SHA256": "sha256",
        "SHA384": "sha384",
        "SHA512": "sha512",
        "SSDEEP": "ssdeep",
    }
    json = {"upload_type": "metadata"}
    hashes = util.rgetattr(observable.object_.properties, ["hashes"])
    if hashes:
        for hash in hashes:
            hash_type = util.rgetattr(hash, ["type_", "value"])
            hash_value = util.rgetattr(hash, ["simple_hash_value", "value"])
            if hash_type and hash_value:
                json[crits_types[hash_type]] = hash_value
    file_name = util.rgetattr(observable.object_.properties, ["file_name", "value"])
    if file_name:
        json["filename"] = file_name
    file_format = util.rgetattr(observable.object_.properties, ["file_format", "value"])
    if file_format:
        json["filetype"] = file_format
    file_size = util.rgetattr(observable.object_.properties, ["size_in_bytes", "value"])
    if file_size:
        json["size"] = file_size
    json["stix_id"] = observable.id_
    return json
Ejemplo n.º 14
0
def cybox_email_to_json(config, observable):
    """translate a cybox email object to crits json"""
    crits_types = {
        "subject": "subject",
        "to": "to",
        "cc": "cc",
        "from_": "from_address",
        "sender": "sender",
        "date": "date",
        "message_id": "message_id",
        "reply_to": "reply_to",
        "boundary": "boundary",
        "x_mailer": "x_mailer",
        "x_originating_ip": "x_originating_ip",
    }
    json = {"upload_type": "fields"}
    subject = util.rgetattr(observable.object_.properties, ["header", "subject", "value"])
    if subject:
        json["subject"] = subject
    to = util.rgetattr(observable.object_.properties, ["header", "to"])
    if to:
        json["to"] = []
        for i in to:
            addr = util.rgetattr(i, ["address_value", "values"])
            if addr:
                json["to"].append(addr)
    cc = util.rgetattr(observable.object_.properties, ["header", "cc"])
    if cc:
        json["cc"] = []
        for i in cc:
            addr = util.rgetattr(i, ["address_value", "values"])
            if addr:
                json["cc"].append(addr)
    from_ = util.rgetattr(observable.object_.properties, ["header", "from_", "address_value", "value"])
    if from_:
        json["from_address"] = [from_]
    sender = util.rgetattr(observable.object_.properties, ["header", "sender", "address_value", "value"])
    if sender:
        json["sender"] = sender
    date = util.rgetattr(observable.object_.properties, ["header", "date", "value"])
    if date:
        json["date"] = date
    message_id = util.rgetattr(observable.object_.properties, ["header", "message_id", "value"])
    if message_id:
        json["message_id"] = message_id
    reply_to = util.rgetattr(observable.object_.properties, ["header", "reply_to", "address_value", "value"])
    if reply_to:
        json["reply_to"] = reply_to
    boundary = util.rgetattr(observable.object_.properties, ["header", "boundary", "value"])
    if boundary:
        json["boundary"] = boundary
    x_mailer = util.rgetattr(observable.object_.properties, ["header", "x_mailer", "value"])
    if x_mailer:
        json["x_mailer"] = x_mailer
    x_originating_ip = util.rgetattr(observable.object_.properties, ["header", "x_originating_ip", "value"])
    if x_originating_ip:
        json["x_originating_ip"] = x_originating_ip
    json["stix_id"] = observable.id_
    return json
Ejemplo n.º 15
0
def process_indicators(config, src, dest, indicators):
    '''handle incoming stix indicators'''
    xmlns_name = config['edge']['sites'][src]['stix']['xmlns_name']
    for i in indicators.keys():
        json = dict()
        json['type'] = 'Related_To'
        json['value'] = util.rgetattr(indicators[i], ['title'],
                                       default_='unknown')
        json['indicator_confidence'] = \
            util.rgetattr(indicators[i], ['confidence', 'value', 'value'],
                           default_='unknown')
        # TODO lookup the corresponding stix prop for indicator_impact
        json['indicator_impact'] = {'rating': 'unknown'}
        # inbox the indicator (we need to crits id!)
        config['edge_tally']['indicators']['incoming'] += 1
        config['edge_tally']['all']['incoming'] += 1
        (crits_indicator_id, success) = crits.crits_inbox(config, dest,
                                                           'indicators',
                                                           json, src=src)
        if not success:
            config['logger'].error(
                log.log_messages['obj_inbox_error'].format(
                    src_type='edge', id_=i, 
                    dest_type='crits indicators api endpoint'))
            continue
        else:
            # successfully inboxed indicator...
            config['edge_tally']['indicators']['processed'] += 1
            config['edge_tally']['all']['processed'] += 1
            if config['daemon']['debug']:
                config['logger'].debug(
                    log.log_messages['obj_inbox_success'].format(
                        src_type='edge', id_=i,
                        dest_type='crits indicators api endpoint'))
        if util.rgetattr(indicators[i], ['observables']):
            for o in indicators[i].observables:
                if util.rgetattr(o, ['idref']) and \
                   not util.rgetattr(o, ['object_']):
                    # TODO need to delete observable compositions from
                    #      mongo once we've processed them
                    obs_comp = \
                        config['db'].get_obs_comp(src, dest, obs_id=o.idref)
                    if not obs_comp:
                        # [ o == embedded observable]
                        config['db'].set_pending_crits_link(src, dest,
                                                            lhs_id=(xmlns_name + ':' + 
                                                                      'indicators' + '-' + 
                                                                      crits_indicator_id),
                                                            rhs_id=o.idref)
                    elif obs_comp:
                        # [o == idref observable composition]
                        # try to fetch the observable composition o.idref
                        # points to
                        # assumption: the observable composition was
                        # previously ingested. TODO what about when
                        # the observable composition comes in *after*
                        # the indicator?
                        observables_list = util.rgetattr(obs_comp,
                                                          ['observables'])
                        if not observables_list:
                            config['logger'].error(
                                log.log_messages['obs_comp_dereference_error'
                                              ].format(id_=i))
                            continue
                        else:
                            for j in observables_list:
                                # store the pending relationship in
                                # the db for later processing
                                config['db'].set_pending_crits_link(src, dest,
                                                                    lhs_id=(xmlns_name + ':' + 
                                                                              'indicators' + '-' +
                                                                              crits_indicator_id),
                                                                    rhs_id=j.idref)
                    # TODO (need to dig up suitable sample data)
                    # if it's an observable composition with inline
                    # observables, pass them to observable composition with
                    # inline observables, pass them to process_observables(),
                    # (which will store the edge/crits id indicator pairing
                    # for later processing.
                    else:
                        config['logger'].error(
                            log.log_messages['obs_comp_dereference_error'
                                          ].format(id_=i))
                        continue
        # as we've now successfully processed the indicator, track
        # the related crits/json ids (by src/dest)
        if util.rgetattr(indicators[i], ['related_indicators']) and len(indicators[i].related_indicators):
            for j in indicators[i].related_indicators:
                if util.rgetattr(j, ['item', 'idref']):
                    # store the pending relationship in the db for
                    # later processing 

                    # TODO for some reason, the crits relationship api
                    # is rejecting _some_ (but not _all_
                    # indicator-to-indicator relationships. the
                    # indicator ids are valid and the api post looks
                    # correct but...sometimes this fails :-/
                    config['db'].set_pending_crits_link(src, dest,
                                                        lhs_id=(xmlns_name + ':' + 
                                                                  'indicators' + '-' +
                                                                  crits_indicator_id),
                                                        rhs_id=j.item.idref)
                                        
        config['db'].set_object_id(src, dest,
                                   edge_id=i,
                                   crits_id=(xmlns_name + ':' + 'indicators' + '-' +
                                             crits_indicator_id))
Ejemplo n.º 16
0
def process_incidents(config, src, dest, incidents):
    '''handle incoming stix incidents'''
    xmlns_name = config['edge']['sites'][src]['stix']['xmlns_name']
    status_trans = {'New': 'New', 'Open': 'In Progress',
                    'Closed': 'Analyzed', 'Rejected': 'Deprecated'}
    for i in incidents.keys():
        json = dict()
        json['event_type'] = 'Threat Report'
        json['title'] = incidents[i].title
        json['description'] = util.rgetattr(incidents[i], ['description', 'value'])
        json['status'] = status_trans[incidents[i].status.value]
        # inbox the incident (we need to crits id!)
        config['edge_tally']['events']['incoming'] += 1
        config['edge_tally']['all']['incoming'] += 1
        (crits_event_id, success) = crits.crits_inbox(config, dest,
                                                       'events',
                                                       json, src=src)
        if not success:
            config['logger'].error(
                log.log_messages['obj_inbox_error'].format(
                    src_type='edge', id_=i, 
                    dest_type='crits events api endpoint'))
            continue
        else:
            # successfully inboxed event...
            config['edge_tally']['events']['processed'] += 1
            config['edge_tally']['all']['processed'] += 1
            if config['daemon']['debug']:
                config['logger'].debug(
                    log.log_messages['obj_inbox_success'].format(
                        src_type='edge', id_=i,
                        dest_type='crits events api endpoint'))
        # as we've now successfully processed the event, track
        # the related crits/json ids (by src/dest)
        if util.rgetattr(incidents[i], ['related_observables']) and len(incidents[i].related_observables):
            for j in incidents[i].related_observables:
                if util.rgetattr(j, ['item', 'idref']):
                    # store the pending relationship in the db for
                    # later processing 
                    config['db'].set_pending_crits_link(src, dest,
                                                        lhs_id=(xmlns_name + ':' + 
                                                                  'events' + '-' +
                                                                  crits_event_id),
                                                        rhs_id=j.item.idref)
        if util.rgetattr(incidents[i], ['related_indicators']) and len(incidents[i].related_indicators):
            for j in incidents[i].related_indicators:
                if util.rgetattr(j, ['item', 'idref']):
                    # store the pending relationship in the db for
                    # later processing 
                    config['db'].set_pending_crits_link(src, dest,
                                                        lhs_id=(xmlns_name + ':' + 
                                                                  'events' + '-' +
                                                                  crits_event_id),
                                                        rhs_id=j.item.idref)
        if util.rgetattr(incidents[i], ['related_incidents']) and len(incidents[i].related_incidents):
            for j in incidents[i].related_incidents:
                if util.rgetattr(j, ['item', 'idref']):
                    # store the pending relationship in the db for
                    # later processing 
                    config['db'].set_pending_crits_link(src, dest,
                                                        lhs_id=(xmlns_name + ':' + 
                                                                  'events' + '-' +
                                                                  crits_event_id),
                                                        rhs_id=j.item.idref)
                                        
        config['db'].set_object_id(src, dest,
                                   edge_id=i,
                                   crits_id=(xmlns_name + ':' + 'events' + '-' +
                                             crits_event_id))
Ejemplo n.º 17
0
def process_indicators(config, src, dest, indicators):
    """handle incoming stix indicators"""
    xmlns_name = config["edge"]["sites"][src]["stix"]["xmlns_name"]
    for i in indicators.keys():
        json = dict()
        json["type"] = "Related_To"
        json["value"] = util.rgetattr(indicators[i], ["title"], default_="unknown")
        json["indicator_confidence"] = util.rgetattr(
            indicators[i], ["confidence", "value", "value"], default_="unknown"
        )
        # TODO lookup the corresponding stix prop for indicator_impact
        json["indicator_impact"] = {"rating": "unknown"}
        # inbox the indicator (we need to crits id!)
        config["edge_tally"]["indicators"]["incoming"] += 1
        config["edge_tally"]["all"]["incoming"] += 1
        (crits_indicator_id, success) = crits.crits_inbox(config, dest, "indicators", json, src=src)
        if not success:
            config["logger"].error(
                log.log_messages["obj_inbox_error"].format(
                    src_type="edge", id_=i, dest_type="crits indicators api endpoint"
                )
            )
            continue
        else:
            # successfully inboxed indicator...
            config["edge_tally"]["indicators"]["processed"] += 1
            config["edge_tally"]["all"]["processed"] += 1
            if config["daemon"]["debug"]:
                config["logger"].debug(
                    log.log_messages["obj_inbox_success"].format(
                        src_type="edge", id_=i, dest_type="crits indicators api endpoint"
                    )
                )
        if util.rgetattr(indicators[i], ["observables"]):
            for o in indicators[i].observables:
                if util.rgetattr(o, ["idref"]) and not util.rgetattr(o, ["object_"]):
                    # TODO need to delete observable compositions from
                    #      mongo once we've processed them
                    obs_comp = config["db"].get_obs_comp(src, dest, obs_id=o.idref)
                    if not obs_comp:
                        # [ o == embedded observable]
                        config["db"].set_pending_crits_link(
                            src,
                            dest,
                            lhs_id=(xmlns_name + ":" + "indicators" + "-" + crits_indicator_id),
                            rhs_id=o.idref,
                        )
                    elif obs_comp:
                        # [o == idref observable composition]
                        # try to fetch the observable composition o.idref
                        # points to
                        # assumption: the observable composition was
                        # previously ingested. TODO what about when
                        # the observable composition comes in *after*
                        # the indicator?
                        observables_list = util.rgetattr(obs_comp, ["observables"])
                        if not observables_list:
                            config["logger"].error(log.log_messages["obs_comp_dereference_error"].format(id_=i))
                            continue
                        else:
                            for j in observables_list:
                                # store the pending relationship in
                                # the db for later processing
                                config["db"].set_pending_crits_link(
                                    src,
                                    dest,
                                    lhs_id=(xmlns_name + ":" + "indicators" + "-" + crits_indicator_id),
                                    rhs_id=j.idref,
                                )
                    # TODO (need to dig up suitable sample data)
                    # if it's an observable composition with inline
                    # observables, pass them to observable composition with
                    # inline observables, pass them to process_observables(),
                    # (which will store the edge/crits id indicator pairing
                    # for later processing.
                    else:
                        config["logger"].error(log.log_messages["obs_comp_dereference_error"].format(id_=i))
                        continue
        # as we've now successfully processed the indicator, track
        # the related crits/json ids (by src/dest)
        if util.rgetattr(indicators[i], ["related_indicators"]) and len(indicators[i].related_indicators):
            for j in indicators[i].related_indicators:
                if util.rgetattr(j, ["item", "idref"]):
                    # store the pending relationship in the db for
                    # later processing

                    # TODO for some reason, the crits relationship api
                    # is rejecting _some_ (but not _all_
                    # indicator-to-indicator relationships. the
                    # indicator ids are valid and the api post looks
                    # correct but...sometimes this fails :-/
                    config["db"].set_pending_crits_link(
                        src,
                        dest,
                        lhs_id=(xmlns_name + ":" + "indicators" + "-" + crits_indicator_id),
                        rhs_id=j.item.idref,
                    )

        config["db"].set_object_id(
            src, dest, edge_id=i, crits_id=(xmlns_name + ":" + "indicators" + "-" + crits_indicator_id)
        )
Ejemplo n.º 18
0
def cybox_email_to_json(config, observable):
    '''translate a cybox email object to crits json'''
    crits_types = {'subject': 'subject', 'to': 'to', 'cc': 'cc',
                   'from_': 'from_address', 'sender': 'sender', 'date': 'date',
                   'message_id': 'message_id', 'reply_to': 'reply_to',
                   'boundary': 'boundary', 'x_mailer': 'x_mailer',
                   'x_originating_ip': 'x_originating_ip'}
    json = {'upload_type': 'fields'}
    subject = util.rgetattr(observable.object_.properties,
                             ['header', 'subject', 'value'])
    if subject:
        json['subject'] = subject
    to = util.rgetattr(observable.object_.properties, ['header', 'to'])
    if to:
        json['to'] = []
        for i in to:
            addr = util.rgetattr(i, ['address_value', 'values'])
            if addr:
                json['to'].append(addr)
    cc = util.rgetattr(observable.object_.properties, ['header', 'cc'])
    if cc:
        json['cc'] = []
        for i in cc:
            addr = util.rgetattr(i, ['address_value', 'values'])
            if addr:
                json['cc'].append(addr)
    from_ = util.rgetattr(observable.object_.properties,
                           ['header', 'from_', 'address_value', 'value'])
    if from_:
        json['from_address'] = [from_]
    sender = util.rgetattr(observable.object_.properties,
                            ['header', 'sender', 'address_value', 'value'])
    if sender:
        json['sender'] = sender
    date = util.rgetattr(observable.object_.properties,
                          ['header', 'date', 'value'])
    if date:
        json['date'] = date
    message_id = util.rgetattr(observable.object_.properties,
                                ['header', 'message_id', 'value'])
    if message_id:
        json['message_id'] = message_id
    reply_to = util.rgetattr(observable.object_.properties,
                              ['header', 'reply_to', 'address_value', 'value'])
    if reply_to:
        json['reply_to'] = reply_to
    boundary = util.rgetattr(observable.object_.properties,
                              ['header', 'boundary', 'value'])
    if boundary:
        json['boundary'] = boundary
    x_mailer = util.rgetattr(observable.object_.properties,
                              ['header', 'x_mailer', 'value'])
    if x_mailer:
        json['x_mailer'] = x_mailer
    x_originating_ip = util.rgetattr(observable.object_.properties,
                                      ['header', 'x_originating_ip', 'value'])
    if x_originating_ip:
        json['x_originating_ip'] = x_originating_ip
    json['stix_id'] = observable.id_
    return(json)