def cybox_address_to_json(config, observable):
"""translate a cybox address object to crits json"""
crits_types = {
"cidr": "Address - cidr",
"ipv4-addr": "Address - ipv4-addr",
"ipv4-net": "Address - ipv4-net",
"ipv4-netmask": "Address - ipv4-net-mask",
"ipv6-addr": "Address - ipv6-addr",
"ipv6-net": "Address - ipv6-net",
"ipv6-netmask": "Address - ipv6-net-mask",
}
condition = util.rgetattr(observable.object_.properties, ["condition"])
if condition in ["Equals", None]:
# currently not handling other observable conditions as
# it's not clear that crits even supports these...
ip_category = util.rgetattr(observable.object_.properties, ["category"])
ip_value = util.rgetattr(observable.object_.properties, ["address_value", "value"])
if ip_value and ip_category:
if ip_category not in crits_types.keys():
config["logger"].error(
log.log_messages["unsupported_object_error"].format(
type_="edge",
obj_type=(str(type(observable.object_.properties)) + ", %s" % ip_category),
id_=observable.id_,
)
)
return None
json = {"ip": ip_value, "ip_type": crits_types[ip_category]}
json["stix_id"] = observable.id_
return json
def cybox_address_to_json(config, observable):
'''translate a cybox address object to crits json'''
crits_types = {'cidr': 'Address - cidr',
'ipv4-addr': 'Address - ipv4-addr',
'ipv4-net': 'Address - ipv4-net',
'ipv4-netmask': 'Address - ipv4-net-mask',
'ipv6-addr': 'Address - ipv6-addr',
'ipv6-net': 'Address - ipv6-net',
'ipv6-netmask': 'Address - ipv6-net-mask'}
condition = util.rgetattr(observable.object_.properties, ['condition'])
if condition in ['Equals', None]:
# currently not handling other observable conditions as
# it's not clear that crits even supports these...
ip_category = util.rgetattr(observable.object_.properties,
['category'])
ip_value = util.rgetattr(observable.object_.properties,
['address_value', 'value'])
if ip_value and ip_category:
if ip_category not in crits_types.keys():
config['logger'].error(
log.log_messages['unsupported_object_error'].format(
type_='edge', obj_type=(str(type(observable.object_.properties))
+ ', %s' % ip_category), id_=observable.id_))
return(None)
json = {'ip': ip_value, 'ip_type': crits_types[ip_category]}
json['stix_id'] = observable.id_
return(json)
def cybox_domain_to_json(config, observable):
"""translate a cybox domain object to crits json"""
crits_types = {"FQDN": "A"}
# crits doesn't appear to support tlds...
domain_category = util.rgetattr(observable.object_.properties, ["type_"])
domain_value = util.rgetattr(observable.object_.properties, ["value", "value"])
if domain_category and domain_value:
json = {"domain": domain_value, "type": crits_types[domain_category]}
json["stix_id"] = observable.id_
return json
def cybox_domain_to_json(config, observable):
'''translate a cybox domain object to crits json'''
crits_types = {'FQDN': 'A'}
# crits doesn't appear to support tlds...
domain_category = util.rgetattr(observable.object_.properties, ['type_'])
domain_value = util.rgetattr(observable.object_.properties,
['value', 'value'])
if domain_category and domain_value:
json = {'domain': domain_value, 'type': crits_types[domain_category]}
json['stix_id'] = observable.id_
return(json)
def process_observables(config, src, dest, observables):
'''handle incoming cybox observables and observable compositions'''
# TODO some of the hailataxii date uses the cybox ###comma###
# construct, which is currently unsupported
for o_id, o in observables.iteritems():
json = dict()
if util.rgetattr(o, ['observable_composition']) \
and not util.rgetattr(o, ['object_']):
# it's an observable composition
# store it in the db...maybe the indicator will only come
# across in a subsequent run so we can't rely on passing
# this around in memory
config['db'].store_obs_comp(src, dest,
obs_id=o_id,
obs_comp=o.observable_composition)
continue
elif util.rgetattr(o, ['object_']):
# it's a normal observable
(json, endpoint) = cybox_observable_to_json(config, o)
if not json:
config['logger'].error(
log.log_messages[
'obj_convert_error'].format(src_type='cybox',
src_obj='observable',
id_=o_id,
dest_type='crits',
dest_obj='json'))
continue
# inbox the observable to crits
config['edge_tally'][endpoint]['incoming'] += 1
config['edge_tally']['all']['incoming'] += 1
(id_, success) = \
crits.crits_inbox(config, dest, endpoint, json,
src=src, edge_id=o_id)
if not success:
config['logger'].error(
log.log_messages['obj_inbox_error'].format(
src_type='edge', id_=o, dest_type='crits ' + endpoint + ' api endpoint'))
continue
# successfully inboxed observable
config['edge_tally'][endpoint]['processed'] += 1
config['edge_tally']['all']['processed'] += 1
if config['daemon']['debug']:
config['logger'].debug(
log.log_messages['obj_inbox_success'].format(
src_type='edge', id_=o_id,
dest_type='crits ' + endpoint + ' api endpoint'))
def cybox_observable_to_json(config, observable):
'''translate a cybox observable to crits json'''
props = util.rgetattr(observable.object_, ['properties'])
endpoint = None
json = None
if props and isinstance(props, Address):
endpoint = 'ips'
json = cybox_address_to_json(config, observable)
elif props and isinstance(props, DomainName):
endpoint = 'domains'
json = cybox_domain_to_json(config, observable)
elif props and isinstance(props, URI):
endpoint = 'domains'
json = cybox_uri_to_json(config, observable)
elif props and isinstance(props, File):
endpoint = 'samples'
json = cybox_file_to_json(config, observable)
elif props and isinstance(props, EmailMessage):
endpoint = 'emails'
json = cybox_email_to_json(config, observable)
if endpoint and json:
# TODO: this would all be a helluva lot easier if the crits
# api supported manually setting an _id
#
# json['_id'] = observable.id_.split('-')[1]
return(json, endpoint)
else:
config['logger'].error(
log.log_messages['unsupported_object_error'].format(
type_='edge', obj_type=type(props), id_=observable.id_))
return(None, None)
def cybox_observable_to_json(config, observable):
"""translate a cybox observable to crits json"""
props = util.rgetattr(observable.object_, ["properties"])
endpoint = None
json = None
if props and isinstance(props, Address):
endpoint = "ips"
json = cybox_address_to_json(config, observable)
elif props and isinstance(props, DomainName):
endpoint = "domains"
json = cybox_domain_to_json(config, observable)
elif props and isinstance(props, URI):
endpoint = "domains"
json = cybox_uri_to_json(config, observable)
elif props and isinstance(props, File):
endpoint = "samples"
json = cybox_file_to_json(config, observable)
elif props and isinstance(props, EmailMessage):
endpoint = "emails"
json = cybox_email_to_json(config, observable)
if endpoint and json:
# TODO: this would all be a helluva lot easier if the crits
# api supported manually setting an _id
#
# json['_id'] = observable.id_.split('-')[1]
return (json, endpoint)
else:
config["logger"].error(
log.log_messages["unsupported_object_error"].format(type_="edge", obj_type=type(props), id_=observable.id_)
)
return (None, None)
def process_observables(config, src, dest, observables):
"""handle incoming cybox observables and observable compositions"""
# TODO some of the hailataxii date uses the cybox ###comma###
# construct, which is currently unsupported
for o_id, o in observables.iteritems():
json = dict()
if util.rgetattr(o, ["observable_composition"]) and not util.rgetattr(o, ["object_"]):
# it's an observable composition
# store it in the db...maybe the indicator will only come
# across in a subsequent run so we can't rely on passing
# this around in memory
config["db"].store_obs_comp(src, dest, obs_id=o_id, obs_comp=o.observable_composition)
continue
elif util.rgetattr(o, ["object_"]):
# it's a normal observable
(json, endpoint) = cybox_observable_to_json(config, o)
if not json:
config["logger"].error(
log.log_messages["obj_convert_error"].format(
src_type="cybox", src_obj="observable", id_=o_id, dest_type="crits", dest_obj="json"
)
)
continue
# inbox the observable to crits
config["edge_tally"][endpoint]["incoming"] += 1
config["edge_tally"]["all"]["incoming"] += 1
(id_, success) = crits.crits_inbox(config, dest, endpoint, json, src=src, edge_id=o_id)
if not success:
config["logger"].error(
log.log_messages["obj_inbox_error"].format(
src_type="edge", id_=o, dest_type="crits " + endpoint + " api endpoint"
)
)
continue
# successfully inboxed observable
config["edge_tally"][endpoint]["processed"] += 1
config["edge_tally"]["all"]["processed"] += 1
if config["daemon"]["debug"]:
config["logger"].debug(
log.log_messages["obj_inbox_success"].format(
src_type="edge", id_=o_id, dest_type="crits " + endpoint + " api endpoint"
)
)
def cybox_uri_to_json(config, observable):
'''translate a cybox uri object to crits json'''
crits_types = {'Domain Name': 'A'}
# urls currently not supported...
domain_category = util.rgetattr(observable.object_.properties,
['type_'])
domain_value = util.rgetattr(observable.object_.properties,
['value', 'value'])
if domain_category and domain_value:
if domain_category not in crits_types.keys():
config['logger'].error(
log.log_messages['unsupported_object_error'].format(
type_='edge', obj_type=(str(type(observable.object_.properties))
+ ', %s' % domain_category), id_=observable.id_))
return(None)
json = {'domain': domain_value, 'type': crits_types[domain_category]}
json['stix_id'] = observable.id_
return(json)
def process_incidents(config, src, dest, incidents):
"""handle incoming stix incidents"""
xmlns_name = config["edge"]["sites"][src]["stix"]["xmlns_name"]
status_trans = {"New": "New", "Open": "In Progress", "Closed": "Analyzed", "Rejected": "Deprecated"}
for i in incidents.keys():
json = dict()
json["event_type"] = "Threat Report"
json["title"] = incidents[i].title
json["description"] = util.rgetattr(incidents[i], ["description", "value"])
json["status"] = status_trans[incidents[i].status.value]
# inbox the incident (we need to crits id!)
config["edge_tally"]["events"]["incoming"] += 1
config["edge_tally"]["all"]["incoming"] += 1
(crits_event_id, success) = crits.crits_inbox(config, dest, "events", json, src=src)
if not success:
config["logger"].error(
log.log_messages["obj_inbox_error"].format(
src_type="edge", id_=i, dest_type="crits events api endpoint"
)
)
continue
else:
# successfully inboxed event...
config["edge_tally"]["events"]["processed"] += 1
config["edge_tally"]["all"]["processed"] += 1
if config["daemon"]["debug"]:
config["logger"].debug(
log.log_messages["obj_inbox_success"].format(
src_type="edge", id_=i, dest_type="crits events api endpoint"
)
)
# as we've now successfully processed the event, track
# the related crits/json ids (by src/dest)
if util.rgetattr(incidents[i], ["related_observables"]) and len(incidents[i].related_observables):
for j in incidents[i].related_observables:
if util.rgetattr(j, ["item", "idref"]):
# store the pending relationship in the db for
# later processing
config["db"].set_pending_crits_link(
src, dest, lhs_id=(xmlns_name + ":" + "events" + "-" + crits_event_id), rhs_id=j.item.idref
)
if util.rgetattr(incidents[i], ["related_indicators"]) and len(incidents[i].related_indicators):
for j in incidents[i].related_indicators:
if util.rgetattr(j, ["item", "idref"]):
# store the pending relationship in the db for
# later processing
config["db"].set_pending_crits_link(
src, dest, lhs_id=(xmlns_name + ":" + "events" + "-" + crits_event_id), rhs_id=j.item.idref
)
if util.rgetattr(incidents[i], ["related_incidents"]) and len(incidents[i].related_incidents):
for j in incidents[i].related_incidents:
if util.rgetattr(j, ["item", "idref"]):
# store the pending relationship in the db for
# later processing
config["db"].set_pending_crits_link(
src, dest, lhs_id=(xmlns_name + ":" + "events" + "-" + crits_event_id), rhs_id=j.item.idref
)
config["db"].set_object_id(src, dest, edge_id=i, crits_id=(xmlns_name + ":" + "events" + "-" + crits_event_id))
def cybox_uri_to_json(config, observable):
"""translate a cybox uri object to crits json"""
crits_types = {"Domain Name": "A"}
# urls currently not supported...
domain_category = util.rgetattr(observable.object_.properties, ["type_"])
domain_value = util.rgetattr(observable.object_.properties, ["value", "value"])
if domain_category and domain_value:
if domain_category not in crits_types.keys():
config["logger"].error(
log.log_messages["unsupported_object_error"].format(
type_="edge",
obj_type=(str(type(observable.object_.properties)) + ", %s" % domain_category),
id_=observable.id_,
)
)
return None
json = {"domain": domain_value, "type": crits_types[domain_category]}
json["stix_id"] = observable.id_
return json
def cybox_file_to_json(config, observable):
'''translate a cybox file object to crits json'''
crits_types = {'MD5': 'md5',
'SHA1': 'sha1',
'SHA224': 'sha224',
'SHA256': 'sha256',
'SHA384': 'sha384',
'SHA512': 'sha512',
'SSDEEP': 'ssdeep'}
json = {'upload_type': 'metadata'}
hashes = util.rgetattr(observable.object_.properties, ['hashes'])
if hashes:
for hash in hashes:
hash_type = util.rgetattr(hash, ['type_', 'value'])
hash_value = util.rgetattr(hash, ['simple_hash_value', 'value'])
if hash_type and hash_value:
json[crits_types[hash_type]] = hash_value
file_name = util.rgetattr(observable.object_.properties,
['file_name', 'value'])
if file_name:
json['filename'] = file_name
file_format = util.rgetattr(observable.object_.properties,
['file_format', 'value'])
if file_format:
json['filetype'] = file_format
file_size = util.rgetattr(observable.object_.properties,
['size_in_bytes', 'value'])
if file_size:
json['size'] = file_size
json['stix_id'] = observable.id_
return(json)
def cybox_file_to_json(config, observable):
"""translate a cybox file object to crits json"""
crits_types = {
"MD5": "md5",
"SHA1": "sha1",
"SHA224": "sha224",
"SHA256": "sha256",
"SHA384": "sha384",
"SHA512": "sha512",
"SSDEEP": "ssdeep",
}
json = {"upload_type": "metadata"}
hashes = util.rgetattr(observable.object_.properties, ["hashes"])
if hashes:
for hash in hashes:
hash_type = util.rgetattr(hash, ["type_", "value"])
hash_value = util.rgetattr(hash, ["simple_hash_value", "value"])
if hash_type and hash_value:
json[crits_types[hash_type]] = hash_value
file_name = util.rgetattr(observable.object_.properties, ["file_name", "value"])
if file_name:
json["filename"] = file_name
file_format = util.rgetattr(observable.object_.properties, ["file_format", "value"])
if file_format:
json["filetype"] = file_format
file_size = util.rgetattr(observable.object_.properties, ["size_in_bytes", "value"])
if file_size:
json["size"] = file_size
json["stix_id"] = observable.id_
return json
def cybox_email_to_json(config, observable):
"""translate a cybox email object to crits json"""
crits_types = {
"subject": "subject",
"to": "to",
"cc": "cc",
"from_": "from_address",
"sender": "sender",
"date": "date",
"message_id": "message_id",
"reply_to": "reply_to",
"boundary": "boundary",
"x_mailer": "x_mailer",
"x_originating_ip": "x_originating_ip",
}
json = {"upload_type": "fields"}
subject = util.rgetattr(observable.object_.properties, ["header", "subject", "value"])
if subject:
json["subject"] = subject
to = util.rgetattr(observable.object_.properties, ["header", "to"])
if to:
json["to"] = []
for i in to:
addr = util.rgetattr(i, ["address_value", "values"])
if addr:
json["to"].append(addr)
cc = util.rgetattr(observable.object_.properties, ["header", "cc"])
if cc:
json["cc"] = []
for i in cc:
addr = util.rgetattr(i, ["address_value", "values"])
if addr:
json["cc"].append(addr)
from_ = util.rgetattr(observable.object_.properties, ["header", "from_", "address_value", "value"])
if from_:
json["from_address"] = [from_]
sender = util.rgetattr(observable.object_.properties, ["header", "sender", "address_value", "value"])
if sender:
json["sender"] = sender
date = util.rgetattr(observable.object_.properties, ["header", "date", "value"])
if date:
json["date"] = date
message_id = util.rgetattr(observable.object_.properties, ["header", "message_id", "value"])
if message_id:
json["message_id"] = message_id
reply_to = util.rgetattr(observable.object_.properties, ["header", "reply_to", "address_value", "value"])
if reply_to:
json["reply_to"] = reply_to
boundary = util.rgetattr(observable.object_.properties, ["header", "boundary", "value"])
if boundary:
json["boundary"] = boundary
x_mailer = util.rgetattr(observable.object_.properties, ["header", "x_mailer", "value"])
if x_mailer:
json["x_mailer"] = x_mailer
x_originating_ip = util.rgetattr(observable.object_.properties, ["header", "x_originating_ip", "value"])
if x_originating_ip:
json["x_originating_ip"] = x_originating_ip
json["stix_id"] = observable.id_
return json
def process_indicators(config, src, dest, indicators):
'''handle incoming stix indicators'''
xmlns_name = config['edge']['sites'][src]['stix']['xmlns_name']
for i in indicators.keys():
json = dict()
json['type'] = 'Related_To'
json['value'] = util.rgetattr(indicators[i], ['title'],
default_='unknown')
json['indicator_confidence'] = \
util.rgetattr(indicators[i], ['confidence', 'value', 'value'],
default_='unknown')
# TODO lookup the corresponding stix prop for indicator_impact
json['indicator_impact'] = {'rating': 'unknown'}
# inbox the indicator (we need to crits id!)
config['edge_tally']['indicators']['incoming'] += 1
config['edge_tally']['all']['incoming'] += 1
(crits_indicator_id, success) = crits.crits_inbox(config, dest,
'indicators',
json, src=src)
if not success:
config['logger'].error(
log.log_messages['obj_inbox_error'].format(
src_type='edge', id_=i,
dest_type='crits indicators api endpoint'))
continue
else:
# successfully inboxed indicator...
config['edge_tally']['indicators']['processed'] += 1
config['edge_tally']['all']['processed'] += 1
if config['daemon']['debug']:
config['logger'].debug(
log.log_messages['obj_inbox_success'].format(
src_type='edge', id_=i,
dest_type='crits indicators api endpoint'))
if util.rgetattr(indicators[i], ['observables']):
for o in indicators[i].observables:
if util.rgetattr(o, ['idref']) and \
not util.rgetattr(o, ['object_']):
# TODO need to delete observable compositions from
# mongo once we've processed them
obs_comp = \
config['db'].get_obs_comp(src, dest, obs_id=o.idref)
if not obs_comp:
# [ o == embedded observable]
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'indicators' + '-' +
crits_indicator_id),
rhs_id=o.idref)
elif obs_comp:
# [o == idref observable composition]
# try to fetch the observable composition o.idref
# points to
# assumption: the observable composition was
# previously ingested. TODO what about when
# the observable composition comes in *after*
# the indicator?
observables_list = util.rgetattr(obs_comp,
['observables'])
if not observables_list:
config['logger'].error(
log.log_messages['obs_comp_dereference_error'
].format(id_=i))
continue
else:
for j in observables_list:
# store the pending relationship in
# the db for later processing
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'indicators' + '-' +
crits_indicator_id),
rhs_id=j.idref)
# TODO (need to dig up suitable sample data)
# if it's an observable composition with inline
# observables, pass them to observable composition with
# inline observables, pass them to process_observables(),
# (which will store the edge/crits id indicator pairing
# for later processing.
else:
config['logger'].error(
log.log_messages['obs_comp_dereference_error'
].format(id_=i))
continue
# as we've now successfully processed the indicator, track
# the related crits/json ids (by src/dest)
if util.rgetattr(indicators[i], ['related_indicators']) and len(indicators[i].related_indicators):
for j in indicators[i].related_indicators:
if util.rgetattr(j, ['item', 'idref']):
# store the pending relationship in the db for
# later processing
# TODO for some reason, the crits relationship api
# is rejecting _some_ (but not _all_
# indicator-to-indicator relationships. the
# indicator ids are valid and the api post looks
# correct but...sometimes this fails :-/
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'indicators' + '-' +
crits_indicator_id),
rhs_id=j.item.idref)
config['db'].set_object_id(src, dest,
edge_id=i,
crits_id=(xmlns_name + ':' + 'indicators' + '-' +
crits_indicator_id))
def process_incidents(config, src, dest, incidents):
'''handle incoming stix incidents'''
xmlns_name = config['edge']['sites'][src]['stix']['xmlns_name']
status_trans = {'New': 'New', 'Open': 'In Progress',
'Closed': 'Analyzed', 'Rejected': 'Deprecated'}
for i in incidents.keys():
json = dict()
json['event_type'] = 'Threat Report'
json['title'] = incidents[i].title
json['description'] = util.rgetattr(incidents[i], ['description', 'value'])
json['status'] = status_trans[incidents[i].status.value]
# inbox the incident (we need to crits id!)
config['edge_tally']['events']['incoming'] += 1
config['edge_tally']['all']['incoming'] += 1
(crits_event_id, success) = crits.crits_inbox(config, dest,
'events',
json, src=src)
if not success:
config['logger'].error(
log.log_messages['obj_inbox_error'].format(
src_type='edge', id_=i,
dest_type='crits events api endpoint'))
continue
else:
# successfully inboxed event...
config['edge_tally']['events']['processed'] += 1
config['edge_tally']['all']['processed'] += 1
if config['daemon']['debug']:
config['logger'].debug(
log.log_messages['obj_inbox_success'].format(
src_type='edge', id_=i,
dest_type='crits events api endpoint'))
# as we've now successfully processed the event, track
# the related crits/json ids (by src/dest)
if util.rgetattr(incidents[i], ['related_observables']) and len(incidents[i].related_observables):
for j in incidents[i].related_observables:
if util.rgetattr(j, ['item', 'idref']):
# store the pending relationship in the db for
# later processing
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'events' + '-' +
crits_event_id),
rhs_id=j.item.idref)
if util.rgetattr(incidents[i], ['related_indicators']) and len(incidents[i].related_indicators):
for j in incidents[i].related_indicators:
if util.rgetattr(j, ['item', 'idref']):
# store the pending relationship in the db for
# later processing
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'events' + '-' +
crits_event_id),
rhs_id=j.item.idref)
if util.rgetattr(incidents[i], ['related_incidents']) and len(incidents[i].related_incidents):
for j in incidents[i].related_incidents:
if util.rgetattr(j, ['item', 'idref']):
# store the pending relationship in the db for
# later processing
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'events' + '-' +
crits_event_id),
rhs_id=j.item.idref)
config['db'].set_object_id(src, dest,
edge_id=i,
crits_id=(xmlns_name + ':' + 'events' + '-' +
crits_event_id))
def process_indicators(config, src, dest, indicators):
"""handle incoming stix indicators"""
xmlns_name = config["edge"]["sites"][src]["stix"]["xmlns_name"]
for i in indicators.keys():
json = dict()
json["type"] = "Related_To"
json["value"] = util.rgetattr(indicators[i], ["title"], default_="unknown")
json["indicator_confidence"] = util.rgetattr(
indicators[i], ["confidence", "value", "value"], default_="unknown"
)
# TODO lookup the corresponding stix prop for indicator_impact
json["indicator_impact"] = {"rating": "unknown"}
# inbox the indicator (we need to crits id!)
config["edge_tally"]["indicators"]["incoming"] += 1
config["edge_tally"]["all"]["incoming"] += 1
(crits_indicator_id, success) = crits.crits_inbox(config, dest, "indicators", json, src=src)
if not success:
config["logger"].error(
log.log_messages["obj_inbox_error"].format(
src_type="edge", id_=i, dest_type="crits indicators api endpoint"
)
)
continue
else:
# successfully inboxed indicator...
config["edge_tally"]["indicators"]["processed"] += 1
config["edge_tally"]["all"]["processed"] += 1
if config["daemon"]["debug"]:
config["logger"].debug(
log.log_messages["obj_inbox_success"].format(
src_type="edge", id_=i, dest_type="crits indicators api endpoint"
)
)
if util.rgetattr(indicators[i], ["observables"]):
for o in indicators[i].observables:
if util.rgetattr(o, ["idref"]) and not util.rgetattr(o, ["object_"]):
# TODO need to delete observable compositions from
# mongo once we've processed them
obs_comp = config["db"].get_obs_comp(src, dest, obs_id=o.idref)
if not obs_comp:
# [ o == embedded observable]
config["db"].set_pending_crits_link(
src,
dest,
lhs_id=(xmlns_name + ":" + "indicators" + "-" + crits_indicator_id),
rhs_id=o.idref,
)
elif obs_comp:
# [o == idref observable composition]
# try to fetch the observable composition o.idref
# points to
# assumption: the observable composition was
# previously ingested. TODO what about when
# the observable composition comes in *after*
# the indicator?
observables_list = util.rgetattr(obs_comp, ["observables"])
if not observables_list:
config["logger"].error(log.log_messages["obs_comp_dereference_error"].format(id_=i))
continue
else:
for j in observables_list:
# store the pending relationship in
# the db for later processing
config["db"].set_pending_crits_link(
src,
dest,
lhs_id=(xmlns_name + ":" + "indicators" + "-" + crits_indicator_id),
rhs_id=j.idref,
)
# TODO (need to dig up suitable sample data)
# if it's an observable composition with inline
# observables, pass them to observable composition with
# inline observables, pass them to process_observables(),
# (which will store the edge/crits id indicator pairing
# for later processing.
else:
config["logger"].error(log.log_messages["obs_comp_dereference_error"].format(id_=i))
continue
# as we've now successfully processed the indicator, track
# the related crits/json ids (by src/dest)
if util.rgetattr(indicators[i], ["related_indicators"]) and len(indicators[i].related_indicators):
for j in indicators[i].related_indicators:
if util.rgetattr(j, ["item", "idref"]):
# store the pending relationship in the db for
# later processing
# TODO for some reason, the crits relationship api
# is rejecting _some_ (but not _all_
# indicator-to-indicator relationships. the
# indicator ids are valid and the api post looks
# correct but...sometimes this fails :-/
config["db"].set_pending_crits_link(
src,
dest,
lhs_id=(xmlns_name + ":" + "indicators" + "-" + crits_indicator_id),
rhs_id=j.item.idref,
)
config["db"].set_object_id(
src, dest, edge_id=i, crits_id=(xmlns_name + ":" + "indicators" + "-" + crits_indicator_id)
)
def cybox_email_to_json(config, observable):
'''translate a cybox email object to crits json'''
crits_types = {'subject': 'subject', 'to': 'to', 'cc': 'cc',
'from_': 'from_address', 'sender': 'sender', 'date': 'date',
'message_id': 'message_id', 'reply_to': 'reply_to',
'boundary': 'boundary', 'x_mailer': 'x_mailer',
'x_originating_ip': 'x_originating_ip'}
json = {'upload_type': 'fields'}
subject = util.rgetattr(observable.object_.properties,
['header', 'subject', 'value'])
if subject:
json['subject'] = subject
to = util.rgetattr(observable.object_.properties, ['header', 'to'])
if to:
json['to'] = []
for i in to:
addr = util.rgetattr(i, ['address_value', 'values'])
if addr:
json['to'].append(addr)
cc = util.rgetattr(observable.object_.properties, ['header', 'cc'])
if cc:
json['cc'] = []
for i in cc:
addr = util.rgetattr(i, ['address_value', 'values'])
if addr:
json['cc'].append(addr)
from_ = util.rgetattr(observable.object_.properties,
['header', 'from_', 'address_value', 'value'])
if from_:
json['from_address'] = [from_]
sender = util.rgetattr(observable.object_.properties,
['header', 'sender', 'address_value', 'value'])
if sender:
json['sender'] = sender
date = util.rgetattr(observable.object_.properties,
['header', 'date', 'value'])
if date:
json['date'] = date
message_id = util.rgetattr(observable.object_.properties,
['header', 'message_id', 'value'])
if message_id:
json['message_id'] = message_id
reply_to = util.rgetattr(observable.object_.properties,
['header', 'reply_to', 'address_value', 'value'])
if reply_to:
json['reply_to'] = reply_to
boundary = util.rgetattr(observable.object_.properties,
['header', 'boundary', 'value'])
if boundary:
json['boundary'] = boundary
x_mailer = util.rgetattr(observable.object_.properties,
['header', 'x_mailer', 'value'])
if x_mailer:
json['x_mailer'] = x_mailer
x_originating_ip = util.rgetattr(observable.object_.properties,
['header', 'x_originating_ip', 'value'])
if x_originating_ip:
json['x_originating_ip'] = x_originating_ip
json['stix_id'] = observable.id_
return(json)