def cybox_address_to_json(config, observable):
'''translate a cybox address object to crits json'''
crits_types = {'cidr': 'Address - cidr',
'ipv4-addr': 'Address - ipv4-addr',
'ipv4-net': 'Address - ipv4-net',
'ipv4-netmask': 'Address - ipv4-net-mask',
'ipv6-addr': 'Address - ipv6-addr',
'ipv6-net': 'Address - ipv6-net',
'ipv6-netmask': 'Address - ipv6-net-mask'}
condition = util_.rgetattr(observable.object_.properties, ['condition'])
if condition in ['Equals', None]:
# currently not handling other observable conditions as
# it's not clear that crits even supports these...
ip_category = util_.rgetattr(observable.object_.properties,
['category'])
ip_value = util_.rgetattr(observable.object_.properties,
['address_value', 'value'])
if ip_value and ip_category:
if ip_category not in crits_types.keys():
config['logger'].error(
log_.log_messages['unsupported_object_error'].format(
type_='edge', obj_type=(str(type(observable.object_.properties))
+ ', %s' % ip_category), id_=observable.id_))
return(None)
json = {'ip': ip_value, 'ip_type': crits_types[ip_category]}
json['stix_id'] = observable.id_
return(json)
def cybox_domain_to_json(config, observable):
'''translate a cybox domain object to crits json'''
crits_types = {'FQDN': 'A'}
# crits doesn't appear to support tlds...
domain_category = util_.rgetattr(observable.object_.properties, ['type_'])
domain_value = util_.rgetattr(observable.object_.properties,
['value', 'value'])
if domain_category and domain_value:
json = {'domain': domain_value, 'type': crits_types[domain_category]}
json['stix_id'] = observable.id_
return(json)
def process_observables(config, src, dest, observables):
'''handle incoming cybox observables and observable compositions'''
# TODO some of the hailataxii date uses the cybox ###comma###
# construct, which is currently unsupported
for o in observables.keys():
json = dict()
if util_.rgetattr(observables[o], ['observable_composition']) \
and not util_.rgetattr(observables[o], ['object_']):
# it's an observable composition
# store it in the db...maybe the indicator will only come
# across in a subsequent run so we can't rely on passing
# this around in memory
config['db'].store_obs_comp(src, dest,
obs_id=o,
obs_comp=observables[o].observable_composition)
continue
elif util_.rgetattr(observables[o], ['object_']):
# it's a normal observable
(json, endpoint) = cybox_observable_to_json(config, observables[o])
if json:
# mark crits releasability
json.update(mark_crits_releasability(config, dest))
else:
config['logger'].error(
log_.log_messages[
'obj_convert_error'].format(src_type='cybox',
src_obj='observable',
id_=o,
dest_type='crits',
dest_obj='json'))
continue
# inbox the observable to crits
config['edge_tally'][endpoint]['incoming'] += 1
config['edge_tally']['all']['incoming'] += 1
(id_, success) = \
crits_.crits_inbox(config, dest, endpoint, json,
src=src, edge_id=o)
if not success:
config['logger'].error(
log_.log_messages['obj_inbox_error'].format(
src_type='edge', id_=o, dest_type='crits ' + endpoint + ' api endpoint'))
continue
else:
# successfully inboxed observable
config['edge_tally'][endpoint]['processed'] += 1
config['edge_tally']['all']['processed'] += 1
if config['daemon']['debug']:
config['logger'].debug(
log_.log_messages['obj_inbox_success'].format(
src_type='edge', id_=o,
dest_type='crits ' + endpoint + ' api endpoint'))
def cybox_observable_to_json(config, observable):
'''translate a cybox observable to crits json'''
props = util_.rgetattr(observable.object_, ['properties'])
if props and isinstance(props, Address):
endpoint = 'ips'
json = cybox_address_to_json(config, observable)
elif props and isinstance(props, DomainName):
endpoint = 'domains'
json = cybox_domain_to_json(config, observable)
elif props and isinstance(props, URI):
endpoint = 'domains'
json = cybox_uri_to_json(config, observable)
elif props and isinstance(props, File):
endpoint = 'samples'
json = cybox_file_to_json(config, observable)
elif props and isinstance(props, EmailMessage):
endpoint = 'emails'
json = cybox_email_to_json(config, observable)
if json and endpoint:
# TODO: this would all be a helluva lot easier if the crits
# api supported manually setting an _id
#
# json['_id'] = observable.id_.split('-')[1]
return(json, endpoint)
else:
config['logger'].error(
log_.log_messages['unsupported_object_error'].format(
type_='edge', obj_type=type(props), id_=observable.id_))
return(None, None)
def cybox_uri_to_json(config, observable):
'''translate a cybox uri object to crits json'''
crits_types = {'Domain Name': 'A'}
# urls currently not supported...
domain_category = util_.rgetattr(observable.object_.properties,
['type_'])
domain_value = util_.rgetattr(observable.object_.properties,
['value', 'value'])
if domain_category and domain_value:
if domain_category not in crits_types.keys():
config['logger'].error(
log_.log_messages['unsupported_object_error'].format(
type_='edge', obj_type=(str(type(observable.object_.properties))
+ ', %s' % domain_category), id_=observable.id_))
return(None)
json = {'domain': domain_value, 'type': crits_types[domain_category]}
json['stix_id'] = observable.id_
return(json)
def cybox_file_to_json(config, observable):
'''translate a cybox file object to crits json'''
crits_types = {'MD5': 'md5',
'SHA1': 'sha1',
'SHA224': 'sha224',
'SHA256': 'sha256',
'SHA384': 'sha384',
'SHA512': 'sha512',
'SSDEEP': 'ssdeep'}
json = {'upload_type': 'metadata'}
hashes = util_.rgetattr(observable.object_.properties, ['hashes'])
if hashes:
for hash in hashes:
hash_type = util_.rgetattr(hash, ['type_', 'value'])
hash_value = util_.rgetattr(hash, ['simple_hash_value', 'value'])
if hash_type and hash_value:
json[crits_types[hash_type]] = hash_value
file_name = util_.rgetattr(observable.object_.properties,
['file_name', 'value'])
if file_name:
json['filename'] = file_name
file_format = util_.rgetattr(observable.object_.properties,
['file_format', 'value'])
if file_format:
json['filetype'] = file_format
file_size = util_.rgetattr(observable.object_.properties,
['size_in_bytes', 'value'])
if file_size:
json['size'] = file_size
json['stix_id'] = observable.id_
return(json)
def process_indicators(config, src, dest, indicators):
'''handle incoming stix indicators'''
xmlns_name = config['edge']['sites'][src]['stix']['xmlns_name']
for i in indicators.keys():
json = dict()
json['type'] = 'Related_To'
json['value'] = util_.rgetattr(indicators[i], ['title'],
default_='unknown')
json['indicator_confidence'] = \
util_.rgetattr(indicators[i], ['confidence', 'value', 'value'],
default_='unknown')
# TODO lookup the corresponding stix prop for indicator_impact
json['indicator_impact'] = {'rating': 'unknown'}
# inbox the indicator (we need to crits id!)
config['edge_tally']['indicators']['incoming'] += 1
config['edge_tally']['all']['incoming'] += 1
(crits_indicator_id, success) = crits_.crits_inbox(config, dest,
'indicators',
json, src=src)
if not success:
config['logger'].error(
log_.log_messages['obj_inbox_error'].format(
src_type='edge', id_=i,
dest_type='crits indicators api endpoint'))
continue
else:
# successfully inboxed indicator...
config['edge_tally']['indicators']['processed'] += 1
config['edge_tally']['all']['processed'] += 1
if config['daemon']['debug']:
config['logger'].debug(
log_.log_messages['obj_inbox_success'].format(
src_type='edge', id_=i,
dest_type='crits indicators api endpoint'))
if util_.rgetattr(indicators[i], ['observables']):
for o in indicators[i].observables:
if util_.rgetattr(o, ['idref']) and \
not util_.rgetattr(o, ['object_']):
# TODO need to delete observable compositions from
# mongo once we've processed them
obs_comp = \
config['db'].get_obs_comp(src, dest, obs_id=o.idref)
if not obs_comp:
# [ o == embedded observable]
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'indicators' + '-' +
crits_indicator_id),
rhs_id=o.idref)
elif obs_comp:
# [o == idref observable composition]
# try to fetch the observable composition o.idref
# points to
# assumption: the observable composition was
# previously ingested. TODO what about when
# the observable composition comes in *after*
# the indicator?
observables_list = util_.rgetattr(obs_comp,
['observables'])
if not observables_list:
config['logger'].error(
log_.log_messages['obs_comp_dereference_error'
].format(id_=i))
continue
else:
for j in observables_list:
# store the pending relationship in
# the db for later processing
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'indicators' + '-' +
crits_indicator_id),
rhs_id=j.idref)
# TODO (need to dig up suitable sample data)
# if it's an observable composition with inline
# observables, pass them to observable composition with
# inline observables, pass them to process_observables(),
# (which will store the edge/crits id indicator pairing
# for later processing.
else:
config['logger'].error(
log_.log_messages['obs_comp_dereference_error'
].format(id_=i))
continue
# as we've now successfully processed the indicator, track
# the related crits/json ids (by src/dest)
if util_.rgetattr(indicators[i], ['related_indicators']) and len(indicators[i].related_indicators):
for j in indicators[i].related_indicators:
if util_.rgetattr(j, ['item', 'idref']):
# store the pending relationship in the db for
# later processing
# TODO for some reason, the crits relationship api
# is rejecting _some_ (but not _all_
# indicator-to-indicator relationships. the
# indicator ids are valid and the api post looks
# correct but...sometimes this fails :-/
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'indicators' + '-' +
crits_indicator_id),
rhs_id=j.item.idref)
config['db'].set_object_id(src, dest,
edge_id=i,
crits_id=(xmlns_name + ':' + 'indicators' + '-' +
crits_indicator_id))
def process_incidents(config, src, dest, incidents):
'''handle incoming stix incidents'''
xmlns_name = config['edge']['sites'][src]['stix']['xmlns_name']
status_trans = {'New': 'New', 'Open': 'In Progress',
'Closed': 'Analyzed', 'Rejected': 'Deprecated'}
for i in incidents.keys():
json = dict()
json['event_type'] = 'Threat Report'
json['title'] = incidents[i].title
json['description'] = util_.rgetattr(incidents[i], ['description', 'value'])
json['status'] = status_trans[incidents[i].status.value]
# inbox the incident (we need to crits id!)
config['edge_tally']['events']['incoming'] += 1
config['edge_tally']['all']['incoming'] += 1
(crits_event_id, success) = crits_.crits_inbox(config, dest,
'events',
json, src=src)
if not success:
config['logger'].error(
log_.log_messages['obj_inbox_error'].format(
src_type='edge', id_=i,
dest_type='crits events api endpoint'))
continue
else:
# successfully inboxed event...
config['edge_tally']['events']['processed'] += 1
config['edge_tally']['all']['processed'] += 1
if config['daemon']['debug']:
config['logger'].debug(
log_.log_messages['obj_inbox_success'].format(
src_type='edge', id_=i,
dest_type='crits events api endpoint'))
# as we've now successfully processed the event, track
# the related crits/json ids (by src/dest)
if util_.rgetattr(incidents[i], ['related_observables']) and len(incidents[i].related_observables):
for j in incidents[i].related_observables:
if util_.rgetattr(j, ['item', 'idref']):
# store the pending relationship in the db for
# later processing
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'events' + '-' +
crits_event_id),
rhs_id=j.item.idref)
if util_.rgetattr(incidents[i], ['related_indicators']) and len(incidents[i].related_indicators):
for j in incidents[i].related_indicators:
if util_.rgetattr(j, ['item', 'idref']):
# store the pending relationship in the db for
# later processing
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'events' + '-' +
crits_event_id),
rhs_id=j.item.idref)
if util_.rgetattr(incidents[i], ['related_incidents']) and len(incidents[i].related_incidents):
for j in incidents[i].related_incidents:
if util_.rgetattr(j, ['item', 'idref']):
# store the pending relationship in the db for
# later processing
config['db'].set_pending_crits_link(src, dest,
lhs_id=(xmlns_name + ':' +
'events' + '-' +
crits_event_id),
rhs_id=j.item.idref)
config['db'].set_object_id(src, dest,
edge_id=i,
crits_id=(xmlns_name + ':' + 'events' + '-' +
crits_event_id))
def cybox_email_to_json(config, observable):
'''translate a cybox email object to crits json'''
crits_types = {'subject': 'subject', 'to': 'to', 'cc': 'cc',
'from_': 'from_address', 'sender': 'sender', 'date': 'date',
'message_id': 'message_id', 'reply_to': 'reply_to',
'boundary': 'boundary', 'x_mailer': 'x_mailer',
'x_originating_ip': 'x_originating_ip'}
json = {'upload_type': 'fields'}
subject = util_.rgetattr(observable.object_.properties,
['header', 'subject', 'value'])
if subject:
json['subject'] = subject
to = util_.rgetattr(observable.object_.properties, ['header', 'to'])
if to:
json['to'] = []
for i in to:
addr = util_.rgetattr(i, ['address_value', 'values'])
if addr:
json['to'].append(addr)
cc = util_.rgetattr(observable.object_.properties, ['header', 'cc'])
if cc:
json['cc'] = []
for i in cc:
addr = util_.rgetattr(i, ['address_value', 'values'])
if addr:
json['cc'].append(addr)
from_ = util_.rgetattr(observable.object_.properties,
['header', 'from_', 'address_value', 'value'])
if from_:
json['from_address'] = [from_]
sender = util_.rgetattr(observable.object_.properties,
['header', 'sender', 'address_value', 'value'])
if sender:
json['sender'] = sender
date = util_.rgetattr(observable.object_.properties,
['header', 'date', 'value'])
if date:
json['date'] = date
message_id = util_.rgetattr(observable.object_.properties,
['header', 'message_id', 'value'])
if message_id:
json['message_id'] = message_id
reply_to = util_.rgetattr(observable.object_.properties,
['header', 'reply_to', 'address_value', 'value'])
if reply_to:
json['reply_to'] = reply_to
boundary = util_.rgetattr(observable.object_.properties,
['header', 'boundary', 'value'])
if boundary:
json['boundary'] = boundary
x_mailer = util_.rgetattr(observable.object_.properties,
['header', 'x_mailer', 'value'])
if x_mailer:
json['x_mailer'] = x_mailer
x_originating_ip = util_.rgetattr(observable.object_.properties,
['header', 'x_originating_ip', 'value'])
if x_originating_ip:
json['x_originating_ip'] = x_originating_ip
json['stix_id'] = observable.id_
return(json)
def process_observables(config, src, dest, observables):
'''handle incoming cybox observables and observable compositions'''
# TODO some of the hailataxii date uses the cybox ###comma###
# construct, which is currently unsupported
for o_id, o in observables.iteritems():
json = dict()
if util_.rgetattr(o, ['observable_composition']) \
and not util_.rgetattr(o, ['object_']):
# it's an observable composition
# store it in the db...maybe the indicator will only come
# across in a subsequent run so we can't rely on passing
# this around in memory
config['db'].store_obs_comp(src, dest,
obs_id=o_id,
obs_comp=o.observable_composition)
continue
elif util_.rgetattr(o, ['object_']):
# it's a normal observable
(json, endpoint) = cybox_observable_to_json(config, o)
if not json:
config['logger'].error(
log_.log_messages[
'obj_convert_error'].format(src_type='cybox',
src_obj='observable',
id_=o_id,
dest_type='crits',
dest_obj='json'))
continue
# mark crits releasability
# TODO: Maybe remove this? Not sure if it works with
# the Crits PATCH API method for setting releasability.
json.update(mark_crits_releasability(config, dest))
# inbox the observable to crits
config['edge_tally'][endpoint]['incoming'] += 1
config['edge_tally']['all']['incoming'] += 1
(id_, success) = \
crits_.crits_inbox(config, dest, endpoint, json,
src=src, edge_id=o_id)
if not success:
config['logger'].error(
log_.log_messages['obj_inbox_error'].format(
src_type='edge', id_=o, dest_type='crits ' + endpoint + ' api endpoint'))
continue
# Successfully inboxed observable
# Send Patch request to set crits releasability
patch_endpoint = '{}/{}'.format(endpoint, id_)
releasability_json = {
'action': 'add_releasability',
'name': config['crits']['sites'][dest]['api']['releasability'],
}
releasability_success = crits_.crits_patch(config, dest,
patch_endpoint, releasability_json)
if not releasability_success:
config['logger'].error(
log_.log_messages['obj_inbox_error'].format(
src_type='edge', id_=o, dest_type='crits ' + patch_endpoint + ' api endpoint'))
continue
config['edge_tally'][endpoint]['processed'] += 1
config['edge_tally']['all']['processed'] += 1
if config['daemon']['debug']:
config['logger'].debug(
log_.log_messages['obj_inbox_success'].format(
src_type='edge', id_=o_id,
dest_type='crits ' + endpoint + ' api endpoint'))
