def strip_observables(pkg_path):
'''Strips observable from a package, support multiple structures'''
result = Observables()
pkg = STIXPackage.from_xml(pkg_path)
processed = []
for ind in pkg.indicators:
if ind.composite_indicator_expression:
"""The indicator is a compsite structure, this references other indicators, which reference the observables..."""
cyboxobject = ObservableComposition()
cyboxobject.operator = str(ind.observable_composition_operator)
for x in ind.composite_indicator_expression:
"""For every indicator in the composite list, get referenced indicator"""
ref_ind = getindicator_by_id(pkg, str(x._idref))
if ref_ind.observables:
for y in ref_ind.observables:
"""For every referenced observable, get the object"""
ref_obs = getobservable_by_id(pkg, y._idref)
if ref_obs:
cyboxobject.add(ref_obs)
processed.append(ref_obs.id_)
result.add(cyboxobject)
if ind.observables:
for x in ind.observables:
if x is not None:
if x.id_ not in processed:
result.add(x)
processed.append(x.id_)
if pkg.observables:
for x in pkg.observables:
if x is not None:
if x.id_ not in processed:
result.add(x)
scanfile = open(os.path.join(iocname,"scan.json"),'w')
scanfile.write(json.dumps(walkobservables(result).to_dict(), indent=4))
scanfile.close()
def _merge_observables(self, observables):
observable_composition = ObservableComposition()
observable_composition.operator = self.observable_composition_operator
for observable in observables:
observable_composition.add(observable)
root_observable = Observable()
root_observable.observable_composition = observable_composition
return root_observable
def _merge_observables(self, observables, operator='AND'):
observable_composition = ObservableComposition()
observable_composition.operator = operator
for observable_ in observables:
observable_composition.add(observable_)
root_observable = Observable()
root_observable.observable_composition = observable_composition
return root_observable
def returnAttachmentComposition(attribute):
file_object = File()
file_object.file_name = attribute["value"]
file_object.parent.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":FileObject-" + attribute[
"uuid"]
observable = Observable()
if "data" in attribute:
artifact = Artifact(data=attribute["data"])
artifact.parent.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":ArtifactObject-" + attribute[
"uuid"]
observable_artifact = Observable(artifact)
observable_artifact.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-artifact-" + attribute[
"uuid"]
observable_file = Observable(file_object)
observable_file.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-file-" + attribute[
"uuid"]
composition = ObservableComposition(
observables=[observable_artifact, observable_file])
observable.observable_composition = composition
else:
observable = Observable(file_object)
observable.id_ = cybox.utils.idgen.__generator.namespace.prefix + ":observable-" + attribute[
"uuid"]
if attribute["comment"] != "":
observable.description = attribute["comment"]
return observable
def generate_domain_ip_observable(self, indicator, attribute):
indicator.add_indicator_type("Domain Watchlist")
domain, ip = attribute.value.split('|')
address_object = self.resolve_ip_type(attribute.type, ip)
address_object.parent.id_ = "{}:AddressObject-{}".format(self.namespace_prefix, attribute.uuid)
address_observable = Observable(address_object)
address_observable.id_ = "{}:Address-{}".format(self.namespace_prefix, attribute.uuid)
domain_object = DomainName()
domain_object.value = domain
domain_object.value.condition = "Equals"
domain_object.parent.id_ = "{}:DomainNameObject-{}".format(self.namespace_prefix, attribute.uuid)
domain_observable = Observable(domain_object)
domain_observable.id_ = "{}:DomainName-{}".format(self.namespace_prefix, attribute.uuid)
composite_object = ObservableComposition(observables=[address_observable, domain_observable])
composite_object.operator = "AND"
observable = Observable(id_="{}:ObservableComposition-{}".format(self.namespace_prefix, attribute.uuid))
observable.observable_composition = composite_object
return observable
def returnAttachmentComposition(attribute):
file_object = File()
file_object.file_name = attribute["value"]
observable = Observable()
if "data" in attribute:
artifact = Artifact(data=attribute["data"])
composition = ObservableComposition(
observables=[artifact, file_object])
observable.observable_composition = composition
else:
observable = Observable(file_object)
return observable
def generate_ip_observable(self, indicator, attribute):
indicator.add_indicator_type("IP Watchlist")
address_object = self.resolve_ip_type(attribute.type, attribute.value)
address_object.parent.id_ = "{}:AddressObject-{}".format(self.namespace_prefix, attribute.uuid)
if '|' in attribute.value:
port = attribute.value.split('|')[1]
address_observable = Observable(address_object)
address_observable.id_ = "{}:Address-{}".format(self.namespace_prefix, attribute.uuid)
port_object = Port()
port_object.port_value = port
port_object.port_value.condition = "Equals"
port_object.parent.id_ = "{}:PortObject-{}".format(self.namespace_prefix, attribute.uuid)
port_observable = Observable(port_object)
port_observable.id_ = "{}:Port-{}".format(self.namespace_prefix, attribute.uuid)
compositeObject = ObservableComposition(observables=[address_observable, port_observable])
compositeObject.operator = "AND"
observable = Observable(id_ = "{}:ObservableComposition-{}".format(self.namespace_prefix, attribute.uuid))
observable.observable_composition = compositeObject
return observable
else:
return address_object
def test_invalid_arguments(self):
obj = Object()
e = Event()
oc = ObservableComposition()
o1 = Observable()
self.assertRaises(TypeError, _set_event, o1, obj)
self.assertRaises(TypeError, _set_oc, o1, obj)
self.assertRaises(TypeError, _set_obj, o1, e)
self.assertRaises(TypeError, _set_oc, o1, e)
self.assertRaises(TypeError, _set_obj, o1, oc)
self.assertRaises(TypeError, _set_event, o1, oc)
def observables(self, value):
"""
The method will automatically create a top ``cybox.core.Observable`` and
append all ``cybox.core.Observable`` using ``observable_composition``
property when a ``list`` is given with length greater than 1.
Note:
The top level ``cybox.core.Observable`` will set the ``operator``
property for the ``cybox.core.ObservableComposition`` via the
``observable_composition_operator`` property. The value of
``operator`` can be changed via ``observable_composition_operator``
property. By default, the composition layer will be set to ``"OR"``.
Args:
value: A ``list`` of ``cybox.core.Observable`` instances or a single
``cybox.core.Observable`` instance.
Raises:
ValueError: If set to a value that cannot be converted to an
instance of ``cybox.core.Observable``.
"""
if not value:
return
if isinstance(value, Observable):
self.observable = value
elif utils.is_sequence(value):
if len(value) == 1:
self.observable = value
return
observable_comp = ObservableComposition()
observable_comp.operator = self.observable_composition_operator
for element in value:
observable_comp.add(element)
self.observable = Observable()
self.observable.observable_composition = observable_comp
def observables(self, value):
"""
The method will automatically create a top ``cybox.core.Observable`` and
append all ``cybox.core.Observable`` using ``observable_composition``
property when a ``list`` is given with length greater than 1.
Note:
The top level ``cybox.core.Observable`` will set the ``operator``
property for the ``cybox.core.ObservableComposition`` via the
``observable_composition_operator`` property. The value of
``operator`` can be changed via ``observable_composition_operator``
property. By default, the composition layer will be set to ``"OR"``.
Args:
value: A ``list`` of ``cybox.core.Observable`` instances or a single
``cybox.core.Observable`` instance.
Raises:
ValueError: If set to a value that cannot be converted to an
instance of ``cybox.core.Observable``.
"""
if not value:
return
if isinstance(value, Observable):
self.observable = value
elif utils.is_sequence(value):
if len(value) == 1:
self.add_observable(value[0])
return
observable_comp = ObservableComposition()
observable_comp.operator = self.observable_composition_operator
for element in value:
observable_comp.add(element)
self.observable = Observable()
self.observable.observable_composition = observable_comp
def generateDomainIPObservable(indicator, attribute):
indicator.add_indicator_type("Domain Watchlist")
compositeObject = ObservableComposition()
compositeObject.operator = "AND"
domain = attribute["value"].split('|')[0]
ip = attribute["value"].split('|')[1]
address_object = resolveIPType(ip, attribute["type"])
domain_object = DomainName()
domain_object.value = domain
compositeObject.add(address_object)
compositeObject.add(domain_object)
return compositeObject
def add_observable(self, observable):
"""Adds an observable to the ``observable`` property of the
:class:`Indicator`.
If the `observable` parameter is ``None``, no item will be added
to the ``observable`` property.
Note:
The STIX Language dictates that an :class:`Indicator` can have only
one ``Observable`` under it. Because of this, when a user adds
another ``Observable`` a new, empty ``Observable`` will be crated
and append the existing and new ``observable`` using the
``ObservableComposition`` property. To access the top level
``Observable`` can be achieved by the ``observable`` property .By
default, the ``operator`` of the composition layer will be set to
``"OR"``. The ``operator`` value can be changed via the
``observable_composition_operator`` property.
Setting ``observable`` or ``observables`` with re-initialize the
property and lose all ``Observable`` in the composition layer.
Args:
observable: An instance of ``cybox.core.Observable`` or an object
type that can be converted into one.
Raises:
ValueError: If the `observable` param cannot be converted into an
instance of ``cybox.core.Observable``.
"""
if not observable:
return
# Sets the first observable.
elif not self.observable:
self.observable = observable
# When another is inserted. A "root" Observable is created and the
# user's Observables are appended to the composition.
elif not self.observable.observable_composition:
observable_comp = ObservableComposition()
observable_comp.operator = self.observable_composition_operator
observable_comp.add(self.observable)
observable_comp.add(observable)
self.observable = Observable()
self.observable.observable_composition = observable_comp
# Keep appending to "root" Observable.
else:
self.observable.observable_composition.add(observable)
def test_observables_property_composition(self):
f1 = File()
f1.file_name = "README.txt"
f2 = File()
f2.file_name = "README2.txt"
obs1 = Observable(f1)
obs2 = Observable(f2)
comp = Observable(ObservableComposition('AND', [obs1, obs2]))
ind = Indicator()
ind.observable = comp
ind2 = Indicator.from_dict(ind.to_dict())
self.assertEqual([obs1.to_dict(), obs2.to_dict()],
[x.to_dict() for x in ind2.observables])
def return_attachment_composition(self, attribute):
file_object = File()
file_object.file_name = attribute.value
file_object.parent.id_ = "{}:FileObject-{}".format(self.namespace_prefix, attribute.uuid)
if 'data' in attribute:
observable_artifact = self.create_artifact_object(attribute, artifact="a")
observable_file = Observable(file_object)
observable_file.id_ = "{}:observable-file-{}".format(self.namespace_prefix, attribute.uuid)
observable = Observable()
composition = ObservableComposition(observables=[observable_artifact, observable_file])
observable.observable_composition = composition
else:
observable = Observable(file_object)
observable.id_ = "{}:observable-{}".format(self.namespace_prefix, attribute.uuid)
if attribute.comment:
observable.description = attribute.comment
return observable
def add_observable(self, observable):
"""Adds an observable to the ``observable`` property of the
:class:`Indicator`.
If the `observable` parameter is ``None``, no item will be added
to the ``observable`` property.
Note:
The STIX Language dictates that an :class:`Indicator` can have only
one ``Observable`` under it. Because of this, when a user adds
another ``Observable`` a new, empty ``Observable`` will be crated
and append the existing and new ``observable`` using the
``ObservableComposition`` property. To access the top level
``Observable`` can be achieved by the ``observable`` property .By
default, the ``operator`` of the composition layer will be set to
``"OR"``. The ``operator`` value can be changed via the
``observable_composition_operator`` property.
Setting ``observable`` or ``observables`` with re-initialize the
property and lose all ``Observable`` in the composition layer.
Args:
observable: An instance of ``cybox.core.Observable`` or an object
type that can be converted into one.
Raises:
ValueError: If the `observable` param cannot be converted into an
instance of ``cybox.core.Observable``.
"""
if not observable:
return
# Sets the first observable.
elif not self.observable:
self.observable = observable
# When another is inserted. A "root" Observable is created and the
# user's Observables are appended to the composition.
elif not self.observable.observable_composition:
observable_comp = ObservableComposition()
observable_comp.operator = self.observable_composition_operator
observable_comp.add(self.observable)
observable_comp.add(observable)
self.observable = Observable()
self.observable.observable_composition = observable_comp
# Keep appending to "root" Observable.
else:
self.observable.observable_composition.add(observable)
def test_observable_init(self):
# Can pass an Object into the Observable constructor
o = Object()
obs = Observable(o)
# Can pass an Event into the Observable constructor
e = Event()
obs = Observable(e)
# Can pass an ObservableComposition into the Observable constructor
oc = ObservableComposition()
obs = Observable(oc)
# Can pass an ObjectProperties subclass into the Observable constructor
a = Address()
obs = Observable(a)
# Cannot pass a String into the Observable constructor.
s = String()
self.assertRaises(TypeError, Observable, s)
def _indicator_to_observable(indicator):
"""Process indicator item(s), that can be nested, and create a composite object with observables.
Args:
indicator: Indicator(s) that will be translated
Returns:
A cybox.core.Observable object if `indicator` can be translated.
None is returned if `indicator` contains invalid or untranslatable items.
"""
items = openioc.get_items(indicator)
nested = openioc.get_indicators(indicator)
if not (nested or items):
return None
# If the openioc indicator has only one IndicatorItem, return an Observable
# object which has a single CybOX Object child.
if not nested and len(items) == 1:
return _translate_item(items[0])
# The openioc Indicator had more than one item or nested indicators, so
# we need to create an Observable Composition.
# Initialize the parent Observable
id_ = _translate_id(indicator.attrib.get("id"))
root = Observable(id_=id_)
operator = indicator.attrib.get("operator", "AND")
composite = ObservableComposition(operator=operator)
root.observable_composition = composite
# Translate all the IndicatorItem and nested Indicator children
observables = _translate_items(items) + _translate_indicators(nested)
# Add the translated Observable objects to the composite
composite.observables.extend(observables)
return root
def main():
# NOTE: ID values will differ due to being regenerated on each script execution
pkg = STIXPackage()
pkg.title = "Examples of Observable Composition"
# USE CASE: single obj with single condition
obs = File()
obs.file_name = "foo.exe"
obs.file_name.condition = "Contains"
pkg.add_observable(obs)
# USE CASE: single obj with multiple conditions
obs = File()
obs.file_name = "foo"
obs.file_name.condition = "Contains"
obs.size_in_bytes = '1896000'
obs.size_in_bytes.condition = "Equals"
pkg.add_observable(obs)
# USE CASE: multiple obj with individual conditions
obs = EmailMessage()
obs.subject = "Syria strategic plans leaked"
obs.subject.condition = "Equals"
file_obj = File()
file_obj.file_name = "bombISIS.pdf"
file_obj.file_name.condition = "Equals"
obs.add_related(file_obj, "Contains")
pkg.add_observable(obs)
# USE CASE: multiple objects with complex condition like (A OR B) AND C
# orcomp = either of a mutex or file are present
orcomp = ObservableComposition()
orcomp.operator = "OR"
obs = Mutex()
obs.name = 'foo'
obs.name.condition = "Contains"
orcomp.add(obs)
obs = File()
obs.file_name = "barfoobar"
obs.file_name.condition = "Equals"
orcomp.add(obs)
# andcomp = the above is true AND a network connection is present
andcomp = ObservableComposition()
andcomp.operator = "AND"
andcomp.add(orcomp)
obs = NetworkConnection()
sock = SocketAddress()
sock.ip_address = "46.123.99.25"
sock.ip_address.category = "ipv4-addr"
sock.ip_address.condition = "Equals"
obs.destination_socket_address = sock
andcomp.add(obs)
pkg.add_observable(andcomp)
# USE CASE: single object, one property with multiple values
obs = SocketAddress()
obs.ip_address = ['10.0.0.0', '10.0.0.1', '10.0.0.2'] # comma delimiter automagically added
obs.ip_address.condition = "Equals"
obs.ip_address.apply_condition = "ANY"
pkg.add_observable(obs)
print pkg.to_xml()
def publish_indicator(self, indicator_data, user):
indicator = DBIndicator(
id_=indicator_data['id'],
title=indicator_data.get('title'),
description=indicator_data.get('description'),
short_description=indicator_data.get('short_description'),
)
indicator.add_indicator_type(indicator_data.get('indicatorType'))
indicator.confidence = indicator_data.get('confidence', '')
indicator.id_ns = indicator_data.get('id_ns', '')
ident = EdgeIdentity(name=indicator_data.get('producer', ''))
indicator.producer = EdgeInformationSource(identity=ident)
indicator.handling = handling_from_draft('ind', indicator_data)
if 'test_mechanisms' in indicator_data:
for test_mechanism in indicator_data['test_mechanisms']:
indicator.test_mechanisms.append(
IndicatorBuilderPatch.test_mechanism_from_draft(
test_mechanism))
api_objects = {}
observable_composition = ObservableComposition(
operator=indicator_data.get('composition_type'))
# If this is an Update rather than a create,
# we only need to copy the id and ns to the composition
reedit_flag = 0
# New Observables via Build or Edit
for data in indicator_data['observables']:
observable_id = data.get('id')
if observable_id is None:
# No id for this observable, therefore it is new and must be created:
object_type = data['objectType']
object_type_info = self.observable_object_generator.get_object_type_information(
object_type)
observable_object = self.observable_object_generator.generate_observable_object_from_data(
object_type, data)
object_container = Object(observable_object)
object_container.id_ = self.generate_id(
object_type_info['id_prefix'])
# Create the observable, and store it in a collection so they can be saved to the database later on:
observable_id = self.generate_id('observable')
observable = Observable(title=data['title'],
description=data.get(
'description', ''),
item=object_container,
id_=observable_id)
observable.id_ns = indicator_data['id_ns']
api_objects[observable.id_] = ApiObject('obs', observable)
elif reedit_flag == 0 and self._is_re_edit_mode(data):
reedit_flag = 1
# Create a reference to the observable, and add it to the observable composition.
# The observable composition will be added to the indicator.
# Adding references to the composition saves duplication of data.
observable_reference = Observable(idref=observable_id,
idref_ns=indicator_data['id_ns'])
observable_composition.add(observable_reference)
# For some reason, the observable composition must be wrapped up in another observable.
# So to clarify, at this point, we have an observable that contains an observable composition. This composition
# contains a collection of references to the actual observables.
# We only want a new ID for the observable composition (obs) if it is first-time Build
# otherwise, get it from...?the database?
if reedit_flag:
user_action_log = logging.getLogger('user_actions')
user_action_log.info(
"%s updated STIX item %s (%s)" %
(user.username, indicator.id_, indicator.title))
# EOIndicator = self.edge_object_loader.load(indicator.id_) # Get the parent indicator
# find_ob_comp = lambda edges: [x.fetch() for x in edges if x.ty == 'obs'][0]
# # Find the observable composition among it's edges and return only the first hit.
# # The call to fetch() resolves the idref into an instance of the object itself
# existing_obs_comp = find_ob_comp(EOIndicator.edges)
# parent_observable = Observable(item=observable_composition, id_=existing_obs_comp.id_)
# else:
parent_observable = Observable(item=observable_composition,
id_=self.generate_id('observable'))
parent_observable.id_ns = indicator.id_ns
parent_observable.timestamp = datetime.utcnow(
) # needed for versioning Observable Composition
# ...and store this observable so we can actually write its contents to the database later...
api_objects[parent_observable.id_] = ApiObject('obs',
parent_observable)
# Add a reference of the 'parent' observable to the indicator
parent_observable_reference = Observable(idref=parent_observable.id_,
idref_ns=indicator.id_ns)
indicator.add_observable(parent_observable_reference)
indicator_api_object = ApiObject('ind', indicator)
# Process related items/correlations
[
relate.correlateIndtoTtp(indicator_api_object, item['idref'])
for item in indicator_data.get('indicated_ttps', [])
]
[
relate.correlateIndtoInd(indicator_api_object, item['idref'])
for item in indicator_data.get('related_indicators', [])
]
[
relate.correlateIndtoCoa(indicator_api_object, item['idref'])
for item in indicator_data.get('suggested_coas', [])
]
# Store the indicator so we can write it to the database later
api_objects[indicator_api_object.id_] = indicator_api_object
# Finally lets add everything to our Mongo collection...
tlp = (indicator_data.get('tlp') or 'NULL').upper()
esms = lines2list(indicator_data.get('markings', ""))
inbox_processor = InboxProcessorForBuilders(
user=user,
trustgroups=indicator_data.get('trustgroups', []),
)
for api_obj in api_objects.itervalues():
inbox_processor.add(
InboxItem(api_object=api_obj, etlp=tlp, esms=esms))
inbox_processor.run()
self.delete_draft(user, indicator_data['id'])
def main():
# NOTE: ID values will differ due to being regenerated on each script execution
pkg = STIXPackage()
pkg.title="Examples of Observable Composition"
# USE CASE: single obj with single condition
obs = File()
obs.file_name = "foo.exe"
obs.file_name.condition = "Contains"
pkg.add_observable(obs)
# USE CASE: single obj with multiple conditions
obs = File()
obs.file_name = "foo"
obs.file_name.condition = "Contains"
obs.size_in_bytes = '1896000'
obs.size_in_bytes.condition = "Equals"
pkg.add_observable(obs)
# USE CASE: multiple obj with individual conditions
obs = EmailMessage()
obs.subject = "Syria strategic plans leaked"
obs.subject.condition= "Equals"
file_obj = File()
file_obj.file_name = "bombISIS.pdf"
file_obj.file_name.condition = "Equals"
obs.add_related(file_obj, "Contains")
pkg.add_observable(obs)
# USE CASE: multiple objects with complex condition like (A OR B) AND C
# orcomp = either of a mutex or file are present
orcomp = ObservableComposition()
orcomp.operator = "OR"
obs = Mutex()
obs.name = 'foo'
obs.name.condition= "Contains"
orcomp.add(obs)
obs = File()
obs.file_name = "barfoobar"
obs.file_name.condition = "Equals"
orcomp.add(obs)
# andcomp = the above is true AND a network connection is present
andcomp = ObservableComposition()
andcomp.operator = "AND"
andcomp.add(orcomp)
obs = NetworkConnection()
sock = SocketAddress()
sock.ip_address = "46.123.99.25"
sock.ip_address.category = "ipv4-addr"
sock.ip_address.condition = "Equals"
obs.destination_socket_address = sock
andcomp.add (obs)
pkg.add_observable(andcomp)
# USE CASE: single object, one property with multiple values
obs = SocketAddress()
obs.ip_address = ['10.0.0.0','10.0.0.1','10.0.0.2'] # comma delimiter automagically added
obs.ip_address.condition = "Equals"
obs.ip_address.apply_condition = "ANY"
pkg.add_observable(obs)
print pkg.to_xml()
# to add logic: # normally you'd probably have logic for all items, but this is just a demo, not reality oproc_ref = Observable() oproc_ref.id_ = None oproc_ref.idref = obs1.id_ ofile_ref = Observable() ofile_ref.id_ = None ofile_ref.idref = obs2.id_ omutex_ref = Observable() omutex_ref.id_ = None omutex_ref.idref = obs3.id_ o_comp = Observable(ObservableComposition(operator = "OR")) o_comp.observable_composition.add(oproc_ref) o_comp.observable_composition.add(ofile_ref) o_comp2 = Observable(ObservableComposition(operator = "AND")) o_comp2.observable_composition.add(omutex_ref) o_comp.observable_composition.add(o_comp2) # add our composition to the observables: observables_doc.add(o_comp) # output to stdout or file or whatever: outfd.write(observables_doc.to_xml())