def test_clean(self):
root = utils.get_etree_root(self.xml)
results = self.UPDATER().clean(root)
doc = results.document.as_element()
disallowed = self.DISALLOWED_KLASS.find(doc)
self.assertEqual(len(disallowed), 0)
def test_clean(self):
root = utils.get_etree_root(self.xml)
results = self.UPDATER().clean(root)
doc = results.document.as_element()
disallowed = self.DISALLOWED_KLASS.find(doc)
self.assertEqual(len(disallowed), 0)
def check_update(self, root, options=None):
"""Determines if the input document can be upgraded.
Args:
root: The XML document. This can be a filename, a file-like object,
an instance of ``etree._Element`` or an instance of
``etree._ElementTree``.
options (optional): A ``ramrod.UpdateOptions`` instance. If
``None``, ``ramrod.DEFAULT_UPDATE_OPTIONS`` will be used.
Raises:
.UnknownVersionError: If the input document does not have a
version.
.InvalidVersionError: If the version of the input document
does not match the `VERSION` class-level attribute value.
.UpdateError: If the input document contains fields which
cannot be updated or constructs with non-unique IDs are discovered.
"""
root = utils.get_etree_root(root)
options = options or DEFAULT_UPDATE_OPTIONS
if options.check_versions:
self._check_version(root)
self._cybox_updater._check_version(root) # noqa
disallowed = self._get_disallowed(root)
if not disallowed:
return
raise errors.UpdateError(
message="Found untranslatable fields in source document.",
disallowed=disallowed)
def check_update(self, root, options=None):
"""Determines if the input document can be upgraded.
Args:
root: The XML document. This can be a filename, a file-like object,
an instance of ``etree._Element`` or an instance of
``etree._ElementTree``.
options (optional): A ``ramrod.UpdateOptions`` instance. If
``None``, ``ramrod.DEFAULT_UPDATE_OPTIONS`` will be used.
Raises:
.UnknownVersionError: If the input document does not have a
version.
.InvalidVersionError: If the version of the input document
does not match the `VERSION` class-level attribute value.
.UpdateError: If the input document contains fields which
cannot be updated or constructs with non-unique IDs are discovered.
"""
root = utils.get_etree_root(root)
options = options or DEFAULT_UPDATE_OPTIONS
if options.check_versions:
self._check_version(root)
self._cybox_updater._check_version(root) # noqa
duplicates = self._get_duplicates(root)
disallowed = self._get_disallowed(root)
if not (disallowed or duplicates):
return
error = "Found duplicate or untranslatable fields in source document."
raise errors.UpdateError(message=error, disallowed=disallowed, duplicates=duplicates)
def update(doc, from_=None, to_=None, options=None, force=False):
"""Updates a CybOX document to align with a given version of the CybOX
Language.
Args:
doc: A CybOX document filename, file-like object, ``etree._Element``, or
``etree._ElementTree``.
from_ (optional, string): The base version for the update process. If
``None``, an attempt will be made to extract the version number
from `doc`.
to_ (optional, string): The version to update to. If ``None``, the
latest version of CybOX is assumed.
options (optional): A :class:`ramrod.UpdateOptions` instance. If
``None``, ``ramrod.DEFAULT_UPDATE_OPTIONS`` will be used.
force (boolean): Forces the update process. This may result in content
being removed during the update process and could result in
schema-invalid content. **Use at your own risk!**
Returns:
An instance of ``ramrod.UpdateResults``.
Raises:
.UpdateError: If any of the following conditions are encountered:
* The `from_` or `to_` versions are invalid.
* An untranslatable field is encountered and `force` is ``False``.
* A non-unique ID is encountered and `force` is ``False``.
.InvalidVersionError: If the source document version and the
`from_` value do not match and `force` is ``False``.
.UnknownVersionError: If the source document does not contain
version information and `force` is ``False``.
"""
root = utils.get_etree_root(doc)
versions = common.CYBOX_VERSIONS
from_ = from_ or BaseCyboxUpdater.get_version(root)
to_ = to_ or versions[-1] # The latest version if not specified
utils.validate_versions(from_, to_, versions)
removed, remapped = [], {}
idx = versions.index
for version in versions[idx(from_):idx(to_)]:
updater = CYBOX_UPDATERS[version]
result = updater().update(root, options=options, force=force)
root = result.document.as_element()
# Update record of removed and remapped fields
removed.extend(result.removed)
remapped.update(result.remapped_ids)
result = results.UpdateResults(
document=root,
removed=removed,
remapped_ids=remapped
)
return result
def test_lists(self):
nsmap = {'AddressObj': 'http://cybox.mitre.org/objects#AddressObject-2'}
updater = UPDATER()
root = utils.get_etree_root(self._xml)
updater._update_lists(root)
address_value = root.xpath('.//AddressObj:Address_Value', namespaces=nsmap)[0].text
self.assertEqual(address_value, self.NEW_DELIMITER.join(self.ATTACKERS))
def test_commas(self):
nsmap = {'EmailMessageObj': 'http://cybox.mitre.org/objects#EmailMessageObject-2'}
updater = UPDATER()
root = utils.get_etree_root(self._xml)
updater._update_lists(root)
subject = root.xpath('.//EmailMessageObj:Subject', namespaces=nsmap)[0].text
self.assertEqual(subject, self.UNESCAPED)
def test_attrib_value(self):
root = utils.get_etree_root(self.xml)
self.TRANS_KLASS.translate(root)
updated_nodes = root.xpath(self.TRANS_XPATH,
namespaces=self.UPDATER.NSMAP)
for node in updated_nodes:
self.assertEqual(node.attrib['datatype'], 'string')
def test_attrib_value(self):
root = utils.get_etree_root(self.xml)
self.TRANS_KLASS.translate(root)
updated_nodes = root.xpath(self.TRANS_XPATH,
namespaces=self.UPDATER.NSMAP)
for node in updated_nodes:
self.assertEqual(node.attrib['datatype'], 'string')
def update(doc, from_=None, to_=None, options=None, force=False):
"""Updates a CybOX document to align with a given version of the CybOX
Language.
Args:
doc: A CybOX document filename, file-like object, ``etree._Element``, or
``etree._ElementTree``.
from_ (optional, string): The base version for the update process. If
``None``, an attempt will be made to extract the version number
from `doc`.
to_ (optional, string): The version to update to. If ``None``, the
latest version of CybOX is assumed.
options (optional): A :class:`ramrod.UpdateOptions` instance. If
``None``, ``ramrod.DEFAULT_UPDATE_OPTIONS`` will be used.
force (boolean): Forces the update process. This may result in content
being removed during the update process and could result in
schema-invalid content. **Use at your own risk!**
Returns:
An instance of ``ramrod.UpdateResults``.
Raises:
.UpdateError: If any of the following conditions are encountered:
* The `from_` or `to_` versions are invalid.
* An untranslatable field is encountered and `force` is ``False``.
* A non-unique ID is encountered and `force` is ``False``.
.InvalidVersionError: If the source document version and the
`from_` value do not match and `force` is ``False``.
.UnknownVersionError: If the source document does not contain
version information and `force` is ``False``.
"""
root = utils.get_etree_root(doc)
versions = common.CYBOX_VERSIONS
from_ = from_ or BaseCyboxUpdater.get_version(root)
to_ = to_ or versions[-1] # The latest version if not specified
utils.validate_versions(from_, to_, versions)
removed, remapped = [], {}
idx = versions.index
for version in versions[idx(from_):idx(to_)]:
updater = CYBOX_UPDATERS[version]
result = updater().update(root, options=options, force=force)
root = result.document.as_element()
# Update record of removed and remapped fields
removed.extend(result.removed)
remapped.update(result.remapped_ids)
result = results.UpdateResults(document=root,
removed=removed,
remapped_ids=remapped)
return result
def test_trans(self):
root = utils.get_etree_root(self.xml)
self.TRANS_KLASS.translate(root)
updated_nodes = root.xpath(self.TRANS_XPATH,
namespaces=self.UPDATER.NSMAP)
for node in updated_nodes:
if self.TRANS_VALUE:
self.assertEqual(node.text, self.TRANS_VALUE)
else:
self.assertTrue(node != None)
def test_trans(self):
root = utils.get_etree_root(self.xml)
self.TRANS_KLASS.translate(root)
updated_nodes = root.xpath(self.TRANS_XPATH,
namespaces=self.UPDATER.NSMAP)
for node in updated_nodes:
if self.TRANS_VALUE:
self.assertEqual(node.text, self.TRANS_VALUE)
else:
self.assertTrue(node != None)
def test_lists(self):
nsmap = {
'AddressObj': 'http://cybox.mitre.org/objects#AddressObject-2'
}
updater = UPDATER()
root = utils.get_etree_root(self._xml)
updater._update_lists(root)
address_value = root.xpath('.//AddressObj:Address_Value',
namespaces=nsmap)[0].text
self.assertEqual(address_value,
self.NEW_DELIMITER.join(self.ATTACKERS))
def test_commas(self):
nsmap = {
'EmailMessageObj':
'http://cybox.mitre.org/objects#EmailMessageObject-2'
}
updater = UPDATER()
root = utils.get_etree_root(self._xml)
updater._update_lists(root)
subject = root.xpath('.//EmailMessageObj:Subject',
namespaces=nsmap)[0].text
self.assertEqual(subject, self.UNESCAPED)
def check_update(self, root, options=None):
"""Determines if the input document can be upgraded.
Args:
root: The XML document. This can be a filename, a file-like object,
an instance of ``etree._Element`` or an instance of
``etree._ElementTree``.
options (optional): A ``ramrod.UpdateOptions`` instance. If
``None``, ``ramrod.DEFAULT_UPDATE_OPTIONS`` will be used.
Raises:
.UnknownVersionError: If the input document does not have a
version.
.InvalidVersionError: If the version of the input document
does not match the `VERSION` class-level attribute value.
"""
root = utils.get_etree_root(root)
options = options or DEFAULT_UPDATE_OPTIONS
if options.check_versions:
self._check_version(root)
def check_update(self, root, options=None):
"""Determines if the input document can be upgraded.
Args:
root: The XML document. This can be a filename, a file-like object,
an instance of ``etree._Element`` or an instance of
``etree._ElementTree``.
options (optional): A ``ramrod.UpdateOptions`` instance. If
``None``, ``ramrod.DEFAULT_UPDATE_OPTIONS`` will be used.
Raises:
.UnknownVersionError: If the input document does not have a
version.
.InvalidVersionError: If the version of the input document
does not match the `VERSION` class-level attribute value.
"""
root = utils.get_etree_root(root)
options = options or DEFAULT_UPDATE_OPTIONS
if options.check_versions:
self._check_version(root)
def clean(self, root, options=None):
"""Removes disallowed elements from `root` and remaps non-unique
IDs to unique IDs for the sake of schema-validation.
Removed items can be retrieved via the ``removed`` attribute on the
return value:
>>> results = updater.clean(root)
>>> print results.removed
(<Element at 0xffdcf234>, <Element at 0xffdcf284>)
Items which have been reassigned IDs can be retrieved via the
``remappped_ids`` attribute on the return value:
>>> results = updater.clean(root)
>>> print results.remapped_ids
{'example:Observable-duplicate': [<Element {http://cybox.mitre.org/cybox-2}Observable at 0xffd67e64>, <Element {http://cybox.mitre.org/cybox-2}Observable at 0xffd67f2c>, <Element {http://cybox.mitre.org/cybox-2}Observable at 0xffd67f54>, <Element {http://cybox.mitre.org/cybox-2}Observable at 0xffd67f7c>, <Element {http://cybox.mitre.org/cybox-2}Observable at 0xffd67fa4>]}
Note:
This does not remap ``idref`` attributes to new ID values because
it is impossible to determine which entity the ``idref`` was
pointing to.
Args:
root: The XML document. This can be a filename, a file-like object,
an instance of ``etree._Element`` or an instance of
``etree._ElementTree``.
options (optional): A :class:`ramrod.UpdateOptions` instance. If
``None``, ``ramrod.DEFAULT_UPDATE_OPTIONS`` will be used.
Returns:
An instance of ``ramrod.UpdateResults``.
"""
root = utils.get_etree_root(root, make_copy=True)
results = self._clean(root, options)
return results
def test_vocab_update(self):
root = utils.get_etree_root(self.xml)
self.VOCAB_KLASS.update(root)
vocabs = self.VOCAB_KLASS.find(root)
self.assertEqual(len(vocabs), 0)
def test_get_version(self):
root = utils.get_etree_root(self._versions)
version = UPDATER.get_version(root)
self.assertEqual(version, UPDATER.VERSION)
def test_get_version(self):
root = utils.get_etree_root(self._versions)
version = UPDATER.get_version(root)
self.assertEqual(version, UPDATER.VERSION)
def get_version(doc):
"""Returns the version number for input CybOX document."""
root = utils.get_etree_root(doc)
return BaseCyboxUpdater.get_version(root)
def test_optional_find(self):
root = utils.get_etree_root(self.xml)
optionals = self.OPTIONAL_KLASS.find(root)
self.assertEqual(len(optionals), self.OPTIONAL_COUNT)
def test_disallowed_find(self):
root = utils.get_etree_root(self.xml)
disallowed = self.DISALLOWED_KLASS.find(root)
self.assertEqual(len(disallowed), self.DISALLOWED_COUNT)
def test_optional_removal(self):
root = utils.get_etree_root(self.xml)
updater = self.UPDATER()
updater._update_optionals(root)
optionals = self.OPTIONAL_KLASS.find(root)
self.assertEqual(len(optionals), 0)
def test_trans_find(self):
root = utils.get_etree_root(self.xml)
to_trans = self.TRANS_KLASS._find(root)
self.assertEqual(len(to_trans), self.TRANS_COUNT)
def test_optional_removal(self):
root = utils.get_etree_root(self.xml)
updater = self.UPDATER()
updater._update_optionals(root)
optionals = self.OPTIONAL_KLASS.find(root)
self.assertEqual(len(optionals), 0)
def test_trans_find(self):
root = utils.get_etree_root(self.xml)
to_trans = self.TRANS_KLASS._find(root)
self.assertEqual(len(to_trans), self.TRANS_COUNT)
def test_vocab_update(self):
root = utils.get_etree_root(self.xml)
self.VOCAB_KLASS.update(root)
vocabs = self.VOCAB_KLASS.find(root)
self.assertEqual(len(vocabs), 0)
def test_disallowed_find(self):
root = utils.get_etree_root(self.xml)
disallowed = self.DISALLOWED_KLASS.find(root)
self.assertEqual(len(disallowed), self.DISALLOWED_COUNT)
def test_find(self):
root = utils.get_etree_root(self.xml)
vocabs = self.VOCAB_KLASS.find(root)
self.assertEqual(len(vocabs), self.VOCAB_COUNT)
def test_find(self):
root = utils.get_etree_root(self.xml)
vocabs = self.VOCAB_KLASS.find(root)
self.assertEqual(len(vocabs), self.VOCAB_COUNT)
def update(doc, from_=None, to_=None, options=None, force=False):
"""Updates an input STIX or CybOX document to align with a newer version
of the STIX/CybOX schemas.
This will perform the following updates:
* Update namespaces
* Update schemalocations
* Update construct versions (``STIX_Package``, ``Observables``, etc.)
* Update controlled vocabularies and fix typos
* Translate structures to new XSD datatype instances where possible.
* Remove empty instances of attributes and elements which were required
in one version of the language and declared optional in another.
Args:
doc: A STIX or CybOX document filename, file-like object,
``etree._Element`` or ``etree._ElementTree`` object instance.
to_ (optional, string): The expected output version of the update
process. If not specified, the latest language version will be
assumed.
from_ (optional, string): The version to update from. If not specified,
the `from_` version will be retrieved from the input document.
options (optional): A :class:`ramrod.UpdateOptions` instance. If
``None``, ``ramrod.DEFAULT_UPDATE_OPTIONS`` will be used.
force (boolean): Attempt to force the update process if the document
contains untranslatable fields.
Returns:
An instance of ``ramrod.UpdateResults``
Raises:
ramrod.UpdateError: If any of the following occur:
* The input `doc` does not contain a ``STIX_Package``
or ``Observables`` root-level node.
* If`force` is ``False`` and an untranslatable field or
non-unique ID is found in the input `doc`.
ramrod.InvalidVersionError: If the input document contains a version
attribute that is incompatible with a STIX/CybOX Updater class
instance.
ramrod.UnknownVersionError: If `from_` was not specified and the input
document does not contain a version attribute.
"""
import ramrod.stix as stix
import ramrod.cybox as cybox
root = utils.get_etree_root(doc)
name = QName(root).localname
update_methods = {
'STIX_Package': stix.update,
'Observables': cybox.update,
}
try:
options = options or DEFAULT_UPDATE_OPTIONS
from_ = from_ or _get_version(root) # _get_version raises KeyError
func = update_methods[name]
except KeyError:
error = (
"Document root node must be one of %s. Found: '%s'" %
(update_methods.keys(), name)
)
raise UpdateError(error)
updated = func(root, from_, to_, options, force)
return updated
def test_optional_find(self):
root = utils.get_etree_root(self.xml)
optionals = self.OPTIONAL_KLASS.find(root)
self.assertEqual(len(optionals), self.OPTIONAL_COUNT)
def get_version(doc):
"""Returns the version number for input CybOX document."""
root = utils.get_etree_root(doc)
return BaseCyboxUpdater.get_version(root)
def update(self, root, options=None, force=False):
"""Attempts to update `root` to the next version of its language
specification.
If `force` is set to True, items may be removed during the
translation process and IDs may be reassigned if they are not
unique within the document.
Note:
This does not remap ``idref`` attributes to new ID values because
it is impossible to determine which entity the ``idref`` was
pointing to.
Removed items can be retrieved via the ``removed`` attribute on the
return value:
>>> results = updater.update(root, force=True)
>>> print results.removed
(<Element at 0xffdcf234>, <Element at 0xffdcf284>)
Items which have been reassigned IDs can be retrieved via the
``remappped_ids`` attribute on the return value:
>>> results = updater.update(root, force=True)
>>> print results.remapped_ids
{'example:Observable-duplicate-id-1': [<Element {http://cybox.mitre.org/cybox-2}Observable at 0xffd67e64>, <Element {http://cybox.mitre.org/cybox-2}Observable at 0xffd67f2c>, <Element {http://cybox.mitre.org/cybox-2}Observable at 0xffd67f54>, <Element {http://cybox.mitre.org/cybox-2}Observable at 0xffd67f7c>, <Element {http://cybox.mitre.org/cybox-2}Observable at 0xffd67fa4>]}
Args:
root: The XML document. This can be a filename, a file-like object,
an instance of ``etree._Element`` or an instance of
``etree._ElementTree``.
options: A :class:`ramrod.UpdateOptions` instance. If ``None``,
``ramrod.DEFAULT_UPDATE_OPTIONS`` will be used.
force: Forces the update process to complete by potentially
removing untranslatable xml nodes and/or remapping non-unique
IDs. This may result in non-schema=conformant XML. **USE AT
YOUR OWN RISK!**
Returns:
An instance of ``ramrod.UpdateResults``.
Raises:
ramrod.UpdateError: If untranslatable fields or non-unique IDs are
discovered in `root` and `force` is ``False``.
ramrod.UnknownVersionError: If the `root` node contains no version
information.
ramrod.InvalidVersionError: If the `root` node contains invalid
version information (e.g., the class expects v1.0 content and
the `root` node contains v1.1 content).
"""
root = utils.get_etree_root(root, make_copy=True)
options = options or DEFAULT_UPDATE_OPTIONS
try:
self.check_update(root, options)
updated = self._update(root, options)
results = self._create_update_results(updated)
except (UpdateError, UnknownVersionError, InvalidVersionError):
if force:
results = self._force_update(root, options)
else:
raise
return results
