def main():
parser = _get_arg_parser()
args = parser.parse_args()
try:
# Validate the input commandline arguments
_validate_args(args)
# Build UpdateOptions from commandline arguments
options = _get_options(args)
# Run the update process.
updated = ramrod.update(
args.infile,
from_=args.from_,
to_=args.to_,
options=options,
force=args.force
)
# Write results
_write_xml(updated.document, args.outfile)
_write_removed(updated.removed)
_write_remapped_ids(updated.remapped_ids)
except errors.UpdateError as ex:
_print_update_error(ex)
sys.exit(EXIT_FAILURE)
except errors.InvalidVersionError as ex:
_print_invalid_version_error(ex)
sys.exit(EXIT_FAILURE)
except errors.UnknownVersionError as ex:
_print_unknown_version_error(str(ex))
sys.exit(EXIT_FAILURE)
def main():
parser = _get_arg_parser()
args = parser.parse_args()
try:
options = _get_options(args)
updated = ramrod.update(args.infile,
from_=args.from_,
to_=args.to_,
options=options,
force=args.force)
_write_xml(updated.document, args.outfile)
_write_removed(updated.removed)
_write_remapped_ids(updated.remapped_ids)
except ramrod.UpdateError as ex:
_print_update_error(ex)
sys.exit(EXIT_FAILURE)
except ramrod.InvalidVersionError as ex:
_print_invalid_version_error(ex)
sys.exit(EXIT_FAILURE)
except ramrod.UnknownVersionError as ex:
_print_unknown_version_error(str(ex))
sys.exit(EXIT_FAILURE)
def main():
parser = _get_arg_parser()
args = parser.parse_args()
try:
# Validate the input commandline arguments
_validate_args(args)
# Build UpdateOptions from commandline arguments
options = _get_options(args)
# Run the update process.
updated = ramrod.update(args.infile,
from_=args.from_,
to_=args.to_,
options=options,
force=args.force)
# Write results
_write_xml(updated.document, args.outfile)
_write_removed(updated.removed)
_write_remapped_ids(updated.remapped_ids)
except errors.UpdateError as ex:
_print_update_error(ex)
sys.exit(EXIT_FAILURE)
except errors.InvalidVersionError as ex:
_print_invalid_version_error(ex)
sys.exit(EXIT_FAILURE)
except errors.UnknownVersionError as ex:
_print_unknown_version_error(str(ex))
sys.exit(EXIT_FAILURE)
def file_to_stix(file_):
'''transform files into stix packages'''
try:
stix_package = STIXPackage.from_xml(file_)
except UnsupportedVersionError as ex:
updated = ramrod.update(file_)
updated_xml = updated.document.as_stringio()
stix_package = STIXPackage.from_xml(updated_xml)
return stix_package
def file_to_stix(file_):
'''transform files into stix packages'''
try:
stix_package = STIXPackage.from_xml(file_)
except UnsupportedVersionError as ex:
updated = ramrod.update(file_)
updated_xml = updated.document.as_stringio()
stix_package = STIXPackage.from_xml(updated_xml)
return stix_package
def test_update_version(self):
valid_versions = ramrod.cybox.CYBOX_VERSIONS
idx = valid_versions.index
version_to = valid_versions[idx(UPDATER.VERSION)+1:]
for version in version_to:
updated = ramrod.update(self._versions, to_=version)
updated_root = updated.document.as_element()
updated_version = UPDATER.get_version(updated_root)
self.assertEqual(version, updated_version)
def test_update_version(self):
valid_versions = ramrod.stix.STIX_VERSIONS
idx = valid_versions.index
version_to = valid_versions[idx(UPDATER.VERSION)+1:]
for version in version_to:
updated = ramrod.update(self._versions, to_=version)
updated_root = updated.document.as_element()
updated_version = UPDATER.get_version(updated_root)
self.assertEqual(version, updated_version)
def __init__(self, source_item):
self.source_item = source_item
try:
self.stix_package = STIXPackage.from_xml(self.io())
except UnsupportedVersionError:
updated = ramrod.update(self.io(), to_='1.1.1')
document = updated.document.as_stringio()
self.stix_package = STIXPackage.from_xml(document)
except Exception:
logging.error('error parsing STIX package (%s)', self.file_name())
self.stix_package = None
def taxii_content_block_to_stix(content_block):
'''transform taxii content blocks into stix packages'''
xml = StringIO(content_block.content)
try:
stix_package = STIXPackage.from_xml(xml)
except UnsupportedVersionError as ex:
updated = ramrod.update(xml)
updated_xml = updated.document.as_stringio()
stix_package = STIXPackage.from_xml(updated_xml)
xml.close()
return stix_package
def taxii_content_block_to_stix(content_block):
'''transform taxii content blocks into stix packages'''
xml = StringIO(content_block.content)
try:
stix_package = STIXPackage.from_xml(xml)
except UnsupportedVersionError as ex:
updated = ramrod.update(xml)
updated_xml = updated.document.as_stringio()
stix_package = STIXPackage.from_xml(updated_xml)
xml.close()
return stix_package
def __init__(self, source_item):
self.source_item = source_item
with warnings.catch_warnings():
warnings.filterwarnings("ignore","The use of this field has been deprecated",UserWarning)
try:
self.stix_package = STIXPackage.from_xml(self.io())
except UnsupportedVersionError:
updated = ramrod.update(self.io(), to_=LATEST_STIX_VERSION)
document = updated.document.as_stringio()
self.stix_package = STIXPackage.from_xml(document)
except Exception:
logging.error('error parsing STIX package (%s)', self.file_name())
self.stix_package = None
self.stix_version = self.stix_package.version
def load_stix_package(self, stix_file):
"""Helper for loading and updating (if required) a STIX package."""
try:
package = STIXPackage.from_xml(stix_file)
except UnsupportedVersionError:
updated = ramrod.update(stix_file, to_='1.1.1')
document = updated.document.as_stringio()
try:
package = STIXPackage.from_xml(document)
except Exception:
package = None
except Exception:
package = None
return package
def __check_info_source(self, datafile):
"""
Parses the STIX Header element looking for an Information source element matching the configured value.
Currently only matches to STIX Header, long-term could check every Information Source element.
:param datafile: The file sent to be parsed, expected to be STIX XML format.
:return Boolean: True if match is found, otherwise False.
"""
ret = False
try:
# TODO: Parse only header or look for all Information_Source elements?
# attempt to map as a STIX Package, if fails force an update with ramrod
try:
stix_package = STIXPackage.from_xml(datafile)
except Exception as e:
self._logger.warning("Unable to parse XML to STIX, attempting version change Exception={0}".format(e))
updated = ramrod.update(datafile, force=True)
stix_package = STIXPackage.from_xml(updated)
# Compare the information source from the stix_header
info_src = stix_package.stix_header.information_source
if info_src.identity:
if info_src.identity.name in self._sources:
return True
# If there are contributing sources, parse and compare to source list
if info_src.contributing_sources:
if info_src.contributing_sources._inner_name == 'sources':
for x in info_src.contributing_sources._inner:
if type(x) is information_source.InformationSource:
if x.identity.name in self._sources:
return True
except Exception as e:
self._logger.warning(
"Unable to force XML to STIX with ramrod. Unable to check source Exception={0}".format(e))
return ret
if ret is False:
self._logger.info("Did not identify the requested source in STIX Header.")
return ret
def parse_stix_package(self, data):
'''Parse an XML string representing a STIX package object.
If the content cannot be parsed, attempt to update the content to the
current STIX version.
Params:
data - an XML string
Returns:
A STIXPackage object.
'''
try:
return STIXPackage.from_xml(StringIO.StringIO(data))
except (parser.UnknownVersionError, parser.UnsupportedVersionError,
parser.UnsupportedRootElementError):
try:
# If we arrived here, we did not find a valid STIX version in this
# document, meaning it is likely a lower STIX version. Try to
# up-convert to latest version of STIX.
updated = ramrod.update(data)
return STIXPackage.from_xml(updated.document.as_stringio())
except (ramrod.errors.UpdateError,
ramrod.errors.InvalidVersionError,
ramrod.errors.UnknownVersionError):
self._logger.exception(
'msg="UpdateError when parsing a STIX document" filename="%s"',
self.filename)
except lxml.etree.XMLSyntaxError:
# Exceptions are not re-raised here because multiple STIXPackage
# objects can result from a single TAXII PollResponse.
# Just log the error in this case as the likely case
# is that a single STIXPackage is in error, usually
# due to a missing CDATA tag.
self._logger.exception(
'msg="XMLSyntaxError when parsing a STIX document" filename="%s"',
self.filename)
def parse_stix(self, reference='', make_event=False, source=''):
"""
Parse the document.
:param reference: The reference to the data.
:type reference: str
:param make_event: Whether or not to create an Event for this document.
:type make_event: bool
:param source: The source of this document.
:type source: str
:raises: :class:`taxii_service.parsers.STIXParserException`
Until we have a way to map source strings in a STIX document to
a source in CRITs, we are being safe and using the source provided
as the true source.
"""
with closing(StringIO(self.data)) as f:
try:
try:
self.package = STIXPackage.from_xml(f)
if not self.package:
raise STIXParserException("STIX package failure")
except UnsupportedVersionError:
v = stix.__version__
v = v[0:-2] if len(v.split('.')) > 3 else v
updated = ramrod.update(f, to_=v)
doc = updated.document.as_stringio()
self.package = STIXPackage.from_xml(doc)
except Exception as e:
msg = "Failed to create STIX/CybOX from XML"
self.failed.append((e.message, "STIX Package (%s)" % msg,
'')) # note for display in UI
return
if not self.preview:
self.stix_version = self.package.version
stix_header = self.package.stix_header
if stix_header and stix_header.information_source and stix_header.information_source.identity:
self.information_source = stix_header.information_source.identity.name
if self.information_source:
info_src = "STIX Source: %s" % self.information_source
if not reference:
reference = ''
else:
reference += ", "
reference += info_src
if source:
if does_source_exist(source):
self.source.name = source
else:
raise STIXParserException(
'Source "%s" does not exist in CRITs.' % source)
elif does_source_exist(self.information_source):
self.source.name = self.information_source
else:
raise STIXParserException("No source to attribute data to.")
self.source_instance.reference = reference
self.source.instances.append(self.source_instance)
if make_event:
title = "STIX Document %s" % self.package.id_
event_type = EventTypes.INTEL_SHARING
date = datetime.datetime.now()
description = str(date)
if self.package.incidents:
incdnt = self.package.incidents[0]
title = incdnt.title
if incdnt.description:
description = incdnt.description
if isinstance(description, StructuredText):
try:
description = description.to_dict()
except:
pass
if incdnt.short_description in EventTypes.values():
event_type = incdnt.short_description
elif incdnt.categories and incdnt.categories[0].value:
event_type = get_crits_event_type(
incdnt.categories[0].value)
else: #package contains no incidents
header = self.package.stix_header
if isinstance(header, STIXHeader):
if header.title:
title = header.title
if header.package_intents:
try:
stix_type = str(header.package_intents[0])
event_type = get_crits_event_type(stix_type)
except:
pass
if header.description:
description = header.description
if isinstance(description, StructuredText):
try:
description = description.to_dict()
except:
pass
if self.preview:
self.imported[self.package.id_] = ('Event', None, title)
else:
res = add_new_event(title, description, event_type,
self.source.name,
self.source_instance.method,
self.source_instance.reference, date,
self.source_instance.analyst)
self.parsed.append(self.package.id_)
if res['success']:
self.event = res['object']
self.imported[self.package.id_] = ('Event',
res['object'].id, title
or res['object'].id)
self.updates[res['object'].id] = res['object']
# Get relationships to the Event
if self.package.incidents:
incdnts = self.package.incidents
for rel in getattr(incdnts[0], 'related_indicators',
()):
if rel.relationship or rel.confidence:
r = rel.relationship.value or RelationshipTypes.RELATED_TO
c = getattr(rel.confidence.value, 'value',
'Unknown')
self.event_rels[rel.item.idref] = (r, c)
else:
self.failed.append((res['message'], "Event (%s)" % title,
self.package.id_))
if self.package.indicators:
self.parse_indicators(self.package.indicators)
if self.package.observables and self.package.observables.observables:
self.parse_observables(self.package.observables.observables)
if self.package.threat_actors:
self.parse_threat_actors(self.package.threat_actors)
def Read(self, stixfile, config, xmlparser=None):
'''
Parse STIX XML document. Return a dictionary object with the data from the document.
'''
# FIXME: Handle composite indicators and related indicators hierarchically
# FIXME: Handle STIX indicators with multiple possible values where the apply_condition is ANY
# Upgrade old versions of STIX documents to the latest supported release (currently 1.1.1)
try:
stix_package = STIXPackage.from_xml(stixfile)
except UnsupportedVersionError:
self.logging.warning("Updating stix document to version 1.1.1")
if not isinstance(stixfile, str):
stixfile.close()
updated = ramrod.update(stixfile.name, force=True)
else:
updated = ramrod.update(stixfile, force=True)
document = updated.document.as_stringio()
stix_package = STIXPackage.from_xml(document)
stix_dict = stix_package.to_dict() # parse to dictionary
ParsedData = {}
if 'stix_header' in stix_dict:
ParsedData['DocumentHeaderData'] = copy.deepcopy(
stix_dict['stix_header'])
else:
ParsedData['DocumentHeaderData'] = {}
if 'profiles' in stix_dict['stix_header'] and stix_dict['stix_header'][
'profiles']:
ParsedData['DocumentHeaderData']['profiles'] = []
for row in stix_dict['stix_header']['profiles']:
ParsedData['DocumentHeaderData']['profiles'].append(
dict(value=row))
if 'id' in stix_dict:
ParsedData['DocumentHeaderData']['id'] = stix_dict['id']
if 'version' in stix_dict:
ParsedData['DocumentHeaderData']['version'] = stix_dict['version']
if 'observables' in stix_dict:
# Rename observables as indicators and build basic indicator dictionary, resolves issue with STIX documents from Soltra with no Indicators.
observables = stix_dict.pop('observables')
if 'indicators' not in stix_dict:
stix_dict['indicators'] = []
if 'observables' in observables:
for row in observables['observables']:
newindicator = {}
if 'description' in row:
newindicator['description'] = row.pop('description')
if 'title' in row:
newindicator['title'] = row.pop('title')
newindicator['observable'] = row
newindicator['indicator_types'] = []
indicator_type = {}
indicator_type['value'] = 'Unknown'
indicator_type[
'xsi:type'] = 'stixVocabs:IndicatorTypeVocab-1.1'
newindicator['indicator_types'].append(indicator_type)
stix_dict['indicators'].append(newindicator)
if 'indicators' in stix_dict:
ParsedData['IndicatorData'] = []
for row in stix_dict['indicators']:
# Transform lists in indicators into usable data
if 'observable' not in row:
self.logging.info(
'Indicator has no observable, skipping: %s', row)
continue
newrow = copy.deepcopy(row)
self._ValidateURIType(newrow)
newrows = self._ExtractRelatedObjects(newrow)
ParsedData['IndicatorData'].append(newrow)
if newrows.__len__() > 0:
stix_dict['indicators'].extend(newrows)
return ParsedData
def parse_stix(self, reference='', make_event=False, source=''):
"""
Parse the document.
:param reference: The reference to the data.
:type reference: str
:param make_event: Whether or not to create an Event for this document.
:type make_event: bool
:param source: The source of this document.
:type source: str
:raises: :class:`taxii_service.parsers.STIXParserException`
Until we have a way to map source strings in a STIX document to
a source in CRITs, we are being safe and using the source provided
as the true source.
"""
with closing(StringIO(self.data)) as f:
try:
try:
self.package = STIXPackage.from_xml(f)
if not self.package:
raise STIXParserException("STIX package failure")
except UnsupportedVersionError:
v = stix.__version__
v = v[0:-2] if len(v.split('.')) > 3 else v
updated = ramrod.update(f, to_=v)
doc = updated.document.as_stringio()
self.package = STIXPackage.from_xml(doc)
except Exception as e:
msg = "Failed to create STIX/CybOX from XML"
self.failed.append((e.message,
"STIX Package (%s)" % msg,
'')) # note for display in UI
return
if not self.preview:
self.stix_version = self.package.version
stix_header = self.package.stix_header
if stix_header and stix_header.information_source and stix_header.information_source.identity:
self.information_source = stix_header.information_source.identity.name
if self.information_source:
info_src = "STIX Source: %s" % self.information_source
if not reference:
reference = ''
else:
reference += ", "
reference += info_src
if source:
if does_source_exist(source):
self.source.name = source
else:
raise STIXParserException('Source "%s" does not exist in CRITs.' % source)
elif does_source_exist(self.information_source):
self.source.name = self.information_source
else:
raise STIXParserException("No source to attribute data to.")
self.source_instance.reference = reference
self.source.instances.append(self.source_instance)
if make_event:
title = "STIX Document %s" % self.package.id_
event_type = EventTypes.INTEL_SHARING
date = datetime.datetime.now()
description = str(date)
if self.package.incidents:
incdnt = self.package.incidents[0]
title = incdnt.title
if incdnt.description:
description = incdnt.description
if isinstance(description, StructuredText):
try:
description = description.to_dict()
except:
pass
if incdnt.short_description in EventTypes.values():
event_type = incdnt.short_description
elif incdnt.categories and incdnt.categories[0].value:
event_type = get_crits_event_type(incdnt.categories[0].value)
else: #package contains no incidents
header = self.package.stix_header
if isinstance(header, STIXHeader):
if header.title:
title = header.title
if header.package_intents:
try:
stix_type = str(header.package_intents[0])
event_type = get_crits_event_type(stix_type)
except:
pass
if header.description:
description = header.description
if isinstance(description, StructuredText):
try:
description = description.to_dict()
except:
pass
if self.preview:
self.imported[self.package.id_] = ('Event',
None,
title)
else:
res = add_new_event(title,
description,
event_type,
self.source.name,
self.source_instance.method,
self.source_instance.reference,
date,
self.source_instance.analyst)
self.parsed.append(self.package.id_)
if res['success']:
self.event = res['object']
self.imported[self.package.id_] = ('Event',
res['object'].id,
title or res['object'].id)
self.updates[res['object'].id] = res['object']
# Get relationships to the Event
if self.package.incidents:
incdnts = self.package.incidents
for rel in getattr(incdnts[0], 'related_indicators', ()):
if rel.relationship or rel.confidence:
r = rel.relationship.value or RelationshipTypes.RELATED_TO
c = getattr(rel.confidence.value, 'value', 'Unknown')
self.event_rels[rel.item.idref] = (r, c)
else:
self.failed.append((res['message'],
"Event (%s)" % title,
self.package.id_))
if self.package.indicators:
self.parse_indicators(self.package.indicators)
if self.package.observables and self.package.observables.observables:
self.parse_observables(self.package.observables.observables)
if self.package.threat_actors:
self.parse_threat_actors(self.package.threat_actors)
def custom_parser(self, datafile):
"""
Custom parser for the XML STIX file.
Successful parsing results in a STIX File object for a tool containing the raw file, requested elements,
and rule signatures found.
:param datafile: The file sent to be parsed, expected to be STIX XML format.
:return alert: A Stix File object containing the parsed elements for tools
"""
ret = []
rule_list = []
el_list = []
try:
alert = StixFile()
# open to set the raw file text in the alert object
if os.path.exists(datafile):
with open(datafile, 'r') as file:
data = file.read()
alert.setRawFile(data)
try:
# attempt to map as a STIX Package, if fails force an update with ramrod
stix_package = STIXPackage.from_xml(datafile)
except Exception as e:
self._logger.warning("Unable to parse XML to STIX, attempting version change Exception={0}".format(e))
updated = ramrod.update(datafile, force=True)
stix_package = STIXPackage.from_xml(updated)
stix_dict = stix_package.to_dict()
# parse as a STIX package to find matching elements
if self._elements:
for i in self._elements:
test = jsonpath_rw.parse('$.' + str(i).lower()) # returns all JSON key matches
for match in test.find(stix_dict):
el_list.append(match.value)
alert.setStixElements(el_list)
# create dictionary with a key for each entry in rules list
rules_dict = {}
for y in self._rules:
rules_dict[y] = []
# parse the STIX package to find elements containing Rules
if self._rules:
# search STIX for all test mechanisms
test = jsonpath_rw.parse('$..test_mechanisms') # STIX defined element that contains rules
for match in test.find(stix_dict):
rule_list.append(match.value[0])
# compare the XSI type to see if it's in the rules list 'xsi:type' = 'yaraTM:YaraTestMechanismType'
for y in self._rules:
if y.lower() in str(match.value[0]['xsi:type']).lower(): # STIX way to determine rule type
# append to list of rules for dictionary entry for "rule[s]" -- 'rule[s]' -> 'value'
if 'rule' in match.value[0]:
rules_dict[y].append(match.value[0]['rule']['value'])
elif 'rules' in match.value[0]:
for z in match.value[0]['rules']:
rules_dict[y].append(z['value']) # TODO: remove hardcoding
# TODO: STIX Snort Specification allows -- Event Filters / Rate Filters / Event Suppression
alert.setRules(rules_dict)
alert.setFullRules(rule_list)
# append the final alert
ret.append(alert)
self._logger.info("Identified {0} matching elements and {1} rule objects.".format(len(el_list), len(rule_list)))
except Exception as e:
self._logger.error("Error occurred parsing STIX file. Exception={0}".format(e))
return ret
return ret
def test_cybox_observables(self):
updated = ramrod.update(self._cybox_observables)
self.assertTrue(updated.document)
def test_stix_package(self):
updated = ramrod.update(self._stix_package)
self.assertTrue(updated.document)
def test_cybox_observables(self):
updated = ramrod.update(self._cybox_observables)
self.assertTrue(updated.document)
def test_stix_package(self):
updated = ramrod.update(self._stix_package)
self.assertTrue(updated.document)
