def main(): parser = _get_arg_parser() args = parser.parse_args() try: # Validate the input commandline arguments _validate_args(args) # Build UpdateOptions from commandline arguments options = _get_options(args) # Run the update process. updated = ramrod.update( args.infile, from_=args.from_, to_=args.to_, options=options, force=args.force ) # Write results _write_xml(updated.document, args.outfile) _write_removed(updated.removed) _write_remapped_ids(updated.remapped_ids) except errors.UpdateError as ex: _print_update_error(ex) sys.exit(EXIT_FAILURE) except errors.InvalidVersionError as ex: _print_invalid_version_error(ex) sys.exit(EXIT_FAILURE) except errors.UnknownVersionError as ex: _print_unknown_version_error(str(ex)) sys.exit(EXIT_FAILURE)
def main(): parser = _get_arg_parser() args = parser.parse_args() try: options = _get_options(args) updated = ramrod.update(args.infile, from_=args.from_, to_=args.to_, options=options, force=args.force) _write_xml(updated.document, args.outfile) _write_removed(updated.removed) _write_remapped_ids(updated.remapped_ids) except ramrod.UpdateError as ex: _print_update_error(ex) sys.exit(EXIT_FAILURE) except ramrod.InvalidVersionError as ex: _print_invalid_version_error(ex) sys.exit(EXIT_FAILURE) except ramrod.UnknownVersionError as ex: _print_unknown_version_error(str(ex)) sys.exit(EXIT_FAILURE)
def main(): parser = _get_arg_parser() args = parser.parse_args() try: # Validate the input commandline arguments _validate_args(args) # Build UpdateOptions from commandline arguments options = _get_options(args) # Run the update process. updated = ramrod.update(args.infile, from_=args.from_, to_=args.to_, options=options, force=args.force) # Write results _write_xml(updated.document, args.outfile) _write_removed(updated.removed) _write_remapped_ids(updated.remapped_ids) except errors.UpdateError as ex: _print_update_error(ex) sys.exit(EXIT_FAILURE) except errors.InvalidVersionError as ex: _print_invalid_version_error(ex) sys.exit(EXIT_FAILURE) except errors.UnknownVersionError as ex: _print_unknown_version_error(str(ex)) sys.exit(EXIT_FAILURE)
def file_to_stix(file_): '''transform files into stix packages''' try: stix_package = STIXPackage.from_xml(file_) except UnsupportedVersionError as ex: updated = ramrod.update(file_) updated_xml = updated.document.as_stringio() stix_package = STIXPackage.from_xml(updated_xml) return stix_package
def test_update_version(self): valid_versions = ramrod.cybox.CYBOX_VERSIONS idx = valid_versions.index version_to = valid_versions[idx(UPDATER.VERSION)+1:] for version in version_to: updated = ramrod.update(self._versions, to_=version) updated_root = updated.document.as_element() updated_version = UPDATER.get_version(updated_root) self.assertEqual(version, updated_version)
def test_update_version(self): valid_versions = ramrod.stix.STIX_VERSIONS idx = valid_versions.index version_to = valid_versions[idx(UPDATER.VERSION)+1:] for version in version_to: updated = ramrod.update(self._versions, to_=version) updated_root = updated.document.as_element() updated_version = UPDATER.get_version(updated_root) self.assertEqual(version, updated_version)
def __init__(self, source_item): self.source_item = source_item try: self.stix_package = STIXPackage.from_xml(self.io()) except UnsupportedVersionError: updated = ramrod.update(self.io(), to_='1.1.1') document = updated.document.as_stringio() self.stix_package = STIXPackage.from_xml(document) except Exception: logging.error('error parsing STIX package (%s)', self.file_name()) self.stix_package = None
def taxii_content_block_to_stix(content_block): '''transform taxii content blocks into stix packages''' xml = StringIO(content_block.content) try: stix_package = STIXPackage.from_xml(xml) except UnsupportedVersionError as ex: updated = ramrod.update(xml) updated_xml = updated.document.as_stringio() stix_package = STIXPackage.from_xml(updated_xml) xml.close() return stix_package
def __init__(self, source_item): self.source_item = source_item with warnings.catch_warnings(): warnings.filterwarnings("ignore","The use of this field has been deprecated",UserWarning) try: self.stix_package = STIXPackage.from_xml(self.io()) except UnsupportedVersionError: updated = ramrod.update(self.io(), to_=LATEST_STIX_VERSION) document = updated.document.as_stringio() self.stix_package = STIXPackage.from_xml(document) except Exception: logging.error('error parsing STIX package (%s)', self.file_name()) self.stix_package = None self.stix_version = self.stix_package.version
def load_stix_package(self, stix_file): """Helper for loading and updating (if required) a STIX package.""" try: package = STIXPackage.from_xml(stix_file) except UnsupportedVersionError: updated = ramrod.update(stix_file, to_='1.1.1') document = updated.document.as_stringio() try: package = STIXPackage.from_xml(document) except Exception: package = None except Exception: package = None return package
def __check_info_source(self, datafile): """ Parses the STIX Header element looking for an Information source element matching the configured value. Currently only matches to STIX Header, long-term could check every Information Source element. :param datafile: The file sent to be parsed, expected to be STIX XML format. :return Boolean: True if match is found, otherwise False. """ ret = False try: # TODO: Parse only header or look for all Information_Source elements? # attempt to map as a STIX Package, if fails force an update with ramrod try: stix_package = STIXPackage.from_xml(datafile) except Exception as e: self._logger.warning("Unable to parse XML to STIX, attempting version change Exception={0}".format(e)) updated = ramrod.update(datafile, force=True) stix_package = STIXPackage.from_xml(updated) # Compare the information source from the stix_header info_src = stix_package.stix_header.information_source if info_src.identity: if info_src.identity.name in self._sources: return True # If there are contributing sources, parse and compare to source list if info_src.contributing_sources: if info_src.contributing_sources._inner_name == 'sources': for x in info_src.contributing_sources._inner: if type(x) is information_source.InformationSource: if x.identity.name in self._sources: return True except Exception as e: self._logger.warning( "Unable to force XML to STIX with ramrod. Unable to check source Exception={0}".format(e)) return ret if ret is False: self._logger.info("Did not identify the requested source in STIX Header.") return ret
def parse_stix_package(self, data): '''Parse an XML string representing a STIX package object. If the content cannot be parsed, attempt to update the content to the current STIX version. Params: data - an XML string Returns: A STIXPackage object. ''' try: return STIXPackage.from_xml(StringIO.StringIO(data)) except (parser.UnknownVersionError, parser.UnsupportedVersionError, parser.UnsupportedRootElementError): try: # If we arrived here, we did not find a valid STIX version in this # document, meaning it is likely a lower STIX version. Try to # up-convert to latest version of STIX. updated = ramrod.update(data) return STIXPackage.from_xml(updated.document.as_stringio()) except (ramrod.errors.UpdateError, ramrod.errors.InvalidVersionError, ramrod.errors.UnknownVersionError): self._logger.exception( 'msg="UpdateError when parsing a STIX document" filename="%s"', self.filename) except lxml.etree.XMLSyntaxError: # Exceptions are not re-raised here because multiple STIXPackage # objects can result from a single TAXII PollResponse. # Just log the error in this case as the likely case # is that a single STIXPackage is in error, usually # due to a missing CDATA tag. self._logger.exception( 'msg="XMLSyntaxError when parsing a STIX document" filename="%s"', self.filename)
def parse_stix(self, reference='', make_event=False, source=''): """ Parse the document. :param reference: The reference to the data. :type reference: str :param make_event: Whether or not to create an Event for this document. :type make_event: bool :param source: The source of this document. :type source: str :raises: :class:`taxii_service.parsers.STIXParserException` Until we have a way to map source strings in a STIX document to a source in CRITs, we are being safe and using the source provided as the true source. """ with closing(StringIO(self.data)) as f: try: try: self.package = STIXPackage.from_xml(f) if not self.package: raise STIXParserException("STIX package failure") except UnsupportedVersionError: v = stix.__version__ v = v[0:-2] if len(v.split('.')) > 3 else v updated = ramrod.update(f, to_=v) doc = updated.document.as_stringio() self.package = STIXPackage.from_xml(doc) except Exception as e: msg = "Failed to create STIX/CybOX from XML" self.failed.append((e.message, "STIX Package (%s)" % msg, '')) # note for display in UI return if not self.preview: self.stix_version = self.package.version stix_header = self.package.stix_header if stix_header and stix_header.information_source and stix_header.information_source.identity: self.information_source = stix_header.information_source.identity.name if self.information_source: info_src = "STIX Source: %s" % self.information_source if not reference: reference = '' else: reference += ", " reference += info_src if source: if does_source_exist(source): self.source.name = source else: raise STIXParserException( 'Source "%s" does not exist in CRITs.' % source) elif does_source_exist(self.information_source): self.source.name = self.information_source else: raise STIXParserException("No source to attribute data to.") self.source_instance.reference = reference self.source.instances.append(self.source_instance) if make_event: title = "STIX Document %s" % self.package.id_ event_type = EventTypes.INTEL_SHARING date = datetime.datetime.now() description = str(date) if self.package.incidents: incdnt = self.package.incidents[0] title = incdnt.title if incdnt.description: description = incdnt.description if isinstance(description, StructuredText): try: description = description.to_dict() except: pass if incdnt.short_description in EventTypes.values(): event_type = incdnt.short_description elif incdnt.categories and incdnt.categories[0].value: event_type = get_crits_event_type( incdnt.categories[0].value) else: #package contains no incidents header = self.package.stix_header if isinstance(header, STIXHeader): if header.title: title = header.title if header.package_intents: try: stix_type = str(header.package_intents[0]) event_type = get_crits_event_type(stix_type) except: pass if header.description: description = header.description if isinstance(description, StructuredText): try: description = description.to_dict() except: pass if self.preview: self.imported[self.package.id_] = ('Event', None, title) else: res = add_new_event(title, description, event_type, self.source.name, self.source_instance.method, self.source_instance.reference, date, self.source_instance.analyst) self.parsed.append(self.package.id_) if res['success']: self.event = res['object'] self.imported[self.package.id_] = ('Event', res['object'].id, title or res['object'].id) self.updates[res['object'].id] = res['object'] # Get relationships to the Event if self.package.incidents: incdnts = self.package.incidents for rel in getattr(incdnts[0], 'related_indicators', ()): if rel.relationship or rel.confidence: r = rel.relationship.value or RelationshipTypes.RELATED_TO c = getattr(rel.confidence.value, 'value', 'Unknown') self.event_rels[rel.item.idref] = (r, c) else: self.failed.append((res['message'], "Event (%s)" % title, self.package.id_)) if self.package.indicators: self.parse_indicators(self.package.indicators) if self.package.observables and self.package.observables.observables: self.parse_observables(self.package.observables.observables) if self.package.threat_actors: self.parse_threat_actors(self.package.threat_actors)
def Read(self, stixfile, config, xmlparser=None): ''' Parse STIX XML document. Return a dictionary object with the data from the document. ''' # FIXME: Handle composite indicators and related indicators hierarchically # FIXME: Handle STIX indicators with multiple possible values where the apply_condition is ANY # Upgrade old versions of STIX documents to the latest supported release (currently 1.1.1) try: stix_package = STIXPackage.from_xml(stixfile) except UnsupportedVersionError: self.logging.warning("Updating stix document to version 1.1.1") if not isinstance(stixfile, str): stixfile.close() updated = ramrod.update(stixfile.name, force=True) else: updated = ramrod.update(stixfile, force=True) document = updated.document.as_stringio() stix_package = STIXPackage.from_xml(document) stix_dict = stix_package.to_dict() # parse to dictionary ParsedData = {} if 'stix_header' in stix_dict: ParsedData['DocumentHeaderData'] = copy.deepcopy( stix_dict['stix_header']) else: ParsedData['DocumentHeaderData'] = {} if 'profiles' in stix_dict['stix_header'] and stix_dict['stix_header'][ 'profiles']: ParsedData['DocumentHeaderData']['profiles'] = [] for row in stix_dict['stix_header']['profiles']: ParsedData['DocumentHeaderData']['profiles'].append( dict(value=row)) if 'id' in stix_dict: ParsedData['DocumentHeaderData']['id'] = stix_dict['id'] if 'version' in stix_dict: ParsedData['DocumentHeaderData']['version'] = stix_dict['version'] if 'observables' in stix_dict: # Rename observables as indicators and build basic indicator dictionary, resolves issue with STIX documents from Soltra with no Indicators. observables = stix_dict.pop('observables') if 'indicators' not in stix_dict: stix_dict['indicators'] = [] if 'observables' in observables: for row in observables['observables']: newindicator = {} if 'description' in row: newindicator['description'] = row.pop('description') if 'title' in row: newindicator['title'] = row.pop('title') newindicator['observable'] = row newindicator['indicator_types'] = [] indicator_type = {} indicator_type['value'] = 'Unknown' indicator_type[ 'xsi:type'] = 'stixVocabs:IndicatorTypeVocab-1.1' newindicator['indicator_types'].append(indicator_type) stix_dict['indicators'].append(newindicator) if 'indicators' in stix_dict: ParsedData['IndicatorData'] = [] for row in stix_dict['indicators']: # Transform lists in indicators into usable data if 'observable' not in row: self.logging.info( 'Indicator has no observable, skipping: %s', row) continue newrow = copy.deepcopy(row) self._ValidateURIType(newrow) newrows = self._ExtractRelatedObjects(newrow) ParsedData['IndicatorData'].append(newrow) if newrows.__len__() > 0: stix_dict['indicators'].extend(newrows) return ParsedData
def parse_stix(self, reference='', make_event=False, source=''): """ Parse the document. :param reference: The reference to the data. :type reference: str :param make_event: Whether or not to create an Event for this document. :type make_event: bool :param source: The source of this document. :type source: str :raises: :class:`taxii_service.parsers.STIXParserException` Until we have a way to map source strings in a STIX document to a source in CRITs, we are being safe and using the source provided as the true source. """ with closing(StringIO(self.data)) as f: try: try: self.package = STIXPackage.from_xml(f) if not self.package: raise STIXParserException("STIX package failure") except UnsupportedVersionError: v = stix.__version__ v = v[0:-2] if len(v.split('.')) > 3 else v updated = ramrod.update(f, to_=v) doc = updated.document.as_stringio() self.package = STIXPackage.from_xml(doc) except Exception as e: msg = "Failed to create STIX/CybOX from XML" self.failed.append((e.message, "STIX Package (%s)" % msg, '')) # note for display in UI return if not self.preview: self.stix_version = self.package.version stix_header = self.package.stix_header if stix_header and stix_header.information_source and stix_header.information_source.identity: self.information_source = stix_header.information_source.identity.name if self.information_source: info_src = "STIX Source: %s" % self.information_source if not reference: reference = '' else: reference += ", " reference += info_src if source: if does_source_exist(source): self.source.name = source else: raise STIXParserException('Source "%s" does not exist in CRITs.' % source) elif does_source_exist(self.information_source): self.source.name = self.information_source else: raise STIXParserException("No source to attribute data to.") self.source_instance.reference = reference self.source.instances.append(self.source_instance) if make_event: title = "STIX Document %s" % self.package.id_ event_type = EventTypes.INTEL_SHARING date = datetime.datetime.now() description = str(date) if self.package.incidents: incdnt = self.package.incidents[0] title = incdnt.title if incdnt.description: description = incdnt.description if isinstance(description, StructuredText): try: description = description.to_dict() except: pass if incdnt.short_description in EventTypes.values(): event_type = incdnt.short_description elif incdnt.categories and incdnt.categories[0].value: event_type = get_crits_event_type(incdnt.categories[0].value) else: #package contains no incidents header = self.package.stix_header if isinstance(header, STIXHeader): if header.title: title = header.title if header.package_intents: try: stix_type = str(header.package_intents[0]) event_type = get_crits_event_type(stix_type) except: pass if header.description: description = header.description if isinstance(description, StructuredText): try: description = description.to_dict() except: pass if self.preview: self.imported[self.package.id_] = ('Event', None, title) else: res = add_new_event(title, description, event_type, self.source.name, self.source_instance.method, self.source_instance.reference, date, self.source_instance.analyst) self.parsed.append(self.package.id_) if res['success']: self.event = res['object'] self.imported[self.package.id_] = ('Event', res['object'].id, title or res['object'].id) self.updates[res['object'].id] = res['object'] # Get relationships to the Event if self.package.incidents: incdnts = self.package.incidents for rel in getattr(incdnts[0], 'related_indicators', ()): if rel.relationship or rel.confidence: r = rel.relationship.value or RelationshipTypes.RELATED_TO c = getattr(rel.confidence.value, 'value', 'Unknown') self.event_rels[rel.item.idref] = (r, c) else: self.failed.append((res['message'], "Event (%s)" % title, self.package.id_)) if self.package.indicators: self.parse_indicators(self.package.indicators) if self.package.observables and self.package.observables.observables: self.parse_observables(self.package.observables.observables) if self.package.threat_actors: self.parse_threat_actors(self.package.threat_actors)
def custom_parser(self, datafile): """ Custom parser for the XML STIX file. Successful parsing results in a STIX File object for a tool containing the raw file, requested elements, and rule signatures found. :param datafile: The file sent to be parsed, expected to be STIX XML format. :return alert: A Stix File object containing the parsed elements for tools """ ret = [] rule_list = [] el_list = [] try: alert = StixFile() # open to set the raw file text in the alert object if os.path.exists(datafile): with open(datafile, 'r') as file: data = file.read() alert.setRawFile(data) try: # attempt to map as a STIX Package, if fails force an update with ramrod stix_package = STIXPackage.from_xml(datafile) except Exception as e: self._logger.warning("Unable to parse XML to STIX, attempting version change Exception={0}".format(e)) updated = ramrod.update(datafile, force=True) stix_package = STIXPackage.from_xml(updated) stix_dict = stix_package.to_dict() # parse as a STIX package to find matching elements if self._elements: for i in self._elements: test = jsonpath_rw.parse('$.' + str(i).lower()) # returns all JSON key matches for match in test.find(stix_dict): el_list.append(match.value) alert.setStixElements(el_list) # create dictionary with a key for each entry in rules list rules_dict = {} for y in self._rules: rules_dict[y] = [] # parse the STIX package to find elements containing Rules if self._rules: # search STIX for all test mechanisms test = jsonpath_rw.parse('$..test_mechanisms') # STIX defined element that contains rules for match in test.find(stix_dict): rule_list.append(match.value[0]) # compare the XSI type to see if it's in the rules list 'xsi:type' = 'yaraTM:YaraTestMechanismType' for y in self._rules: if y.lower() in str(match.value[0]['xsi:type']).lower(): # STIX way to determine rule type # append to list of rules for dictionary entry for "rule[s]" -- 'rule[s]' -> 'value' if 'rule' in match.value[0]: rules_dict[y].append(match.value[0]['rule']['value']) elif 'rules' in match.value[0]: for z in match.value[0]['rules']: rules_dict[y].append(z['value']) # TODO: remove hardcoding # TODO: STIX Snort Specification allows -- Event Filters / Rate Filters / Event Suppression alert.setRules(rules_dict) alert.setFullRules(rule_list) # append the final alert ret.append(alert) self._logger.info("Identified {0} matching elements and {1} rule objects.".format(len(el_list), len(rule_list))) except Exception as e: self._logger.error("Error occurred parsing STIX file. Exception={0}".format(e)) return ret return ret
def test_cybox_observables(self): updated = ramrod.update(self._cybox_observables) self.assertTrue(updated.document)
def test_stix_package(self): updated = ramrod.update(self._stix_package) self.assertTrue(updated.document)