def all_versions(self, stix_id, _composite_filters=None): """Retrieve STIX objects from in-memory dict via STIX ID, all versions of it. Args: stix_id (str): The STIX ID of the STIX 2 object to retrieve. _composite_filters (FilterSet): collection of filters passed from the parent CompositeDataSource, not user supplied Returns: (list): list of STIX objects that have the supplied ID. """ results = [] stix_objs_to_filter = None if is_marking(stix_id): stix_obj = self._data.get(stix_id) if stix_obj: stix_objs_to_filter = [stix_obj] else: object_family = self._data.get(stix_id) if object_family: stix_objs_to_filter = object_family.all_versions.values() if stix_objs_to_filter: all_filters = list( itertools.chain( _composite_filters or [], self.filters, ), ) results.extend( apply_common_filters(stix_objs_to_filter, all_filters), ) return results
def get(self, stix_id, version=None, _composite_filters=None): """Retrieve STIX object from file directory via STIX ID. Args: stix_id (str): The STIX ID of the STIX object to be retrieved. _composite_filters (FilterSet): collection of filters passed from the parent CompositeDataSource, not user supplied version (str): If present, it forces the parser to use the version provided. Otherwise, the library will make the best effort based on checking the "spec_version" property. Returns: (STIX object): STIX object that has the supplied STIX ID. The STIX object is loaded from its json file, parsed into a python STIX object and then returned """ all_data = self.all_versions(stix_id, version=version, _composite_filters=_composite_filters) if all_data: if is_marking(stix_id): # Markings are unversioned; there shouldn't be more than one # result. stix_obj = all_data[0] else: stix_obj = sorted(all_data, key=lambda k: k['modified'])[-1] else: stix_obj = None return stix_obj
def get(self, stix_id, _composite_filters=None): """Retrieve STIX object from in-memory dict via STIX ID. Args: stix_id (str): The STIX ID of the STIX object to be retrieved. _composite_filters (FilterSet): collection of filters passed from the parent CompositeDataSource, not user supplied Returns: (STIX object): STIX object that has the supplied ID. """ stix_obj = None if is_marking(stix_id): stix_obj = self._data.get(stix_id) else: object_family = self._data.get(stix_id) if object_family: stix_obj = object_family.latest_version if stix_obj: all_filters = list( itertools.chain( _composite_filters or [], self.filters, ), ) stix_obj = next(apply_common_filters([stix_obj], all_filters), None) return stix_obj
def _check_path_and_write(self, stix_obj, encoding='utf-8'): """Write the given STIX object to a file in the STIX file directory. """ type_dir = os.path.join(self._stix_dir, stix_obj["type"]) if is_marking(stix_obj): filename = stix_obj["id"] obj_dir = type_dir else: filename = _timestamp2filename(stix_obj["modified"]) obj_dir = os.path.join(type_dir, stix_obj["id"]) file_path = os.path.join(obj_dir, filename + ".json") if not os.path.exists(obj_dir): os.makedirs(obj_dir) if self.bundlify: if 'spec_version' in stix_obj: # Assuming future specs will allow multiple SDO/SROs # versions in a single bundle we won't need to check this # and just use the latest supported Bundle version. stix_obj = v21.Bundle(stix_obj, allow_custom=self.allow_custom) else: stix_obj = v20.Bundle(stix_obj, allow_custom=self.allow_custom) if os.path.isfile(file_path): raise DataSourceError( "Attempted to overwrite file (!) at: {}".format(file_path)) else: with io.open(file_path, 'w', encoding=encoding) as f: stix_obj = stix_obj.serialize(pretty=True, encoding=encoding, ensure_ascii=False) f.write(stix_obj)
def get(self, stix_id, version=None, _composite_filters=None): """Retrieve STIX object from file directory via STIX ID. Args: stix_id (str): The STIX ID of the STIX object to be retrieved. _composite_filters (FilterSet): collection of filters passed from the parent CompositeDataSource, not user supplied version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. Returns: (STIX object): STIX object that has the supplied STIX ID. The STIX object is loaded from its json file, parsed into a python STIX object and then returned """ all_data = self.all_versions(stix_id, version=version, _composite_filters=_composite_filters) if all_data: if is_marking(stix_id): # Markings are unversioned; there shouldn't be more than one # result. stix_obj = all_data[0] else: stix_obj = sorted(all_data, key=lambda k: k['modified'])[-1] else: stix_obj = None return stix_obj
def remove_markings(obj, marking, selectors): """ Remove a granular marking from the granular_markings collection. The method makes a best-effort attempt to distinguish between a marking-definition or language granular marking. Args: obj: An SDO or SRO object. marking: identifier or list of marking identifiers that apply to the properties selected by `selectors`. selectors: string or list of selectors strings relative to the SDO or SRO in which the properties appear. Raises: InvalidSelectorError: If `selectors` fail validation. MarkingNotFoundError: If markings to remove are not found on the provided SDO or SRO. Returns: A new version of the given SDO or SRO with specified markings removed. """ selectors = utils.convert_to_list(selectors) marking = utils.convert_to_marking_list(marking) utils.validate(obj, selectors) granular_markings = obj.get('granular_markings') if not granular_markings: return obj granular_markings = utils.expand_markings(granular_markings) to_remove = [] for m in marking: if is_marking(m): to_remove.append({'marking_ref': m, 'selectors': selectors}) else: to_remove.append({'lang': m, 'selectors': selectors}) remove = utils.build_granular_marking(to_remove).get('granular_markings') if not any(marking in granular_markings for marking in remove): raise exceptions.MarkingNotFoundError(obj, remove) granular_markings = [m for m in granular_markings if m not in remove] granular_markings = utils.compress_markings(granular_markings) if granular_markings: return new_version(obj, granular_markings=granular_markings, allow_custom=True) else: return new_version(obj, granular_markings=None, allow_custom=True)
def _add(store, stix_data, allow_custom=True, version=None): """Add STIX objects to MemoryStore/Sink. Adds STIX objects to an in-memory dictionary for fast lookup. Recursive function, breaks down STIX Bundles and lists. Args: store: A MemoryStore, MemorySink or MemorySource object. stix_data (list OR dict OR STIX object): STIX objects to be added allow_custom (bool): Whether to allow custom properties as well unknown custom objects. Note that unknown custom objects cannot be parsed into STIX objects, and will be returned as is. Default: False. version (str): Which STIX2 version to lock the parser to. (e.g. "2.0", "2.1"). If None, the library makes the best effort to figure out the spec representation of the object. """ if isinstance(stix_data, list): # STIX objects are in a list- recurse on each object for stix_obj in stix_data: _add(store, stix_obj, allow_custom, version) elif stix_data["type"] == "bundle": # adding a json bundle - so just grab STIX objects for stix_obj in stix_data.get("objects", []): _add(store, stix_obj, allow_custom, version) else: # Adding a single non-bundle object if isinstance(stix_data, _STIXBase): stix_obj = stix_data else: stix_obj = parse(stix_data, allow_custom, version) # Map ID directly to the object, if it is a marking. Otherwise, # map to a family, so we can track multiple versions. if is_marking(stix_obj): store._data[stix_obj["id"]] = stix_obj else: if stix_obj["id"] in store._data: obj_family = store._data[stix_obj["id"]] else: obj_family = _ObjectFamily() store._data[stix_obj["id"]] = obj_family obj_family.add(stix_obj)
def add_markings(obj, marking, selectors): """ Append a granular marking to the granular_markings collection. The method makes a best-effort attempt to distinguish between a marking-definition or language granular marking. Args: obj: An SDO or SRO object. marking: identifier or list of marking identifiers that apply to the properties selected by `selectors`. selectors: list of type string, selectors must be relative to the TLO in which the properties appear. Raises: InvalidSelectorError: If `selectors` fail validation. Returns: A new version of the given SDO or SRO with specified markings added. """ selectors = utils.convert_to_list(selectors) marking = utils.convert_to_marking_list(marking) utils.validate(obj, selectors) granular_marking = [] for m in marking: if is_marking(m): granular_marking.append({ 'marking_ref': m, 'selectors': sorted(selectors) }) else: granular_marking.append({ 'lang': m, 'selectors': sorted(selectors) }) if obj.get('granular_markings'): granular_marking.extend(obj.get('granular_markings')) granular_marking = utils.expand_markings(granular_marking) granular_marking = utils.compress_markings(granular_marking) return new_version(obj, granular_markings=granular_marking, allow_custom=True)
def _add(store, stix_data=None, allow_custom=True, version=None): """Add STIX objects to MemoryStore/Sink. Adds STIX objects to an in-memory dictionary for fast lookup. Recursive function, breaks down STIX Bundles and lists. Args: stix_data (list OR dict OR STIX object): STIX objects to be added version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. """ if isinstance(stix_data, list): # STIX objects are in a list- recurse on each object for stix_obj in stix_data: _add(store, stix_obj, allow_custom, version) elif stix_data["type"] == "bundle": # adding a json bundle - so just grab STIX objects for stix_obj in stix_data.get("objects", []): _add(store, stix_obj, allow_custom, version) else: # Adding a single non-bundle object if isinstance(stix_data, _STIXBase): stix_obj = stix_data else: stix_obj = parse(stix_data, allow_custom, version) # Map ID directly to the object, if it is a marking. Otherwise, # map to a family, so we can track multiple versions. if is_marking(stix_obj): store._data[stix_obj["id"]] = stix_obj else: if stix_obj["id"] in store._data: obj_family = store._data[stix_obj["id"]] else: obj_family = _ObjectFamily() store._data[stix_obj["id"]] = obj_family obj_family.add(stix_obj)
def _check_path_and_write(self, stix_obj): """Write the given STIX object to a file in the STIX file directory. """ type_dir = os.path.join(self._stix_dir, stix_obj["type"]) if is_marking(stix_obj): filename = stix_obj["id"] obj_dir = type_dir else: filename = _timestamp2filename(stix_obj["modified"]) obj_dir = os.path.join(type_dir, stix_obj["id"]) file_path = os.path.join(obj_dir, filename + ".json") if not os.path.exists(obj_dir): os.makedirs(obj_dir) if self.bundlify: stix_obj = Bundle(stix_obj, allow_custom=self.allow_custom) with open(file_path, "w") as f: f.write(str(stix_obj))
def compress_markings(granular_markings): """Compress granular markings list. If there is more than one marking identifier matches. It will collapse into a single granular marking. Example: >>> compress_markings([ ... { ... "selectors": [ ... "description" ... ], ... "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" ... }, ... { ... "selectors": [ ... "name" ... ], ... "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" ... } ... ]) [ { "selectors": [ "description", "name" ], "marking_ref": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9" } ] Args: granular_markings: The granular markings list property present in a SDO or SRO. Returns: list: A list with all markings collapsed. """ if not granular_markings: return map_ = collections.defaultdict(set) for granular_marking in granular_markings: if granular_marking.get('marking_ref'): map_[granular_marking.get('marking_ref')].update( granular_marking.get('selectors')) if granular_marking.get('lang'): map_[granular_marking.get('lang')].update( granular_marking.get('selectors')) compressed = \ [ {'marking_ref': item, 'selectors': sorted(selectors)} if utils.is_marking(item) else {'lang': item, 'selectors': sorted(selectors)} for item, selectors in six.iteritems(map_) ] return compressed