def test_composite_datasource_operations(stix_objs1, stix_objs2): BUNDLE1 = dict( id="bundle--%s" % make_id(), objects=stix_objs1, spec_version="2.0", type="bundle", ) cds1 = CompositeDataSource() ds1_1 = MemorySource(stix_data=BUNDLE1) ds1_2 = MemorySource(stix_data=stix_objs2) cds2 = CompositeDataSource() ds2_1 = MemorySource(stix_data=BUNDLE1) ds2_2 = MemorySource(stix_data=stix_objs2) cds1.add_data_sources([ds1_1, ds1_2]) cds2.add_data_sources([ds2_1, ds2_2]) indicators = cds1.all_versions("indicator--00000000-0000-4000-8000-000000000001") # In STIX_OBJS2 changed the 'modified' property to a later time... assert len(indicators) == 3 cds1.add_data_sources([cds2]) indicator = cds1.get("indicator--00000000-0000-4000-8000-000000000001") assert indicator["id"] == "indicator--00000000-0000-4000-8000-000000000001" assert indicator["modified"] == parse_into_datetime("2017-01-31T13:49:53.935Z") assert indicator["type"] == "indicator" query1 = [ Filter("type", "=", "indicator"), ] query2 = [ Filter("valid_from", "=", "2017-01-27T13:49:53.935382Z"), ] cds1.filters.add(query2) results = cds1.query(query1) # STIX_OBJS2 has indicator with later time, one with different id, one with # original time in STIX_OBJS1 assert len(results) == 4 indicator = cds1.get("indicator--00000000-0000-4000-8000-000000000001") assert indicator["id"] == "indicator--00000000-0000-4000-8000-000000000001" assert indicator["modified"] == parse_into_datetime("2017-01-31T13:49:53.935Z") assert indicator["type"] == "indicator" results = cds1.all_versions("indicator--00000000-0000-4000-8000-000000000001") assert len(results) == 3 # Since we have filters already associated with our CompositeSource providing # nothing returns the same as cds1.query(query1) (the associated query is query2) results = cds1.query([]) assert len(results) == 4
def test_add_get_remove_filter(collection): ds = stix2.TAXIICollectionSource(collection) # First 3 filters are valid, remaining properties are erroneous in some way valid_filters = [ Filter('type', '=', 'malware'), Filter('id', '!=', 'stix object id'), Filter('labels', 'in', ["heartbleed", "malicious-activity"]), ] assert len(ds.filters) == 0 ds.filters.add(valid_filters[0]) assert len(ds.filters) == 1 # Addin the same filter again will have no effect since `filters` acts # like a set ds.filters.add(valid_filters[0]) assert len(ds.filters) == 1 ds.filters.add(valid_filters[1]) assert len(ds.filters) == 2 ds.filters.add(valid_filters[2]) assert len(ds.filters) == 3 assert valid_filters == [f for f in ds.filters] # remove ds.filters.remove(valid_filters[0]) assert len(ds.filters) == 2 ds.filters.add(valid_filters)
def all_versions(self, stix_id, version=None, _composite_filters=None): """Retrieve STIX object from local/remote TAXII Collection endpoint, all versions of it Args: stix_id (str): The STIX ID of the STIX objects to be retrieved. _composite_filters (FilterSet): collection of filters passed from the parent CompositeDataSource, not user supplied version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. Returns: (see query() as all_versions() is just a wrapper) """ # make query in TAXII query format since 'id' is TAXII field query = [ Filter("match[id]", "=", stix_id), Filter("match[version]", "=", "all") ] all_data = self.query(query=query, _composite_filters=_composite_filters) # parse STIX objects from TAXII returned json all_data = [parse(stix_obj, allow_custom=self.allow_custom, version=version) for stix_obj in all_data] # check - was added to handle erroneous TAXII servers all_data_clean = [stix_obj for stix_obj in all_data if stix_obj.id == stix_id] return all_data_clean
def all_versions(self, stix_id, version=None, _composite_filters=None): """Retrieve STIX object from local/remote TAXII Collection endpoint, all versions of it Args: stix_id (str): The STIX ID of the STIX objects to be retrieved. version (str): If present, it forces the parser to use the version provided. Otherwise, the library will make the best effort based on checking the "spec_version" property. _composite_filters (FilterSet): collection of filters passed from the parent CompositeDataSource, not user supplied Returns: (see query() as all_versions() is just a wrapper) """ # make query in TAXII query format since 'id' is TAXII field query = [ Filter('id', '=', stix_id), Filter('version', '=', 'all'), ] all_data = self.query(query=query, _composite_filters=_composite_filters) # parse STIX objects from TAXII returned json all_data = [parse(stix_obj, allow_custom=self.allow_custom, version=version) for stix_obj in all_data] # check - was added to handle erroneous TAXII servers all_data_clean = [stix_obj for stix_obj in all_data if stix_obj.id == stix_id] return all_data_clean
def test_datetime_filter_behavior(): """if a filter is initialized with its value being a datetime object OR the STIX object property being filtered on is a datetime object, all resulting comparisons executed are done on the string representations of the datetime objects, as the Filter functionality will convert all datetime objects to there string forms using format_datetim() This test makes sure all datetime comparisons are carried out correctly """ filter_with_dt_obj = Filter( "created", "=", parse_into_datetime("2016-02-14T00:00:00.000Z", "millisecond")) filter_with_str = Filter("created", "=", "2016-02-14T00:00:00.000Z") # compare datetime obj to filter w/ datetime obj resp = list(apply_common_filters(real_stix_objs, [filter_with_dt_obj])) assert len(resp) == 1 assert resp[0][ "id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef" assert isinstance(resp[0].created, STIXdatetime) # make sure original object not altered # compare datetime string to filter w/ str resp = list(apply_common_filters(stix_objs, [filter_with_str])) assert len(resp) == 1 assert resp[0][ "id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef" # compare datetime obj to filter w/ str resp = list(apply_common_filters(real_stix_objs, [filter_with_str])) assert len(resp) == 1 assert resp[0][ "id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef" assert isinstance(resp[0].created, STIXdatetime) # make sure original object not altered
def test_filter_value_type_check(): # invalid filters - non supported value types with pytest.raises(TypeError) as excinfo: Filter('created', '=', object()) # On Python 2, the type of object() is `<type 'object'>` On Python 3, it's `<class 'object'>`. assert any([ s in str(excinfo.value) for s in ["<type 'object'>", "'<class 'object'>'"] ]) assert "is not supported. The type must be a Python immutable type or dictionary" in str( excinfo.value) with pytest.raises(TypeError) as excinfo: Filter("type", "=", complex(2, -1)) assert any([ s in str(excinfo.value) for s in ["<type 'complex'>", "'<class 'complex'>'"] ]) assert "is not supported. The type must be a Python immutable type or dictionary" in str( excinfo.value) with pytest.raises(TypeError) as excinfo: Filter("type", "=", set([16, 23])) assert any( [s in str(excinfo.value) for s in ["<type 'set'>", "'<class 'set'>'"]]) assert "is not supported. The type must be a Python immutable type or dictionary" in str( excinfo.value)
def test_composite_datasource_operations(stix_objs1, stix_objs2): BUNDLE1 = dict(id="bundle--%s" % make_id(), objects=stix_objs1, spec_version="2.0", type="bundle") cds1 = CompositeDataSource() ds1_1 = MemorySource(stix_data=BUNDLE1) ds1_2 = MemorySource(stix_data=stix_objs2) cds2 = CompositeDataSource() ds2_1 = MemorySource(stix_data=BUNDLE1) ds2_2 = MemorySource(stix_data=stix_objs2) cds1.add_data_sources([ds1_1, ds1_2]) cds2.add_data_sources([ds2_1, ds2_2]) indicators = cds1.all_versions( "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") # In STIX_OBJS2 changed the 'modified' property to a later time... assert len(indicators) == 2 cds1.add_data_sources([cds2]) indicator = cds1.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" assert indicator["modified"] == "2017-01-31T13:49:53.935Z" assert indicator["type"] == "indicator" query1 = [Filter("type", "=", "indicator")] query2 = [Filter("valid_from", "=", "2017-01-27T13:49:53.935382Z")] cds1.filters.add(query2) results = cds1.query(query1) # STIX_OBJS2 has indicator with later time, one with different id, one with # original time in STIX_OBJS1 assert len(results) == 3 indicator = cds1.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") assert indicator["id"] == "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f" assert indicator["modified"] == "2017-01-31T13:49:53.935Z" assert indicator["type"] == "indicator" # There is only one indicator with different ID. Since we use the same data # when deduplicated, only two indicators (one with different modified). results = cds1.all_versions( "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") assert len(results) == 2 # Since we have filters already associated with our CompositeSource providing # nothing returns the same as cds1.query(query1) (the associated query is query2) results = cds1.query([]) assert len(results) == 3
def test_filters6(stix_objs2, real_stix_objs2): # Test filtering on non-common property resp = list(apply_common_filters(stix_objs2, [Filter("name", "=", "Malicious site hosting downloader")])) assert resp[0]['id'] == stix_objs2[0]['id'] assert len(resp) == 3 resp = list(apply_common_filters(real_stix_objs2, [Filter("name", "=", "Malicious site hosting downloader")])) assert resp[0].id == real_stix_objs2[0].id assert len(resp) == 3
def test_filters5(stix_objs2, real_stix_objs2): # "Return any object whose id is not indicator--00000000-0000-4000-8000-000000000002" resp = list(apply_common_filters(stix_objs2, [Filter("id", "!=", "indicator--00000000-0000-4000-8000-000000000002")])) assert resp[0]['id'] == stix_objs2[0]['id'] assert len(resp) == 1 resp = list(apply_common_filters(real_stix_objs2, [Filter("id", "!=", "indicator--00000000-0000-4000-8000-000000000002")])) assert resp[0].id == real_stix_objs2[0].id assert len(resp) == 1
def test_filters2(stix_objs2, real_stix_objs2): # "Return any object modified after or on 2017-01-28T13:49:53.935Z" resp = list(apply_common_filters(stix_objs2, [Filter("modified", ">=", "2017-01-27T13:49:53.935Z")])) assert resp[0]['id'] == stix_objs2[0]['id'] assert len(resp) == 3 resp = list(apply_common_filters(real_stix_objs2, [Filter("modified", ">=", parse_into_datetime("2017-01-27T13:49:53.935Z"))])) assert resp[0].id == real_stix_objs2[0].id assert len(resp) == 3
def test_filters7(stix_objs2, real_stix_objs2): # Test filtering on embedded property obsvd_data_obj = { "type": "observed-data", "spec_version": "2.1", "id": OBSERVED_DATA_ID, "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", "created": "2016-04-06T19:58:16.000Z", "modified": "2016-04-06T19:58:16.000Z", "first_observed": "2015-12-21T19:00:00Z", "last_observed": "2015-12-21T19:00:00Z", "number_observed": 50, "objects": { "0": { "type": "file", "hashes": { "SHA-256": "35a01331e9ad96f751278b891b6ea09699806faedfa237d40513d92ad1b7100f", }, "extensions": { "pdf-ext": { "version": "1.7", "document_info_dict": { "Title": "Sample document", "Author": "Adobe Systems Incorporated", "Creator": "Adobe FrameMaker 5.5.3 for Power Macintosh", "Producer": "Acrobat Distiller 3.01 for Power Macintosh", "CreationDate": "20070412090123-02", }, "pdfid0": "DFCE52BD827ECF765649852119D", "pdfid1": "57A1E0F9ED2AE523E313C", }, }, }, }, } stix_objects = list(stix_objs2) + [obsvd_data_obj] real_stix_objects = list(real_stix_objs2) + [parse(obsvd_data_obj)] resp = list( apply_common_filters( stix_objects, [Filter("objects.0.extensions.pdf-ext.version", ">", "1.2")])) assert resp[0]['id'] == stix_objects[3]['id'] assert len(resp) == 1 resp = list( apply_common_filters( real_stix_objects, [Filter("objects.0.extensions.pdf-ext.version", ">", "1.2")])) assert resp[0].id == real_stix_objects[3].id assert len(resp) == 1
def test_filter_ops_check(): # invalid filters - non supported operators with pytest.raises(ValueError) as excinfo: # create Filter that has an operator that is not allowed Filter('modified', '*', 'not supported operator') assert str(excinfo.value) == "Filter operator '*' not supported for specified property: 'modified'" with pytest.raises(ValueError) as excinfo: Filter("type", "%", "4") assert "Filter operator '%' not supported for specified property" in str(excinfo.value)
def relationships(self, obj, relationship_type=None, source_only=False, target_only=False): """Retrieve Relationships involving the given STIX object. Only one of `source_only` and `target_only` may be `True`. Args: obj (STIX object OR dict OR str): The STIX object (or its ID) whose relationships will be looked up. relationship_type (str): Only retrieve Relationships of this type. If None, all relationships will be returned, regardless of type. source_only (bool): Only retrieve Relationships for which this object is the source_ref. Default: False. target_only (bool): Only retrieve Relationships for which this object is the target_ref. Default: False. Returns: list: The Relationship objects involving the given STIX object. """ results = [] filters = [Filter('type', '=', 'relationship')] try: obj_id = obj['id'] except KeyError: raise ValueError("STIX object has no 'id' property") except TypeError: # Assume `obj` is an ID string obj_id = obj if relationship_type: filters.append(Filter('relationship_type', '=', relationship_type)) if source_only and target_only: raise ValueError( "Search either source only or target only, but not both") if not target_only: results.extend( self.query(filters + [Filter('source_ref', '=', obj_id)])) if not source_only: results.extend( self.query(filters + [Filter('target_ref', '=', obj_id)])) return results
def test_store_mixed(indicator): mstore = MemoryStore([TLP_GREEN, indicator]) assert mstore.get(TLP_GREEN.id) == TLP_GREEN assert mstore.all_versions(TLP_GREEN.id) == [TLP_GREEN] assert mstore.query(Filter("id", "=", TLP_GREEN.id)) == [TLP_GREEN] assert mstore.get(indicator.id) == indicator assert mstore.all_versions(indicator.id) == [indicator] assert mstore.query(Filter("id", "=", indicator.id)) == [indicator] all_objs = mstore.query() assert TLP_GREEN in all_objs assert indicator in all_objs assert len(all_objs) == 2
def get(self, stix_id, version=None, _composite_filters=None): """Retrieve STIX object from file directory via STIX ID. Args: stix_id (str): The STIX ID of the STIX object to be retrieved. _composite_filters (FilterSet): collection of filters passed from the parent CompositeDataSource, not user supplied version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. Returns: (STIX object): STIX object that has the supplied STIX ID. The STIX object is loaded from its json file, parsed into a python STIX object and then returned """ query = [Filter("id", "=", stix_id)] all_data = self.query(query=query, version=version, _composite_filters=_composite_filters) if all_data: stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] else: stix_obj = None return stix_obj
def test_filters4(): # Assert invalid Filter cannot be created with pytest.raises(ValueError) as excinfo: Filter("modified", "?", "2017-01-27T13:49:53.935Z") assert str(excinfo.value) == ( "Filter operator '?' not supported " "for specified property: 'modified'" )
def test_filter_value_type_check(): # invalid filters - non supported value types with pytest.raises(TypeError) as excinfo: Filter('created', '=', object()) assert "'<class 'object'>'" in str(excinfo.value) assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value) with pytest.raises(TypeError) as excinfo: Filter("type", "=", complex(2, -1)) assert "'<class 'complex'>'" in str(excinfo.value) assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value) with pytest.raises(TypeError) as excinfo: Filter("type", "=", set([16, 23])) assert "'<class 'set'>'" in str(excinfo.value) assert "is not supported. The type must be a Python immutable type or dictionary" in str(excinfo.value)
def test_query_404(collection): """ a TAXIICollectionSource.query() call that recieves an HTTP 404 response code from the taxii2client should be returned as an exception""" ds = stix2.TAXIICollectionStore(collection) query = [Filter("type", "=", "malware")] with pytest.raises(DataSourceError) as excinfo: ds.query(query=query) assert "are either not found or access is denied" in str(excinfo.value) assert "404" in str(excinfo.value)
def related_to(self, obj, relationship_type=None, source_only=False, target_only=False, filters=None): """Retrieve STIX Objects that have a Relationship involving the given STIX object. Only one of `source_only` and `target_only` may be `True`. Args: obj (STIX object OR dict OR str): The STIX object (or its ID) whose related objects will be looked up. relationship_type (str): Only retrieve objects related by this Relationships type. If None, all related objects will be returned, regardless of type. source_only (bool): Only examine Relationships for which this object is the source_ref. Default: False. target_only (bool): Only examine Relationships for which this object is the target_ref. Default: False. filters (list): list of additional filters the related objects must match. Returns: list: The STIX objects related to the given STIX object. """ results = [] rels = self.relationships(obj, relationship_type, source_only, target_only) try: obj_id = obj['id'] except TypeError: # Assume `obj` is an ID string obj_id = obj # Get all unique ids from the relationships except that of the object ids = set() for r in rels: ids.update((r.source_ref, r.target_ref)) ids.discard(obj_id) # Assemble filters filter_list = FilterSet(filters) for i in ids: results.extend( self.query([f for f in filter_list] + [Filter('id', '=', i)])) return results
def test_parse_taxii_filters(collection): query = [ Filter("added_after", "=", "2016-02-01T00:00:01.000Z"), Filter("id", "=", "taxii stix object ID"), Filter("type", "=", "taxii stix object ID"), Filter("version", "=", "first"), Filter("created_by_ref", "=", "Bane"), ] taxii_filters_expected = [ Filter("added_after", "=", "2016-02-01T00:00:01.000Z"), Filter("id", "=", "taxii stix object ID"), Filter("type", "=", "taxii stix object ID"), Filter("version", "=", "first"), ] ds = stix2.TAXIICollectionSource(collection) taxii_filters = ds._parse_taxii_filters(query) assert taxii_filters == taxii_filters_expected
def all_versions(self, stix_id, version=None, _composite_filters=None): """Retrieve STIX object from file directory via STIX ID, all versions. Note: Since FileSystem sources/sinks don't handle multiple versions of a STIX object, this operation is unnecessary. Pass call to get(). Args: stix_id (str): The STIX ID of the STIX objects to be retrieved. _composite_filters (FilterSet): collection of filters passed from the parent CompositeDataSource, not user supplied version (str): If present, it forces the parser to use the version provided. Otherwise, the library will make the best effort based on checking the "spec_version" property. Returns: (list): of STIX objects that has the supplied STIX ID. The STIX objects are loaded from their json files, parsed into a python STIX objects and then returned """ query = [Filter("id", "=", stix_id)] return self.query(query, version=version, _composite_filters=_composite_filters)
def all_versions(self, stix_id, version=None, _composite_filters=None): """Retrieve STIX object from file directory via STIX ID, all versions. Note: Since FileSystem sources/sinks don't handle multiple versions of a STIX object, this operation is unnecessary. Pass call to get(). Args: stix_id (str): The STIX ID of the STIX objects to be retrieved. _composite_filters (FilterSet): collection of filters passed from the parent CompositeDataSource, not user supplied version (str): Which STIX2 version to use. (e.g. "2.0", "2.1"). If None, use latest version. Returns: (list): of STIX objects that has the supplied STIX ID. The STIX objects are loaded from their json files, parsed into a python STIX objects and then returned """ query = [Filter("id", "=", stix_id)] return self.query(query, version=version, _composite_filters=_composite_filters)
def get(self, stix_id, _composite_filters=None): """Retrieve STIX object from in-memory dict via STIX ID. Args: stix_id (str): The STIX ID of the STIX object to be retrieved. _composite_filters (FilterSet): collection of filters passed from the parent CompositeDataSource, not user supplied Returns: (dict OR STIX object): STIX object that has the supplied ID. As the MemoryStore(i.e. MemorySink) adds STIX objects to memory as they are supplied (either as python dictionary or STIX object), it is returned in the same form as it as added """ if _composite_filters is None: # if get call is only based on 'id', no need to search, just retrieve from dict try: stix_obj = self._data[stix_id] except KeyError: stix_obj = None return stix_obj # if there are filters from the composite level, process full query query = [Filter("id", "=", stix_id)] all_data = self.query(query=query, _composite_filters=_composite_filters) if all_data: # reduce to most recent version stix_obj = sorted(all_data, key=lambda k: k['modified'])[0] return stix_obj else: return None
def test_source_markings(): msrc = MemorySource(TLP_GREEN) assert msrc.get(TLP_GREEN.id) == TLP_GREEN assert msrc.all_versions(TLP_GREEN.id) == [TLP_GREEN] assert msrc.query(Filter("id", "=", TLP_GREEN.id)) == [TLP_GREEN]
def test_datastore_query_raises(): with pytest.raises(AttributeError) as excinfo: DataStoreMixin().query([Filter("type", "=", "indicator")]) assert "DataStoreMixin has no data source to query" == str(excinfo.value)
def test_composite_datastore_query_raises_error(): with pytest.raises(AttributeError) as excinfo: CompositeDataSource().query([Filter("type", "=", "indicator")]) assert "CompositeDataSource has no data sources" == str(excinfo.value)
def test_store_markings(): mstore = MemoryStore(TLP_GREEN) assert mstore.get(TLP_GREEN.id) == TLP_GREEN assert mstore.all_versions(TLP_GREEN.id) == [TLP_GREEN] assert mstore.query(Filter("id", "=", TLP_GREEN.id)) == [TLP_GREEN]
def test_filter_type_underscore_check(): # check that Filters where property="type", value (name) doesnt have underscores with pytest.raises(ValueError) as excinfo: Filter("type", "=", "oh_underscore") assert "Filter for property 'type' cannot have its value 'oh_underscore'" in str( excinfo.value)
"created": "2016-04-06T19:58:16.000Z", "modified": "2016-04-06T19:58:16.000Z", "first_observed": "2015-12-21T19:00:00Z", "last_observed": "2015-12-21T19:00:00Z", "number_observed": 1, "objects": { "0": { "type": "file", "name": "HAL 9000.exe", }, }, }, ] filters = [ Filter("type", "!=", "relationship"), Filter("id", "=", "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463"), Filter("labels", "in", "remote-access-trojan"), Filter("created", ">", "2015-01-01T01:00:00.000Z"), Filter("revoked", "=", True), Filter("revoked", "!=", True), Filter("object_marking_refs", "=", "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"), Filter("granular_markings.selectors", "in", "relationship_type"), Filter("granular_markings.marking_ref", "=", "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"), Filter("external_references.external_id", "in", "CVE-2014-0160,CVE-2017-6608"), Filter("created_by_ref", "=", "identity--f1350682-3290-4e0d-be58-69e290537647"), Filter("object_marking_refs", "=",