def add(self, stix_data=None, version=None):
"""Add STIX objects to file directory.
Args:
stix_data (STIX object OR dict OR str OR list): valid STIX 2.0 content
in a STIX object (or list of), dict (or list of), or a STIX 2.0
json encoded string.
version (str): If present, it forces the parser to use the version
provided. Otherwise, the library will make the best effort based
on checking the "spec_version" property.
Note:
``stix_data`` can be a Bundle object, but each object in it will be
saved separately; you will be able to retrieve any of the objects
the Bundle contained, but not the Bundle itself.
"""
if isinstance(stix_data, (v20.Bundle, v21.Bundle)):
# recursively add individual STIX objects
for stix_obj in stix_data.get("objects", []):
self.add(stix_obj, version=version)
elif isinstance(stix_data, _STIXBase):
# adding python STIX object
self._check_path_and_write(stix_data)
elif isinstance(stix_data, (str, dict)):
parsed_data = parse(stix_data,
allow_custom=self.allow_custom,
version=version)
if isinstance(parsed_data, _STIXBase):
self.add(parsed_data, version=version)
else:
# custom unregistered object type
self._check_path_and_write(parsed_data)
elif isinstance(stix_data, list):
# recursively add individual STIX objects
for stix_obj in stix_data:
self.add(stix_obj)
else:
raise TypeError(
"stix_data must be a STIX object (or list of), "
"JSON formatted STIX (or list of), "
"or a JSON formatted STIX bundle", )
def _check_object_from_file(query, filepath, allow_custom, version, encoding):
"""
Read a STIX object from the given file, and check it against the given
filters.
Args:
query: Iterable of filters
filepath (str): Path to file to read
allow_custom (bool): Whether to allow custom properties as well unknown
custom objects.
version (str): If present, it forces the parser to use the version
provided. Otherwise, the library will make the best effort based
on checking the "spec_version" property.
encoding (str): The encoding to use when reading a file from the
filesystem.
Returns:
The (parsed) STIX object, if the object passes the filters. If
not, None is returned.
Raises:
TypeError: If the file had invalid JSON
IOError: If there are problems opening/reading the file
stix2.exceptions.STIXError: If there were problems creating a STIX
object from the JSON
"""
try:
with io.open(filepath, "r", encoding=encoding) as f:
stix_json = json.load(f)
except ValueError: # not a JSON file
raise TypeError(
"STIX JSON object at '{0}' could either not be parsed "
"to JSON or was not valid STIX JSON".format(filepath),
)
stix_obj = parse(stix_json, allow_custom, version)
if stix_obj["type"] == "bundle":
stix_obj = stix_obj["objects"][0]
# check against other filters, add if match
result = next(apply_common_filters([stix_obj], query), None)
return result
def all_versions(self, stix_id, version=None, _composite_filters=None):
"""Retrieve STIX object from local/remote TAXII Collection
endpoint, all versions of it
Args:
stix_id (str): The STIX ID of the STIX objects to be retrieved.
version (str): If present, it forces the parser to use the version
provided. Otherwise, the library will make the best effort based
on checking the "spec_version" property.
_composite_filters (FilterSet): collection of filters passed from the parent
CompositeDataSource, not user supplied
Returns:
(see query() as all_versions() is just a wrapper)
"""
# make query in TAXII query format since 'id' is TAXII field
query = [
Filter('id', '=', stix_id),
Filter('version', '=', 'all'),
]
all_data = self.query(query=query,
_composite_filters=_composite_filters)
# parse STIX objects from TAXII returned json
all_data = [
parse(stix_obj, allow_custom=self.allow_custom, version=version)
for stix_obj in all_data
]
# check - was added to handle erroneous TAXII servers
all_data_clean = [
stix_obj for stix_obj in all_data if stix_obj['id'] == stix_id
]
return all_data_clean
def add(self, stix_data, version=None):
"""Add/push STIX content to TAXII Collection endpoint
Args:
stix_data (STIX object OR dict OR str OR list): valid STIX2
content in a STIX object (or Bundle), STIX object dict (or
Bundle dict), or a STIX2 json encoded string, or list of
any of the following.
version (str): If present, it forces the parser to use the version
provided. Otherwise, the library will make the best effort based
on checking the "spec_version" property.
"""
if isinstance(stix_data, _STIXBase):
# adding python STIX object
if stix_data['type'] == 'bundle':
bundle = stix_data.serialize(encoding='utf-8',
ensure_ascii=False)
elif 'spec_version' in stix_data:
# If the spec_version is present, use new Bundle object...
bundle = v21.Bundle(stix_data,
allow_custom=self.allow_custom).serialize(
encoding='utf-8', ensure_ascii=False)
else:
bundle = v20.Bundle(stix_data,
allow_custom=self.allow_custom).serialize(
encoding='utf-8', ensure_ascii=False)
elif isinstance(stix_data, dict):
# adding python dict (of either Bundle or STIX obj)
if stix_data['type'] == 'bundle':
bundle = parse(stix_data,
allow_custom=self.allow_custom,
version=version).serialize(encoding='utf-8',
ensure_ascii=False)
elif 'spec_version' in stix_data:
# If the spec_version is present, use new Bundle object...
bundle = v21.Bundle(stix_data,
allow_custom=self.allow_custom).serialize(
encoding='utf-8', ensure_ascii=False)
else:
bundle = v20.Bundle(stix_data,
allow_custom=self.allow_custom).serialize(
encoding='utf-8', ensure_ascii=False)
elif isinstance(stix_data, list):
# adding list of something - recurse on each
for obj in stix_data:
self.add(obj, version=version)
return
elif isinstance(stix_data, str):
# adding json encoded string of STIX content
stix_data = parse(stix_data,
allow_custom=self.allow_custom,
version=version)
if stix_data['type'] == 'bundle':
bundle = stix_data.serialize(encoding='utf-8',
ensure_ascii=False)
elif 'spec_version' in stix_data:
# If the spec_version is present, use new Bundle object...
bundle = v21.Bundle(stix_data,
allow_custom=self.allow_custom).serialize(
encoding='utf-8', ensure_ascii=False)
else:
bundle = v20.Bundle(stix_data,
allow_custom=self.allow_custom).serialize(
encoding='utf-8', ensure_ascii=False)
else:
raise TypeError(
"stix_data must be as STIX object(or list of),json formatted STIX (or list of), or a json formatted STIX bundle"
)
self.collection.add_objects(bundle)
def query(self, query=None, version=None, _composite_filters=None):
"""Search and retreive STIX objects based on the complete query
A "complete query" includes the filters from the query, the filters
attached to MemorySource, and any filters passed from a
CompositeDataSource (i.e. _composite_filters)
Args:
query (list): list of filters to search on
version (str): If present, it forces the parser to use the version
provided. Otherwise, the library will make the best effort based
on checking the "spec_version" property.
_composite_filters (FilterSet): collection of filters passed from
the CompositeDataSource, not user supplied
Returns:
(list): list of STIX objects that matches the supplied
query. The STIX objects are received from TAXII as dicts,
parsed into python STIX objects and then returned.
"""
query = FilterSet(query)
# combine all query filters
if self.filters:
query.add(self.filters)
if _composite_filters:
query.add(_composite_filters)
# parse taxii query params (that can be applied remotely)
taxii_filters = self._parse_taxii_filters(query)
# taxii2client requires query params as keywords
taxii_filters_dict = dict((f.property, f.value) for f in taxii_filters)
# query TAXII collection
all_data = []
paged_request = tcv21.as_pages if isinstance(
self.collection, tcv21.Collection) else tcv20.as_pages
try:
for resource in paged_request(self.collection.get_objects,
per_request=self.items_per_page,
**taxii_filters_dict):
all_data.extend(resource.get("objects", []))
except HTTPError as e:
# if resources not found or access is denied from TAXII server, return empty list
if e.response.status_code == 404:
raise DataSourceError(
"The requested STIX objects for the TAXII Collection resource defined in"
" the supplied TAXII Collection object are either not found or access is"
" denied. Received error: ",
e,
)
# TAXII 2.0 paging can result in a 416 (Range Not Satisfiable) if
# the server isn't sending Content-Range headers, so the pager just
# goes until it runs out of pages. So 416 can't be treated as a
# real error, just an end-of-pages condition. For other codes,
# propagate the exception.
elif e.response.status_code != 416:
raise
# deduplicate data (before filtering as reduces wasted filtering)
all_data = deduplicate(all_data)
# apply local (CompositeDataSource, TAXIICollectionSource and query) filters
query.remove(taxii_filters)
all_data = list(apply_common_filters(all_data, query))
# parse python STIX objects from the STIX object dicts
stix_objs = [
parse(stix_obj_dict,
allow_custom=self.allow_custom,
version=version) for stix_obj_dict in all_data
]
return stix_objs
def query(self, query=None, version=None, _composite_filters=None):
"""Search and retreive STIX objects based on the complete query
A "complete query" includes the filters from the query, the filters
attached to MemorySource, and any filters passed from a
CompositeDataSource (i.e. _composite_filters)
Args:
query (list): list of filters to search on
version (str): If present, it forces the parser to use the version
provided. Otherwise, the library will make the best effort based
on checking the "spec_version" property.
_composite_filters (FilterSet): collection of filters passed from
the CompositeDataSource, not user supplied
Returns:
(list): list of STIX objects that matches the supplied
query. The STIX objects are received from TAXII as dicts,
parsed into python STIX objects and then returned.
"""
query = FilterSet(query)
# combine all query filters
if self.filters:
query.add(self.filters)
if _composite_filters:
query.add(_composite_filters)
# parse taxii query params (that can be applied remotely)
taxii_filters = self._parse_taxii_filters(query)
# taxii2client requires query params as keywords
taxii_filters_dict = dict((f.property, f.value) for f in taxii_filters)
# query TAXII collection
try:
all_data = self.collection.get_objects(**taxii_filters_dict).get(
'objects', [])
# deduplicate data (before filtering as reduces wasted filtering)
all_data = deduplicate(all_data)
# apply local (CompositeDataSource, TAXIICollectionSource and query) filters
query.remove(taxii_filters)
all_data = list(apply_common_filters(all_data, query))
except HTTPError as e:
# if resources not found or access is denied from TAXII server, return empty list
if e.response.status_code == 404:
raise DataSourceError(
"The requested STIX objects for the TAXII Collection resource defined in"
" the supplied TAXII Collection object are either not found or access is"
" denied. Received error: ",
e,
)
# parse python STIX objects from the STIX object dicts
stix_objs = [
parse(stix_obj_dict,
allow_custom=self.allow_custom,
version=version) for stix_obj_dict in all_data
]
return stix_objs
