def test_composite_datasource_operations(stix_objs1, stix_objs2):
BUNDLE1 = dict(
id="bundle--%s" % make_id(),
objects=stix_objs1,
spec_version="2.0",
type="bundle",
)
cds1 = CompositeDataSource()
ds1_1 = MemorySource(stix_data=BUNDLE1)
ds1_2 = MemorySource(stix_data=stix_objs2)
cds2 = CompositeDataSource()
ds2_1 = MemorySource(stix_data=BUNDLE1)
ds2_2 = MemorySource(stix_data=stix_objs2)
cds1.add_data_sources([ds1_1, ds1_2])
cds2.add_data_sources([ds2_1, ds2_2])
indicators = cds1.all_versions("indicator--00000000-0000-4000-8000-000000000001")
# In STIX_OBJS2 changed the 'modified' property to a later time...
assert len(indicators) == 3
cds1.add_data_sources([cds2])
indicator = cds1.get("indicator--00000000-0000-4000-8000-000000000001")
assert indicator["id"] == "indicator--00000000-0000-4000-8000-000000000001"
assert indicator["modified"] == parse_into_datetime("2017-01-31T13:49:53.935Z")
assert indicator["type"] == "indicator"
query1 = [
Filter("type", "=", "indicator"),
]
query2 = [
Filter("valid_from", "=", "2017-01-27T13:49:53.935382Z"),
]
cds1.filters.add(query2)
results = cds1.query(query1)
# STIX_OBJS2 has indicator with later time, one with different id, one with
# original time in STIX_OBJS1
assert len(results) == 4
indicator = cds1.get("indicator--00000000-0000-4000-8000-000000000001")
assert indicator["id"] == "indicator--00000000-0000-4000-8000-000000000001"
assert indicator["modified"] == parse_into_datetime("2017-01-31T13:49:53.935Z")
assert indicator["type"] == "indicator"
results = cds1.all_versions("indicator--00000000-0000-4000-8000-000000000001")
assert len(results) == 3
# Since we have filters already associated with our CompositeSource providing
# nothing returns the same as cds1.query(query1) (the associated query is query2)
results = cds1.query([])
assert len(results) == 4
def test_memory_store_query_empty_query(mem_store):
resp = mem_store.query()
# sort since returned in random order
resp = sorted(resp, key=lambda k: (k['id'], k['modified']))
assert len(resp) == 3
assert resp[0]['id'] == 'indicator--00000000-0000-4000-8000-000000000001'
assert resp[0]['modified'] == parse_into_datetime('2017-01-27T13:49:53.935Z')
assert resp[1]['id'] == 'indicator--00000000-0000-4000-8000-000000000001'
assert resp[1]['modified'] == parse_into_datetime('2017-01-27T13:49:53.936Z')
assert resp[2]['id'] == 'indicator--00000000-0000-4000-8000-000000000002'
assert resp[2]['modified'] == parse_into_datetime('2017-01-27T13:49:53.935Z')
def format_stix_to_db(stix_objs: List[Dict]) -> List[Dict]:
"""
Returns a list of stix objects formatted for the DB, also moves relationships to the end of the list
to make
:type stix_objs: List[Dict]
:rtype: List[Dict]
"""
db_stix: List[Dict] = []
relations: List[Dict] = []
for obj in stix_objs:
if obj['type'] == 'observed-data':
if 'objects' in obj and isinstance(obj['objects'], list):
obj['object_refs'] = []
for o in obj['objects']:
o['id_'] = f'{o["type"]}--{uuid.uuid4()}'
obj['object_refs'] = o['id_']
print(o)
db_stix.insert(0, o)
del obj['objects']
_ob = {}
for prop, val in obj.items():
if prop == 'id':
_ob['id_'] = val
elif prop == 'objects':
_ob['objects_'] = val
elif prop in [
'created', 'modified', 'last_seen', 'first_seen',
'valid_from', 'valid_until', 'first_observed',
'last_observed', 'published'
]:
#hacky way to fix incorrectly formatted STIX times
correct_format = True
try:
datetime.datetime.strptime(val, '%Y/%m/%dT%H:%M:%SZ')
except ValueError:
correct_format = False
if not val or len(val) == 0 or correct_format == False:
now = datetime.datetime.now(tz=pytz.UTC)
_ob[prop] = format_datetime(
parse_into_datetime(now, precision='millisecond'))
else:
_ob[prop] = format_datetime(
parse_into_datetime(val, precision='millisecond'))
# elif prop == 'objects':
# _ob['objects_'] = json.dumps(val)
else:
_ob[prop] = val
if _ob['type'] != 'relationship':
db_stix.append(_ob)
else:
relations.append(_ob)
return db_stix + relations
def test_datetime_filter_behavior():
"""if a filter is initialized with its value being a datetime object
OR the STIX object property being filtered on is a datetime object, all
resulting comparisons executed are done on the string representations
of the datetime objects, as the Filter functionality will convert
all datetime objects to there string forms using format_datetim()
This test makes sure all datetime comparisons are carried out correctly
"""
filter_with_dt_obj = Filter(
"created", "=",
parse_into_datetime("2016-02-14T00:00:00.000Z", "millisecond"))
filter_with_str = Filter("created", "=", "2016-02-14T00:00:00.000Z")
# compare datetime obj to filter w/ datetime obj
resp = list(apply_common_filters(real_stix_objs, [filter_with_dt_obj]))
assert len(resp) == 1
assert resp[0][
"id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef"
assert isinstance(resp[0].created,
STIXdatetime) # make sure original object not altered
# compare datetime string to filter w/ str
resp = list(apply_common_filters(stix_objs, [filter_with_str]))
assert len(resp) == 1
assert resp[0][
"id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef"
# compare datetime obj to filter w/ str
resp = list(apply_common_filters(real_stix_objs, [filter_with_str]))
assert len(resp) == 1
assert resp[0][
"id"] == "vulnerability--ee916c28-c7a4-4d0d-ad56-a8d357f89fef"
assert isinstance(resp[0].created,
STIXdatetime) # make sure original object not altered
def test_filters2(stix_objs2, real_stix_objs2):
# "Return any object modified after or on 2017-01-28T13:49:53.935Z"
resp = list(apply_common_filters(stix_objs2, [Filter("modified", ">=", "2017-01-27T13:49:53.935Z")]))
assert resp[0]['id'] == stix_objs2[0]['id']
assert len(resp) == 3
resp = list(apply_common_filters(real_stix_objs2, [Filter("modified", ">=", parse_into_datetime("2017-01-27T13:49:53.935Z"))]))
assert resp[0].id == real_stix_objs2[0].id
assert len(resp) == 3
def _timestamp2filename(timestamp):
"""
Encapsulates a way to create unique filenames based on an object's
"modified" property value. This should not include an extension.
Args:
timestamp: A timestamp, as a datetime.datetime object or string.
"""
# The format_datetime will determine the correct level of precision.
if isinstance(timestamp, str):
timestamp = parse_into_datetime(timestamp)
ts = format_datetime(timestamp)
ts = re.sub(r"[-T:\.Z ]", "", ts)
return ts
def test_filters3():
# "Return any object modified before or on 2017-01-28T13:49:53.935Z"
resp = list(
apply_common_filters(
STIX_OBJS2,
[Filter("modified", "<=", "2017-01-27T13:49:53.935Z")]))
assert resp[0]['id'] == STIX_OBJS2[1]['id']
assert len(resp) == 2
# "Return any object modified before or on 2017-01-28T13:49:53.935Z"
fv = Filter("modified", "<=",
parse_into_datetime("2017-01-27T13:49:53.935Z"))
resp = list(apply_common_filters(REAL_STIX_OBJS2, [fv]))
assert resp[0].id == REAL_STIX_OBJS2[1].id
assert len(resp) == 2
def test_filters1():
# "Return any object modified after 2017-01-28T13:49:53.935Z"
resp = list(
apply_common_filters(
STIX_OBJS2, [Filter("modified", ">", "2017-01-28T13:49:53.935Z")]))
assert resp[0]['id'] == STIX_OBJS2[0]['id']
assert len(resp) == 1
resp = list(
apply_common_filters(REAL_STIX_OBJS2, [
Filter("modified", ">",
parse_into_datetime("2017-01-28T13:49:53.935Z"))
]))
assert resp[0].id == REAL_STIX_OBJS2[0].id
assert len(resp) == 1
def test_parse_datetime(
us, precision, precision_constraint, expected_truncated_us,
):
# complete the datetime string with microseconds
dt_us_str = "{}.{:06d}Z".format(_DT_STR, us)
sdt = parse_into_datetime(
dt_us_str,
precision=precision,
precision_constraint=precision_constraint,
)
assert sdt.precision is precision
assert sdt.precision_constraint is precision_constraint
assert sdt.microsecond == expected_truncated_us
def new_version(data, allow_custom=None, **kwargs):
"""
Create a new version of a STIX object, by modifying properties and
updating the ``modified`` property.
:param data: The object to create a new version of. Maybe a stix2 object
or dict.
:param allow_custom: Whether to allow custom properties on the new object.
If True, allow them (regardless of whether the original had custom
properties); if False disallow them; if None, propagate the preference
from the original object.
:param kwargs: The properties to change. Setting to None requests property
removal.
:return: The new object.
"""
is_versionable, is_21 = _is_versionable(data)
if not is_versionable:
raise ValueError(
"cannot create new version of object of this type! "
"Try a dictionary or instance of an SDO or SRO class.", )
if data.get('revoked'):
raise RevokeError("new_version")
try:
new_obj_inner = copy.deepcopy(data._inner)
except AttributeError:
new_obj_inner = copy.deepcopy(data)
# Make sure certain properties aren't trying to change
# ID contributing properties of 2.1+ SCOs may also not change if a UUIDv5
# is in use (depending on whether they were used to create it... but they
# probably were). That would imply an ID change, which is not allowed
# across versions.
sco_locked_props = []
if is_21 and isinstance(data, stix2.base._Observable):
uuid_ = uuid.UUID(data["id"][-36:])
if uuid_.variant == uuid.RFC_4122 and uuid_.version == 5:
sco_locked_props = data._id_contributing_properties
unchangable_properties = set()
for prop in itertools.chain(STIX_UNMOD_PROPERTIES, sco_locked_props):
if prop in kwargs:
unchangable_properties.add(prop)
if unchangable_properties:
raise UnmodifiablePropertyError(unchangable_properties)
# Different versioning precision rules in STIX 2.0 vs 2.1, so we need
# to know which rules to apply.
precision_constraint = "min" if is_21 else "exact"
cls = type(data)
if 'modified' not in kwargs:
old_modified = parse_into_datetime(
data["modified"],
precision="millisecond",
precision_constraint=precision_constraint,
)
new_modified = get_timestamp()
new_modified = _fudge_modified(old_modified, new_modified, is_21)
kwargs['modified'] = new_modified
elif 'modified' in data:
old_modified_property = parse_into_datetime(
data.get('modified'),
precision='millisecond',
precision_constraint=precision_constraint,
)
new_modified_property = parse_into_datetime(
kwargs['modified'],
precision='millisecond',
precision_constraint=precision_constraint,
)
if new_modified_property <= old_modified_property:
raise InvalidValueError(
cls,
'modified',
"The new modified datetime cannot be before than or equal to the current modified datetime."
"It cannot be equal, as according to STIX 2 specification, objects that are different "
"but have the same id and modified timestamp do not have defined consumer behavior.",
)
new_obj_inner.update(kwargs)
# Set allow_custom appropriately if versioning an object. We will ignore
# it for dicts.
if isinstance(data, stix2.base._STIXBase):
if allow_custom is None:
new_obj_inner["allow_custom"] = data._allow_custom
else:
new_obj_inner["allow_custom"] = allow_custom
# Exclude properties with a value of 'None' in case data is not an instance of a _STIXBase subclass
return cls(**{k: v for k, v in new_obj_inner.items() if v is not None})
def get_datetime() -> str:
"""Get date time
:rtype: str
"""
now = datetime.datetime.now(tz=pytz.UTC)
return format_datetime(parse_into_datetime(now, precision='millisecond'))
