def test_valid_snapshot_envelope_encoding(self):
encoded_envelope_indicator = json.dumps(
self.snapshot_envelope_indicator, cls=SnapshotEnvelopeEncoder)
encoded_envelope_sighting = json.dumps(self.snapshot_envelope_sighting,
cls=SnapshotEnvelopeEncoder)
# encoding cannot be compared as string, because order of the fileds in
# the string representation is not fixed.
# We cast the JSON back to a simple python dict and them compare values
py_dict = json.loads(encoded_envelope_indicator)
self.assertEqual(py_dict["snapshot_type"], MessageType.INDICATOR.value)
self.assertEqual(py_dict["snapshot_id"], self.snapshot_id)
self.assertEqual(py_dict["body"]["created"],
format_datetime(self.created))
self.assertEqual(py_dict["body"]["id"], self.indicator_id)
self.assertEqual(py_dict["body"]["pattern"], self.pattern)
self.assertEqual(py_dict["body"]["pattern_type"], self.pattern_type)
self.assertEqual(py_dict["type"], SnapshotEnvelope.__name__.lower())
py_dict = json.loads(encoded_envelope_sighting)
self.assertEqual(py_dict["snapshot_type"], MessageType.SIGHTING.value)
self.assertEqual(py_dict["snapshot_id"], self.snapshot_id)
self.assertEqual(py_dict["body"]["created"],
format_datetime(self.created))
self.assertEqual(py_dict["body"]["sighting_of_ref"], self.indicator_id)
self.assertEqual(
py_dict["body"][
ThreatBusSTIX2Constants.X_THREATBUS_SIGHTING_CONTEXT.value],
self.sighting_context,
)
self.assertEqual(py_dict["type"], SnapshotEnvelope.__name__.lower())
def format_stix_to_db(stix_objs: List[Dict]) -> List[Dict]:
"""
Returns a list of stix objects formatted for the DB, also moves relationships to the end of the list
to make
:type stix_objs: List[Dict]
:rtype: List[Dict]
"""
db_stix: List[Dict] = []
relations: List[Dict] = []
for obj in stix_objs:
if obj['type'] == 'observed-data':
if 'objects' in obj and isinstance(obj['objects'], list):
obj['object_refs'] = []
for o in obj['objects']:
o['id_'] = f'{o["type"]}--{uuid.uuid4()}'
obj['object_refs'] = o['id_']
print(o)
db_stix.insert(0, o)
del obj['objects']
_ob = {}
for prop, val in obj.items():
if prop == 'id':
_ob['id_'] = val
elif prop == 'objects':
_ob['objects_'] = val
elif prop in [
'created', 'modified', 'last_seen', 'first_seen',
'valid_from', 'valid_until', 'first_observed',
'last_observed', 'published'
]:
#hacky way to fix incorrectly formatted STIX times
correct_format = True
try:
datetime.datetime.strptime(val, '%Y/%m/%dT%H:%M:%SZ')
except ValueError:
correct_format = False
if not val or len(val) == 0 or correct_format == False:
now = datetime.datetime.now(tz=pytz.UTC)
_ob[prop] = format_datetime(
parse_into_datetime(now, precision='millisecond'))
else:
_ob[prop] = format_datetime(
parse_into_datetime(val, precision='millisecond'))
# elif prop == 'objects':
# _ob['objects_'] = json.dumps(val)
else:
_ob[prop] = val
if _ob['type'] != 'relationship':
db_stix.append(_ob)
else:
relations.append(_ob)
return db_stix + relations
def _check_property(self, stix_obj_property):
"""Check a property of a STIX Object against this filter.
Args:
stix_obj_property: value to check this filter against
Returns:
True if property matches the filter,
False otherwise.
"""
if isinstance(stix_obj_property, datetime):
# if a datetime obj, convert to str format before comparison
# NOTE: this check seems like it should be done upstream
# but will put here for now
stix_obj_property = format_datetime(stix_obj_property)
if self.op == "=":
return stix_obj_property == self.value
elif self.op == "!=":
return stix_obj_property != self.value
elif self.op == "in":
return stix_obj_property in self.value
elif self.op == ">":
return stix_obj_property > self.value
elif self.op == "<":
return stix_obj_property < self.value
elif self.op == ">=":
return stix_obj_property >= self.value
elif self.op == "<=":
return stix_obj_property <= self.value
else:
raise ValueError(
"Filter operator: {0} not supported for specified property: {1}"
.format(self.op, self.property))
def test_format_datetime(us, precision, precision_constraint, expected_us_str):
dt = _DT.replace(microsecond=us)
expected_dt_str = "{}{}Z".format(_DT_STR, expected_us_str)
sdt = STIXdatetime(
dt,
precision=precision,
precision_constraint=precision_constraint,
)
s = format_datetime(sdt)
assert s == expected_dt_str
def _timestamp2filename(timestamp):
"""
Encapsulates a way to create unique filenames based on an object's
"modified" property value. This should not include an extension.
Args:
timestamp: A timestamp, as a datetime.datetime object.
"""
# The format_datetime will determine the correct level of precision.
ts = format_datetime(timestamp)
ts = re.sub(r"[-T:\.Z ]", "", ts)
return ts
def __new__(cls, prop, op, value):
# If value is a list, convert it to a tuple so it is hashable.
if isinstance(value, list):
value = tuple(value)
if isinstance(value, datetime):
# if value is a datetime obj, convert to str
value = format_datetime(value)
_check_filter_components(prop, op, value)
self = super(Filter, cls).__new__(cls, prop, op, value)
return self
def check_tlp_marking(marking_obj, spec_version):
# Specific TLP Marking validation case.
if marking_obj["definition_type"] == "tlp":
color = marking_obj["definition"]["tlp"]
if color == "white":
if spec_version == '2.0':
w = (
'{"created": "2017-01-20T00:00:00.000Z", "definition": {"tlp": "white"}, "definition_type": "tlp",'
' "id": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", "type": "marking-definition"}'
)
else:
w = (
'{"created": "2017-01-20T00:00:00.000Z", "definition": {"tlp": "white"}, "definition_type": "tlp",'
' "id": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", "type": "marking-definition",'
' "spec_version": "2.1"}')
if marking_obj[
"id"] != "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9":
raise exceptions.TLPMarkingDefinitionError(
marking_obj["id"], w)
elif utils.format_datetime(
marking_obj["created"]) != "2017-01-20T00:00:00.000Z":
raise exceptions.TLPMarkingDefinitionError(
utils.format_datetime(marking_obj["created"]), w)
elif color == "green":
if spec_version == '2.0':
g = (
'{"created": "2017-01-20T00:00:00.000Z", "definition": {"tlp": "green"}, "definition_type": "tlp",'
' "id": "marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da", "type": "marking-definition"}'
)
else:
g = (
'{"created": "2017-01-20T00:00:00.000Z", "definition": {"tlp": "green"}, "definition_type": "tlp",'
' "id": "marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da", "type": "marking-definition",'
' "spec_version": "2.1"}')
if marking_obj[
"id"] != "marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da":
raise exceptions.TLPMarkingDefinitionError(
marking_obj["id"], g)
elif utils.format_datetime(
marking_obj["created"]) != "2017-01-20T00:00:00.000Z":
raise exceptions.TLPMarkingDefinitionError(
utils.format_datetime(marking_obj["created"]), g)
elif color == "amber":
if spec_version == '2.0':
a = (
'{"created": "2017-01-20T00:00:00.000Z", "definition": {"tlp": "amber"}, "definition_type": "tlp",'
' "id": "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82", "type": "marking-definition"}'
)
else:
a = (
'{"created": "2017-01-20T00:00:00.000Z", "definition": {"tlp": "amber"}, "definition_type": "tlp",'
' "id": "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82", "type": "marking-definition",'
' "spec_version": "2.1"}')
if marking_obj[
"id"] != "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82":
raise exceptions.TLPMarkingDefinitionError(
marking_obj["id"], a)
elif utils.format_datetime(
marking_obj["created"]) != "2017-01-20T00:00:00.000Z":
raise exceptions.TLPMarkingDefinitionError(
utils.format_datetime(marking_obj["created"]), a)
elif color == "red":
if spec_version == '2.0':
r = (
'{"created": "2017-01-20T00:00:00.000Z", "definition": {"tlp": "red"}, "definition_type": "tlp",'
' "id": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", "type": "marking-definition"}'
)
else:
r = (
'{"created": "2017-01-20T00:00:00.000Z", "definition": {"tlp": "red"}, "definition_type": "tlp",'
' "id": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", "type": "marking-definition",'
' "spec_version": "2.1"}')
if marking_obj[
"id"] != "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed":
raise exceptions.TLPMarkingDefinitionError(
marking_obj["id"], r)
elif utils.format_datetime(
marking_obj["created"]) != "2017-01-20T00:00:00.000Z":
raise exceptions.TLPMarkingDefinitionError(
utils.format_datetime(marking_obj["created"]), r)
else:
raise exceptions.TLPMarkingDefinitionError(
marking_obj["id"], "Does not match any TLP Marking definition")
def get_datetime() -> str:
"""Get date time
:rtype: str
"""
now = datetime.datetime.now(tz=pytz.UTC)
return format_datetime(parse_into_datetime(now, precision='millisecond'))