def test_bundle_objs_ids_found(bundle_data):
bundle = stix2.parse(bundle_data)
mal_list = bundle.get_obj("malware--00000000-0000-4000-8000-000000000003")
assert bundle.objects[1] == mal_list[0]
assert bundle.objects[2] == mal_list[1]
assert len(mal_list) == 2
async def match_intel(
vast_binary: str,
vast_endpoint: str,
indicator_queue: asyncio.Queue,
sightings_queue: asyncio.Queue,
live_match: bool,
retro_match: bool,
retro_match_max_events: int,
retro_match_timeout: float,
):
"""
Reads from the indicator_queue and matches all IoCs, either via VAST's
live-matching or retro-matching.
@param vast_binary The vast binary command to use with PyVAST
@param vast_endpoint The endpoint of a running vast node ('host:port')
@param indicator_queue The queue to read new IoCs from
@param sightings_queue The queue to put new sightings into
@param live_match Boolean flag to use retro-matching
@param retro_match Boolean flag to use live-matching
@param retro_match_max_events Max amount of retro match results
@param retro_match_timeout Interval after which to terminate the retro-query
"""
global logger, open_tasks
while True:
msg = await indicator_queue.get()
try:
indicator = parse(msg, allow_custom=True)
except Exception as e:
logger.warning(
f"Failed to decode STIX-2 Indicator item {msg}: {e}")
continue
if type(indicator) is not Indicator:
logger.warning(
f"Ignoring unknown message type, expected STIX-2 Indicator: {type(indicator)}"
)
continue
if (ThreatBusSTIX2Constants.X_THREATBUS_UPDATE.value in indicator
and indicator.x_threatbus_update == Operation.REMOVE.value):
g_iocs_removed.inc()
if live_match:
asyncio.create_task(
remove_vast_ioc(vast_binary, vast_endpoint, indicator))
else:
# add new Indicator to matcher / query Indicator retrospectively
g_iocs_added.inc()
if retro_match:
g_retro_match_backlog.inc()
asyncio.create_task(
retro_match_vast(
vast_binary,
vast_endpoint,
retro_match_max_events,
retro_match_timeout,
indicator,
sightings_queue,
))
if live_match:
asyncio.create_task(
ingest_vast_ioc(vast_binary, vast_endpoint, indicator))
indicator_queue.task_done()
def test_parse_report(data):
rept = stix2.parse(data, version="2.0")
assert rept.type == 'report'
assert rept.id == REPORT_ID
assert rept.created == dt.datetime(2015,
12,
21,
19,
59,
11,
tzinfo=pytz.utc)
assert rept.modified == dt.datetime(2015,
12,
21,
19,
59,
11,
tzinfo=pytz.utc)
assert rept.created_by_ref == "identity--a463ffb3-1bd9-4d94-b02d-74e4f1658283"
assert rept.object_refs == [
"indicator--26ffb872-1dd9-446e-b6f5-d58527e5b5d2",
"campaign--83422c77-904c-4dc1-aff5-5c38f3a2c55c",
"relationship--f82356ae-fe6c-437c-9c24-6b64314ae68a",
]
assert rept.description == "A simple report with an indicator and campaign"
assert rept.labels == ["campaign"]
assert rept.name == "The Black Vine Cyberespionage Group"
def test_parse_autonomous_system_valid(data):
odata_str = OBJECTS_REGEX.sub('"objects": { %s }' % data, EXPECTED)
odata = stix2.parse(odata_str, version="2.1")
assert odata.objects["0"].type == "autonomous-system"
assert odata.objects["0"].number == 15139
assert odata.objects["0"].name == "Slime Industries"
assert odata.objects["0"].rir == "ARIN"
def slide_file(fn, encoding="utf-8"):
cybox.utils.caches.cache_clear()
setup_logger(fn)
validator_options = get_validator_options()
with io.open(fn, "r", encoding=encoding) as json_data:
json_content = json.load(json_data)
obj = stix2.parse(json_content, allow_custom=True)
stix_package = convert_bundle(obj)
if stix_package:
xml = stix_package.to_xml(encoding=None)
validator_options.in_files = io.StringIO(xml)
try:
scripts.set_output_level(validator_options)
validation_results = scripts.validate_file(
validator_options.in_files, validator_options)
results = {stix_package.id_: validation_results}
# Print stix-validator results
scripts.print_results(results, validator_options)
except (errors.ValidationError, IOError) as ex:
scripts.error("Validation error occurred: '%s'" % str(ex),
codes.EXIT_VALIDATION_ERROR)
except Exception:
log.exception("Fatal error occurred", extra={'ecode': 0})
sys.exit(codes.EXIT_FAILURE)
return xml
def test_parse_observed_data(data):
odata = stix2.parse(data)
assert odata.type == 'observed-data'
assert odata.id == OBSERVED_DATA_ID
assert odata.created == dt.datetime(2016,
4,
6,
19,
58,
16,
tzinfo=pytz.utc)
assert odata.modified == dt.datetime(2016,
4,
6,
19,
58,
16,
tzinfo=pytz.utc)
assert odata.first_observed == dt.datetime(2015,
12,
21,
19,
0,
0,
tzinfo=pytz.utc)
assert odata.last_observed == dt.datetime(2015,
12,
21,
19,
0,
0,
tzinfo=pytz.utc)
assert odata.created_by_ref == "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff"
assert odata.objects["0"].type == "file"
def loadEvent(self, args, pathname):
try:
filename = os.path.join(pathname, args[1])
tempFile = open(filename, 'r', encoding='utf-8')
self.filename = filename
event = stix2.get_dict(tempFile)
self.stix_version = 'stix {}'.format(event.get('spec_version'))
for o in event.get('objects'):
try:
try:
self.event.append(stix2.parse(o))
except:
self.parse_custom(o)
except:
pass
if not self.event:
print(
json.dumps({
'success':
0,
'message':
'There is no valid STIX object to import'
}))
sys.exit(1)
except:
print(
json.dumps({
'success': 0,
'message': 'The STIX file could not be read'
}))
sys.exit(1)
def slide_string(string):
cybox.utils.caches.cache_clear()
obj = stix2.parse(string)
setup_logger(obj["id"])
validator_options = get_validator_options()
stix_package = convert_bundle(obj)
if stix_package:
xml = stix_package.to_xml(encoding=None)
validator_options.in_files = io.StringIO(xml)
try:
scripts.set_output_level(validator_options)
validation_results = scripts.validate_file(
validator_options.in_files, validator_options)
results = {stix_package.id_: validation_results}
# Print stix-validator results
scripts.print_results(results, validator_options)
except (errors.ValidationError, IOError) as ex:
scripts.error("Validation error occurred: '%s'" % str(ex),
codes.EXIT_VALIDATION_ERROR)
except Exception:
log.exception("Fatal error occurred", extra={'ecode': 0})
sys.exit(codes.EXIT_FAILURE)
return xml
def test_bundle_getitem_overload_obj_id_not_found():
bundle = stix2.parse(EXPECTED_BUNDLE)
with pytest.raises(KeyError) as excinfo:
bundle['non existent']
assert "neither a property on the bundle nor does it match the id property" in str(
excinfo.value)
def test_import(self):
# That an imported campaign object
with open('test_camp.json') as fp:
camp = stix2.parse(fp.read())
# and an imported translation object
with open('test_trans.json') as fp:
transobj = stix2.parse(fp.read())
# when added to a stixlangobj:
o = stixlangwrap(['th', 'en'], camp)
o.addtranslationobject(transobj)
# work as expected
self.assertEqual(o.getlangtext('description'),
('th', 'a Thai description'))
def test_filesystem_sink_json_stix_bundle(fs_sink, fs_source):
# add json-encoded stix bundle
bund2 = '{"type": "bundle", "id": "bundle--3d267103-8475-4d8f-b321-35ec6eccfa37",' \
' "spec_version": "2.0", "objects": [{"type": "campaign", "id": "campaign--2c03b8bf-82ee-433e-9918-ca2cb6e9534b",' \
' "created":"2017-05-31T21:31:53.197755Z",'\
' "modified":"2017-05-31T21:31:53.197755Z",'\
' "name": "Spartacus", "objective": "Oppressive regimes of Africa and Middle East"}]}'
fs_sink.add(bund2)
bund2obj = stix2.parse(bund2)
camp_obj = bund2obj["objects"][0]
filepath = os.path.join(
FS_PATH,
"campaign",
"campaign--2c03b8bf-82ee-433e-9918-ca2cb6e9534b",
_timestamp2filename(camp_obj["modified"]) + ".json",
)
assert os.path.exists(filepath)
camp5_r = fs_source.get("campaign--2c03b8bf-82ee-433e-9918-ca2cb6e9534b")
assert camp5_r.id == "campaign--2c03b8bf-82ee-433e-9918-ca2cb6e9534b"
assert camp5_r.name == "Spartacus"
os.remove(filepath)
def test_bundle_obj_id_not_found():
bundle = stix2.parse(EXPECTED_BUNDLE)
with pytest.raises(KeyError) as excinfo:
bundle.get_obj('non existent')
assert "does not match the id property of any of the bundle" in str(
excinfo.value)
def test_filesystem_sink_add_stix_bundle_dict(fs_sink, fs_source):
# add stix bundle dict
bund = {
"type": "bundle",
"id": "bundle--040ae5ec-2e91-4e94-b075-bc8b368e8ca3",
"spec_version": "2.0",
"objects": [
{
"name": "Atilla",
"type": "campaign",
"objective": "Bulgarian, Albanian and Romanian Intelligence Services",
"aliases": ["Huns"],
"id": "campaign--b8f86161-ccae-49de-973a-4ca320c62478",
"created": "2017-05-31T21:31:53.197755Z",
"modified": "2017-05-31T21:31:53.197755Z",
},
],
}
fs_sink.add(bund)
camp_obj = stix2.parse(bund["objects"][0])
filepath = os.path.join(
FS_PATH, "campaign", camp_obj["id"],
_timestamp2filename(camp_obj["modified"]) + ".json",
)
assert os.path.exists(filepath)
camp3_r = fs_source.get(bund["objects"][0]["id"])
assert camp3_r.id == bund["objects"][0]["id"]
assert camp3_r.name == bund["objects"][0]["name"]
assert "Huns" in camp3_r.aliases
os.remove(filepath)
def test_filesystem_sink_add_stix_object_dict(fs_sink, fs_source):
# add stix object dict
camp2 = {
"name": "Aurelius",
"type": "campaign",
"objective": "German and French Intelligence Services",
"aliases": ["Purple Robes"],
"id": "campaign--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f",
"created": "2017-05-31T21:31:53.197755Z",
"modified": "2017-05-31T21:31:53.197755Z",
}
fs_sink.add(camp2)
# Need to get the exact "modified" timestamp which would have been
# in effect at the time the object was saved to the sink, which determines
# the filename it would have been saved as. It may not be exactly the same
# as what's in the dict, since the parsing process can enforce a precision
# constraint (e.g. truncate to milliseconds), which results in a slightly
# different name.
camp2obj = stix2.parse(camp2)
filepath = os.path.join(
FS_PATH, "campaign", camp2obj["id"],
_timestamp2filename(camp2obj["modified"]) + ".json",
)
assert os.path.exists(filepath)
camp2_r = fs_source.get(camp2["id"])
assert camp2_r.id == camp2["id"]
assert camp2_r.name == camp2["name"]
assert "Purple Robes" in camp2_r.aliases
os.remove(filepath)
def test_parse_report(data):
rept = stix2.parse(data, version="2.1")
assert rept.type == 'report'
assert rept.spec_version == '2.1'
assert rept.id == REPORT_ID
assert rept.created == dt.datetime(2015,
12,
21,
19,
59,
11,
tzinfo=pytz.utc)
assert rept.modified == dt.datetime(2015,
12,
21,
19,
59,
11,
tzinfo=pytz.utc)
assert rept.created_by_ref == IDENTITY_ID
assert rept.object_refs == [
INDICATOR_ID,
CAMPAIGN_ID,
RELATIONSHIP_ID,
]
assert rept.description == "A simple report with an indicator and campaign"
assert rept.report_types == ["campaign"]
assert rept.name == "The Black Vine Cyberespionage Group"
def test_parse_opinion(data):
opinion = stix2.parse(data, version="2.1")
assert opinion.type == 'opinion'
assert opinion.spec_version == '2.1'
assert opinion.id == OPINION_ID
assert opinion.created == dt.datetime(2016,
5,
12,
8,
17,
27,
tzinfo=pytz.utc)
assert opinion.modified == dt.datetime(2016,
5,
12,
8,
17,
27,
tzinfo=pytz.utc)
assert opinion.opinion == 'strongly-disagree'
assert opinion.object_refs[
0] == 'relationship--16d2358f-3b0d-4c88-b047-0da2f7ed4471'
assert opinion.explanation == EXPLANATION
rep = re.sub(r"(\[|=| )u('|\"|\\\'|\\\")", r"\g<1>\g<2>", repr(opinion))
assert rep == EXPECTED_OPINION_REPR
async def transform_context(sighting: Sighting, transform_cmd: str) -> Sighting:
"""
Transforms the context of a sighting using the command configured in
`transform_context`
@param sighting the sighting as it was reported by VAST
@param transform_cmd The command to use to pipe sightings to. Treated
as template string: occurrences of '%ioc' in the cmd string get replaced
with the matched IoC.
@return a copy of the original sighting with the x_threatbus_context field
set and transformed accordingly
"""
context = (
sighting.x_threatbus_sighting_context
if ThreatBusSTIX2Constants.X_THREATBUS_SIGHTING_CONTEXT.value
in sighting.object_properties()
else None
)
if not context:
logger.error(
f"Cannot invoke `transform_context` command because no context data is found in the sighting {sighting}"
)
return
indicator = (
sighting.x_threatbus_indicator
if ThreatBusSTIX2Constants.X_THREATBUS_INDICATOR.value
in sighting.object_properties()
else None
)
if indicator:
_, ioc_value = split_object_path_and_value(indicator.pattern)
else:
# try to find the indicator value instead
ioc_value = (
sighting.x_threatbus_indicator_value
if ThreatBusSTIX2Constants.X_THREATBUS_INDICATOR_VALUE.value
in sighting.object_properties()
else None
)
if not ioc_value:
logger.error(
f"Cannot invoke `transform_context` command because no indicator value is found in the sighting {sighting}"
)
return
transformed_context_raw = await invoke_cmd_for_context(
transform_cmd, context, ioc_value
)
try:
transformed_context = json.loads(transformed_context_raw)
# recreate the sighting with the new transformed context
ser = json.loads(sighting.serialize())
ser[
ThreatBusSTIX2Constants.X_THREATBUS_SIGHTING_CONTEXT.value
] = transformed_context
return parse(json.dumps(ser), allow_custom=True)
except Exception as e:
logger.error(
f"Cannot parse transformed sighting context (expecting JSON): {transformed_context_raw}",
e,
)
def test_custom_subobject_obj():
ident = stix2.v20.Identity(
name="alice", identity_class=123, x_foo=123, allow_custom=True,
)
obj_dict = {
"type": "bundle",
"spec_version": "2.0",
"objects": [ident],
}
obj = stix2.parse(obj_dict, allow_custom=True)
assert obj["objects"][0]["x_foo"] == 123
assert obj.has_custom
with pytest.raises(InvalidValueError):
stix2.parse(obj_dict, allow_custom=False)
def test_parse_identity(data):
identity = stix2.parse(data, version="2.0")
assert identity.type == 'identity'
assert identity.id == IDENTITY_ID
assert identity.created == dt.datetime(2015, 12, 21, 19, 59, 11, tzinfo=pytz.utc)
assert identity.modified == dt.datetime(2015, 12, 21, 19, 59, 11, tzinfo=pytz.utc)
assert identity.name == "John Smith"
def slide_string(string):
obj = stix2.parse(string)
setup_logger(obj["id"])
stix_package = convert_bundle(obj)
if stix_package:
return stix_package.to_xml(encoding=None)
def test_parse_marking_definition(data):
gm = stix2.parse(data)
assert gm.type == 'marking-definition'
assert gm.id == MARKING_DEFINITION_ID
assert gm.created == dt.datetime(2017, 1, 20, 0, 0, 0, tzinfo=pytz.utc)
assert gm.definition.tlp == "white"
assert gm.definition_type == "tlp"
def test_parse_custom_observable_object():
nt_string = """{
"type": "x-new-observable",
"property1": "something"
}"""
nt = stix2.parse(nt_string, [], version='2.1')
assert isinstance(nt, stix2.base._STIXBase)
assert nt.property1 == 'something'
def test_granularnomarkings(self):
with open('test_grannolang.json') as fp:
campobj = stix2.parse(fp.read())
o = stixlangwrap('en', campobj)
self.assertEqual(o.getlangtext('description'),
('en', u'Weitere Informationen über Banküberfall'))
def test_malware_with_os_refs():
software = stix2.parse({
"type": "software",
"name": "SuperOS",
"spec_version": "2.1",
})
malware = stix2.parse({
"type": "malware",
"id": MALWARE_ID,
"spec_version": "2.1",
"is_family": False,
"malware_types": ["something"],
"operating_system_refs": [software],
})
assert malware["operating_system_refs"][0] == software["id"]
def test_parse_unknown_type():
unknown = {
"type": "other",
"id": "other--8e2e2d2b-17d4-4cbf-938f-98ee46b3cd3f",
"created": "2016-04-06T20:03:00Z",
"modified": "2016-04-06T20:03:00Z",
"created_by_ref": IDENTITY_ID,
"description":
"Campaign by Green Group against a series of targets in the financial services sector.",
"name": "Green Group Attacks Against Finance",
}
with pytest.raises(stix2.exceptions.ParseError) as excinfo:
stix2.parse(unknown, version="2.0")
assert str(
excinfo.value
) == "Can't parse unknown object type 'other'! For custom types, use the CustomObject decorator."
def test_generation_min_props(generator_min_props, spec_name):
obj_dict = generator_min_props.generate(spec_name)
# Ensure json-serializability
json.dumps(obj_dict, ensure_ascii=False)
# Distinguish between a STIX object spec and a "helper" spec used
# by STIX object specs. Only makes sense to stix2.parse() the former.
if spec_name[0].isupper():
try:
stix2.parse(obj_dict, version="2.1")
except stix2.exceptions.ParseError:
# Maybe we can use this to mean this was an SCO?
# Try a re-parse as an SCO. Need a better way to make the
# distinction...
stix2.parse_observable(obj_dict, version="2.1")
def test_parse_custom_object_type():
nt_string = """{
"type": "x-new-type",
"created": "2015-12-21T19:59:11Z",
"property1": "something"
}"""
nt = stix2.parse(nt_string, allow_custom=True)
assert nt["property1"] == 'something'
def test_parse_bundle(version):
bundle = stix2.parse(EXPECTED_BUNDLE, version=version)
assert bundle.type == "bundle"
assert bundle.id.startswith("bundle--")
assert type(bundle.objects[0]) is stix2.v20.Indicator
assert bundle.objects[0].type == 'indicator'
assert bundle.objects[1].type == 'malware'
assert bundle.objects[2].type == 'relationship'
def test_parse_malware(data):
mal = stix2.parse(data, version="2.0")
assert mal.type == 'malware'
assert mal.id == MALWARE_ID
assert mal.created == dt.datetime(2016, 5, 12, 8, 17, 27, tzinfo=pytz.utc)
assert mal.modified == dt.datetime(2016, 5, 12, 8, 17, 27, tzinfo=pytz.utc)
assert mal.labels == ['ransomware']
assert mal.name == "Cryptolocker"
def test_parse_basic_tcp_traffic(data):
odata = stix2.parse(
data, version='2.1',
)
assert odata.type == "network-traffic"
assert odata.src_ref == "ipv4-addr--e535b017-cc1c-566b-a3e2-f69f92ed9c4c"
assert odata.dst_ref == "ipv4-addr--78327430-9ad9-5632-ae3d-8e2fce8f5483"
assert odata.protocols == ["tcp"]
