def main():
directory = os.path.dirname(__file__)
if len(sys.argv) > 1:
json_dir = sys.argv[1]
else:
json_dir = find_dir(directory, "idioms-json")
if len(sys.argv) > 2:
xml_dir = sys.argv[2]
else:
xml_dir = find_dir(directory, "idioms-xml")
if not os.path.exists(json_dir):
os.makedirs(json_dir)
for filename in os.listdir(xml_dir):
file_and_ext = filename.split(".")
xml_path = os.path.join(xml_dir, filename)
destination = os.path.join(json_dir, file_and_ext[0] + ".json")
destination = os.path.abspath(destination)
initialize_options()
if file_and_ext[1] == "xml":
sys.stdout.write(xml_path + "\n")
json_output = elevate_file(xml_path)
output_file = open(destination, "w")
output_file.write(json_output)
output_file.close()
def main():
elevator_parser = _get_arg_parser(False)
elevator_parser.add_argument(
"dir_",
help="A directory contaning STIX 1.x documents to be elevated.",
metavar="dir")
elevator_parser.add_argument("--output-directory",
help="output logs",
dest="output_directory",
action="store",
default=None)
elevator_args = elevator_parser.parse_args()
initialize_options(elevator_args)
for filename in os.listdir(elevator_args.dir_):
path = os.path.join(elevator_args.dir_, filename)
if path.endswith(".xml"):
sys.stdout.write(path + "\n")
file_and_ext = filename.split(".")
set_option_value("file_", file_and_ext[0])
result = elevate_file(path)
if elevator_args.output_directory:
destination = os.path.join(elevator_args.output_directory,
file_and_ext[0] + ".json")
destination = os.path.abspath(destination)
output_file = open(destination, "w")
output_file.write(result)
output_file.close()
else:
print(result + "\n")
def idiom_mappings(xml_file_path, stored_json):
"""Test fresh conversion from XML to JSON matches stored JSON samples."""
print("Checking - " + xml_file_path)
initialize_options()
converted_json = elevate_file(xml_file_path)
io = StringIO(converted_json)
converted_json = json.load(io)
for good, to_check in zip(iterpath(stored_json), iterpath(converted_json)):
good_path, good_value = good
last_good_field = good_path[-1]
if isinstance(good_value, (dict, list)):
# No need to verify iterable types. Since we will deal
# with individual values in the future.
continue
if last_good_field in IGNORE:
# Since fresh conversion may create dynamic values.
# Some fields are omitted for verification. Currently
# fields with: identifier and timestamp values.
continue
yield good, to_check
def main():
# Parse stix-elevator command-line args
elevator_parser = _get_arg_parser()
elevator_args = elevator_parser.parse_args()
initialize_options(elevator_args)
print(elevate_file(elevator_args.file_))
def main():
# elevate the file & return into a new file
initialize_options()
results = elevate_file(testfrag)
fi = open("testcache2.json", "w")
fi.write(results)
def main():
directory = os.path.dirname(__file__)
if len(sys.argv) > 1:
json_dir = sys.argv[1]
else:
json_dir = find_dir(directory, "idioms-json")
if len(sys.argv) > 2:
xml_dir = sys.argv[2]
else:
xml_dir = find_dir(directory, "idioms-xml")
if not os.path.exists(json_dir):
os.makedirs(json_dir)
sys.setrecursionlimit(2000)
for filename in sorted(os.listdir(xml_dir)):
file_and_ext = filename.split(".")
xml_path = os.path.join(xml_dir, filename)
destination = os.path.join(json_dir, file_and_ext[0] + ".json")
destination = os.path.abspath(destination)
initialize_options()
if file_and_ext[1] == "xml":
sys.stdout.write(xml_path + "\n")
json_output = elevate_file(xml_path)
with io.open(destination, "w", encoding="utf-8") as f:
f.write(json_output)
def _process_message(self, data: Dict) -> str:
file_fetch = data["file_fetch"]
file_uri = self.helper.opencti_url + file_fetch
self.helper.log_info(f"Importing the file {file_uri}")
file_content = self.helper.api.fetch_opencti_file(file_uri)
if data["file_mime"] == "text/xml":
self.helper.log_debug("Stix1 file. Attempting conversion")
initialize_options()
file_content = elevate(file_content)
entity_id = data.get("entity_id", None)
if entity_id:
self.helper.log_debug("Contextual import.")
bundle = parse(file_content)["objects"]
if self._contains_report(bundle):
self.helper.log_debug("Bundle contains report.")
else:
self.helper.log_debug("No Report in Stix file. Updating current report")
bundle = self._update_report(bundle, entity_id)
file_content = Bundle(objects=bundle).serialize()
bundles_sent = self.helper.send_stix2_bundle(file_content)
return "Sent " + str(len(bundles_sent)) + " stix bundle(s) for worker import"
def test_setup_options(opts):
options.ALL_OPTIONS = None # To make sure we can set it again
initialize_options(opts)
assert get_option_value("policy") == "no_policy"
assert get_option_value("spec_version") == "2.1"
assert get_option_value("log_level") == "DEBUG"
assert get_option_value("disabled") == [212, 901]
def elevatefile(self, filename):
"""
Convert a STIX 1 XML file to a STIX 2 JSON
:param filename:
"""
initialize_options()
with open(filename, 'r') as stixfile:
results = elevate(stixfile.read())
print(results)
def main():
for filename in os.listdir(sys.argv[1]):
path = os.path.join(sys.argv[1], filename)
initialize_options()
set_option_value("incidents", False)
if path.endswith(".xml"):
sys.stdout.write(path + "\n")
print(elevate_file(path) + "\n")
def setup_options():
version = get_environment_variable_value('VERSION', "2.1")
policy = get_environment_variable_value("MISSING_POLICY", "ignore")
initialize_options()
set_option_value("missing_policy", policy)
set_option_value("log_level", "DEBUG")
set_option_value("spec_version", version)
set_option_value("validator_args", "--version " + version)
set_option_value("policy", "no_policy")
def setup_options():
version = os.environ["VERSION"]
policy = os.environ["MISSING_POLICY"]
initialize_options()
set_option_value("missing_policy", policy)
set_option_value("log_level", "DEBUG")
set_option_value("spec_version", version)
set_option_value("validator_args", "--version " + version)
set_option_value("policy", "no_policy")
def _process_message(self, data):
file_fetch = data["file_fetch"]
file_uri = self.helper.opencti_url + file_fetch
self.helper.log_info("Importing the file " + file_uri)
file_content = self.helper.api.fetch_opencti_file(file_uri)
if data["file_mime"] == "text/xml":
initialize_options()
file_content = elevate(file_content)
bundles_sent = self.helper.send_stix2_bundle(file_content)
return "Sent " + str(len(bundles_sent)) + " stix bundle(s) for worker import"
def main():
# Parse stix-elevator command-line args
elevator_parser = _get_arg_parser()
elevator_args = elevator_parser.parse_args()
initialize_options(elevator_args)
result = elevate_file(elevator_args.file_)
if result:
print(result + "\n")
else:
sys.exit(1)
def main():
# Parse stix2-elevator command-line args
elevator_parser = _get_arg_parser()
elevator_args = elevator_parser.parse_args()
sys.setrecursionlimit(3000)
initialize_options(options=elevator_args)
result = elevate(elevator_args.file_)
if result:
sys.stdout.write(result + "\n")
else:
sys.exit(1)
def convert_to_stix2_from_stix_file_path(stix_file_path):
# 1.x -> 2.0
initialize_options()
set_option_value('validator_args', '--silent')
set_option_value('silent', 'True')
stix20_str = elevate_file(stix_file_path)
stix20 = parse(replace_stix2_tlp(stix20_str))
# 2.0 -> 2.1
stix20_json = json.loads(str(stix20))
stix21_json_str = step_bundle(stix20_json)
stix2 = parse(stix21_json_str)
return stix2
def main():
stepper_arg_parser = _get_arg_parser()
stepper_args = stepper_arg_parser.parse_args()
initialize_options()
set_option_value("missing_policy", "use-extensions")
set_option_value("custom_property_prefix",
stepper_args.custom_property_prefix)
validator_options = stix2validator.parse_args(
shlex.split(stepper_args.validator_args))
stix2validator.output.set_level(validator_options.verbose)
stix2validator.output.set_silent(validator_options.silent)
step_file(stepper_args.file_, validator_options)
def idiom_elevator_mappings(before_file_path, stored_json, version):
"""Test fresh conversion from XML to JSON matches stored JSON samples."""
print("Checking - " + before_file_path)
print("With Master - " + stored_json["id"])
initialize_options()
set_option_value("log_level", "CRITICAL")
set_option_value("spec_version", version)
set_option_value("validator_args", "--no-cache --version " + version)
if not get_option_value("policy") == "no_policy":
print("'no_policy' is not allowed for testing")
set_option_value("policy", "no_policy")
sys.setrecursionlimit(3000)
converted_json = elevate_file(before_file_path)
converted_json = json.loads(converted_json)
return idiom_mappings(converted_json, stored_json)
def idiom_stepper_mappings(before_file_path, stored_json):
"""Test fresh conversion from XML to JSON matches stored JSON samples."""
validator_options = stix2validator.parse_args("")
initialize_options()
set_option_value("missing_policy", "use-extensions")
set_option_value("custom_property_prefix", "elevator")
stix2validator.output.set_level(validator_options.verbose)
stix2validator.output.set_silent(validator_options.silent)
print("Checking - " + before_file_path)
print("With Master - " + stored_json["id"])
converted_json = step_file(before_file_path, validator_options)
converted_json = json.loads(converted_json)
return idiom_mappings(converted_json, stored_json, _IGNORE)
def main():
elevator_parser = _get_arg_parser(False)
elevator_parser.add_argument(
"dir_",
help="A directory containing STIX 1.x documents to be elevated.",
metavar="dir")
elevator_parser.add_argument("--output-directory",
help="output logs",
dest="output_directory",
action="store",
default=None)
elevator_args = elevator_parser.parse_args()
initialize_options(elevator_args)
set_option_value(
"validator_args",
get_option_value("validator_args") + " --version " +
get_option_value("spec_version"))
all_succeeded = True
sys.setrecursionlimit(2000)
for filename in sorted(os.listdir(elevator_args.dir_)):
path = os.path.join(elevator_args.dir_, filename)
if path.endswith(".xml"):
sys.stdout.write(path + "\n")
file_and_ext = filename.split(".")
set_option_value("file_", file_and_ext[0])
result = elevate_file(path)
if result:
if elevator_args.output_directory:
destination = os.path.join(elevator_args.output_directory,
file_and_ext[0] + ".json")
destination = os.path.abspath(destination)
with io.open(destination, "w", encoding="utf-8") as f:
f.write(result)
else:
sys.stdout.write(result + "\n")
else:
all_succeeded = False
if not all_succeeded:
sys.exit(1)
def idiom_elevator_mappings(before_file_path, stored_json, version, missing_policy, ignore):
"""Test fresh conversion from XML to JSON matches stored JSON samples."""
print("Checking - " + before_file_path)
print("With Master - " + stored_json["id"])
initialize_options()
set_option_value("missing_policy", missing_policy)
set_option_value("log_level", "WARN")
set_option_value("incidents", True)
set_option_value("spec_version", version)
set_option_value("validator_args", "--version " + version)
if not get_option_value("policy") == "no_policy":
print("'no_policy' is the default for testing")
set_option_value("policy", "no_policy")
sys.setrecursionlimit(3000)
converted_json = elevate(before_file_path)
print(converted_json)
converted_json = json.loads(converted_json)
return idiom_mappings(converted_json, stored_json, ignore)
def convert_to_stix_20_to_21(stix20_json):
# 2.0 -> 2.1
initialize_options()
set_option_value('validator_args', '--silent')
set_option_value('silent', 'True')
stix21 = step_bundle(stix20_json)
# for stix2-elevator issue
stix21_new = {}
stix21_new['id'] = stix21['id']
stix21_new['type'] = stix21['type']
stix21_new['objects'] = []
for o_ in stix21['objects']:
if o_['type'] == 'indicator':
if 'pattern_type' not in o_:
o_['pattern_type'] = 'stix'
if o_['type'] == 'malware':
if 'is_family' not in o_:
o_['is_family'] = False
stix21_new['objects'].append(o_)
stix2 = parse(stix21_new, allow_custom=True)
return stix2
def convert_to_stix_1x_to_21(stix_file_path):
# 1.x -> 2.0
initialize_options()
set_option_value('validator_args', '--silent')
set_option_value('silent', 'True')
stix20_str = ELEVATE(stix_file_path)
stix20 = _replace_stix2_tlp(stix20_str)
# for stix2-elevator issue
stix20_new = {}
stix20_new['id'] = stix20['id']
stix20_new['spec_version'] = stix20['spec_version']
stix20_new['type'] = stix20['type']
stix20_new['objects'] = []
for o_ in stix20['objects']:
if o_['type'] == 'identity':
if 'identity_class' not in o_:
o_['identity_class'] = 'unknown'
stix20_new['objects'].append(o_)
stix20_str = json.dumps(stix20_new)
stix20_o = parse(stix20_str)
stix20_json = json.loads(str(stix20_o))
return convert_to_stix_20_to_21(stix20_json)
def _process_message(self, data: Dict) -> str:
file_fetch = data["file_fetch"]
bypass_validation = data["bypass_validation"]
file_uri = self.helper.opencti_url + file_fetch
self.helper.log_info(f"Importing the file {file_uri}")
file_content = self.helper.api.fetch_opencti_file(file_uri)
if data["file_mime"] == "text/xml":
self.helper.log_debug("Stix1 file. Attempting conversion")
initialize_options()
file_content = elevate(file_content)
entity_id = data.get("entity_id", None)
if entity_id:
self.helper.log_info("Contextual import.")
bundle = parse(file_content, allow_custom=True)["objects"]
if self._contains_report(bundle):
self.helper.log_info("Bundle contains report.")
else:
self.helper.log_info(
"No Report in Stix file. Updating current report")
bundle = self._update_report(bundle, entity_id)
file_content = Bundle(objects=bundle,
allow_custom=True).serialize()
bundles_sent = self.helper.send_stix2_bundle(
file_content,
bypass_validation=bypass_validation,
file_name=data["file_id"],
entity_id=entity_id,
)
if self.helper.get_validate_before_import() and not bypass_validation:
return "Generated bundle sent for validation"
else:
return str(
len(bundles_sent)) + " generated bundle(s) for worker import"
def idiom_mappings(xml_file_path, stored_json):
"""Test fresh conversion from XML to JSON matches stored JSON samples."""
print("Checking - " + xml_file_path)
print("With Master - " + stored_json["id"])
initialize_options()
set_option_value("log_level", "CRITICAL")
set_option_value("validator_args", "--no-cache")
if not get_option_value("policy") == "no_policy":
print("'no_policy' is not allowed for testing")
set_option_value("policy", "no_policy")
sys.setrecursionlimit(3000)
converted_json = elevate_file(xml_file_path)
converted_json = json.loads(converted_json)
for good, to_check in zip(iterpath(stored_json), iterpath(converted_json)):
good_path, good_value = good
last_good_field = good_path[-1]
if isinstance(good_value, (dict, list)):
# Rule #1: No need to verify iterable types. Since we will deal
# with individual values in the future.
continue
if (any(s in (u"object_marking_refs", u"granular_markings")
for s in good_path)):
# Exception to Rule #1: object_marking_refs and granular_markings
# are not verifiable because they contain identifiers per rule #2.
continue
if last_good_field in IGNORE:
# Rule #2: Since fresh conversion may create dynamic values.
# Some fields are omitted for verification. Currently
# fields with: identifier and timestamp values.
continue
yield good, to_check
def stix_to_collection(bundle, name, version, description=None):
"""Enhance an existing stix bundle with a ATT&CK Collection object
:param bundle: dictionary representation of a stix bundle
:param name: name for the generated collection object
:param version: parameter indicating the ATT&CK version for the generated collection object
:param description: optional parameter describing the collection
:returns: updated bundle, now containing a ATT&CK Collection object
"""
working_bundle = copy.deepcopy(bundle)
for obj in working_bundle[
"objects"]: # check to see if this bundle already contains a collection
if obj["type"] == 'x-mitre-collection':
return bundle
if bundle.get("spec_version", "") == "2.0":
try:
print(
"[NOTE] - version 2.0 spec detected. Forcibly upgrading the bundle to 2.1 to support "
"collections.")
initialize_options(
ElevatorOptions(custom_property_prefix='mitre',
silent=True))
working_bundle = step_bundle(working_bundle)
print(
"[NOTE] - NOTICE: ATT&CK in STIX 2.1 includes additional fields which were not present on the "
"STIX 2.0 data. These fields have not been added automatically and their absence may affect "
"compatibility with ingesting software. Please see "
"https://github.com/mitre-attack/attack-stix-data/blob/master/USAGE.md for more information."
)
except Exception as e:
print(
f"[ERROR] - Unexpected issue encountered when trying to upgrade from 2.0 to 2.1: {e}. "
f"Terminating...")
print(f"[ERROR] - Full Error trace: {traceback.print_exc(e)}")
return None
if bundle.get("spec_version", "") != "2.1" and bundle.get(
"spec_version", "") != "2.0":
print(
f"[ERROR] - version {working_bundle.get('spec_version', '[NOT FOUND]')} is not one of [2.0, 2.1]. "
f"This module only processes stix 2.0 and stix 2.1 bundles.")
time = datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ")
if not description:
description = "This collection was autogenerated by STIXToCollection, as part of mitreattack-python"
raw_collection = dict(type="x-mitre-collection",
id=f"x-mitre-collection--{uuid4()}",
spec_version="2.1",
name=name,
x_mitre_version=version,
x_mitre_attack_spec_version=X_MITRE_SPEC_VERSION,
description=description,
created_by_ref="",
created=time,
modified=time,
object_marking_refs=[],
x_mitre_contents=[])
for obj in working_bundle['objects']:
if obj['type'] != 'marking-definition':
try:
raw_collection['x_mitre_contents'].append(
dict(object_ref=obj["id"],
object_modified=obj["modified"]))
except KeyError as e:
print(
f"[ERROR] - object {obj} is missing a necessary field: {e}. Exiting this script..."
)
return None
if "object_marking_refs" in obj.keys():
for omr in obj["object_marking_refs"]:
if omr not in raw_collection["object_marking_refs"]:
raw_collection["object_marking_refs"].append(omr)
if "created_by_ref" in obj.keys():
if obj["created_by_ref"] != raw_collection[
"created_by_ref"]:
if raw_collection["created_by_ref"] != "":
print(
f"[NOTE] multiple 'created_by_ref' values detected. "
f"{raw_collection['created_by_ref']} (first encountered) will take precedence over "
f"{obj['created_by_ref']}")
continue
raw_collection["created_by_ref"] = obj[
"created_by_ref"]
working_bundle["objects"].insert(0, raw_collection)
return working_bundle
