def main(): directory = os.path.dirname(__file__) if len(sys.argv) > 1: json_dir = sys.argv[1] else: json_dir = find_dir(directory, "idioms-json") if len(sys.argv) > 2: xml_dir = sys.argv[2] else: xml_dir = find_dir(directory, "idioms-xml") if not os.path.exists(json_dir): os.makedirs(json_dir) for filename in os.listdir(xml_dir): file_and_ext = filename.split(".") xml_path = os.path.join(xml_dir, filename) destination = os.path.join(json_dir, file_and_ext[0] + ".json") destination = os.path.abspath(destination) initialize_options() if file_and_ext[1] == "xml": sys.stdout.write(xml_path + "\n") json_output = elevate_file(xml_path) output_file = open(destination, "w") output_file.write(json_output) output_file.close()
def main(): elevator_parser = _get_arg_parser(False) elevator_parser.add_argument( "dir_", help="A directory contaning STIX 1.x documents to be elevated.", metavar="dir") elevator_parser.add_argument("--output-directory", help="output logs", dest="output_directory", action="store", default=None) elevator_args = elevator_parser.parse_args() initialize_options(elevator_args) for filename in os.listdir(elevator_args.dir_): path = os.path.join(elevator_args.dir_, filename) if path.endswith(".xml"): sys.stdout.write(path + "\n") file_and_ext = filename.split(".") set_option_value("file_", file_and_ext[0]) result = elevate_file(path) if elevator_args.output_directory: destination = os.path.join(elevator_args.output_directory, file_and_ext[0] + ".json") destination = os.path.abspath(destination) output_file = open(destination, "w") output_file.write(result) output_file.close() else: print(result + "\n")
def idiom_mappings(xml_file_path, stored_json): """Test fresh conversion from XML to JSON matches stored JSON samples.""" print("Checking - " + xml_file_path) initialize_options() converted_json = elevate_file(xml_file_path) io = StringIO(converted_json) converted_json = json.load(io) for good, to_check in zip(iterpath(stored_json), iterpath(converted_json)): good_path, good_value = good last_good_field = good_path[-1] if isinstance(good_value, (dict, list)): # No need to verify iterable types. Since we will deal # with individual values in the future. continue if last_good_field in IGNORE: # Since fresh conversion may create dynamic values. # Some fields are omitted for verification. Currently # fields with: identifier and timestamp values. continue yield good, to_check
def main(): # Parse stix-elevator command-line args elevator_parser = _get_arg_parser() elevator_args = elevator_parser.parse_args() initialize_options(elevator_args) print(elevate_file(elevator_args.file_))
def main(): # elevate the file & return into a new file initialize_options() results = elevate_file(testfrag) fi = open("testcache2.json", "w") fi.write(results)
def main(): directory = os.path.dirname(__file__) if len(sys.argv) > 1: json_dir = sys.argv[1] else: json_dir = find_dir(directory, "idioms-json") if len(sys.argv) > 2: xml_dir = sys.argv[2] else: xml_dir = find_dir(directory, "idioms-xml") if not os.path.exists(json_dir): os.makedirs(json_dir) sys.setrecursionlimit(2000) for filename in sorted(os.listdir(xml_dir)): file_and_ext = filename.split(".") xml_path = os.path.join(xml_dir, filename) destination = os.path.join(json_dir, file_and_ext[0] + ".json") destination = os.path.abspath(destination) initialize_options() if file_and_ext[1] == "xml": sys.stdout.write(xml_path + "\n") json_output = elevate_file(xml_path) with io.open(destination, "w", encoding="utf-8") as f: f.write(json_output)
def _process_message(self, data: Dict) -> str: file_fetch = data["file_fetch"] file_uri = self.helper.opencti_url + file_fetch self.helper.log_info(f"Importing the file {file_uri}") file_content = self.helper.api.fetch_opencti_file(file_uri) if data["file_mime"] == "text/xml": self.helper.log_debug("Stix1 file. Attempting conversion") initialize_options() file_content = elevate(file_content) entity_id = data.get("entity_id", None) if entity_id: self.helper.log_debug("Contextual import.") bundle = parse(file_content)["objects"] if self._contains_report(bundle): self.helper.log_debug("Bundle contains report.") else: self.helper.log_debug("No Report in Stix file. Updating current report") bundle = self._update_report(bundle, entity_id) file_content = Bundle(objects=bundle).serialize() bundles_sent = self.helper.send_stix2_bundle(file_content) return "Sent " + str(len(bundles_sent)) + " stix bundle(s) for worker import"
def test_setup_options(opts): options.ALL_OPTIONS = None # To make sure we can set it again initialize_options(opts) assert get_option_value("policy") == "no_policy" assert get_option_value("spec_version") == "2.1" assert get_option_value("log_level") == "DEBUG" assert get_option_value("disabled") == [212, 901]
def elevatefile(self, filename): """ Convert a STIX 1 XML file to a STIX 2 JSON :param filename: """ initialize_options() with open(filename, 'r') as stixfile: results = elevate(stixfile.read()) print(results)
def main(): for filename in os.listdir(sys.argv[1]): path = os.path.join(sys.argv[1], filename) initialize_options() set_option_value("incidents", False) if path.endswith(".xml"): sys.stdout.write(path + "\n") print(elevate_file(path) + "\n")
def setup_options(): version = get_environment_variable_value('VERSION', "2.1") policy = get_environment_variable_value("MISSING_POLICY", "ignore") initialize_options() set_option_value("missing_policy", policy) set_option_value("log_level", "DEBUG") set_option_value("spec_version", version) set_option_value("validator_args", "--version " + version) set_option_value("policy", "no_policy")
def setup_options(): version = os.environ["VERSION"] policy = os.environ["MISSING_POLICY"] initialize_options() set_option_value("missing_policy", policy) set_option_value("log_level", "DEBUG") set_option_value("spec_version", version) set_option_value("validator_args", "--version " + version) set_option_value("policy", "no_policy")
def _process_message(self, data): file_fetch = data["file_fetch"] file_uri = self.helper.opencti_url + file_fetch self.helper.log_info("Importing the file " + file_uri) file_content = self.helper.api.fetch_opencti_file(file_uri) if data["file_mime"] == "text/xml": initialize_options() file_content = elevate(file_content) bundles_sent = self.helper.send_stix2_bundle(file_content) return "Sent " + str(len(bundles_sent)) + " stix bundle(s) for worker import"
def main(): # Parse stix-elevator command-line args elevator_parser = _get_arg_parser() elevator_args = elevator_parser.parse_args() initialize_options(elevator_args) result = elevate_file(elevator_args.file_) if result: print(result + "\n") else: sys.exit(1)
def main(): # Parse stix2-elevator command-line args elevator_parser = _get_arg_parser() elevator_args = elevator_parser.parse_args() sys.setrecursionlimit(3000) initialize_options(options=elevator_args) result = elevate(elevator_args.file_) if result: sys.stdout.write(result + "\n") else: sys.exit(1)
def convert_to_stix2_from_stix_file_path(stix_file_path): # 1.x -> 2.0 initialize_options() set_option_value('validator_args', '--silent') set_option_value('silent', 'True') stix20_str = elevate_file(stix_file_path) stix20 = parse(replace_stix2_tlp(stix20_str)) # 2.0 -> 2.1 stix20_json = json.loads(str(stix20)) stix21_json_str = step_bundle(stix20_json) stix2 = parse(stix21_json_str) return stix2
def main(): stepper_arg_parser = _get_arg_parser() stepper_args = stepper_arg_parser.parse_args() initialize_options() set_option_value("missing_policy", "use-extensions") set_option_value("custom_property_prefix", stepper_args.custom_property_prefix) validator_options = stix2validator.parse_args( shlex.split(stepper_args.validator_args)) stix2validator.output.set_level(validator_options.verbose) stix2validator.output.set_silent(validator_options.silent) step_file(stepper_args.file_, validator_options)
def idiom_elevator_mappings(before_file_path, stored_json, version): """Test fresh conversion from XML to JSON matches stored JSON samples.""" print("Checking - " + before_file_path) print("With Master - " + stored_json["id"]) initialize_options() set_option_value("log_level", "CRITICAL") set_option_value("spec_version", version) set_option_value("validator_args", "--no-cache --version " + version) if not get_option_value("policy") == "no_policy": print("'no_policy' is not allowed for testing") set_option_value("policy", "no_policy") sys.setrecursionlimit(3000) converted_json = elevate_file(before_file_path) converted_json = json.loads(converted_json) return idiom_mappings(converted_json, stored_json)
def idiom_stepper_mappings(before_file_path, stored_json): """Test fresh conversion from XML to JSON matches stored JSON samples.""" validator_options = stix2validator.parse_args("") initialize_options() set_option_value("missing_policy", "use-extensions") set_option_value("custom_property_prefix", "elevator") stix2validator.output.set_level(validator_options.verbose) stix2validator.output.set_silent(validator_options.silent) print("Checking - " + before_file_path) print("With Master - " + stored_json["id"]) converted_json = step_file(before_file_path, validator_options) converted_json = json.loads(converted_json) return idiom_mappings(converted_json, stored_json, _IGNORE)
def main(): elevator_parser = _get_arg_parser(False) elevator_parser.add_argument( "dir_", help="A directory containing STIX 1.x documents to be elevated.", metavar="dir") elevator_parser.add_argument("--output-directory", help="output logs", dest="output_directory", action="store", default=None) elevator_args = elevator_parser.parse_args() initialize_options(elevator_args) set_option_value( "validator_args", get_option_value("validator_args") + " --version " + get_option_value("spec_version")) all_succeeded = True sys.setrecursionlimit(2000) for filename in sorted(os.listdir(elevator_args.dir_)): path = os.path.join(elevator_args.dir_, filename) if path.endswith(".xml"): sys.stdout.write(path + "\n") file_and_ext = filename.split(".") set_option_value("file_", file_and_ext[0]) result = elevate_file(path) if result: if elevator_args.output_directory: destination = os.path.join(elevator_args.output_directory, file_and_ext[0] + ".json") destination = os.path.abspath(destination) with io.open(destination, "w", encoding="utf-8") as f: f.write(result) else: sys.stdout.write(result + "\n") else: all_succeeded = False if not all_succeeded: sys.exit(1)
def idiom_elevator_mappings(before_file_path, stored_json, version, missing_policy, ignore): """Test fresh conversion from XML to JSON matches stored JSON samples.""" print("Checking - " + before_file_path) print("With Master - " + stored_json["id"]) initialize_options() set_option_value("missing_policy", missing_policy) set_option_value("log_level", "WARN") set_option_value("incidents", True) set_option_value("spec_version", version) set_option_value("validator_args", "--version " + version) if not get_option_value("policy") == "no_policy": print("'no_policy' is the default for testing") set_option_value("policy", "no_policy") sys.setrecursionlimit(3000) converted_json = elevate(before_file_path) print(converted_json) converted_json = json.loads(converted_json) return idiom_mappings(converted_json, stored_json, ignore)
def convert_to_stix_20_to_21(stix20_json): # 2.0 -> 2.1 initialize_options() set_option_value('validator_args', '--silent') set_option_value('silent', 'True') stix21 = step_bundle(stix20_json) # for stix2-elevator issue stix21_new = {} stix21_new['id'] = stix21['id'] stix21_new['type'] = stix21['type'] stix21_new['objects'] = [] for o_ in stix21['objects']: if o_['type'] == 'indicator': if 'pattern_type' not in o_: o_['pattern_type'] = 'stix' if o_['type'] == 'malware': if 'is_family' not in o_: o_['is_family'] = False stix21_new['objects'].append(o_) stix2 = parse(stix21_new, allow_custom=True) return stix2
def convert_to_stix_1x_to_21(stix_file_path): # 1.x -> 2.0 initialize_options() set_option_value('validator_args', '--silent') set_option_value('silent', 'True') stix20_str = ELEVATE(stix_file_path) stix20 = _replace_stix2_tlp(stix20_str) # for stix2-elevator issue stix20_new = {} stix20_new['id'] = stix20['id'] stix20_new['spec_version'] = stix20['spec_version'] stix20_new['type'] = stix20['type'] stix20_new['objects'] = [] for o_ in stix20['objects']: if o_['type'] == 'identity': if 'identity_class' not in o_: o_['identity_class'] = 'unknown' stix20_new['objects'].append(o_) stix20_str = json.dumps(stix20_new) stix20_o = parse(stix20_str) stix20_json = json.loads(str(stix20_o)) return convert_to_stix_20_to_21(stix20_json)
def _process_message(self, data: Dict) -> str: file_fetch = data["file_fetch"] bypass_validation = data["bypass_validation"] file_uri = self.helper.opencti_url + file_fetch self.helper.log_info(f"Importing the file {file_uri}") file_content = self.helper.api.fetch_opencti_file(file_uri) if data["file_mime"] == "text/xml": self.helper.log_debug("Stix1 file. Attempting conversion") initialize_options() file_content = elevate(file_content) entity_id = data.get("entity_id", None) if entity_id: self.helper.log_info("Contextual import.") bundle = parse(file_content, allow_custom=True)["objects"] if self._contains_report(bundle): self.helper.log_info("Bundle contains report.") else: self.helper.log_info( "No Report in Stix file. Updating current report") bundle = self._update_report(bundle, entity_id) file_content = Bundle(objects=bundle, allow_custom=True).serialize() bundles_sent = self.helper.send_stix2_bundle( file_content, bypass_validation=bypass_validation, file_name=data["file_id"], entity_id=entity_id, ) if self.helper.get_validate_before_import() and not bypass_validation: return "Generated bundle sent for validation" else: return str( len(bundles_sent)) + " generated bundle(s) for worker import"
def idiom_mappings(xml_file_path, stored_json): """Test fresh conversion from XML to JSON matches stored JSON samples.""" print("Checking - " + xml_file_path) print("With Master - " + stored_json["id"]) initialize_options() set_option_value("log_level", "CRITICAL") set_option_value("validator_args", "--no-cache") if not get_option_value("policy") == "no_policy": print("'no_policy' is not allowed for testing") set_option_value("policy", "no_policy") sys.setrecursionlimit(3000) converted_json = elevate_file(xml_file_path) converted_json = json.loads(converted_json) for good, to_check in zip(iterpath(stored_json), iterpath(converted_json)): good_path, good_value = good last_good_field = good_path[-1] if isinstance(good_value, (dict, list)): # Rule #1: No need to verify iterable types. Since we will deal # with individual values in the future. continue if (any(s in (u"object_marking_refs", u"granular_markings") for s in good_path)): # Exception to Rule #1: object_marking_refs and granular_markings # are not verifiable because they contain identifiers per rule #2. continue if last_good_field in IGNORE: # Rule #2: Since fresh conversion may create dynamic values. # Some fields are omitted for verification. Currently # fields with: identifier and timestamp values. continue yield good, to_check
def stix_to_collection(bundle, name, version, description=None): """Enhance an existing stix bundle with a ATT&CK Collection object :param bundle: dictionary representation of a stix bundle :param name: name for the generated collection object :param version: parameter indicating the ATT&CK version for the generated collection object :param description: optional parameter describing the collection :returns: updated bundle, now containing a ATT&CK Collection object """ working_bundle = copy.deepcopy(bundle) for obj in working_bundle[ "objects"]: # check to see if this bundle already contains a collection if obj["type"] == 'x-mitre-collection': return bundle if bundle.get("spec_version", "") == "2.0": try: print( "[NOTE] - version 2.0 spec detected. Forcibly upgrading the bundle to 2.1 to support " "collections.") initialize_options( ElevatorOptions(custom_property_prefix='mitre', silent=True)) working_bundle = step_bundle(working_bundle) print( "[NOTE] - NOTICE: ATT&CK in STIX 2.1 includes additional fields which were not present on the " "STIX 2.0 data. These fields have not been added automatically and their absence may affect " "compatibility with ingesting software. Please see " "https://github.com/mitre-attack/attack-stix-data/blob/master/USAGE.md for more information." ) except Exception as e: print( f"[ERROR] - Unexpected issue encountered when trying to upgrade from 2.0 to 2.1: {e}. " f"Terminating...") print(f"[ERROR] - Full Error trace: {traceback.print_exc(e)}") return None if bundle.get("spec_version", "") != "2.1" and bundle.get( "spec_version", "") != "2.0": print( f"[ERROR] - version {working_bundle.get('spec_version', '[NOT FOUND]')} is not one of [2.0, 2.1]. " f"This module only processes stix 2.0 and stix 2.1 bundles.") time = datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ") if not description: description = "This collection was autogenerated by STIXToCollection, as part of mitreattack-python" raw_collection = dict(type="x-mitre-collection", id=f"x-mitre-collection--{uuid4()}", spec_version="2.1", name=name, x_mitre_version=version, x_mitre_attack_spec_version=X_MITRE_SPEC_VERSION, description=description, created_by_ref="", created=time, modified=time, object_marking_refs=[], x_mitre_contents=[]) for obj in working_bundle['objects']: if obj['type'] != 'marking-definition': try: raw_collection['x_mitre_contents'].append( dict(object_ref=obj["id"], object_modified=obj["modified"])) except KeyError as e: print( f"[ERROR] - object {obj} is missing a necessary field: {e}. Exiting this script..." ) return None if "object_marking_refs" in obj.keys(): for omr in obj["object_marking_refs"]: if omr not in raw_collection["object_marking_refs"]: raw_collection["object_marking_refs"].append(omr) if "created_by_ref" in obj.keys(): if obj["created_by_ref"] != raw_collection[ "created_by_ref"]: if raw_collection["created_by_ref"] != "": print( f"[NOTE] multiple 'created_by_ref' values detected. " f"{raw_collection['created_by_ref']} (first encountered) will take precedence over " f"{obj['created_by_ref']}") continue raw_collection["created_by_ref"] = obj[ "created_by_ref"] working_bundle["objects"].insert(0, raw_collection) return working_bundle