def main(): stepper_arg_parser = _get_arg_parser() stepper_args = stepper_arg_parser.parse_args() initialize_options() set_option_value("missing_policy", "use-extensions") set_option_value("custom_property_prefix", stepper_args.custom_property_prefix) validator_options = stix2validator.parse_args( shlex.split(stepper_args.validator_args)) stix2validator.output.set_level(validator_options.verbose) stix2validator.output.set_silent(validator_options.silent) step_file(stepper_args.file_, validator_options)
def elevate_package(package): global MESSAGES_GENERATED clear_id_mapping() clear_1x_markings_map() clear_pattern_cache() clear_object_id_mapping() clear_observable_mappings() cybox.utils.caches.cache_clear() MESSAGES_GENERATED = False validator_options = get_validator_options() try: output.set_level(validator_options.verbose) output.set_silent(validator_options.silent) # It needs to be re-parsed. container = stixmarx.parse(StringIO(package.to_xml())) stix_package = container.package set_option_value("marking_container", container) if not isinstance(stix_package, STIXPackage): raise TypeError("Must be an instance of stix.core.STIXPackage") setup_logger(stix_package.id_) warn( "Results produced by the stix2-elevator are not for production purposes.", 201) json_string = json.dumps(convert_package( stix_package, get_option_value("package_created_by_id"), get_option_value("default_timestamp")), indent=4, separators=(',', ': '), sort_keys=True) if get_option_value("policy") == "no_policy": return json_string else: validation_results = validate_string(json_string, validator_options) output.print_results(validation_results) if not MESSAGES_GENERATED and validation_results._is_valid: return json_string else: return None except ValidationError as ex: output.error("Validation error occurred: '%s'" % ex, codes.EXIT_VALIDATION_ERROR) except OSError as ex: log.error(ex)
def elevate_file(fn): # TODO: combine elevate_file, elevate_string and elevate_package warnings.warn("This method is deprecated and will be removed in the next major release. Please use elevate() instead.", DeprecationWarning) global MESSAGES_GENERATED MESSAGES_GENERATED = False print("Results produced by the stix2-elevator are not for production purposes.") clear_globals() validator_options = get_validator_options() try: output.set_level(validator_options.verbose) output.set_silent(validator_options.silent) if os.path.isfile(fn) is False: raise IOError("The file '{}' was not found.".format(fn)) container = stixmarx.parse(fn) stix_package = container.package set_option_value("marking_container", container) if not isinstance(stix_package, STIXPackage): raise TypeError("Must be an instance of stix.core.STIXPackage") setup_logger(stix_package.id_) warn("Results produced by the stix2-elevator may generate warning messages which should be investigated.", 201) env = Environment(get_option_value("package_created_by_id")) json_string = json.dumps(convert_package(stix_package, env), ensure_ascii=False, indent=4, separators=(',', ': '), sort_keys=True) validation_results = validate_stix2_string(json_string, validator_options, fn) output.print_results([validation_results]) if get_option_value("policy") == "no_policy": return json_string else: if not MESSAGES_GENERATED and validation_results._is_valid: return json_string else: return None except ValidationError as ex: output.error("Validation error occurred: '{}'".format(ex)) output.error("Error Code: {}".format(codes.EXIT_VALIDATION_ERROR)) except (OSError, IOError, lxml.etree.Error) as ex: log.error("Error occurred: %s", ex)
def idiom_stepper_mappings(before_file_path, stored_json): """Test fresh conversion from XML to JSON matches stored JSON samples.""" validator_options = stix2validator.parse_args("") initialize_options() set_option_value("missing_policy", "use-extensions") set_option_value("custom_property_prefix", "elevator") stix2validator.output.set_level(validator_options.verbose) stix2validator.output.set_silent(validator_options.silent) print("Checking - " + before_file_path) print("With Master - " + stored_json["id"]) converted_json = step_file(before_file_path, validator_options) converted_json = json.loads(converted_json) return idiom_mappings(converted_json, stored_json, _IGNORE)
def main(): elevator_parser = _get_arg_parser(False) elevator_parser.add_argument( "dir_", help="A directory containing STIX 1.x documents to be elevated.", metavar="dir") elevator_parser.add_argument("--output-directory", help="output logs", dest="output_directory", action="store", default=None) elevator_args = elevator_parser.parse_args() initialize_options(elevator_args) set_option_value( "validator_args", get_option_value("validator_args") + " --version " + get_option_value("spec_version")) all_succeeded = True sys.setrecursionlimit(2000) for filename in sorted(os.listdir(elevator_args.dir_)): path = os.path.join(elevator_args.dir_, filename) if path.endswith(".xml"): sys.stdout.write(path + "\n") file_and_ext = filename.split(".") set_option_value("file_", file_and_ext[0]) result = elevate_file(path) if result: if elevator_args.output_directory: destination = os.path.join(elevator_args.output_directory, file_and_ext[0] + ".json") destination = os.path.abspath(destination) with io.open(destination, "w", encoding="utf-8") as f: f.write(result) else: sys.stdout.write(result + "\n") else: all_succeeded = False if not all_succeeded: sys.exit(1)
def idiom_elevator_mappings(before_file_path, stored_json, version, missing_policy, ignore): """Test fresh conversion from XML to JSON matches stored JSON samples.""" print("Checking - " + before_file_path) print("With Master - " + stored_json["id"]) initialize_options() set_option_value("missing_policy", missing_policy) set_option_value("log_level", "INFO") set_option_value("spec_version", version) set_option_value("validator_args", "--version " + version) if not get_option_value("policy") == "no_policy": print("'no_policy' is not allowed for testing") set_option_value("policy", "no_policy") sys.setrecursionlimit(3000) converted_json = elevate(before_file_path) print(converted_json) converted_json = json.loads(converted_json) return idiom_mappings(converted_json, stored_json, ignore)
def main(): elevator_parser = _get_arg_parser(False) elevator_parser.add_argument( "dir_", help="A directory containing STIX 1.x documents to be elevated.", metavar="dir") elevator_parser.add_argument("--output-directory", help="output logs", dest="output_directory", action="store", default=None) elevator_args = elevator_parser.parse_args() initialize_options(elevator_args) all_succeeded = True for filename in sorted(os.listdir(elevator_args.dir_)): path = os.path.join(elevator_args.dir_, filename) if path.endswith(".xml"): sys.stdout.write(path + "\n") file_and_ext = filename.split(".") set_option_value("file_", file_and_ext[0]) result = elevate_file(path) if result: if elevator_args.output_directory: destination = os.path.join(elevator_args.output_directory, file_and_ext[0] + ".json") destination = os.path.abspath(destination) output_file = open(destination, "w") output_file.write(result) output_file.close() else: print(result + "\n") else: all_succeeded = False if not all_succeeded: sys.exit(1)
def convert_to_stix_20_to_21(stix20_json): # 2.0 -> 2.1 initialize_options() set_option_value('validator_args', '--silent') set_option_value('silent', 'True') stix21 = step_bundle(stix20_json) # for stix2-elevator issue stix21_new = {} stix21_new['id'] = stix21['id'] stix21_new['type'] = stix21['type'] stix21_new['objects'] = [] for o_ in stix21['objects']: if o_['type'] == 'indicator': if 'pattern_type' not in o_: o_['pattern_type'] = 'stix' if o_['type'] == 'malware': if 'is_family' not in o_: o_['is_family'] = False stix21_new['objects'].append(o_) stix2 = parse(stix21_new, allow_custom=True) return stix2
def convert_to_stix_1x_to_21(stix_file_path): # 1.x -> 2.0 initialize_options() set_option_value('validator_args', '--silent') set_option_value('silent', 'True') stix20_str = ELEVATE(stix_file_path) stix20 = _replace_stix2_tlp(stix20_str) # for stix2-elevator issue stix20_new = {} stix20_new['id'] = stix20['id'] stix20_new['spec_version'] = stix20['spec_version'] stix20_new['type'] = stix20['type'] stix20_new['objects'] = [] for o_ in stix20['objects']: if o_['type'] == 'identity': if 'identity_class' not in o_: o_['identity_class'] = 'unknown' stix20_new['objects'].append(o_) stix20_str = json.dumps(stix20_new) stix20_o = parse(stix20_str) stix20_json = json.loads(str(stix20_o)) return convert_to_stix_20_to_21(stix20_json)
def idiom_elevator_mappings(before_file_path, stored_json, version, missing_policy, ignore): """Test fresh conversion from XML to JSON matches stored JSON samples.""" print("Checking - " + before_file_path) print("With Master - " + stored_json["id"]) initialize_options( options={ "spec_version": version, "missing_policy": missing_policy, "log_level": "WARN", "incidents": True, "validator_args": "--version " + version }) if not get_option_value("policy") == "no_policy": print("'no_policy' is the default for testing") set_option_value("policy", "no_policy") if version == "2.1": set_option_value("acs", True) sys.setrecursionlimit(3000) converted_json = elevate(before_file_path) print(converted_json) converted_json = json.loads(converted_json) return idiom_mappings(converted_json, stored_json, ignore)
def idiom_mappings(xml_file_path, stored_json): """Test fresh conversion from XML to JSON matches stored JSON samples.""" print("Checking - " + xml_file_path) print("With Master - " + stored_json["id"]) initialize_options() set_option_value("log_level", "CRITICAL") set_option_value("validator_args", "--no-cache") if not get_option_value("policy") == "no_policy": print("'no_policy' is not allowed for testing") set_option_value("policy", "no_policy") sys.setrecursionlimit(3000) converted_json = elevate_file(xml_file_path) converted_json = json.loads(converted_json) for good, to_check in zip(iterpath(stored_json), iterpath(converted_json)): good_path, good_value = good last_good_field = good_path[-1] if isinstance(good_value, (dict, list)): # Rule #1: No need to verify iterable types. Since we will deal # with individual values in the future. continue if (any(s in (u"object_marking_refs", u"granular_markings") for s in good_path)): # Exception to Rule #1: object_marking_refs and granular_markings # are not verifiable because they contain identifiers per rule #2. continue if last_good_field in IGNORE: # Rule #2: Since fresh conversion may create dynamic values. # Some fields are omitted for verification. Currently # fields with: identifier and timestamp values. continue yield good, to_check
def elevate_file(fn): # TODO: combine elevate_file, elevate_string and elevate_package global MESSAGES_GENERATED print( "Results produced by the stix2-elevator are not for production purposes." ) clear_id_mapping() clear_1x_markings_map() clear_pattern_cache() clear_object_id_mapping() clear_observable_mappings() cybox.utils.caches.cache_clear() MESSAGES_GENERATED = False validator_options = get_validator_options() try: output.set_level(validator_options.verbose) output.set_silent(validator_options.silent) container = stixmarx.parse(fn) stix_package = container.package set_option_value("marking_container", container) if not isinstance(stix_package, STIXPackage): raise TypeError("Must be an instance of stix.core.STIXPackage") setup_logger(stix_package.id_) warn( "Results produced by the stix2-elevator are not for production purposes.", 201) if get_option_value("default_timestamp"): timestamp = datetime.strptime( get_option_value("default_timestamp"), "%Y-%m-%dT%H:%M:%S.%fZ"), else: timestamp = None env = Environment(get_option_value("package_created_by_id"), timestamp) json_string = json.dumps(convert_package(stix_package, env), ensure_ascii=False, indent=4, separators=(',', ': '), sort_keys=True) validation_results = validate_stix2_string(json_string, validator_options, fn) output.print_results([validation_results]) if get_option_value("policy") == "no_policy": return json_string else: if not MESSAGES_GENERATED and validation_results._is_valid: return json_string else: return None except ValidationError as ex: output.error("Validation error occurred: '%s'" % ex, codes.EXIT_VALIDATION_ERROR) except OSError as ex: log.error(ex)
def elevate(stix_package): global MESSAGES_GENERATED MESSAGES_GENERATED = False print("Results produced by the stix2-elevator are not for production purposes.") clear_globals() fn = None validator_options = get_validator_options() output.set_level(validator_options.verbose) output.set_silent(validator_options.silent) try: if isinstance(stix_package, MarkingContainer): # No need to re-parse the MarkingContainer. container = stix_package elif isinstance(stix_package, STIXPackage): io = BytesIO(stix_package.to_xml()) container = stixmarx.parse(io) elif isinstance(stix_package, text_type): if stix_package.endswith(".xml") or os.path.isfile(stix_package): # a path-like string was passed fn = stix_package if os.path.exists(fn) is False: raise IOError("The file '{}' was not found.".format(fn)) else: stix_package = StringIO(stix_package) container = stixmarx.parse(stix_package) elif isinstance(stix_package, binary_type): if stix_package.endswith(b".xml") or os.path.isfile(stix_package): # a path-like string was passed fn = stix_package if os.path.exists(fn) is False: raise IOError("The file '{}' was not found.".format(fn)) else: stix_package = BytesIO(stix_package) container = stixmarx.parse(stix_package) else: raise RuntimeError("Unable to resolve object {} of type {}".format(stix_package, type(stix_package))) container_package = container.package set_option_value("marking_container", container) if not isinstance(container_package, STIXPackage): raise TypeError("Must be an instance of stix.core.STIXPackage") except (OSError, IOError, lxml.etree.Error) as ex: log.error("Error occurred: %s", ex) # log.exception(ex) return None try: setup_logger(container_package.id_) warn("Results produced by the stix2-elevator may generate warning messages which should be investigated.", 201) env = Environment(get_option_value("package_created_by_id")) json_string = json.dumps( convert_package(container_package, env), ensure_ascii=False, indent=4, separators=(',', ': '), sort_keys=True ) bundle_id = re.findall( r"bundle--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}", json_string ) validation_results = validate_stix2_string(json_string, validator_options, fn or bundle_id[0]) output.print_results([validation_results]) if get_option_value("policy") == "no_policy": return json_string else: if not MESSAGES_GENERATED and validation_results._is_valid: return json_string except ValidationError as ex: output.error("Validation error occurred: '{}'".format(ex)) output.error("Error Code: {}".format(codes.EXIT_VALIDATION_ERROR))