def test_setup_options(opts): options.ALL_OPTIONS = None # To make sure we can set it again initialize_options(opts) assert get_option_value("policy") == "no_policy" assert get_option_value("spec_version") == "2.1" assert get_option_value("log_level") == "DEBUG" assert get_option_value("disabled") == [212, 901]
def elevate_string(string): warn( "Results produced by the stix2-elevator are not for production purposes.", 201) clear_id_mapping() clear_pattern_mapping() validator_options = get_validator_options() try: output.set_level(validator_options.verbose) io = StringIO(string) stix_package = EntityParser().parse_xml(io) if not isinstance(stix_package, STIXPackage): raise TypeError("Must be an instance of stix.core.STIXPackage") setup_logger(stix_package.id_) json_string = json.dumps(convert_package( stix_package, get_option_value("package_created_by_id"), get_option_value("default_timestamp")), indent=4, separators=(',', ': '), sort_keys=True) validation_results = validate_string(json_string, validator_options) output.print_results(validation_results) return json_string except ValidationError as ex: output.error("Validation error occurred: '%s'" % ex, codes.EXIT_VALIDATION_ERROR) except OSError as ex: log.error(ex, extra={"ecode": 210})
def setup_logger(package_id): if options.ALL_OPTIONS: if not options.get_option_value("message_log_directory"): return global log output_directory = options.get_option_value("message_log_directory") file_directory = options.get_option_value("file_") if file_directory: project_path, filename = os.path.split(file_directory) filename = filename.split(".")[0] filename += ".log" else: filename = package_id.split(":")[1] filename += ".log" if not os.path.exists(output_directory): os.makedirs(output_directory) destination = os.path.join(output_directory, filename) destination = os.path.abspath(destination) fh = logging.FileHandler(destination, mode='w') fh.setFormatter( logging.Formatter( "[%(ecode)d] [%(levelname)-7s] [%(asctime)s] %(message)s")) log.addHandler(fh)
def elevate_string(string): global MESSAGES_GENERATED clear_id_mapping() clear_1x_markings_map() clear_pattern_cache() clear_object_id_mapping() clear_observable_mappings() cybox.utils.caches.cache_clear() MESSAGES_GENERATED = False validator_options = get_validator_options() try: output.set_level(validator_options.verbose) output.set_silent(validator_options.silent) io = StringIO(string) container = stixmarx.parse(io) stix_package = container.package set_option_value("marking_container", container) if not isinstance(stix_package, STIXPackage): raise TypeError("Must be an instance of stix.core.STIXPackage") setup_logger(stix_package.id_) warn( "Results produced by the stix2-elevator are not for production purposes.", 201) if get_option_value("default_timestamp"): timestamp = datetime.strptime( get_option_value("default_timestamp"), "%Y-%m-%dT%H:%M:%S.%fZ"), else: timestamp = None env = Environment(get_option_value("package_created_by_id"), timestamp) json_string = json.dumps(convert_package(stix_package, env), ensure_ascii=False, indent=4, separators=(',', ': '), sort_keys=True) validation_results = validate_stix2_string(json_string, validator_options) output.print_results([validation_results]) if get_option_value("policy") == "no_policy": return json_string else: if not MESSAGES_GENERATED and validation_results._is_valid: return json_string else: return None except ValidationError as ex: output.error("Validation error occurred: '%s'" % ex, codes.EXIT_VALIDATION_ERROR) except OSError as ex: log.error(ex)
def handle_missing_confidence_property(sdo_instance, confidence, parent_property_name=None): if confidence and confidence.value: if get_option_value("missing_policy") == "add-to-description" and confidence: add_confidence_property_to_description(sdo_instance, confidence, parent_property_name) elif get_option_value("missing_policy") == "use-custom-properties": add_confidence_property_as_custom_property(sdo_instance, confidence, parent_property_name) else: warn("Missing property 'confidence' of %s is ignored", 307, sdo_instance["id"])
def handle_missing_string_property(sdo_instance, property_name, property_value, is_list=False, is_sco=False): if property_value: if get_option_value("missing_policy") == "add-to-description" and not is_sco and "description" in sdo_instance: add_string_property_to_description(sdo_instance, property_name, property_value, is_list) elif get_option_value("missing_policy") == "use-custom-properties": add_string_property_as_custom_property(sdo_instance, property_name, property_value, is_list) else: warn("Missing property %s is ignored", 307, ("'" + property_name + "'" + (" of " + sdo_instance["id"] if "id" in sdo_instance else "")))
def handle_missing_statement_properties(sdo_instance, statement, property_name): if statement: if get_option_value("missing_policy") == "add-to-description": add_statement_type_to_description(sdo_instance, statement, property_name) elif get_option_value("missing_policy") == "use-custom-properties": statement_type_as_properties(sdo_instance, statement, property_name) warn("Used custom properties for Statement type content of %s", 308, sdo_instance["id"]) else: warn("Missing property %s of %s is ignored", 307, property_name, sdo_instance["id"])
def elevate_file(fn): # TODO: combine elevate_file, elevate_string and elevate_package warnings.warn( "This method is deprecated and will be removed in the next major release. Please use elevate() instead.", DeprecationWarning) global MESSAGES_GENERATED MESSAGES_GENERATED = False print( "Results produced by the stix2-elevator are not for production purposes." ) clear_globals() validator_options = get_validator_options() try: output.set_level(validator_options.verbose) output.set_silent(validator_options.silent) if os.path.isfile(fn) is False: raise IOError("The file '{}' was not found.".format(fn)) container = stixmarx.parse(fn) stix_package = container.package set_option_value("marking_container", container) if not isinstance(stix_package, STIXPackage): raise TypeError("Must be an instance of stix.core.STIXPackage") setup_logger(stix_package.id_) warn( "Results produced by the stix2-elevator may generate warning messages which should be investigated.", 201) env = Environment(get_option_value("package_created_by_id")) json_string = json.dumps(convert_package(stix_package, env), ensure_ascii=False, indent=4, separators=(',', ': '), sort_keys=True) validation_results = validate_stix2_string(json_string, validator_options, fn) output.print_results([validation_results]) if get_option_value("policy") == "no_policy": return json_string else: if not MESSAGES_GENERATED and validation_results._is_valid: return json_string else: return None except ValidationError as ex: output.error("Validation error occurred: '{}'".format(ex)) output.error("Error Code: {}".format(codes.EXIT_VALIDATION_ERROR)) except (OSError, IOError, lxml.etree.Error) as ex: log.error("Error occurred: %s", ex)
def handle_missing_tool_property(sdo_instance, tool): if tool.name: if get_option_value("missing_policy") == "add-to-description": sdo_instance["description"] += "\n\nTOOL SOURCE:" sdo_instance["description"] += "\n\tname: " + text_type(tool.name) warn("Appended Tool type content to description of %s", 306, sdo_instance["id"]) elif get_option_value("missing_policy") == "use-custom-properties": sdo_instance[convert_to_custom_name("tool_source")] = text_type(tool.name) else: warn("Missing property name of %s is ignored", 307, sdo_instance["id"])
def elevate_file(fn): # TODO: combine elevate_file, elevate_string and elevate_package global MESSAGES_GENERATED print( "Results produced by the stix2-elevator are not for production purposes." ) clear_id_mapping() clear_1x_markings_map() clear_pattern_cache() clear_object_id_mapping() clear_observable_mappings() MESSAGES_GENERATED = False validator_options = get_validator_options() try: output.set_level(validator_options.verbose) output.set_silent(validator_options.silent) container = stixmarx.parse(fn) stix_package = container.package set_option_value("marking_container", container) if not isinstance(stix_package, STIXPackage): raise TypeError("Must be an instance of stix.core.STIXPackage") setup_logger(stix_package.id_) warn( "Results produced by the stix2-elevator are not for production purposes.", 201) json_string = json.dumps(convert_package( stix_package, get_option_value("package_created_by_id"), get_option_value("default_timestamp")), indent=4, separators=(',', ': '), sort_keys=True) if get_option_value("policy") == "no_policy": return json_string else: validation_results = validate_string(json_string, validator_options) output.print_results(validation_results) if not MESSAGES_GENERATED and validation_results._is_valid: return json_string else: return None except ValidationError as ex: output.error("Validation error occurred: '%s'" % ex, codes.EXIT_VALIDATION_ERROR) except OSError as ex: log.error(ex)
def convert_process(process): index = 0 cybox_dict = {} process_dict = {"type": "process"} cybox_dict[text_type(index)] = process_dict index += 1 if process.name and get_option_value("spec_version") == "2.0": process_dict["name"] = text_type(process.name) if process.pid: process_dict["pid"] = process.pid.value if process.creation_time: process_dict["created"] = convert_timestamp_to_string( process.creation_time.value) if process.child_pid_list: for cp in process.child_pid_list: create_process_ref(cp, process_dict, cybox_dict, index, "child_refs") index += 1 if process.parent_pid: create_process_ref(process.parent_pid, process_dict, cybox_dict, index, "parent_ref") index += 1 if process.argument_list and get_option_value("spec_version") == "2.0": process_dict["arguments"] = [] for a in process.argument_list: process_dict["arguments"].append(a.value) if process.network_connection_list: renumbered_nc_dicts = {} process_dict["opened_connection_refs"] = [] for nc in process.network_connection_list: nc_dicts = convert_network_connection(nc) root_obj_index = find_index_of_type(nc_dicts, "network-traffic") current_largest_id, number_mapping = do_renumbering( nc_dicts, index, root_obj_index, renumbered_nc_dicts) add_objects(cybox_dict, renumbered_nc_dicts) process_dict["opened_connection_refs"].append( text_type(number_mapping[root_obj_index])) index = current_largest_id if isinstance(process, WinProcess): extended_properties = {} process_properties = convert_windows_process(process) if process_properties: extended_properties["windows-process-ext"] = process_properties if isinstance(process, WinService): service_properties = convert_windows_service(process) if service_properties: extended_properties["windows-service-ext"] = service_properties if extended_properties: process_dict["extensions"] = extended_properties return cybox_dict
def handle_multiple_missing_statement_properties(sdo_instance, statements, property_name): if statements: if len(statements) == 1: handle_missing_statement_properties(sdo_instance, statements[0], property_name) else: if get_option_value("missing_policy") == "add-to-description": for s in statements: add_statement_type_to_description(sdo_instance, s, property_name) elif get_option_value("missing_policy") == "use-custom-properties": statements_json = [] for s in statements: statements_json.append(add_statement_type_as_custom_property(s)) sdo_instance[convert_to_custom_name(property_name + "s")] = statements_json else: warn("Missing property %s of %s is ignored", 307, property_name, sdo_instance["id"])
def convert_network_socket(socket): cybox_traffic = {"type": "network-traffic"} socket_extension = {} if socket.is_blocking: socket_extension["is_blocking"] = socket.is_blocking if socket.is_listening: socket_extension["is_listening"] = socket.is_listening if socket.address_family: if socket.address_family in ADDRESS_FAMILY_ENUMERATION: socket_extension["address_family"] = socket.address_family.value else: warn("%s is not a member of the %s enumeration", 627, socket.address_family, "address family") if socket.type_: socket_extension["socket_type"] = socket.type_ if socket.domain and get_option_value("spec_version") == "2.0": socket_extension["protocol_family"] = socket.domain if socket.options: socket_extension["options"] = convert_socket_options(socket.options) if socket.socket_descriptor: socket_extension["socket_descriptor"] = socket.socket_descriptor if socket.local_address: warn("Network_Socket.local_address content not supported in STIX 2.0", 424) if socket.remote_address: warn("Network_Socket.remote_address content not supported in STIX 2.0", 424) if socket.protocol: cybox_traffic["protocols"] = [socket.protocol.value] cybox_traffic["extensions"] = {"socket-ext": socket_extension} return cybox_traffic
def main(): elevator_parser = _get_arg_parser(False) elevator_parser.add_argument( "dir_", help="A directory containing STIX 1.x documents to be elevated.", metavar="dir") elevator_parser.add_argument("--output-directory", help="output logs", dest="output_directory", action="store", default=None) elevator_args = elevator_parser.parse_args() initialize_options(elevator_args) set_option_value( "validator_args", get_option_value("validator_args") + " --version " + get_option_value("spec_version")) all_succeeded = True sys.setrecursionlimit(2000) for filename in sorted(os.listdir(elevator_args.dir_)): path = os.path.join(elevator_args.dir_, filename) if path.endswith(".xml"): sys.stdout.write(path + "\n") file_and_ext = filename.split(".") set_option_value("file_", file_and_ext[0]) result = elevate_file(path) if result: if elevator_args.output_directory: destination = os.path.join(elevator_args.output_directory, file_and_ext[0] + ".json") destination = os.path.abspath(destination) with io.open(destination, "w", encoding="utf-8") as f: f.write(result) else: sys.stdout.write(result + "\n") else: all_succeeded = False if not all_succeeded: sys.exit(1)
def setup_logger(package_id): global log global fh if options.ALL_OPTIONS: log.setLevel(options.get_option_value("log_level")) if not options.get_option_value("message_log_directory"): return output_directory = options.get_option_value("message_log_directory") file_directory = options.get_option_value("file_") if file_directory: project_path, filename = os.path.split(file_directory) filename = filename.split(".")[0] filename += ".log" else: filename = package_id.split(":")[1] filename += ".log" if not os.path.exists(output_directory): os.makedirs(output_directory) destination = os.path.join(output_directory, filename) destination = os.path.abspath(destination) # Remove File Handler from root logger if present. if fh in log.handlers: fh.close() log.removeHandler(fh) # The delay=True should prevent the file from being opened until a # message is emitted by the logger. fh = logging.FileHandler(destination, mode='w', delay=True) fh.setFormatter(formatter) log.addHandler(fh)
def idiom_elevator_mappings(before_file_path, stored_json, version): """Test fresh conversion from XML to JSON matches stored JSON samples.""" print("Checking - " + before_file_path) print("With Master - " + stored_json["id"]) initialize_options() set_option_value("log_level", "CRITICAL") set_option_value("spec_version", version) set_option_value("validator_args", "--no-cache --version " + version) if not get_option_value("policy") == "no_policy": print("'no_policy' is not allowed for testing") set_option_value("policy", "no_policy") sys.setrecursionlimit(3000) converted_json = elevate_file(before_file_path) converted_json = json.loads(converted_json) return idiom_mappings(converted_json, stored_json)
def idiom_elevator_mappings(before_file_path, stored_json, version, missing_policy, ignore): """Test fresh conversion from XML to JSON matches stored JSON samples.""" print("Checking - " + before_file_path) print("With Master - " + stored_json["id"]) initialize_options() set_option_value("missing_policy", missing_policy) set_option_value("log_level", "WARN") set_option_value("incidents", True) set_option_value("spec_version", version) set_option_value("validator_args", "--version " + version) if not get_option_value("policy") == "no_policy": print("'no_policy' is the default for testing") set_option_value("policy", "no_policy") sys.setrecursionlimit(3000) converted_json = elevate(before_file_path) print(converted_json) converted_json = json.loads(converted_json) return idiom_mappings(converted_json, stored_json, ignore)
def convert_account(acc): account_dict = {"type": "user-account"} if acc.creation_date: account_dict["account_created"] = acc.creation_date.value # if acc.last_accessed_time: # account_dict["account_last_login"] = acc.last_accessed_time if acc.disabled: account_dict["is_disabled"] = acc.disabled if acc.authentication and get_option_value("spec_version") == "2.1": if acc.authentication.authentication_data: account_dict["credential"] = acc.authentication.authentication_data if isinstance(acc, UserAccount): if acc.username: account_dict["account_login"] = acc.username.value if acc.full_name: account_dict["display_name"] = acc.full_name.value if acc.last_login: account_dict["account_last_login"] = convert_timestamp_to_string( acc.last_login.value) if isinstance(acc, UnixUserAccount): account_dict["account_type"] = "unix" ext_dict = {} if acc.group_id: ext_dict["gid"] = acc.group_id.value if acc.user_id: account_dict["user_id"] = text_type(acc.user_id.value) if acc.login_shell: ext_dict["shell"] = acc.login_shell.value if acc.home_directory: ext_dict["home_dir"] = acc.home_directory.value if acc.group_list: ext_dict["groups"] = [] for g in acc.group_list: ext_dict["groups"].append(text_type(g.group_id.value)) if ext_dict != {}: account_dict["extensions"] = {"unix-account-ext": ext_dict} elif isinstance(acc, WinComputerAccount): if acc.domain: account_dict["account_type"] = "windows-domain" else: account_dict["account_type"] = "windows-local" return account_dict
def idiom_mappings(xml_file_path, stored_json): """Test fresh conversion from XML to JSON matches stored JSON samples.""" print("Checking - " + xml_file_path) print("With Master - " + stored_json["id"]) initialize_options() set_option_value("log_level", "CRITICAL") set_option_value("validator_args", "--no-cache") if not get_option_value("policy") == "no_policy": print("'no_policy' is not allowed for testing") set_option_value("policy", "no_policy") sys.setrecursionlimit(3000) converted_json = elevate_file(xml_file_path) converted_json = json.loads(converted_json) for good, to_check in zip(iterpath(stored_json), iterpath(converted_json)): good_path, good_value = good last_good_field = good_path[-1] if isinstance(good_value, (dict, list)): # Rule #1: No need to verify iterable types. Since we will deal # with individual values in the future. continue if (any(s in (u"object_marking_refs", u"granular_markings") for s in good_path)): # Exception to Rule #1: object_marking_refs and granular_markings # are not verifiable because they contain identifiers per rule #2. continue if last_good_field in IGNORE: # Rule #2: Since fresh conversion may create dynamic values. # Some fields are omitted for verification. Currently # fields with: identifier and timestamp values. continue yield good, to_check
def convert_to_custom_property_name(prop_name): if re.search('[A-Z]', prop_name): warn("Custom property name %s has been converted to all lower case", 727, prop_name) return "x_" + get_option_value( "custom_property_prefix") + "_" + prop_name.lower()
def remove_custom_name(name, separator="_"): prefix = "x" + separator + get_option_value( "custom_property_prefix") + separator return name[len(prefix):]
def convert_to_custom_name(name, separator="_"): if re.search('[A-Z]', name): warn("Custom name %s has been converted to all lower case", 727, name) # use custom_property_prefix for all custom names return "x" + separator + get_option_value( "custom_property_prefix") + separator + name.lower()
def check_for_missing_policy(policy): return get_option_value("missing_policy") == policy
def elevate(stix_package): global MESSAGES_GENERATED MESSAGES_GENERATED = False print("Results produced by the stix2-elevator are not for production purposes.") clear_globals() fn = None validator_options = get_validator_options() output.set_level(validator_options.verbose) output.set_silent(validator_options.silent) try: if isinstance(stix_package, MarkingContainer): # No need to re-parse the MarkingContainer. container = stix_package elif isinstance(stix_package, STIXPackage): io = BytesIO(stix_package.to_xml()) container = stixmarx.parse(io) elif isinstance(stix_package, text_type): if stix_package.endswith(".xml") or os.path.isfile(stix_package): # a path-like string was passed fn = stix_package if os.path.exists(fn) is False: raise IOError("The file '{}' was not found.".format(fn)) else: stix_package = StringIO(stix_package) container = stixmarx.parse(stix_package) elif isinstance(stix_package, binary_type): if stix_package.endswith(b".xml") or os.path.isfile(stix_package): # a path-like string was passed fn = stix_package if os.path.exists(fn) is False: raise IOError("The file '{}' was not found.".format(fn)) else: stix_package = BytesIO(stix_package) container = stixmarx.parse(stix_package) else: raise RuntimeError("Unable to resolve object {} of type {}".format(stix_package, type(stix_package))) container_package = container.package set_option_value("marking_container", container) if not isinstance(container_package, STIXPackage): raise TypeError("Must be an instance of stix.core.STIXPackage") except (OSError, IOError, lxml.etree.Error) as ex: log.error("Error occurred: %s", ex) # log.exception(ex) return None try: setup_logger(container_package.id_) warn("Results produced by the stix2-elevator may generate warning messages which should be investigated.", 201) env = Environment(get_option_value("package_created_by_id")) json_string = json.dumps( convert_package(container_package, env), ensure_ascii=False, indent=4, separators=(',', ': '), sort_keys=True ) bundle_id = re.findall( r"bundle--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}", json_string ) validation_results = validate_stix2_string(json_string, validator_options, fn or bundle_id[0]) output.print_results([validation_results]) if get_option_value("policy") == "no_policy": return json_string else: if not MESSAGES_GENERATED and validation_results._is_valid: return json_string except ValidationError as ex: output.error("Validation error occurred: '{}'".format(ex)) output.error("Error Code: {}".format(codes.EXIT_VALIDATION_ERROR))