def generate_stix2x_id(stix2x_so_name, stix12_id=None, id_used=False): if not stix12_id or id_used: new_id = stix2x_so_name + "--" + str(uuid.uuid4()) add_ids_with_no_1x_object(new_id) if id_used and stix12_id: warn("%s already used, generated new id %s", 726, stix12_id, new_id) return new_id else: # this works for all versions of UUID result = re.search('^(.+)-([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})', stix12_id) if result: current_uuid = result.group(2) if stix2x_so_name is None: stx1x_type = result.group(1).split(":") if stx1x_type[1].lower() == "ttp" or stx1x_type[1].lower() == "et": error("Unable to determine the STIX 2.x type for %s", 604, stix12_id) return None else: return map_1x_type_to_20(stx1x_type[1]) + "--" + current_uuid else: return stix2x_so_name + "--" + current_uuid else: if stix2x_so_name: warn("Malformed id %s. Generated a new uuid", 605, stix12_id) return stix2x_so_name + "--" + str(uuid.uuid4()) else: error("Unable to determine the STIX 2.x type for %s, which is malformed", 629, stix12_id) return None
def convert_controlled_vocabs_to_open_vocabs(new_obj, new_property_name, old_vocabs, vocab_mapping, only_one, required=True): if not old_vocabs and required: if only_one: new_obj[new_property_name] = "unknown" else: new_obj[new_property_name] = ["unknown"] warn("No STIX 1.x vocab value given for %s, using 'unknown'", 509, new_property_name) else: new_obj[new_property_name] = [] for t in old_vocabs: if new_obj[new_property_name] is None or not only_one: if isinstance(t, (text_type, binary_type)): new_obj[new_property_name].append( map_vocabs_to_label(t, vocab_mapping)) else: new_obj[new_property_name].append( map_vocabs_to_label(text_type(t.value), vocab_mapping)) else: warn("Only one %s allowed in STIX 2.0 - used first one", 510, new_property_name)
def determine_container_for_missing_properties(object_type, object_instance, custom_object=False): if check_for_missing_policy("use-extensions"): extension_definition_id = get_extension_definition_id(object_type) if "extensions" in object_instance and extension_definition_id in object_instance[ "extensions"]: return object_instance["extensions"][ extension_definition_id], extension_definition_id elif not extension_definition_id: warn("No extension-definition was found for STIX 1 type %s %s", 312, object_type, (("of " + object_instance["id"]) if "id" in object_instance else "")) if custom_object: new_id = "extension-definition" + "--" + str(uuid.uuid4()) warn("New extension-definition id %s was generated for %s. %s", 315, new_id, object_type, (("See " + object_instance["id"]) if "id" in object_instance else "")) return dict(), new_id else: return None, None else: container = dict() return container, extension_definition_id else: return object_instance, None
def generate_stix20_id(stix20_so_name, stix12_id=None, id_used=False): if not stix12_id or id_used: new_id = stix20_so_name + "--" + text_type(uuid.uuid4()) add_ids_with_no_1x_object(new_id) return new_id else: result = re.search( '^(.+)-([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})', stix12_id) if result: current_uuid = result.group(2) if stix20_so_name is None: stx1x_type = result.group(1).split(":") if stx1x_type[1].lower() == "ttp" or stx1x_type[1].lower( ) == "et": error("Unable to determine the STIX 2.0 type for %s", 604, stix12_id) return None else: return map_1x_type_to_20( stx1x_type[1]) + "--" + current_uuid else: return stix20_so_name + "--" + current_uuid else: warn("Malformed id %s. Generated a new uuid", 605, stix12_id) return stix20_so_name + "--" + text_type(uuid.uuid4())
def convert_windows_service(service): cybox_ws = {} if hasattr(service, "service_name") and service.service_name: cybox_ws["service_name"] = service.service_name.value if hasattr(service, "description_list") and service.description_list: descriptions = [] for d in service.description_list: descriptions.append(d.value) cybox_ws["descriptions"] = descriptions if hasattr(service, "display_name") and service.display_name: cybox_ws["display_name"] = service.display_name.value if hasattr(service, "startup_command_line") and service.startup_command_line: cybox_ws["startup_command_line"] = service.startup_command_line.value if hasattr(service, "start_type") and service.start_type: cybox_ws["start_type"] = map_vocabs_to_label(service.start_type, SERVICE_START_TYPE) if hasattr(service, "service_type") and service.service_type: cybox_ws["service_type"] = map_vocabs_to_label(service.service_type, SERVICE_TYPE) if hasattr(service, "service_status") and service.service_status: cybox_ws["service_status"] = map_vocabs_to_label( service.service_status, SERVICE_STATUS) if hasattr(service, "service_dll") and service.service_dll: warn("WinServiceObject.service_dll is not handled, yet.", 804) return cybox_ws
def handle_missing_string_property(container, property_name, property_value, sdo_id, is_list=False, is_sco=False, is_literal=False, mapping={}): if property_value: if check_for_missing_policy("add-to-description"): if is_sco or "description" not in container: warn( "Missing property %s is ignored, because there is no description property", 309, ("'" + property_name + "'" + (" of " + sdo_id if sdo_id else ""))) else: add_string_property_to_description(container, property_name, property_value, is_list) elif check_for_missing_policy("use-custom-properties"): add_string_property_as_custom_property(container, property_name, property_value, is_list) elif check_for_missing_policy("use-extensions"): add_string_property_as_extension_property(container, property_name, property_value, sdo_id, is_list, is_literal, mapping) else: warn("Missing property %s is ignored", 307, ("'" + property_name + "'" + (" of " + sdo_id if sdo_id else "")))
def add_id_value(key, value): if not value: warn("No object mapped to %s", 610, key) if exists_id_key(key): _IDS_TO_NEW_IDS[key].append(value) else: _IDS_TO_NEW_IDS[key] = [value]
def add_string_property_as_extension_property(container, property_name, property_value, sdo_id, is_list=False, is_literal=False, mapping={}): if is_list: if is_literal: container[property_name] = [] for v in property_value: v_as_string = str(v) if v_as_string in mapping: # conversion in mapping container[property_name].append(mapping[v_as_string]) else: container[property_name].append( convert_to_stix_literal(v_as_string)) else: container[property_name] = [str(v) for v in property_value] else: prop_values_as_string = str(property_value) if is_literal: if prop_values_as_string in mapping: container[property_name] = mapping[prop_values_as_string] else: container[property_name] = convert_to_stix_literal( prop_values_as_string) else: container[property_name] = prop_values_as_string warn("Used extension property for %s", 313, property_name + (" of " + sdo_id if sdo_id else ""))
def add_object_id_value(key, value): if exists_object_id_key(key): warn("This observable %s already is associated with cyber observables", 639, key) else: _IDS_TO_CYBER_OBSERVABLES[key] = value if not value: warn("Can not associate %s with None", 611, key)
def handle_multiple_missing_statement_properties(container, statements, property_name, id, is_literal=True, mapping=None): if mapping is None: mapping = {} if statements: if len(statements) == 1: handle_missing_statement_properties(container, statements[0], property_name, id, is_list=True, is_literal=is_literal) else: if check_for_missing_policy("add-to-description"): for s in statements: add_statement_type_to_description(container, s, singular(property_name)) elif check_for_missing_policy("use-custom-properties"): container[convert_to_custom_name(property_name)] = \ collect_statement_type_as_custom_or_extension_property(statements, is_literal=False) elif check_for_missing_policy("use-extensions"): container[ property_name] = collect_statement_type_as_custom_or_extension_property( statements, is_literal=is_literal, mapping=mapping) else: warn("Missing property %s of %s is ignored", 307, property_name, id)
def convert_control_set(control_set): cs = {} for item in control_set.value: item_parts = item.split(":") token = item_parts[0] if token == "CLS": cs["classification"] = item_parts[1] elif token == "SCI": convert_one_scope(cs, "sci_controls", item_parts[1]) elif token == "LAC": convert_one_scope(cs, "logical_authority_category", item_parts[1]) elif token == "FD": convert_one_scope(cs, "formal_determination", item_parts[1]) elif token == "CVT": convert_one_scope(cs, "caveat", item_parts[1]) elif token == "SENS": convert_one_scope(cs, "sensitivity", item_parts[1]) elif token == "SHAR": convert_one_scope(cs, "shareability", item_parts[1]) elif token == "ENTITY": convert_one_scope(cs, "entity", item_parts[1]) elif token == "CTRY": convert_one_scope(cs, "permitted_nationalities", item_parts[1]) elif token == "ORG": convert_one_scope(cs, "permitted_organizations", item_parts[1]) elif token == "CUI": if item_parts[1] == "FOUO": convert_one_scope(cs, "formal_determination", item_parts[1]) warn("CUI:FOUO is treated as FD:FOUO", 0) else: warn("Token in control set not recognized: %s", 318, token) return cs
def add_confidence_property_as_custom_property(sdo_instance, confidence, parent_property_name=None): prefix = parent_property_name + "_" if parent_property_name else "" if confidence.value is not None: sdo_instance[convert_to_custom_name(prefix + "confidence")] = text_type(confidence.value) if confidence.description is not None: sdo_instance[convert_to_custom_name(prefix + "confidence_description")] = text_type(confidence.description) warn("Used custom properties for Confidence type content of %s", 308, sdo_instance["id"])
def add_id_value(key, value): if not value: warn("Trying to associate %s with None", 610, key) if exists_id_key(key): _IDS_TO_NEW_IDS[key].append(value) else: _IDS_TO_NEW_IDS[key] = [value]
def elevate_string(string): global MESSAGES_GENERATED clear_id_mapping() clear_1x_markings_map() clear_pattern_cache() clear_object_id_mapping() clear_observable_mappings() cybox.utils.caches.cache_clear() MESSAGES_GENERATED = False validator_options = get_validator_options() try: output.set_level(validator_options.verbose) output.set_silent(validator_options.silent) io = StringIO(string) container = stixmarx.parse(io) stix_package = container.package set_option_value("marking_container", container) if not isinstance(stix_package, STIXPackage): raise TypeError("Must be an instance of stix.core.STIXPackage") setup_logger(stix_package.id_) warn( "Results produced by the stix2-elevator are not for production purposes.", 201) if get_option_value("default_timestamp"): timestamp = datetime.strptime( get_option_value("default_timestamp"), "%Y-%m-%dT%H:%M:%S.%fZ"), else: timestamp = None env = Environment(get_option_value("package_created_by_id"), timestamp) json_string = json.dumps(convert_package(stix_package, env), ensure_ascii=False, indent=4, separators=(',', ': '), sort_keys=True) validation_results = validate_stix2_string(json_string, validator_options) output.print_results([validation_results]) if get_option_value("policy") == "no_policy": return json_string else: if not MESSAGES_GENERATED and validation_results._is_valid: return json_string else: return None except ValidationError as ex: output.error("Validation error occurred: '%s'" % ex, codes.EXIT_VALIDATION_ERROR) except OSError as ex: log.error(ex)
def handle_missing_confidence_property(sdo_instance, confidence, parent_property_name=None): if confidence and confidence.value: if get_option_value("missing_policy") == "add-to-description" and confidence: add_confidence_property_to_description(sdo_instance, confidence, parent_property_name) elif get_option_value("missing_policy") == "use-custom-properties": add_confidence_property_as_custom_property(sdo_instance, confidence, parent_property_name) else: warn("Missing property 'confidence' of %s is ignored", 307, sdo_instance["id"])
def handle_missing_string_property(sdo_instance, property_name, property_value, is_list=False, is_sco=False): if property_value: if get_option_value("missing_policy") == "add-to-description" and not is_sco and "description" in sdo_instance: add_string_property_to_description(sdo_instance, property_name, property_value, is_list) elif get_option_value("missing_policy") == "use-custom-properties": add_string_property_as_custom_property(sdo_instance, property_name, property_value, is_list) else: warn("Missing property %s is ignored", 307, ("'" + property_name + "'" + (" of " + sdo_instance["id"] if "id" in sdo_instance else "")))
def convert_edh_marking_to_acs_marking(marking_definition_instance, isa_marking, marking_assertion): acs_marking = {"extension_type": "property-extension"} # name is optional if isa_marking.create_date_time: acs_marking["create_date_time"] = convert_timestamp_to_string(isa_marking.create_date_time) else: warn("Required property %s is not provided for ACS data marking", 641, "create_date_time") if isa_marking.responsible_entity: for entity in isa_marking.responsible_entity.value: responsible_entity_parts = entity.split(":") if responsible_entity_parts[0] == "CUST": acs_marking["responsible_entity_custodian"] = responsible_entity_parts[1] if responsible_entity_parts[0] == "ORIG": acs_marking["responsible_entity_originator"] = responsible_entity_parts[1] if "responsible_entity_custodian" not in acs_marking: warn("Required property %s is not provided for ACS data marking", 641, "responsible_entity_custodian") if isa_marking.identifier: identifier = isa_marking.identifier if not re.match(_ISA_IDENTIFIER_PATTERN, identifier): warn("ACS identifier %s is not valid", 643, identifier) acs_marking["identifier"] = identifier # both auth_ref properties in the XML schema are minOccurs="0" maxOccurs="1" if marking_assertion.auth_ref: acs_marking["authority_reference"] = [marking_assertion.auth_ref] if isa_marking.auth_ref: if acs_marking["authority_reference"]: acs_marking["authority_reference"].append(isa_marking.auth_ref) else: acs_marking["authority_reference"] = isa_marking.auth_ref if marking_assertion.policy_ref: acs_marking["policy_reference"] = marking_assertion.policy_ref else: warn("Required property %s is not provided for ACS data marking", 641, "policy_reference") if marking_assertion.original_classification: acs_marking["original_classification"] = convert_original_classification(marking_assertion.original_classification) if marking_assertion.derivative_classification: acs_marking["derivative_classification"] = convert_derivative_classification(marking_assertion.derivative_classification) if marking_assertion.declassification: acs_marking["declassification"] = convert_declassification(marking_assertion.declassification) if marking_assertion.resource_disposition: acs_marking["resource_disposition"] = convert_resource_disposition(marking_assertion.resource_disposition) if marking_assertion.public_release: acs_marking["public_release"] = convert_public_release(marking_assertion.public_release) if marking_assertion.access_privilege: acs_marking["access_privilege"] = [] for ac in marking_assertion.access_privilege: acs_marking["access_privilege"].append(convert_access_privilege(ac)) if marking_assertion.further_sharing: acs_marking["further_sharing"] = [] for fs in marking_assertion.further_sharing: acs_marking["further_sharing"].append(convert_further_sharing(fs)) if marking_assertion.control_set: acs_marking["control_set"] = convert_control_set(marking_assertion.control_set) else: warn("Required property %s is not provided for ACS data marking", 641, "control_set") marking_definition_instance["extensions"] = {_ACS_EXTENSION_DEFINITION_ID: acs_marking}
def create_icmp_extension(icmp_header): imcp_extension = {} if icmp_header.type_: imcp_extension["icmp_type_hex"] = icmp_header.type_.value if icmp_header.code: imcp_extension["icmp_code_hex"] = icmp_header.code.value if icmp_header.checksum: warn("ICMP_Packet/Checksum content not supported in STIX 2.0", 424) return imcp_extension
def add_confidence_property_to_description(sdo_instance, confidence, parent_property_name): prefix = parent_property_name.upper() + " " if parent_property_name else "" if confidence is not None: sdo_instance["description"] += "\n\n" + prefix + "CONFIDENCE: " if confidence.value is not None: sdo_instance["description"] += text_type(confidence.value) if confidence.description is not None: sdo_instance["description"] += "\n\t" + prefix + "DESCRIPTION: " + text_type(confidence.description) warn("Appended Confidence type content to description of %s", 304, sdo_instance["id"])
def add_string_property_as_custom_property(sdo_instance, property_name, property_value, is_list=False): if is_list: property_values = list() for v in property_value: property_values.append(text_type(v)) sdo_instance[convert_to_custom_name(property_name)] = ",".join(property_values) else: sdo_instance[convert_to_custom_name(property_name)] = text_type(property_value) warn("Used custom property for %s", 308, property_name + (" of " + sdo_instance["id"] if "id" in sdo_instance else ""))
def handle_missing_statement_properties(sdo_instance, statement, property_name): if statement: if get_option_value("missing_policy") == "add-to-description": add_statement_type_to_description(sdo_instance, statement, property_name) elif get_option_value("missing_policy") == "use-custom-properties": statement_type_as_properties(sdo_instance, statement, property_name) warn("Used custom properties for Statement type content of %s", 308, sdo_instance["id"]) else: warn("Missing property %s of %s is ignored", 307, property_name, sdo_instance["id"])
def convert_public_release(public_release): pr = {} if public_release.released_by: pr["released_by"] = str(public_release.released_by) else: warn("Required property %s is not provided for ACS data marking", 641, "released_by") if public_release.released_on: pr["released_on"] = convert_timestamp_to_string(public_release.released_on) return pr
def elevate_file(fn): # TODO: combine elevate_file, elevate_string and elevate_package warnings.warn( "This method is deprecated and will be removed in the next major release. Please use elevate() instead.", DeprecationWarning) global MESSAGES_GENERATED MESSAGES_GENERATED = False print( "Results produced by the stix2-elevator are not for production purposes." ) clear_globals() validator_options = get_validator_options() try: output.set_level(validator_options.verbose) output.set_silent(validator_options.silent) if os.path.isfile(fn) is False: raise IOError("The file '{}' was not found.".format(fn)) container = stixmarx.parse(fn) stix_package = container.package set_option_value("marking_container", container) if not isinstance(stix_package, STIXPackage): raise TypeError("Must be an instance of stix.core.STIXPackage") setup_logger(stix_package.id_) warn( "Results produced by the stix2-elevator may generate warning messages which should be investigated.", 201) env = Environment(get_option_value("package_created_by_id")) json_string = json.dumps(convert_package(stix_package, env), ensure_ascii=False, indent=4, separators=(',', ': '), sort_keys=True) validation_results = validate_stix2_string(json_string, validator_options, fn) output.print_results([validation_results]) if get_option_value("policy") == "no_policy": return json_string else: if not MESSAGES_GENERATED and validation_results._is_valid: return json_string else: return None except ValidationError as ex: output.error("Validation error occurred: '{}'".format(ex)) output.error("Error Code: {}".format(codes.EXIT_VALIDATION_ERROR)) except (OSError, IOError, lxml.etree.Error) as ex: log.error("Error occurred: %s", ex)
def add_string_property_to_description(sdo_instance, property_name, property_value, is_list=False): if is_list: sdo_instance["description"] += "\n\n" + property_name.upper() + ":\n" property_values = [] for v in property_value: property_values.append(text_type(v)) sdo_instance["description"] += ",\n".join(property_values) else: sdo_instance["description"] += "\n\n" + property_name.upper() + ":\n\t" + text_type(property_value) warn("Appended %s to description of %s", 302, property_name, sdo_instance["id"])
def handle_missing_tool_property(sdo_instance, tool): if tool.name: if get_option_value("missing_policy") == "add-to-description": sdo_instance["description"] += "\n\nTOOL SOURCE:" sdo_instance["description"] += "\n\tname: " + text_type(tool.name) warn("Appended Tool type content to description of %s", 306, sdo_instance["id"]) elif get_option_value("missing_policy") == "use-custom-properties": sdo_instance[convert_to_custom_name("tool_source")] = text_type(tool.name) else: warn("Missing property name of %s is ignored", 307, sdo_instance["id"])
def convert_address(add): if add.category == add.CAT_IPV4: return {"type": "ipv4-addr", "value": add.address_value.value} elif add.category == add.CAT_IPV6: return {"type": "ipv6-addr", "value": add.address_value.value} elif add.category == add.CAT_MAC: return {"type": "mac-addr", "value": add.address_value.value} elif add.category == add.CAT_EMAIL: return {"type": "email-addr", "value": add.address_value.value} else: warn("The address type %s is not part of STIX 2.0", 421, add.category)
def elevate_file(fn): # TODO: combine elevate_file, elevate_string and elevate_package global MESSAGES_GENERATED print( "Results produced by the stix2-elevator are not for production purposes." ) clear_id_mapping() clear_1x_markings_map() clear_pattern_cache() clear_object_id_mapping() clear_observable_mappings() MESSAGES_GENERATED = False validator_options = get_validator_options() try: output.set_level(validator_options.verbose) output.set_silent(validator_options.silent) container = stixmarx.parse(fn) stix_package = container.package set_option_value("marking_container", container) if not isinstance(stix_package, STIXPackage): raise TypeError("Must be an instance of stix.core.STIXPackage") setup_logger(stix_package.id_) warn( "Results produced by the stix2-elevator are not for production purposes.", 201) json_string = json.dumps(convert_package( stix_package, get_option_value("package_created_by_id"), get_option_value("default_timestamp")), indent=4, separators=(',', ': '), sort_keys=True) if get_option_value("policy") == "no_policy": return json_string else: validation_results = validate_string(json_string, validator_options) output.print_results(validation_results) if not MESSAGES_GENERATED and validation_results._is_valid: return json_string else: return None except ValidationError as ex: output.error("Validation error occurred: '%s'" % ex, codes.EXIT_VALIDATION_ERROR) except OSError as ex: log.error(ex)
def add_confidence_property_as_extension_property(container, confidence, id, parent_property_name=None): prefix = parent_property_name + "_" if parent_property_name else "" if confidence.value is not None: container[prefix + "confidence"] = str(confidence.value) if confidence.description is not None: container[prefix + "confidence_description"] = str( confidence.description) warn("Used extensions properties for Confidence type content of %s", 313, id)
def handle_missing_tool_property(sdo_instance, tool): if tool.name: if check_for_missing_policy("add-to-description"): sdo_instance["description"] += "\n\nTOOL SOURCE:" sdo_instance["description"] += "\n\tname: " + str(tool.name) warn("Appended Tool type content to description of %s", 306, sdo_instance["id"]) elif check_for_missing_policy("use-custom-properties"): sdo_instance[convert_to_custom_name("tool_source")] = str(tool.name) else: warn("Missing property 'name' %s is ignored", 307, ("of" + sdo_instance["id"] if "id" in sdo_instance else ""))
def convert_derivative_classification(derivative_classification): cd = {} if derivative_classification.classified_by: cd["classified_by"] = str(derivative_classification.classified_by) else: warn("Required property %s is not provided for ACS data marking", 641, "classified_by") if derivative_classification.classified_on: cd["classified_on"] = convert_timestamp_to_string(derivative_classification.classified_on) if derivative_classification.derived_from: cd["derived_from"] = str(derivative_classification.derived_from) else: warn("Required property %s is not provided for ACS data marking", 641, "derived_from") return cd