def generate_stix2x_id(stix2x_so_name, stix12_id=None, id_used=False):
if not stix12_id or id_used:
new_id = stix2x_so_name + "--" + str(uuid.uuid4())
add_ids_with_no_1x_object(new_id)
if id_used and stix12_id:
warn("%s already used, generated new id %s", 726, stix12_id, new_id)
return new_id
else:
# this works for all versions of UUID
result = re.search('^(.+)-([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})',
stix12_id)
if result:
current_uuid = result.group(2)
if stix2x_so_name is None:
stx1x_type = result.group(1).split(":")
if stx1x_type[1].lower() == "ttp" or stx1x_type[1].lower() == "et":
error("Unable to determine the STIX 2.x type for %s", 604, stix12_id)
return None
else:
return map_1x_type_to_20(stx1x_type[1]) + "--" + current_uuid
else:
return stix2x_so_name + "--" + current_uuid
else:
if stix2x_so_name:
warn("Malformed id %s. Generated a new uuid", 605, stix12_id)
return stix2x_so_name + "--" + str(uuid.uuid4())
else:
error("Unable to determine the STIX 2.x type for %s, which is malformed", 629, stix12_id)
return None
def convert_controlled_vocabs_to_open_vocabs(new_obj,
new_property_name,
old_vocabs,
vocab_mapping,
only_one,
required=True):
if not old_vocabs and required:
if only_one:
new_obj[new_property_name] = "unknown"
else:
new_obj[new_property_name] = ["unknown"]
warn("No STIX 1.x vocab value given for %s, using 'unknown'", 509,
new_property_name)
else:
new_obj[new_property_name] = []
for t in old_vocabs:
if new_obj[new_property_name] is None or not only_one:
if isinstance(t, (text_type, binary_type)):
new_obj[new_property_name].append(
map_vocabs_to_label(t, vocab_mapping))
else:
new_obj[new_property_name].append(
map_vocabs_to_label(text_type(t.value), vocab_mapping))
else:
warn("Only one %s allowed in STIX 2.0 - used first one", 510,
new_property_name)
def determine_container_for_missing_properties(object_type,
object_instance,
custom_object=False):
if check_for_missing_policy("use-extensions"):
extension_definition_id = get_extension_definition_id(object_type)
if "extensions" in object_instance and extension_definition_id in object_instance[
"extensions"]:
return object_instance["extensions"][
extension_definition_id], extension_definition_id
elif not extension_definition_id:
warn("No extension-definition was found for STIX 1 type %s %s",
312, object_type,
(("of " +
object_instance["id"]) if "id" in object_instance else ""))
if custom_object:
new_id = "extension-definition" + "--" + str(uuid.uuid4())
warn("New extension-definition id %s was generated for %s. %s",
315, new_id, object_type,
(("See " + object_instance["id"])
if "id" in object_instance else ""))
return dict(), new_id
else:
return None, None
else:
container = dict()
return container, extension_definition_id
else:
return object_instance, None
def generate_stix20_id(stix20_so_name, stix12_id=None, id_used=False):
if not stix12_id or id_used:
new_id = stix20_so_name + "--" + text_type(uuid.uuid4())
add_ids_with_no_1x_object(new_id)
return new_id
else:
result = re.search(
'^(.+)-([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})',
stix12_id)
if result:
current_uuid = result.group(2)
if stix20_so_name is None:
stx1x_type = result.group(1).split(":")
if stx1x_type[1].lower() == "ttp" or stx1x_type[1].lower(
) == "et":
error("Unable to determine the STIX 2.0 type for %s", 604,
stix12_id)
return None
else:
return map_1x_type_to_20(
stx1x_type[1]) + "--" + current_uuid
else:
return stix20_so_name + "--" + current_uuid
else:
warn("Malformed id %s. Generated a new uuid", 605, stix12_id)
return stix20_so_name + "--" + text_type(uuid.uuid4())
def convert_windows_service(service):
cybox_ws = {}
if hasattr(service, "service_name") and service.service_name:
cybox_ws["service_name"] = service.service_name.value
if hasattr(service, "description_list") and service.description_list:
descriptions = []
for d in service.description_list:
descriptions.append(d.value)
cybox_ws["descriptions"] = descriptions
if hasattr(service, "display_name") and service.display_name:
cybox_ws["display_name"] = service.display_name.value
if hasattr(service,
"startup_command_line") and service.startup_command_line:
cybox_ws["startup_command_line"] = service.startup_command_line.value
if hasattr(service, "start_type") and service.start_type:
cybox_ws["start_type"] = map_vocabs_to_label(service.start_type,
SERVICE_START_TYPE)
if hasattr(service, "service_type") and service.service_type:
cybox_ws["service_type"] = map_vocabs_to_label(service.service_type,
SERVICE_TYPE)
if hasattr(service, "service_status") and service.service_status:
cybox_ws["service_status"] = map_vocabs_to_label(
service.service_status, SERVICE_STATUS)
if hasattr(service, "service_dll") and service.service_dll:
warn("WinServiceObject.service_dll is not handled, yet.", 804)
return cybox_ws
def handle_missing_string_property(container,
property_name,
property_value,
sdo_id,
is_list=False,
is_sco=False,
is_literal=False,
mapping={}):
if property_value:
if check_for_missing_policy("add-to-description"):
if is_sco or "description" not in container:
warn(
"Missing property %s is ignored, because there is no description property",
309, ("'" + property_name + "'" +
(" of " + sdo_id if sdo_id else "")))
else:
add_string_property_to_description(container, property_name,
property_value, is_list)
elif check_for_missing_policy("use-custom-properties"):
add_string_property_as_custom_property(container, property_name,
property_value, is_list)
elif check_for_missing_policy("use-extensions"):
add_string_property_as_extension_property(container, property_name,
property_value, sdo_id,
is_list, is_literal,
mapping)
else:
warn("Missing property %s is ignored", 307,
("'" + property_name + "'" +
(" of " + sdo_id if sdo_id else "")))
def add_id_value(key, value):
if not value:
warn("No object mapped to %s", 610, key)
if exists_id_key(key):
_IDS_TO_NEW_IDS[key].append(value)
else:
_IDS_TO_NEW_IDS[key] = [value]
def add_string_property_as_extension_property(container,
property_name,
property_value,
sdo_id,
is_list=False,
is_literal=False,
mapping={}):
if is_list:
if is_literal:
container[property_name] = []
for v in property_value:
v_as_string = str(v)
if v_as_string in mapping:
# conversion in mapping
container[property_name].append(mapping[v_as_string])
else:
container[property_name].append(
convert_to_stix_literal(v_as_string))
else:
container[property_name] = [str(v) for v in property_value]
else:
prop_values_as_string = str(property_value)
if is_literal:
if prop_values_as_string in mapping:
container[property_name] = mapping[prop_values_as_string]
else:
container[property_name] = convert_to_stix_literal(
prop_values_as_string)
else:
container[property_name] = prop_values_as_string
warn("Used extension property for %s", 313,
property_name + (" of " + sdo_id if sdo_id else ""))
def add_object_id_value(key, value):
if exists_object_id_key(key):
warn("This observable %s already is associated with cyber observables", 639, key)
else:
_IDS_TO_CYBER_OBSERVABLES[key] = value
if not value:
warn("Can not associate %s with None", 611, key)
def handle_multiple_missing_statement_properties(container,
statements,
property_name,
id,
is_literal=True,
mapping=None):
if mapping is None:
mapping = {}
if statements:
if len(statements) == 1:
handle_missing_statement_properties(container,
statements[0],
property_name,
id,
is_list=True,
is_literal=is_literal)
else:
if check_for_missing_policy("add-to-description"):
for s in statements:
add_statement_type_to_description(container, s,
singular(property_name))
elif check_for_missing_policy("use-custom-properties"):
container[convert_to_custom_name(property_name)] = \
collect_statement_type_as_custom_or_extension_property(statements, is_literal=False)
elif check_for_missing_policy("use-extensions"):
container[
property_name] = collect_statement_type_as_custom_or_extension_property(
statements, is_literal=is_literal, mapping=mapping)
else:
warn("Missing property %s of %s is ignored", 307,
property_name, id)
def convert_control_set(control_set):
cs = {}
for item in control_set.value:
item_parts = item.split(":")
token = item_parts[0]
if token == "CLS":
cs["classification"] = item_parts[1]
elif token == "SCI":
convert_one_scope(cs, "sci_controls", item_parts[1])
elif token == "LAC":
convert_one_scope(cs, "logical_authority_category", item_parts[1])
elif token == "FD":
convert_one_scope(cs, "formal_determination", item_parts[1])
elif token == "CVT":
convert_one_scope(cs, "caveat", item_parts[1])
elif token == "SENS":
convert_one_scope(cs, "sensitivity", item_parts[1])
elif token == "SHAR":
convert_one_scope(cs, "shareability", item_parts[1])
elif token == "ENTITY":
convert_one_scope(cs, "entity", item_parts[1])
elif token == "CTRY":
convert_one_scope(cs, "permitted_nationalities", item_parts[1])
elif token == "ORG":
convert_one_scope(cs, "permitted_organizations", item_parts[1])
elif token == "CUI":
if item_parts[1] == "FOUO":
convert_one_scope(cs, "formal_determination", item_parts[1])
warn("CUI:FOUO is treated as FD:FOUO", 0)
else:
warn("Token in control set not recognized: %s", 318, token)
return cs
def add_confidence_property_as_custom_property(sdo_instance, confidence, parent_property_name=None):
prefix = parent_property_name + "_" if parent_property_name else ""
if confidence.value is not None:
sdo_instance[convert_to_custom_name(prefix + "confidence")] = text_type(confidence.value)
if confidence.description is not None:
sdo_instance[convert_to_custom_name(prefix + "confidence_description")] = text_type(confidence.description)
warn("Used custom properties for Confidence type content of %s", 308, sdo_instance["id"])
def add_id_value(key, value):
if not value:
warn("Trying to associate %s with None", 610, key)
if exists_id_key(key):
_IDS_TO_NEW_IDS[key].append(value)
else:
_IDS_TO_NEW_IDS[key] = [value]
def elevate_string(string):
global MESSAGES_GENERATED
clear_id_mapping()
clear_1x_markings_map()
clear_pattern_cache()
clear_object_id_mapping()
clear_observable_mappings()
cybox.utils.caches.cache_clear()
MESSAGES_GENERATED = False
validator_options = get_validator_options()
try:
output.set_level(validator_options.verbose)
output.set_silent(validator_options.silent)
io = StringIO(string)
container = stixmarx.parse(io)
stix_package = container.package
set_option_value("marking_container", container)
if not isinstance(stix_package, STIXPackage):
raise TypeError("Must be an instance of stix.core.STIXPackage")
setup_logger(stix_package.id_)
warn(
"Results produced by the stix2-elevator are not for production purposes.",
201)
if get_option_value("default_timestamp"):
timestamp = datetime.strptime(
get_option_value("default_timestamp"),
"%Y-%m-%dT%H:%M:%S.%fZ"),
else:
timestamp = None
env = Environment(get_option_value("package_created_by_id"), timestamp)
json_string = json.dumps(convert_package(stix_package, env),
ensure_ascii=False,
indent=4,
separators=(',', ': '),
sort_keys=True)
validation_results = validate_stix2_string(json_string,
validator_options)
output.print_results([validation_results])
if get_option_value("policy") == "no_policy":
return json_string
else:
if not MESSAGES_GENERATED and validation_results._is_valid:
return json_string
else:
return None
except ValidationError as ex:
output.error("Validation error occurred: '%s'" % ex,
codes.EXIT_VALIDATION_ERROR)
except OSError as ex:
log.error(ex)
def handle_missing_confidence_property(sdo_instance, confidence, parent_property_name=None):
if confidence and confidence.value:
if get_option_value("missing_policy") == "add-to-description" and confidence:
add_confidence_property_to_description(sdo_instance, confidence, parent_property_name)
elif get_option_value("missing_policy") == "use-custom-properties":
add_confidence_property_as_custom_property(sdo_instance, confidence, parent_property_name)
else:
warn("Missing property 'confidence' of %s is ignored", 307, sdo_instance["id"])
def handle_missing_string_property(sdo_instance, property_name, property_value, is_list=False, is_sco=False):
if property_value:
if get_option_value("missing_policy") == "add-to-description" and not is_sco and "description" in sdo_instance:
add_string_property_to_description(sdo_instance, property_name, property_value, is_list)
elif get_option_value("missing_policy") == "use-custom-properties":
add_string_property_as_custom_property(sdo_instance, property_name, property_value, is_list)
else:
warn("Missing property %s is ignored", 307, ("'" + property_name + "'" + (" of " + sdo_instance["id"] if "id" in sdo_instance else "")))
def convert_edh_marking_to_acs_marking(marking_definition_instance, isa_marking, marking_assertion):
acs_marking = {"extension_type": "property-extension"}
# name is optional
if isa_marking.create_date_time:
acs_marking["create_date_time"] = convert_timestamp_to_string(isa_marking.create_date_time)
else:
warn("Required property %s is not provided for ACS data marking", 641, "create_date_time")
if isa_marking.responsible_entity:
for entity in isa_marking.responsible_entity.value:
responsible_entity_parts = entity.split(":")
if responsible_entity_parts[0] == "CUST":
acs_marking["responsible_entity_custodian"] = responsible_entity_parts[1]
if responsible_entity_parts[0] == "ORIG":
acs_marking["responsible_entity_originator"] = responsible_entity_parts[1]
if "responsible_entity_custodian" not in acs_marking:
warn("Required property %s is not provided for ACS data marking", 641, "responsible_entity_custodian")
if isa_marking.identifier:
identifier = isa_marking.identifier
if not re.match(_ISA_IDENTIFIER_PATTERN, identifier):
warn("ACS identifier %s is not valid", 643, identifier)
acs_marking["identifier"] = identifier
# both auth_ref properties in the XML schema are minOccurs="0" maxOccurs="1"
if marking_assertion.auth_ref:
acs_marking["authority_reference"] = [marking_assertion.auth_ref]
if isa_marking.auth_ref:
if acs_marking["authority_reference"]:
acs_marking["authority_reference"].append(isa_marking.auth_ref)
else:
acs_marking["authority_reference"] = isa_marking.auth_ref
if marking_assertion.policy_ref:
acs_marking["policy_reference"] = marking_assertion.policy_ref
else:
warn("Required property %s is not provided for ACS data marking", 641, "policy_reference")
if marking_assertion.original_classification:
acs_marking["original_classification"] = convert_original_classification(marking_assertion.original_classification)
if marking_assertion.derivative_classification:
acs_marking["derivative_classification"] = convert_derivative_classification(marking_assertion.derivative_classification)
if marking_assertion.declassification:
acs_marking["declassification"] = convert_declassification(marking_assertion.declassification)
if marking_assertion.resource_disposition:
acs_marking["resource_disposition"] = convert_resource_disposition(marking_assertion.resource_disposition)
if marking_assertion.public_release:
acs_marking["public_release"] = convert_public_release(marking_assertion.public_release)
if marking_assertion.access_privilege:
acs_marking["access_privilege"] = []
for ac in marking_assertion.access_privilege:
acs_marking["access_privilege"].append(convert_access_privilege(ac))
if marking_assertion.further_sharing:
acs_marking["further_sharing"] = []
for fs in marking_assertion.further_sharing:
acs_marking["further_sharing"].append(convert_further_sharing(fs))
if marking_assertion.control_set:
acs_marking["control_set"] = convert_control_set(marking_assertion.control_set)
else:
warn("Required property %s is not provided for ACS data marking", 641, "control_set")
marking_definition_instance["extensions"] = {_ACS_EXTENSION_DEFINITION_ID: acs_marking}
def create_icmp_extension(icmp_header):
imcp_extension = {}
if icmp_header.type_:
imcp_extension["icmp_type_hex"] = icmp_header.type_.value
if icmp_header.code:
imcp_extension["icmp_code_hex"] = icmp_header.code.value
if icmp_header.checksum:
warn("ICMP_Packet/Checksum content not supported in STIX 2.0", 424)
return imcp_extension
def add_confidence_property_to_description(sdo_instance, confidence, parent_property_name):
prefix = parent_property_name.upper() + " " if parent_property_name else ""
if confidence is not None:
sdo_instance["description"] += "\n\n" + prefix + "CONFIDENCE: "
if confidence.value is not None:
sdo_instance["description"] += text_type(confidence.value)
if confidence.description is not None:
sdo_instance["description"] += "\n\t" + prefix + "DESCRIPTION: " + text_type(confidence.description)
warn("Appended Confidence type content to description of %s", 304, sdo_instance["id"])
def add_string_property_as_custom_property(sdo_instance, property_name, property_value, is_list=False):
if is_list:
property_values = list()
for v in property_value:
property_values.append(text_type(v))
sdo_instance[convert_to_custom_name(property_name)] = ",".join(property_values)
else:
sdo_instance[convert_to_custom_name(property_name)] = text_type(property_value)
warn("Used custom property for %s", 308, property_name + (" of " + sdo_instance["id"] if "id" in sdo_instance else ""))
def handle_missing_statement_properties(sdo_instance, statement, property_name):
if statement:
if get_option_value("missing_policy") == "add-to-description":
add_statement_type_to_description(sdo_instance, statement, property_name)
elif get_option_value("missing_policy") == "use-custom-properties":
statement_type_as_properties(sdo_instance, statement, property_name)
warn("Used custom properties for Statement type content of %s", 308, sdo_instance["id"])
else:
warn("Missing property %s of %s is ignored", 307, property_name, sdo_instance["id"])
def convert_public_release(public_release):
pr = {}
if public_release.released_by:
pr["released_by"] = str(public_release.released_by)
else:
warn("Required property %s is not provided for ACS data marking", 641, "released_by")
if public_release.released_on:
pr["released_on"] = convert_timestamp_to_string(public_release.released_on)
return pr
def elevate_file(fn):
# TODO: combine elevate_file, elevate_string and elevate_package
warnings.warn(
"This method is deprecated and will be removed in the next major release. Please use elevate() instead.",
DeprecationWarning)
global MESSAGES_GENERATED
MESSAGES_GENERATED = False
print(
"Results produced by the stix2-elevator are not for production purposes."
)
clear_globals()
validator_options = get_validator_options()
try:
output.set_level(validator_options.verbose)
output.set_silent(validator_options.silent)
if os.path.isfile(fn) is False:
raise IOError("The file '{}' was not found.".format(fn))
container = stixmarx.parse(fn)
stix_package = container.package
set_option_value("marking_container", container)
if not isinstance(stix_package, STIXPackage):
raise TypeError("Must be an instance of stix.core.STIXPackage")
setup_logger(stix_package.id_)
warn(
"Results produced by the stix2-elevator may generate warning messages which should be investigated.",
201)
env = Environment(get_option_value("package_created_by_id"))
json_string = json.dumps(convert_package(stix_package, env),
ensure_ascii=False,
indent=4,
separators=(',', ': '),
sort_keys=True)
validation_results = validate_stix2_string(json_string,
validator_options, fn)
output.print_results([validation_results])
if get_option_value("policy") == "no_policy":
return json_string
else:
if not MESSAGES_GENERATED and validation_results._is_valid:
return json_string
else:
return None
except ValidationError as ex:
output.error("Validation error occurred: '{}'".format(ex))
output.error("Error Code: {}".format(codes.EXIT_VALIDATION_ERROR))
except (OSError, IOError, lxml.etree.Error) as ex:
log.error("Error occurred: %s", ex)
def add_string_property_to_description(sdo_instance, property_name, property_value, is_list=False):
if is_list:
sdo_instance["description"] += "\n\n" + property_name.upper() + ":\n"
property_values = []
for v in property_value:
property_values.append(text_type(v))
sdo_instance["description"] += ",\n".join(property_values)
else:
sdo_instance["description"] += "\n\n" + property_name.upper() + ":\n\t" + text_type(property_value)
warn("Appended %s to description of %s", 302, property_name, sdo_instance["id"])
def handle_missing_tool_property(sdo_instance, tool):
if tool.name:
if get_option_value("missing_policy") == "add-to-description":
sdo_instance["description"] += "\n\nTOOL SOURCE:"
sdo_instance["description"] += "\n\tname: " + text_type(tool.name)
warn("Appended Tool type content to description of %s", 306, sdo_instance["id"])
elif get_option_value("missing_policy") == "use-custom-properties":
sdo_instance[convert_to_custom_name("tool_source")] = text_type(tool.name)
else:
warn("Missing property name of %s is ignored", 307, sdo_instance["id"])
def convert_address(add):
if add.category == add.CAT_IPV4:
return {"type": "ipv4-addr", "value": add.address_value.value}
elif add.category == add.CAT_IPV6:
return {"type": "ipv6-addr", "value": add.address_value.value}
elif add.category == add.CAT_MAC:
return {"type": "mac-addr", "value": add.address_value.value}
elif add.category == add.CAT_EMAIL:
return {"type": "email-addr", "value": add.address_value.value}
else:
warn("The address type %s is not part of STIX 2.0", 421, add.category)
def elevate_file(fn):
# TODO: combine elevate_file, elevate_string and elevate_package
global MESSAGES_GENERATED
print(
"Results produced by the stix2-elevator are not for production purposes."
)
clear_id_mapping()
clear_1x_markings_map()
clear_pattern_cache()
clear_object_id_mapping()
clear_observable_mappings()
MESSAGES_GENERATED = False
validator_options = get_validator_options()
try:
output.set_level(validator_options.verbose)
output.set_silent(validator_options.silent)
container = stixmarx.parse(fn)
stix_package = container.package
set_option_value("marking_container", container)
if not isinstance(stix_package, STIXPackage):
raise TypeError("Must be an instance of stix.core.STIXPackage")
setup_logger(stix_package.id_)
warn(
"Results produced by the stix2-elevator are not for production purposes.",
201)
json_string = json.dumps(convert_package(
stix_package, get_option_value("package_created_by_id"),
get_option_value("default_timestamp")),
indent=4,
separators=(',', ': '),
sort_keys=True)
if get_option_value("policy") == "no_policy":
return json_string
else:
validation_results = validate_string(json_string,
validator_options)
output.print_results(validation_results)
if not MESSAGES_GENERATED and validation_results._is_valid:
return json_string
else:
return None
except ValidationError as ex:
output.error("Validation error occurred: '%s'" % ex,
codes.EXIT_VALIDATION_ERROR)
except OSError as ex:
log.error(ex)
def add_confidence_property_as_extension_property(container,
confidence,
id,
parent_property_name=None):
prefix = parent_property_name + "_" if parent_property_name else ""
if confidence.value is not None:
container[prefix + "confidence"] = str(confidence.value)
if confidence.description is not None:
container[prefix + "confidence_description"] = str(
confidence.description)
warn("Used extensions properties for Confidence type content of %s", 313,
id)
def handle_missing_tool_property(sdo_instance, tool):
if tool.name:
if check_for_missing_policy("add-to-description"):
sdo_instance["description"] += "\n\nTOOL SOURCE:"
sdo_instance["description"] += "\n\tname: " + str(tool.name)
warn("Appended Tool type content to description of %s", 306,
sdo_instance["id"])
elif check_for_missing_policy("use-custom-properties"):
sdo_instance[convert_to_custom_name("tool_source")] = str(tool.name)
else:
warn("Missing property 'name' %s is ignored", 307,
("of" + sdo_instance["id"] if "id" in sdo_instance else ""))
def convert_derivative_classification(derivative_classification):
cd = {}
if derivative_classification.classified_by:
cd["classified_by"] = str(derivative_classification.classified_by)
else:
warn("Required property %s is not provided for ACS data marking", 641, "classified_by")
if derivative_classification.classified_on:
cd["classified_on"] = convert_timestamp_to_string(derivative_classification.classified_on)
if derivative_classification.derived_from:
cd["derived_from"] = str(derivative_classification.derived_from)
else:
warn("Required property %s is not provided for ACS data marking", 641, "derived_from")
return cd
