def test_setup_options(opts):
options.ALL_OPTIONS = None # To make sure we can set it again
initialize_options(opts)
assert get_option_value("policy") == "no_policy"
assert get_option_value("spec_version") == "2.1"
assert get_option_value("log_level") == "DEBUG"
assert get_option_value("disabled") == [212, 901]
def elevate_string(string):
warn(
"Results produced by the stix2-elevator are not for production purposes.",
201)
clear_id_mapping()
clear_pattern_mapping()
validator_options = get_validator_options()
try:
output.set_level(validator_options.verbose)
io = StringIO(string)
stix_package = EntityParser().parse_xml(io)
if not isinstance(stix_package, STIXPackage):
raise TypeError("Must be an instance of stix.core.STIXPackage")
setup_logger(stix_package.id_)
json_string = json.dumps(convert_package(
stix_package, get_option_value("package_created_by_id"),
get_option_value("default_timestamp")),
indent=4,
separators=(',', ': '),
sort_keys=True)
validation_results = validate_string(json_string, validator_options)
output.print_results(validation_results)
return json_string
except ValidationError as ex:
output.error("Validation error occurred: '%s'" % ex,
codes.EXIT_VALIDATION_ERROR)
except OSError as ex:
log.error(ex, extra={"ecode": 210})
def setup_logger(package_id):
if options.ALL_OPTIONS:
if not options.get_option_value("message_log_directory"):
return
global log
output_directory = options.get_option_value("message_log_directory")
file_directory = options.get_option_value("file_")
if file_directory:
project_path, filename = os.path.split(file_directory)
filename = filename.split(".")[0]
filename += ".log"
else:
filename = package_id.split(":")[1]
filename += ".log"
if not os.path.exists(output_directory):
os.makedirs(output_directory)
destination = os.path.join(output_directory, filename)
destination = os.path.abspath(destination)
fh = logging.FileHandler(destination, mode='w')
fh.setFormatter(
logging.Formatter(
"[%(ecode)d] [%(levelname)-7s] [%(asctime)s] %(message)s"))
log.addHandler(fh)
def elevate_string(string):
global MESSAGES_GENERATED
clear_id_mapping()
clear_1x_markings_map()
clear_pattern_cache()
clear_object_id_mapping()
clear_observable_mappings()
cybox.utils.caches.cache_clear()
MESSAGES_GENERATED = False
validator_options = get_validator_options()
try:
output.set_level(validator_options.verbose)
output.set_silent(validator_options.silent)
io = StringIO(string)
container = stixmarx.parse(io)
stix_package = container.package
set_option_value("marking_container", container)
if not isinstance(stix_package, STIXPackage):
raise TypeError("Must be an instance of stix.core.STIXPackage")
setup_logger(stix_package.id_)
warn(
"Results produced by the stix2-elevator are not for production purposes.",
201)
if get_option_value("default_timestamp"):
timestamp = datetime.strptime(
get_option_value("default_timestamp"),
"%Y-%m-%dT%H:%M:%S.%fZ"),
else:
timestamp = None
env = Environment(get_option_value("package_created_by_id"), timestamp)
json_string = json.dumps(convert_package(stix_package, env),
ensure_ascii=False,
indent=4,
separators=(',', ': '),
sort_keys=True)
validation_results = validate_stix2_string(json_string,
validator_options)
output.print_results([validation_results])
if get_option_value("policy") == "no_policy":
return json_string
else:
if not MESSAGES_GENERATED and validation_results._is_valid:
return json_string
else:
return None
except ValidationError as ex:
output.error("Validation error occurred: '%s'" % ex,
codes.EXIT_VALIDATION_ERROR)
except OSError as ex:
log.error(ex)
def handle_missing_confidence_property(sdo_instance, confidence, parent_property_name=None):
if confidence and confidence.value:
if get_option_value("missing_policy") == "add-to-description" and confidence:
add_confidence_property_to_description(sdo_instance, confidence, parent_property_name)
elif get_option_value("missing_policy") == "use-custom-properties":
add_confidence_property_as_custom_property(sdo_instance, confidence, parent_property_name)
else:
warn("Missing property 'confidence' of %s is ignored", 307, sdo_instance["id"])
def handle_missing_string_property(sdo_instance, property_name, property_value, is_list=False, is_sco=False):
if property_value:
if get_option_value("missing_policy") == "add-to-description" and not is_sco and "description" in sdo_instance:
add_string_property_to_description(sdo_instance, property_name, property_value, is_list)
elif get_option_value("missing_policy") == "use-custom-properties":
add_string_property_as_custom_property(sdo_instance, property_name, property_value, is_list)
else:
warn("Missing property %s is ignored", 307, ("'" + property_name + "'" + (" of " + sdo_instance["id"] if "id" in sdo_instance else "")))
def handle_missing_statement_properties(sdo_instance, statement, property_name):
if statement:
if get_option_value("missing_policy") == "add-to-description":
add_statement_type_to_description(sdo_instance, statement, property_name)
elif get_option_value("missing_policy") == "use-custom-properties":
statement_type_as_properties(sdo_instance, statement, property_name)
warn("Used custom properties for Statement type content of %s", 308, sdo_instance["id"])
else:
warn("Missing property %s of %s is ignored", 307, property_name, sdo_instance["id"])
def elevate_file(fn):
# TODO: combine elevate_file, elevate_string and elevate_package
warnings.warn(
"This method is deprecated and will be removed in the next major release. Please use elevate() instead.",
DeprecationWarning)
global MESSAGES_GENERATED
MESSAGES_GENERATED = False
print(
"Results produced by the stix2-elevator are not for production purposes."
)
clear_globals()
validator_options = get_validator_options()
try:
output.set_level(validator_options.verbose)
output.set_silent(validator_options.silent)
if os.path.isfile(fn) is False:
raise IOError("The file '{}' was not found.".format(fn))
container = stixmarx.parse(fn)
stix_package = container.package
set_option_value("marking_container", container)
if not isinstance(stix_package, STIXPackage):
raise TypeError("Must be an instance of stix.core.STIXPackage")
setup_logger(stix_package.id_)
warn(
"Results produced by the stix2-elevator may generate warning messages which should be investigated.",
201)
env = Environment(get_option_value("package_created_by_id"))
json_string = json.dumps(convert_package(stix_package, env),
ensure_ascii=False,
indent=4,
separators=(',', ': '),
sort_keys=True)
validation_results = validate_stix2_string(json_string,
validator_options, fn)
output.print_results([validation_results])
if get_option_value("policy") == "no_policy":
return json_string
else:
if not MESSAGES_GENERATED and validation_results._is_valid:
return json_string
else:
return None
except ValidationError as ex:
output.error("Validation error occurred: '{}'".format(ex))
output.error("Error Code: {}".format(codes.EXIT_VALIDATION_ERROR))
except (OSError, IOError, lxml.etree.Error) as ex:
log.error("Error occurred: %s", ex)
def handle_missing_tool_property(sdo_instance, tool):
if tool.name:
if get_option_value("missing_policy") == "add-to-description":
sdo_instance["description"] += "\n\nTOOL SOURCE:"
sdo_instance["description"] += "\n\tname: " + text_type(tool.name)
warn("Appended Tool type content to description of %s", 306, sdo_instance["id"])
elif get_option_value("missing_policy") == "use-custom-properties":
sdo_instance[convert_to_custom_name("tool_source")] = text_type(tool.name)
else:
warn("Missing property name of %s is ignored", 307, sdo_instance["id"])
def elevate_file(fn):
# TODO: combine elevate_file, elevate_string and elevate_package
global MESSAGES_GENERATED
print(
"Results produced by the stix2-elevator are not for production purposes."
)
clear_id_mapping()
clear_1x_markings_map()
clear_pattern_cache()
clear_object_id_mapping()
clear_observable_mappings()
MESSAGES_GENERATED = False
validator_options = get_validator_options()
try:
output.set_level(validator_options.verbose)
output.set_silent(validator_options.silent)
container = stixmarx.parse(fn)
stix_package = container.package
set_option_value("marking_container", container)
if not isinstance(stix_package, STIXPackage):
raise TypeError("Must be an instance of stix.core.STIXPackage")
setup_logger(stix_package.id_)
warn(
"Results produced by the stix2-elevator are not for production purposes.",
201)
json_string = json.dumps(convert_package(
stix_package, get_option_value("package_created_by_id"),
get_option_value("default_timestamp")),
indent=4,
separators=(',', ': '),
sort_keys=True)
if get_option_value("policy") == "no_policy":
return json_string
else:
validation_results = validate_string(json_string,
validator_options)
output.print_results(validation_results)
if not MESSAGES_GENERATED and validation_results._is_valid:
return json_string
else:
return None
except ValidationError as ex:
output.error("Validation error occurred: '%s'" % ex,
codes.EXIT_VALIDATION_ERROR)
except OSError as ex:
log.error(ex)
def convert_process(process):
index = 0
cybox_dict = {}
process_dict = {"type": "process"}
cybox_dict[text_type(index)] = process_dict
index += 1
if process.name and get_option_value("spec_version") == "2.0":
process_dict["name"] = text_type(process.name)
if process.pid:
process_dict["pid"] = process.pid.value
if process.creation_time:
process_dict["created"] = convert_timestamp_to_string(
process.creation_time.value)
if process.child_pid_list:
for cp in process.child_pid_list:
create_process_ref(cp, process_dict, cybox_dict, index,
"child_refs")
index += 1
if process.parent_pid:
create_process_ref(process.parent_pid, process_dict, cybox_dict, index,
"parent_ref")
index += 1
if process.argument_list and get_option_value("spec_version") == "2.0":
process_dict["arguments"] = []
for a in process.argument_list:
process_dict["arguments"].append(a.value)
if process.network_connection_list:
renumbered_nc_dicts = {}
process_dict["opened_connection_refs"] = []
for nc in process.network_connection_list:
nc_dicts = convert_network_connection(nc)
root_obj_index = find_index_of_type(nc_dicts, "network-traffic")
current_largest_id, number_mapping = do_renumbering(
nc_dicts, index, root_obj_index, renumbered_nc_dicts)
add_objects(cybox_dict, renumbered_nc_dicts)
process_dict["opened_connection_refs"].append(
text_type(number_mapping[root_obj_index]))
index = current_largest_id
if isinstance(process, WinProcess):
extended_properties = {}
process_properties = convert_windows_process(process)
if process_properties:
extended_properties["windows-process-ext"] = process_properties
if isinstance(process, WinService):
service_properties = convert_windows_service(process)
if service_properties:
extended_properties["windows-service-ext"] = service_properties
if extended_properties:
process_dict["extensions"] = extended_properties
return cybox_dict
def handle_multiple_missing_statement_properties(sdo_instance, statements, property_name):
if statements:
if len(statements) == 1:
handle_missing_statement_properties(sdo_instance, statements[0], property_name)
else:
if get_option_value("missing_policy") == "add-to-description":
for s in statements:
add_statement_type_to_description(sdo_instance, s, property_name)
elif get_option_value("missing_policy") == "use-custom-properties":
statements_json = []
for s in statements:
statements_json.append(add_statement_type_as_custom_property(s))
sdo_instance[convert_to_custom_name(property_name + "s")] = statements_json
else:
warn("Missing property %s of %s is ignored", 307, property_name, sdo_instance["id"])
def convert_network_socket(socket):
cybox_traffic = {"type": "network-traffic"}
socket_extension = {}
if socket.is_blocking:
socket_extension["is_blocking"] = socket.is_blocking
if socket.is_listening:
socket_extension["is_listening"] = socket.is_listening
if socket.address_family:
if socket.address_family in ADDRESS_FAMILY_ENUMERATION:
socket_extension["address_family"] = socket.address_family.value
else:
warn("%s is not a member of the %s enumeration", 627,
socket.address_family, "address family")
if socket.type_:
socket_extension["socket_type"] = socket.type_
if socket.domain and get_option_value("spec_version") == "2.0":
socket_extension["protocol_family"] = socket.domain
if socket.options:
socket_extension["options"] = convert_socket_options(socket.options)
if socket.socket_descriptor:
socket_extension["socket_descriptor"] = socket.socket_descriptor
if socket.local_address:
warn("Network_Socket.local_address content not supported in STIX 2.0",
424)
if socket.remote_address:
warn("Network_Socket.remote_address content not supported in STIX 2.0",
424)
if socket.protocol:
cybox_traffic["protocols"] = [socket.protocol.value]
cybox_traffic["extensions"] = {"socket-ext": socket_extension}
return cybox_traffic
def main():
elevator_parser = _get_arg_parser(False)
elevator_parser.add_argument(
"dir_",
help="A directory containing STIX 1.x documents to be elevated.",
metavar="dir")
elevator_parser.add_argument("--output-directory",
help="output logs",
dest="output_directory",
action="store",
default=None)
elevator_args = elevator_parser.parse_args()
initialize_options(elevator_args)
set_option_value(
"validator_args",
get_option_value("validator_args") + " --version " +
get_option_value("spec_version"))
all_succeeded = True
sys.setrecursionlimit(2000)
for filename in sorted(os.listdir(elevator_args.dir_)):
path = os.path.join(elevator_args.dir_, filename)
if path.endswith(".xml"):
sys.stdout.write(path + "\n")
file_and_ext = filename.split(".")
set_option_value("file_", file_and_ext[0])
result = elevate_file(path)
if result:
if elevator_args.output_directory:
destination = os.path.join(elevator_args.output_directory,
file_and_ext[0] + ".json")
destination = os.path.abspath(destination)
with io.open(destination, "w", encoding="utf-8") as f:
f.write(result)
else:
sys.stdout.write(result + "\n")
else:
all_succeeded = False
if not all_succeeded:
sys.exit(1)
def setup_logger(package_id):
global log
global fh
if options.ALL_OPTIONS:
log.setLevel(options.get_option_value("log_level"))
if not options.get_option_value("message_log_directory"):
return
output_directory = options.get_option_value("message_log_directory")
file_directory = options.get_option_value("file_")
if file_directory:
project_path, filename = os.path.split(file_directory)
filename = filename.split(".")[0]
filename += ".log"
else:
filename = package_id.split(":")[1]
filename += ".log"
if not os.path.exists(output_directory):
os.makedirs(output_directory)
destination = os.path.join(output_directory, filename)
destination = os.path.abspath(destination)
# Remove File Handler from root logger if present.
if fh in log.handlers:
fh.close()
log.removeHandler(fh)
# The delay=True should prevent the file from being opened until a
# message is emitted by the logger.
fh = logging.FileHandler(destination, mode='w', delay=True)
fh.setFormatter(formatter)
log.addHandler(fh)
def idiom_elevator_mappings(before_file_path, stored_json, version):
"""Test fresh conversion from XML to JSON matches stored JSON samples."""
print("Checking - " + before_file_path)
print("With Master - " + stored_json["id"])
initialize_options()
set_option_value("log_level", "CRITICAL")
set_option_value("spec_version", version)
set_option_value("validator_args", "--no-cache --version " + version)
if not get_option_value("policy") == "no_policy":
print("'no_policy' is not allowed for testing")
set_option_value("policy", "no_policy")
sys.setrecursionlimit(3000)
converted_json = elevate_file(before_file_path)
converted_json = json.loads(converted_json)
return idiom_mappings(converted_json, stored_json)
def idiom_elevator_mappings(before_file_path, stored_json, version, missing_policy, ignore):
"""Test fresh conversion from XML to JSON matches stored JSON samples."""
print("Checking - " + before_file_path)
print("With Master - " + stored_json["id"])
initialize_options()
set_option_value("missing_policy", missing_policy)
set_option_value("log_level", "WARN")
set_option_value("incidents", True)
set_option_value("spec_version", version)
set_option_value("validator_args", "--version " + version)
if not get_option_value("policy") == "no_policy":
print("'no_policy' is the default for testing")
set_option_value("policy", "no_policy")
sys.setrecursionlimit(3000)
converted_json = elevate(before_file_path)
print(converted_json)
converted_json = json.loads(converted_json)
return idiom_mappings(converted_json, stored_json, ignore)
def convert_account(acc):
account_dict = {"type": "user-account"}
if acc.creation_date:
account_dict["account_created"] = acc.creation_date.value
# if acc.last_accessed_time:
# account_dict["account_last_login"] = acc.last_accessed_time
if acc.disabled:
account_dict["is_disabled"] = acc.disabled
if acc.authentication and get_option_value("spec_version") == "2.1":
if acc.authentication.authentication_data:
account_dict["credential"] = acc.authentication.authentication_data
if isinstance(acc, UserAccount):
if acc.username:
account_dict["account_login"] = acc.username.value
if acc.full_name:
account_dict["display_name"] = acc.full_name.value
if acc.last_login:
account_dict["account_last_login"] = convert_timestamp_to_string(
acc.last_login.value)
if isinstance(acc, UnixUserAccount):
account_dict["account_type"] = "unix"
ext_dict = {}
if acc.group_id:
ext_dict["gid"] = acc.group_id.value
if acc.user_id:
account_dict["user_id"] = text_type(acc.user_id.value)
if acc.login_shell:
ext_dict["shell"] = acc.login_shell.value
if acc.home_directory:
ext_dict["home_dir"] = acc.home_directory.value
if acc.group_list:
ext_dict["groups"] = []
for g in acc.group_list:
ext_dict["groups"].append(text_type(g.group_id.value))
if ext_dict != {}:
account_dict["extensions"] = {"unix-account-ext": ext_dict}
elif isinstance(acc, WinComputerAccount):
if acc.domain:
account_dict["account_type"] = "windows-domain"
else:
account_dict["account_type"] = "windows-local"
return account_dict
def idiom_mappings(xml_file_path, stored_json):
"""Test fresh conversion from XML to JSON matches stored JSON samples."""
print("Checking - " + xml_file_path)
print("With Master - " + stored_json["id"])
initialize_options()
set_option_value("log_level", "CRITICAL")
set_option_value("validator_args", "--no-cache")
if not get_option_value("policy") == "no_policy":
print("'no_policy' is not allowed for testing")
set_option_value("policy", "no_policy")
sys.setrecursionlimit(3000)
converted_json = elevate_file(xml_file_path)
converted_json = json.loads(converted_json)
for good, to_check in zip(iterpath(stored_json), iterpath(converted_json)):
good_path, good_value = good
last_good_field = good_path[-1]
if isinstance(good_value, (dict, list)):
# Rule #1: No need to verify iterable types. Since we will deal
# with individual values in the future.
continue
if (any(s in (u"object_marking_refs", u"granular_markings")
for s in good_path)):
# Exception to Rule #1: object_marking_refs and granular_markings
# are not verifiable because they contain identifiers per rule #2.
continue
if last_good_field in IGNORE:
# Rule #2: Since fresh conversion may create dynamic values.
# Some fields are omitted for verification. Currently
# fields with: identifier and timestamp values.
continue
yield good, to_check
def convert_to_custom_property_name(prop_name):
if re.search('[A-Z]', prop_name):
warn("Custom property name %s has been converted to all lower case",
727, prop_name)
return "x_" + get_option_value(
"custom_property_prefix") + "_" + prop_name.lower()
def remove_custom_name(name, separator="_"):
prefix = "x" + separator + get_option_value(
"custom_property_prefix") + separator
return name[len(prefix):]
def convert_to_custom_name(name, separator="_"):
if re.search('[A-Z]', name):
warn("Custom name %s has been converted to all lower case", 727, name)
# use custom_property_prefix for all custom names
return "x" + separator + get_option_value(
"custom_property_prefix") + separator + name.lower()
def check_for_missing_policy(policy):
return get_option_value("missing_policy") == policy
def elevate(stix_package):
global MESSAGES_GENERATED
MESSAGES_GENERATED = False
print("Results produced by the stix2-elevator are not for production purposes.")
clear_globals()
fn = None
validator_options = get_validator_options()
output.set_level(validator_options.verbose)
output.set_silent(validator_options.silent)
try:
if isinstance(stix_package, MarkingContainer):
# No need to re-parse the MarkingContainer.
container = stix_package
elif isinstance(stix_package, STIXPackage):
io = BytesIO(stix_package.to_xml())
container = stixmarx.parse(io)
elif isinstance(stix_package, text_type):
if stix_package.endswith(".xml") or os.path.isfile(stix_package):
# a path-like string was passed
fn = stix_package
if os.path.exists(fn) is False:
raise IOError("The file '{}' was not found.".format(fn))
else:
stix_package = StringIO(stix_package)
container = stixmarx.parse(stix_package)
elif isinstance(stix_package, binary_type):
if stix_package.endswith(b".xml") or os.path.isfile(stix_package):
# a path-like string was passed
fn = stix_package
if os.path.exists(fn) is False:
raise IOError("The file '{}' was not found.".format(fn))
else:
stix_package = BytesIO(stix_package)
container = stixmarx.parse(stix_package)
else:
raise RuntimeError("Unable to resolve object {} of type {}".format(stix_package, type(stix_package)))
container_package = container.package
set_option_value("marking_container", container)
if not isinstance(container_package, STIXPackage):
raise TypeError("Must be an instance of stix.core.STIXPackage")
except (OSError, IOError, lxml.etree.Error) as ex:
log.error("Error occurred: %s", ex)
# log.exception(ex)
return None
try:
setup_logger(container_package.id_)
warn("Results produced by the stix2-elevator may generate warning messages which should be investigated.", 201)
env = Environment(get_option_value("package_created_by_id"))
json_string = json.dumps(
convert_package(container_package, env),
ensure_ascii=False,
indent=4,
separators=(',', ': '),
sort_keys=True
)
bundle_id = re.findall(
r"bundle--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}",
json_string
)
validation_results = validate_stix2_string(json_string, validator_options, fn or bundle_id[0])
output.print_results([validation_results])
if get_option_value("policy") == "no_policy":
return json_string
else:
if not MESSAGES_GENERATED and validation_results._is_valid:
return json_string
except ValidationError as ex:
output.error("Validation error occurred: '{}'".format(ex))
output.error("Error Code: {}".format(codes.EXIT_VALIDATION_ERROR))
