def main():
stepper_arg_parser = _get_arg_parser()
stepper_args = stepper_arg_parser.parse_args()
initialize_options()
set_option_value("missing_policy", "use-extensions")
set_option_value("custom_property_prefix",
stepper_args.custom_property_prefix)
validator_options = stix2validator.parse_args(
shlex.split(stepper_args.validator_args))
stix2validator.output.set_level(validator_options.verbose)
stix2validator.output.set_silent(validator_options.silent)
step_file(stepper_args.file_, validator_options)
def elevate_package(package):
global MESSAGES_GENERATED
clear_id_mapping()
clear_1x_markings_map()
clear_pattern_cache()
clear_object_id_mapping()
clear_observable_mappings()
cybox.utils.caches.cache_clear()
MESSAGES_GENERATED = False
validator_options = get_validator_options()
try:
output.set_level(validator_options.verbose)
output.set_silent(validator_options.silent)
# It needs to be re-parsed.
container = stixmarx.parse(StringIO(package.to_xml()))
stix_package = container.package
set_option_value("marking_container", container)
if not isinstance(stix_package, STIXPackage):
raise TypeError("Must be an instance of stix.core.STIXPackage")
setup_logger(stix_package.id_)
warn(
"Results produced by the stix2-elevator are not for production purposes.",
201)
json_string = json.dumps(convert_package(
stix_package, get_option_value("package_created_by_id"),
get_option_value("default_timestamp")),
indent=4,
separators=(',', ': '),
sort_keys=True)
if get_option_value("policy") == "no_policy":
return json_string
else:
validation_results = validate_string(json_string,
validator_options)
output.print_results(validation_results)
if not MESSAGES_GENERATED and validation_results._is_valid:
return json_string
else:
return None
except ValidationError as ex:
output.error("Validation error occurred: '%s'" % ex,
codes.EXIT_VALIDATION_ERROR)
except OSError as ex:
log.error(ex)
def elevate_file(fn):
# TODO: combine elevate_file, elevate_string and elevate_package
warnings.warn("This method is deprecated and will be removed in the next major release. Please use elevate() instead.", DeprecationWarning)
global MESSAGES_GENERATED
MESSAGES_GENERATED = False
print("Results produced by the stix2-elevator are not for production purposes.")
clear_globals()
validator_options = get_validator_options()
try:
output.set_level(validator_options.verbose)
output.set_silent(validator_options.silent)
if os.path.isfile(fn) is False:
raise IOError("The file '{}' was not found.".format(fn))
container = stixmarx.parse(fn)
stix_package = container.package
set_option_value("marking_container", container)
if not isinstance(stix_package, STIXPackage):
raise TypeError("Must be an instance of stix.core.STIXPackage")
setup_logger(stix_package.id_)
warn("Results produced by the stix2-elevator may generate warning messages which should be investigated.", 201)
env = Environment(get_option_value("package_created_by_id"))
json_string = json.dumps(convert_package(stix_package, env),
ensure_ascii=False,
indent=4,
separators=(',', ': '),
sort_keys=True)
validation_results = validate_stix2_string(json_string, validator_options, fn)
output.print_results([validation_results])
if get_option_value("policy") == "no_policy":
return json_string
else:
if not MESSAGES_GENERATED and validation_results._is_valid:
return json_string
else:
return None
except ValidationError as ex:
output.error("Validation error occurred: '{}'".format(ex))
output.error("Error Code: {}".format(codes.EXIT_VALIDATION_ERROR))
except (OSError, IOError, lxml.etree.Error) as ex:
log.error("Error occurred: %s", ex)
def idiom_stepper_mappings(before_file_path, stored_json):
"""Test fresh conversion from XML to JSON matches stored JSON samples."""
validator_options = stix2validator.parse_args("")
initialize_options()
set_option_value("missing_policy", "use-extensions")
set_option_value("custom_property_prefix", "elevator")
stix2validator.output.set_level(validator_options.verbose)
stix2validator.output.set_silent(validator_options.silent)
print("Checking - " + before_file_path)
print("With Master - " + stored_json["id"])
converted_json = step_file(before_file_path, validator_options)
converted_json = json.loads(converted_json)
return idiom_mappings(converted_json, stored_json, _IGNORE)
def main():
elevator_parser = _get_arg_parser(False)
elevator_parser.add_argument(
"dir_",
help="A directory containing STIX 1.x documents to be elevated.",
metavar="dir")
elevator_parser.add_argument("--output-directory",
help="output logs",
dest="output_directory",
action="store",
default=None)
elevator_args = elevator_parser.parse_args()
initialize_options(elevator_args)
set_option_value(
"validator_args",
get_option_value("validator_args") + " --version " +
get_option_value("spec_version"))
all_succeeded = True
sys.setrecursionlimit(2000)
for filename in sorted(os.listdir(elevator_args.dir_)):
path = os.path.join(elevator_args.dir_, filename)
if path.endswith(".xml"):
sys.stdout.write(path + "\n")
file_and_ext = filename.split(".")
set_option_value("file_", file_and_ext[0])
result = elevate_file(path)
if result:
if elevator_args.output_directory:
destination = os.path.join(elevator_args.output_directory,
file_and_ext[0] + ".json")
destination = os.path.abspath(destination)
with io.open(destination, "w", encoding="utf-8") as f:
f.write(result)
else:
sys.stdout.write(result + "\n")
else:
all_succeeded = False
if not all_succeeded:
sys.exit(1)
def idiom_elevator_mappings(before_file_path, stored_json, version,
missing_policy, ignore):
"""Test fresh conversion from XML to JSON matches stored JSON samples."""
print("Checking - " + before_file_path)
print("With Master - " + stored_json["id"])
initialize_options()
set_option_value("missing_policy", missing_policy)
set_option_value("log_level", "INFO")
set_option_value("spec_version", version)
set_option_value("validator_args", "--version " + version)
if not get_option_value("policy") == "no_policy":
print("'no_policy' is not allowed for testing")
set_option_value("policy", "no_policy")
sys.setrecursionlimit(3000)
converted_json = elevate(before_file_path)
print(converted_json)
converted_json = json.loads(converted_json)
return idiom_mappings(converted_json, stored_json, ignore)
def main():
elevator_parser = _get_arg_parser(False)
elevator_parser.add_argument(
"dir_",
help="A directory containing STIX 1.x documents to be elevated.",
metavar="dir")
elevator_parser.add_argument("--output-directory",
help="output logs",
dest="output_directory",
action="store",
default=None)
elevator_args = elevator_parser.parse_args()
initialize_options(elevator_args)
all_succeeded = True
for filename in sorted(os.listdir(elevator_args.dir_)):
path = os.path.join(elevator_args.dir_, filename)
if path.endswith(".xml"):
sys.stdout.write(path + "\n")
file_and_ext = filename.split(".")
set_option_value("file_", file_and_ext[0])
result = elevate_file(path)
if result:
if elevator_args.output_directory:
destination = os.path.join(elevator_args.output_directory,
file_and_ext[0] + ".json")
destination = os.path.abspath(destination)
output_file = open(destination, "w")
output_file.write(result)
output_file.close()
else:
print(result + "\n")
else:
all_succeeded = False
if not all_succeeded:
sys.exit(1)
def convert_to_stix_20_to_21(stix20_json):
# 2.0 -> 2.1
initialize_options()
set_option_value('validator_args', '--silent')
set_option_value('silent', 'True')
stix21 = step_bundle(stix20_json)
# for stix2-elevator issue
stix21_new = {}
stix21_new['id'] = stix21['id']
stix21_new['type'] = stix21['type']
stix21_new['objects'] = []
for o_ in stix21['objects']:
if o_['type'] == 'indicator':
if 'pattern_type' not in o_:
o_['pattern_type'] = 'stix'
if o_['type'] == 'malware':
if 'is_family' not in o_:
o_['is_family'] = False
stix21_new['objects'].append(o_)
stix2 = parse(stix21_new, allow_custom=True)
return stix2
def convert_to_stix_1x_to_21(stix_file_path):
# 1.x -> 2.0
initialize_options()
set_option_value('validator_args', '--silent')
set_option_value('silent', 'True')
stix20_str = ELEVATE(stix_file_path)
stix20 = _replace_stix2_tlp(stix20_str)
# for stix2-elevator issue
stix20_new = {}
stix20_new['id'] = stix20['id']
stix20_new['spec_version'] = stix20['spec_version']
stix20_new['type'] = stix20['type']
stix20_new['objects'] = []
for o_ in stix20['objects']:
if o_['type'] == 'identity':
if 'identity_class' not in o_:
o_['identity_class'] = 'unknown'
stix20_new['objects'].append(o_)
stix20_str = json.dumps(stix20_new)
stix20_o = parse(stix20_str)
stix20_json = json.loads(str(stix20_o))
return convert_to_stix_20_to_21(stix20_json)
def idiom_elevator_mappings(before_file_path, stored_json, version,
missing_policy, ignore):
"""Test fresh conversion from XML to JSON matches stored JSON samples."""
print("Checking - " + before_file_path)
print("With Master - " + stored_json["id"])
initialize_options(
options={
"spec_version": version,
"missing_policy": missing_policy,
"log_level": "WARN",
"incidents": True,
"validator_args": "--version " + version
})
if not get_option_value("policy") == "no_policy":
print("'no_policy' is the default for testing")
set_option_value("policy", "no_policy")
if version == "2.1":
set_option_value("acs", True)
sys.setrecursionlimit(3000)
converted_json = elevate(before_file_path)
print(converted_json)
converted_json = json.loads(converted_json)
return idiom_mappings(converted_json, stored_json, ignore)
def idiom_mappings(xml_file_path, stored_json):
"""Test fresh conversion from XML to JSON matches stored JSON samples."""
print("Checking - " + xml_file_path)
print("With Master - " + stored_json["id"])
initialize_options()
set_option_value("log_level", "CRITICAL")
set_option_value("validator_args", "--no-cache")
if not get_option_value("policy") == "no_policy":
print("'no_policy' is not allowed for testing")
set_option_value("policy", "no_policy")
sys.setrecursionlimit(3000)
converted_json = elevate_file(xml_file_path)
converted_json = json.loads(converted_json)
for good, to_check in zip(iterpath(stored_json), iterpath(converted_json)):
good_path, good_value = good
last_good_field = good_path[-1]
if isinstance(good_value, (dict, list)):
# Rule #1: No need to verify iterable types. Since we will deal
# with individual values in the future.
continue
if (any(s in (u"object_marking_refs", u"granular_markings")
for s in good_path)):
# Exception to Rule #1: object_marking_refs and granular_markings
# are not verifiable because they contain identifiers per rule #2.
continue
if last_good_field in IGNORE:
# Rule #2: Since fresh conversion may create dynamic values.
# Some fields are omitted for verification. Currently
# fields with: identifier and timestamp values.
continue
yield good, to_check
def elevate_file(fn):
# TODO: combine elevate_file, elevate_string and elevate_package
global MESSAGES_GENERATED
print(
"Results produced by the stix2-elevator are not for production purposes."
)
clear_id_mapping()
clear_1x_markings_map()
clear_pattern_cache()
clear_object_id_mapping()
clear_observable_mappings()
cybox.utils.caches.cache_clear()
MESSAGES_GENERATED = False
validator_options = get_validator_options()
try:
output.set_level(validator_options.verbose)
output.set_silent(validator_options.silent)
container = stixmarx.parse(fn)
stix_package = container.package
set_option_value("marking_container", container)
if not isinstance(stix_package, STIXPackage):
raise TypeError("Must be an instance of stix.core.STIXPackage")
setup_logger(stix_package.id_)
warn(
"Results produced by the stix2-elevator are not for production purposes.",
201)
if get_option_value("default_timestamp"):
timestamp = datetime.strptime(
get_option_value("default_timestamp"),
"%Y-%m-%dT%H:%M:%S.%fZ"),
else:
timestamp = None
env = Environment(get_option_value("package_created_by_id"), timestamp)
json_string = json.dumps(convert_package(stix_package, env),
ensure_ascii=False,
indent=4,
separators=(',', ': '),
sort_keys=True)
validation_results = validate_stix2_string(json_string,
validator_options, fn)
output.print_results([validation_results])
if get_option_value("policy") == "no_policy":
return json_string
else:
if not MESSAGES_GENERATED and validation_results._is_valid:
return json_string
else:
return None
except ValidationError as ex:
output.error("Validation error occurred: '%s'" % ex,
codes.EXIT_VALIDATION_ERROR)
except OSError as ex:
log.error(ex)
def elevate(stix_package):
global MESSAGES_GENERATED
MESSAGES_GENERATED = False
print("Results produced by the stix2-elevator are not for production purposes.")
clear_globals()
fn = None
validator_options = get_validator_options()
output.set_level(validator_options.verbose)
output.set_silent(validator_options.silent)
try:
if isinstance(stix_package, MarkingContainer):
# No need to re-parse the MarkingContainer.
container = stix_package
elif isinstance(stix_package, STIXPackage):
io = BytesIO(stix_package.to_xml())
container = stixmarx.parse(io)
elif isinstance(stix_package, text_type):
if stix_package.endswith(".xml") or os.path.isfile(stix_package):
# a path-like string was passed
fn = stix_package
if os.path.exists(fn) is False:
raise IOError("The file '{}' was not found.".format(fn))
else:
stix_package = StringIO(stix_package)
container = stixmarx.parse(stix_package)
elif isinstance(stix_package, binary_type):
if stix_package.endswith(b".xml") or os.path.isfile(stix_package):
# a path-like string was passed
fn = stix_package
if os.path.exists(fn) is False:
raise IOError("The file '{}' was not found.".format(fn))
else:
stix_package = BytesIO(stix_package)
container = stixmarx.parse(stix_package)
else:
raise RuntimeError("Unable to resolve object {} of type {}".format(stix_package, type(stix_package)))
container_package = container.package
set_option_value("marking_container", container)
if not isinstance(container_package, STIXPackage):
raise TypeError("Must be an instance of stix.core.STIXPackage")
except (OSError, IOError, lxml.etree.Error) as ex:
log.error("Error occurred: %s", ex)
# log.exception(ex)
return None
try:
setup_logger(container_package.id_)
warn("Results produced by the stix2-elevator may generate warning messages which should be investigated.", 201)
env = Environment(get_option_value("package_created_by_id"))
json_string = json.dumps(
convert_package(container_package, env),
ensure_ascii=False,
indent=4,
separators=(',', ': '),
sort_keys=True
)
bundle_id = re.findall(
r"bundle--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}",
json_string
)
validation_results = validate_stix2_string(json_string, validator_options, fn or bundle_id[0])
output.print_results([validation_results])
if get_option_value("policy") == "no_policy":
return json_string
else:
if not MESSAGES_GENERATED and validation_results._is_valid:
return json_string
except ValidationError as ex:
output.error("Validation error occurred: '{}'".format(ex))
output.error("Error Code: {}".format(codes.EXIT_VALIDATION_ERROR))
