def format_and_merge_unexportable_objects(data_dict, unexportable_objects):
formatted_objects = []
for unexportable_object in unexportable_objects:
placeholder = {"name": unexportable_object["name"]}
for unexportable_obj_type in placeholder_type_by_obj_type.keys():
if unexportable_obj_type in unexportable_object["type"]:
for field in placeholder_type_by_obj_type[unexportable_obj_type]:
field_value = placeholder_type_by_obj_type[unexportable_obj_type][field]
if field_value:
placeholder[field] = placeholder_type_by_obj_type[unexportable_obj_type][field]
else:
placeholder[field] = generate_new_dummy_ip_address()
if "type" not in placeholder:
placeholder["type"] = "group"
formatted_objects.append(placeholder)
if placeholder["type"] in data_dict:
data_dict[placeholder["type"]].insert(0, placeholder)
else:
data_dict[placeholder["type"]] = [placeholder]
def add_object(line, counter, position_decrement_due_to_rule,
position_decrement_due_to_section, fields, api_type,
generic_type, layer, layers_to_attach, changed_layer_names,
api_call, num_objects, client, args):
global duplicates_dict
global position_decrements_for_sections
global missing_parameter_set
global should_create_imported_nat_top_section
global should_create_imported_nat_bottom_section
global imported_nat_top_section_uid
if "access-rule" in api_type:
position_decrements_for_sections.append(position_decrement_due_to_rule)
payload, _ = create_payload(fields, line, 0, api_type, client.api_version)
# for objects that had collisions, use new name in the imported package
for field in ["members", "source", "destination"]:
if field in payload:
for i, member in enumerate(payload[field]):
if member in name_collision_map:
payload[field][i] = name_collision_map[member]
payload[
"ignore-warnings"] = True # Useful for example when creating two hosts with the same IP
if "nat-rule" in api_type:
# For NAT rules, the 'layer' parameter is the name of the policy package!!!
payload["package"] = layer
# --- NAT rules specific logic ---
# Importing only rules, without sections.
# Rules marked as "__before_auto_rules = TRUE" will be imported at the TOP of the rulebase, inside a new section "IMPORTED UPPER RULES".
# There is an additional new section "Original Upper Rules" at the bottom of "IMPORTED UPPER RULES".
# Rules marked as "__before_auto_rules = FALSE" will be imported at the BOTTOM of the rulebase, inside a new section "IMPORTED LOWER RULES".
# There will be no rule merges!!!
before_auto_rules = payload["__before_auto_rules"]
payload.pop("__before_auto_rules", None)
if "true" in before_auto_rules:
if should_create_imported_nat_top_section:
should_create_imported_nat_top_section = False
nat_section_payload = {}
nat_section_payload["package"] = layer
nat_section_payload["position"] = "top"
# --> we add the footer section first!!!
nat_section_payload["name"] = "Original Upper Rules"
client.api_call("add-nat-section", nat_section_payload)
# <--
nat_section_payload["name"] = "IMPORTED UPPER RULES"
nat_section_reply = client.api_call("add-nat-section",
nat_section_payload)
if nat_section_reply.success:
imported_nat_top_section_uid = nat_section_reply.data[
"uid"]
if imported_nat_top_section_uid is None:
payload["position"] = "bottom"
else:
sub_payload = {}
sub_payload["bottom"] = imported_nat_top_section_uid
payload["position"] = sub_payload
else:
if should_create_imported_nat_bottom_section:
should_create_imported_nat_bottom_section = False
nat_section_payload = {}
nat_section_payload["package"] = layer
nat_section_payload["position"] = "bottom"
nat_section_payload["name"] = "IMPORTED LOWER RULES"
client.api_call("add-nat-section", nat_section_payload)
payload["position"] = "bottom"
else:
if "position" in payload:
if "rule" in api_type:
payload["position"] = str(
int(payload["position"]) - position_decrement_due_to_rule)
if payload["action"] == "Drop":
if "action-settings" in payload:
payload.pop("action-settings")
if "user-check" in payload:
if "frequency" in payload["user-check"]:
payload["user-check"].pop("frequency")
if "custom-frequency" in payload["user-check"]:
payload["user-check"].pop("custom-frequency")
if "confirm" in payload["user-check"]:
payload["user-check"].pop("confirm")
if "section" in api_type:
section_position_decrement = (
position_decrements_for_sections[int(payload["position"]) -
1]
if len(position_decrements_for_sections) > 0 else
0) + position_decrement_due_to_section
payload["position"] = str(
int(payload["position"]) - section_position_decrement)
if generic_type:
payload["create"] = generic_type
if "layer" in api_type:
check_duplicate_layer(payload, changed_layer_names, api_type,
client)
if compare_versions(client.api_version, "1.1") != -1:
payload["add-default-rule"] = "false"
if layer is None:
if "access-layer" in api_type:
#---> This code segment distinguishes between an inline layer and an ordered layer during import
is_ordered_access_control_layer = payload[
"__ordered_access_control_layer"]
payload.pop("__ordered_access_control_layer", None)
if "true" in is_ordered_access_control_layer:
layers_to_attach["access"].append(
payload["name"]) # ordered access layer
#<--- end of code segment
else:
layers_to_attach["threat"].append(payload["name"])
elif "rule" in api_type or "section" in api_type or \
(api_type == "threat-exception" and "exception-group-name" not in payload):
payload["layer"] = layer
if client.api_version != "1" and api_type == "access-rule" and "track-alert" in payload:
payload["track"] = {}
payload["track"]["alert"] = payload["track-alert"]
payload.pop("track-alert", None)
elif api_type == "exception-group" and "applied-threat-rules" in payload:
for applied_rule in payload["applied-threat-rules"]:
if applied_rule["layer"] in changed_layer_names.keys():
applied_rule["layer"] = changed_layer_names[
applied_rule["layer"]]
api_reply = client.api_call(api_call, payload)
if not api_reply.success and "name" in payload and "More than one object" in api_reply.error_message:
i = 0
original_name = payload["name"]
while not api_reply.success:
payload["name"] = "NAME_COLLISION_RESOLVED" + (
"_" if i == 0 else "_%s_" % i) + original_name
api_reply = client.api_call(api_call, payload)
i += 1
if i > 100:
payload["name"] = original_name
break
if api_reply.success:
debug_log(
"Object \"%s\" was renamed to \"%s\" to resolve the name collision"
% (original_name, payload["name"]), True, True)
name_collision_map[original_name] = payload["name"]
if not api_reply.success:
if api_reply.data and "errors" in api_reply.data:
error_msg = api_reply.data["errors"][0]["message"]
elif api_reply.data and "warnings" in api_reply.data:
error_msg = api_reply.data["warnings"][0]["message"]
else:
error_msg = api_reply.error_message
log_err_msg = "Failed to import {0}{1}. Error: {2}".format(
api_type, " with name [" + payload["name"] +
"]" if "name" in payload else "", error_msg)
if "More than one object" in api_reply.error_message:
log_err_msg = api_reply.error_message + ". Cannot import this object"
if "rule" in api_type and ("Requested object"
in api_reply.error_message
and "not found" in api_reply.error_message):
field_value = api_reply.error_message.split("[")[1].split("]")[0]
indices_of_field = [
i for i, x in enumerate(line) if x == field_value
]
field_keys = [
x for x in fields if fields.index(x) in indices_of_field
]
for field_key in field_keys:
if field_key.split(".")[0] in generic_objects_for_rule_fields:
missing_obj_data = generic_objects_for_rule_fields[
field_key.split(".")[0]]
missing_type = missing_obj_data[0]
mandatory_field = missing_obj_data[1] if len(
missing_obj_data) > 1 else None
add_missing_command = "add-" + missing_type
new_name = "import_error_due_to_missing_fields_" + field_value.replace(
" ", "_")
add_succeeded = True
if new_name not in missing_parameter_set:
missing_parameter_set.add(new_name)
add_missing_payload = {"name": new_name}
if mandatory_field == "port":
add_missing_payload["port"] = "8080"
elif mandatory_field == "ip-address":
add_missing_payload[
"ip-address"] = generate_new_dummy_ip_address(
)
add_missing_reply = client.api_call(
add_missing_command, add_missing_payload)
if not add_missing_reply.success:
log_err_msg += "\nAlso failed to generate placeholder object: {0}".format(
add_missing_reply.error_message)
add_succeeded = False
if add_succeeded:
line[fields.index(field_key)] = new_name
return add_object(line, counter,
position_decrement_due_to_rule,
position_decrement_due_to_section,
fields, api_type, generic_type,
layer, layers_to_attach,
changed_layer_names, api_call,
num_objects, client, args)
if "Invalid parameter for [position]" in api_reply.error_message:
if "access-rule" in api_type:
position_decrement_due_to_rule += adjust_position_decrement(
int(payload["position"]), api_reply.error_message)
elif "access-section" in api_type:
position_decrement_due_to_section += adjust_position_decrement(
int(payload["position"]), api_reply.error_message)
return add_object(line, counter, position_decrement_due_to_rule,
position_decrement_due_to_section, fields,
api_type, generic_type, layer, layers_to_attach,
changed_layer_names, api_call, num_objects,
client, args)
elif "is not unique" in api_reply.error_message and "name" in api_reply.error_message:
field_value = api_reply.error_message.partition("name")[2].split(
"[")[1].split("]")[0]
debug_log(
"Not unique name problem \"%s\" - changing payload to use UID instead."
% field_value, True, True)
obj_uid_found_and_used = False
if field_value not in duplicates_dict:
show_objects_reply = client.api_query(
"show-objects",
payload={"in": ["name", "\"" + field_value + "\""]})
if show_objects_reply.success:
for obj in show_objects_reply.data:
if obj["name"] == field_value:
duplicates_dict[field_value] = obj["uid"]
obj_uid_found_and_used = True
if obj_uid_found_and_used:
indices_of_field = [
i for i, x in enumerate(line) if x == field_value
]
field_keys = [
x for x in fields if fields.index(x) in indices_of_field
]
for field_key in field_keys:
line[fields.index(
field_key)] = duplicates_dict[field_value]
return add_object(line, counter,
position_decrement_due_to_rule,
position_decrement_due_to_section, fields,
api_type, generic_type, layer,
layers_to_attach, changed_layer_names,
api_call, num_objects, client, args)
else:
debug_log(
"Not unique name problem \"%s\" - cannot change payload to use UID instead of name."
% field_value, True, True)
elif "will place the exception in an Exception-Group" in api_reply.error_message:
return add_object(line, counter,
position_decrement_due_to_rule - 1,
position_decrement_due_to_section, fields,
api_type, generic_type, layer, layers_to_attach,
changed_layer_names, api_call, num_objects,
client, args)
position_decrement_due_to_rule += 1
debug_log(log_err_msg, True, True)
if args is not None and args.strict:
discard_reply = client.api_call("discard")
if not discard_reply.success:
debug_log(
"Failed to discard changes! Terminating. Error: " +
discard_reply.error_message, True, True)
exit(1)
else:
imported_name = payload["name"] if "name" in payload else ""
debug_log("Imported {0}{1}".format(
api_type, " with name [" + imported_name.encode("utf-8") + "]"))
if counter % 20 == 0 or counter == num_objects:
percentage = int(float(counter) / float(num_objects) * 100)
debug_log(
"Imported {0} out of {1} {2} ({3}%)".format(
counter, num_objects,
singular_to_plural_dictionary[client.api_version][api_type]
if api_type
in singular_to_plural_dictionary[client.api_version] else
"generic objects", percentage), True)
if counter % 100 == 0 or counter == num_objects:
publish_reply = client.api_call("publish", wait_for_task=True)
if not publish_reply.success:
plural = singular_to_plural_dictionary[client.api_version][api_type].replace('_', ' ') \
if api_type in singular_to_plural_dictionary[client.api_version] \
else "generic objects of type " + api_type
try:
debug_log(
"Failed to publish import of " + plural +
" from tar file #" + str((counter / 100) + 1) +
"! " + plural.capitalize() +
" from said file were not imported!. Error: " +
str(publish_reply.error_message), True, True)
except UnicodeEncodeError:
try:
debug_log(
"UnicodeEncodeError: " +
str(publish_reply.error_message), True, True)
except:
debug_log(
"UnicodeEncodeError: .encode('utf-8') FAILED",
True, True)
discard_reply = client.api_call("discard")
if not discard_reply.success:
debug_log(
"Failed to discard changes of unsuccessful publish! Terminating. Error: "
+ discard_reply.error_message, True, True)
exit(1)
return counter + 1, position_decrement_due_to_rule