def format_and_merge_unexportable_objects(data_dict, unexportable_objects): formatted_objects = [] for unexportable_object in unexportable_objects: placeholder = {"name": unexportable_object["name"]} for unexportable_obj_type in placeholder_type_by_obj_type.keys(): if unexportable_obj_type in unexportable_object["type"]: for field in placeholder_type_by_obj_type[unexportable_obj_type]: field_value = placeholder_type_by_obj_type[unexportable_obj_type][field] if field_value: placeholder[field] = placeholder_type_by_obj_type[unexportable_obj_type][field] else: placeholder[field] = generate_new_dummy_ip_address() if "type" not in placeholder: placeholder["type"] = "group" formatted_objects.append(placeholder) if placeholder["type"] in data_dict: data_dict[placeholder["type"]].insert(0, placeholder) else: data_dict[placeholder["type"]] = [placeholder]
def add_object(line, counter, position_decrement_due_to_rule, position_decrement_due_to_section, fields, api_type, generic_type, layer, layers_to_attach, changed_layer_names, api_call, num_objects, client, args): global duplicates_dict global position_decrements_for_sections global missing_parameter_set global should_create_imported_nat_top_section global should_create_imported_nat_bottom_section global imported_nat_top_section_uid if "access-rule" in api_type: position_decrements_for_sections.append(position_decrement_due_to_rule) payload, _ = create_payload(fields, line, 0, api_type, client.api_version) # for objects that had collisions, use new name in the imported package for field in ["members", "source", "destination"]: if field in payload: for i, member in enumerate(payload[field]): if member in name_collision_map: payload[field][i] = name_collision_map[member] payload[ "ignore-warnings"] = True # Useful for example when creating two hosts with the same IP if "nat-rule" in api_type: # For NAT rules, the 'layer' parameter is the name of the policy package!!! payload["package"] = layer # --- NAT rules specific logic --- # Importing only rules, without sections. # Rules marked as "__before_auto_rules = TRUE" will be imported at the TOP of the rulebase, inside a new section "IMPORTED UPPER RULES". # There is an additional new section "Original Upper Rules" at the bottom of "IMPORTED UPPER RULES". # Rules marked as "__before_auto_rules = FALSE" will be imported at the BOTTOM of the rulebase, inside a new section "IMPORTED LOWER RULES". # There will be no rule merges!!! before_auto_rules = payload["__before_auto_rules"] payload.pop("__before_auto_rules", None) if "true" in before_auto_rules: if should_create_imported_nat_top_section: should_create_imported_nat_top_section = False nat_section_payload = {} nat_section_payload["package"] = layer nat_section_payload["position"] = "top" # --> we add the footer section first!!! nat_section_payload["name"] = "Original Upper Rules" client.api_call("add-nat-section", nat_section_payload) # <-- nat_section_payload["name"] = "IMPORTED UPPER RULES" nat_section_reply = client.api_call("add-nat-section", nat_section_payload) if nat_section_reply.success: imported_nat_top_section_uid = nat_section_reply.data[ "uid"] if imported_nat_top_section_uid is None: payload["position"] = "bottom" else: sub_payload = {} sub_payload["bottom"] = imported_nat_top_section_uid payload["position"] = sub_payload else: if should_create_imported_nat_bottom_section: should_create_imported_nat_bottom_section = False nat_section_payload = {} nat_section_payload["package"] = layer nat_section_payload["position"] = "bottom" nat_section_payload["name"] = "IMPORTED LOWER RULES" client.api_call("add-nat-section", nat_section_payload) payload["position"] = "bottom" else: if "position" in payload: if "rule" in api_type: payload["position"] = str( int(payload["position"]) - position_decrement_due_to_rule) if payload["action"] == "Drop": if "action-settings" in payload: payload.pop("action-settings") if "user-check" in payload: if "frequency" in payload["user-check"]: payload["user-check"].pop("frequency") if "custom-frequency" in payload["user-check"]: payload["user-check"].pop("custom-frequency") if "confirm" in payload["user-check"]: payload["user-check"].pop("confirm") if "section" in api_type: section_position_decrement = ( position_decrements_for_sections[int(payload["position"]) - 1] if len(position_decrements_for_sections) > 0 else 0) + position_decrement_due_to_section payload["position"] = str( int(payload["position"]) - section_position_decrement) if generic_type: payload["create"] = generic_type if "layer" in api_type: check_duplicate_layer(payload, changed_layer_names, api_type, client) if compare_versions(client.api_version, "1.1") != -1: payload["add-default-rule"] = "false" if layer is None: if "access-layer" in api_type: #---> This code segment distinguishes between an inline layer and an ordered layer during import is_ordered_access_control_layer = payload[ "__ordered_access_control_layer"] payload.pop("__ordered_access_control_layer", None) if "true" in is_ordered_access_control_layer: layers_to_attach["access"].append( payload["name"]) # ordered access layer #<--- end of code segment else: layers_to_attach["threat"].append(payload["name"]) elif "rule" in api_type or "section" in api_type or \ (api_type == "threat-exception" and "exception-group-name" not in payload): payload["layer"] = layer if client.api_version != "1" and api_type == "access-rule" and "track-alert" in payload: payload["track"] = {} payload["track"]["alert"] = payload["track-alert"] payload.pop("track-alert", None) elif api_type == "exception-group" and "applied-threat-rules" in payload: for applied_rule in payload["applied-threat-rules"]: if applied_rule["layer"] in changed_layer_names.keys(): applied_rule["layer"] = changed_layer_names[ applied_rule["layer"]] api_reply = client.api_call(api_call, payload) if not api_reply.success and "name" in payload and "More than one object" in api_reply.error_message: i = 0 original_name = payload["name"] while not api_reply.success: payload["name"] = "NAME_COLLISION_RESOLVED" + ( "_" if i == 0 else "_%s_" % i) + original_name api_reply = client.api_call(api_call, payload) i += 1 if i > 100: payload["name"] = original_name break if api_reply.success: debug_log( "Object \"%s\" was renamed to \"%s\" to resolve the name collision" % (original_name, payload["name"]), True, True) name_collision_map[original_name] = payload["name"] if not api_reply.success: if api_reply.data and "errors" in api_reply.data: error_msg = api_reply.data["errors"][0]["message"] elif api_reply.data and "warnings" in api_reply.data: error_msg = api_reply.data["warnings"][0]["message"] else: error_msg = api_reply.error_message log_err_msg = "Failed to import {0}{1}. Error: {2}".format( api_type, " with name [" + payload["name"] + "]" if "name" in payload else "", error_msg) if "More than one object" in api_reply.error_message: log_err_msg = api_reply.error_message + ". Cannot import this object" if "rule" in api_type and ("Requested object" in api_reply.error_message and "not found" in api_reply.error_message): field_value = api_reply.error_message.split("[")[1].split("]")[0] indices_of_field = [ i for i, x in enumerate(line) if x == field_value ] field_keys = [ x for x in fields if fields.index(x) in indices_of_field ] for field_key in field_keys: if field_key.split(".")[0] in generic_objects_for_rule_fields: missing_obj_data = generic_objects_for_rule_fields[ field_key.split(".")[0]] missing_type = missing_obj_data[0] mandatory_field = missing_obj_data[1] if len( missing_obj_data) > 1 else None add_missing_command = "add-" + missing_type new_name = "import_error_due_to_missing_fields_" + field_value.replace( " ", "_") add_succeeded = True if new_name not in missing_parameter_set: missing_parameter_set.add(new_name) add_missing_payload = {"name": new_name} if mandatory_field == "port": add_missing_payload["port"] = "8080" elif mandatory_field == "ip-address": add_missing_payload[ "ip-address"] = generate_new_dummy_ip_address( ) add_missing_reply = client.api_call( add_missing_command, add_missing_payload) if not add_missing_reply.success: log_err_msg += "\nAlso failed to generate placeholder object: {0}".format( add_missing_reply.error_message) add_succeeded = False if add_succeeded: line[fields.index(field_key)] = new_name return add_object(line, counter, position_decrement_due_to_rule, position_decrement_due_to_section, fields, api_type, generic_type, layer, layers_to_attach, changed_layer_names, api_call, num_objects, client, args) if "Invalid parameter for [position]" in api_reply.error_message: if "access-rule" in api_type: position_decrement_due_to_rule += adjust_position_decrement( int(payload["position"]), api_reply.error_message) elif "access-section" in api_type: position_decrement_due_to_section += adjust_position_decrement( int(payload["position"]), api_reply.error_message) return add_object(line, counter, position_decrement_due_to_rule, position_decrement_due_to_section, fields, api_type, generic_type, layer, layers_to_attach, changed_layer_names, api_call, num_objects, client, args) elif "is not unique" in api_reply.error_message and "name" in api_reply.error_message: field_value = api_reply.error_message.partition("name")[2].split( "[")[1].split("]")[0] debug_log( "Not unique name problem \"%s\" - changing payload to use UID instead." % field_value, True, True) obj_uid_found_and_used = False if field_value not in duplicates_dict: show_objects_reply = client.api_query( "show-objects", payload={"in": ["name", "\"" + field_value + "\""]}) if show_objects_reply.success: for obj in show_objects_reply.data: if obj["name"] == field_value: duplicates_dict[field_value] = obj["uid"] obj_uid_found_and_used = True if obj_uid_found_and_used: indices_of_field = [ i for i, x in enumerate(line) if x == field_value ] field_keys = [ x for x in fields if fields.index(x) in indices_of_field ] for field_key in field_keys: line[fields.index( field_key)] = duplicates_dict[field_value] return add_object(line, counter, position_decrement_due_to_rule, position_decrement_due_to_section, fields, api_type, generic_type, layer, layers_to_attach, changed_layer_names, api_call, num_objects, client, args) else: debug_log( "Not unique name problem \"%s\" - cannot change payload to use UID instead of name." % field_value, True, True) elif "will place the exception in an Exception-Group" in api_reply.error_message: return add_object(line, counter, position_decrement_due_to_rule - 1, position_decrement_due_to_section, fields, api_type, generic_type, layer, layers_to_attach, changed_layer_names, api_call, num_objects, client, args) position_decrement_due_to_rule += 1 debug_log(log_err_msg, True, True) if args is not None and args.strict: discard_reply = client.api_call("discard") if not discard_reply.success: debug_log( "Failed to discard changes! Terminating. Error: " + discard_reply.error_message, True, True) exit(1) else: imported_name = payload["name"] if "name" in payload else "" debug_log("Imported {0}{1}".format( api_type, " with name [" + imported_name.encode("utf-8") + "]")) if counter % 20 == 0 or counter == num_objects: percentage = int(float(counter) / float(num_objects) * 100) debug_log( "Imported {0} out of {1} {2} ({3}%)".format( counter, num_objects, singular_to_plural_dictionary[client.api_version][api_type] if api_type in singular_to_plural_dictionary[client.api_version] else "generic objects", percentage), True) if counter % 100 == 0 or counter == num_objects: publish_reply = client.api_call("publish", wait_for_task=True) if not publish_reply.success: plural = singular_to_plural_dictionary[client.api_version][api_type].replace('_', ' ') \ if api_type in singular_to_plural_dictionary[client.api_version] \ else "generic objects of type " + api_type try: debug_log( "Failed to publish import of " + plural + " from tar file #" + str((counter / 100) + 1) + "! " + plural.capitalize() + " from said file were not imported!. Error: " + str(publish_reply.error_message), True, True) except UnicodeEncodeError: try: debug_log( "UnicodeEncodeError: " + str(publish_reply.error_message), True, True) except: debug_log( "UnicodeEncodeError: .encode('utf-8') FAILED", True, True) discard_reply = client.api_call("discard") if not discard_reply.success: debug_log( "Failed to discard changes of unsuccessful publish! Terminating. Error: " + discard_reply.error_message, True, True) exit(1) return counter + 1, position_decrement_due_to_rule