def process_json(hardware_vendor, opersys, device_type, policy_file):
if hardware_vendor == 'cisco' and opersys == 'ios' and device_type == 'firewall':
with open(
"{}/superloop_code/policy/cisco/ios/firewall/{}".format(
get_home_directory(), policy_file), 'r') as json_file:
data = commentjson.load(json_file)
elif hardware_vendor == 'juniper' and opersys == 'junos' and device_type == 'vfirewall':
with open(
"{}/superloop_code/policy/juniper/junos/firewall/{}".format(
get_home_directory(), policy_file), 'r') as json_file:
data = commentjson.load(json_file)
return data
def process_policies():
with open(
"{}/superloop_code/database/policy_push.yaml".format(
get_home_directory()), 'r') as yaml_file:
policy_object = yaml.load(yaml_file, yaml.UnsafeLoader)
return policy_object
def process_templates():
with open(
"{}/superloop_code/database/templates.yaml".format(
get_home_directory()), 'r') as yaml_file:
template_object = yaml.load(yaml_file, yaml.UnsafeLoader)
return template_object
def process_models():
with open(
"{}/superloop_code/database/models.yaml".format(
get_home_directory()), 'r') as yaml_file:
models_object = yaml.load(yaml_file, yaml.UnsafeLoader)
return models_object
def remove(args):
database = process_nodes()
index = 0
try:
for element in database:
if element['name'] == args.node or element['mgmt_ip4'] == args.node:
break
else:
index = index + 1
"""
Delete node from list.
"""
database.pop(index)
updated_database = yaml.dump(database,default_flow_style = False)
try:
with open('{}/superloop_code/database/nodes.yaml'.format(get_home_directory()),'w') as file:
file.write('---\n')
file.write(updated_database)
print('- Node successfully removed from database.')
except FileNotFoundError as error:
print("FileNotFoundError: file cannot be found")
print(error)
except IndexError as error:
print('+ Node does not exist in database.')
return None
def process_nodes():
with open(
"{}/superloop_code/database/nodes.yaml".format(
get_home_directory()), 'r') as yaml_file:
node_object = yaml.load(yaml_file, Loader=Loader)
return node_object
def search_template(template_list, match_node, node_template, node_object,
auditcreeper):
"""
This function will take the search results from the list of nodes
and run it against node_object to determine the hardware vendor and type
and compare with the node_template database to match. Once matched,
it will check to verify an existing template is available.
"""
search_result = []
index = 0
element = 0
for node in match_node:
for node_obj in node_object:
if node == node_obj['name']:
"""
index variable gets the position in the list and appends it to the global variable 'element'.
"""
index = node_object.index(node_obj)
initialize.element.append(index)
"""
This section will pull out all the templates belonging to the specific
hardware vendor, operating system and type from the template database.
"""
for node_temp in node_template:
if node_obj['hardware_vendor'] == node_temp[
'hardware_vendor'] and node_obj[
'opersys'] == node_temp['opersys'] and node_obj[
'type'] == node_temp['type']:
if auditcreeper:
template_node_list = []
for template_dir_name in node_temp['templates']:
template_name = template_dir_name.split(
'/')[-1]
template_node_list.append(template_name)
template_list.append(template_node_list)
else:
directory = get_template_directory(
node_obj['hardware_vendor'],
node_obj['opersys'], node_obj['type'])
file = directory + template_list[element]
template_index = 0
for template_path in node_temp['templates']:
node_temp['templates'][
template_index] = template_path.replace(
'~', '{}'.format(get_home_directory()))
template_index = template_index + 1
if file in node_temp['templates']:
search_result.append("MATCH")
else:
print('+ No associating template {}'.format(
template_list[element]) +
' for node {}'.format(node))
search_result.append("NO MATCH")
else:
continue
else:
continue
return search_result
def append(args):
argument_node = args.node
database = process_nodes()
index = 0
match_node = []
mgmt_ip4 = get_resolve_hostname(argument_node)
# try:
"""
Check if new node currently exist in database.
"""
print(database)
for node in database:
if mgmt_ip4 == node['mgmt_ip4']:
match_node.append('MATCH')
break
else:
continue
try:
if 'MATCH' in match_node:
print('+ Node currently present in database.')
else:
device = snmp(argument_node)
new_node = yaml.dump(device,default_flow_style = False)
with open('{}/superloop_code/database/nodes.yaml'.format(get_home_directory()),'a') as file:
file.write(new_node)
database = process_nodes()
sorted_database = sortdb(database)
updated_database = yaml.dump(sorted_database,default_flow_style = False)
with open('{}/superloop_code/database/nodes.yaml'.format(get_home_directory()),'w') as file:
file.write('---\n')
file.write(updated_database)
print('+ SNMP discovery successful.')
print('+ New node appended to database.')
except FileNotFoundError as error:
print('FileNotFoundError: file cannot be found')
print(error)
# except Exception as error:
# print('SNMP query timeout. Please ensure that FQDN or IP address is reachable via SNMP.')
return None
def process_jinja2_template(node_object, index, template, with_remediation):
hardware_vendor_template_directory = get_template_directory(
node_object[index]['hardware_vendor'], node_object[index]['opersys'],
node_object[index]['type'])
standards_directory = get_standards_directory(
node_object[index]['name'], node_object[index]['hardware_vendor'],
node_object[index]['type'])
env = Environment(loader=FileSystemLoader(
[hardware_vendor_template_directory, standards_directory]),
lstrip_blocks=True,
trim_blocks=True)
baseline = env.get_template(template)
os.makedirs('{}/rendered-configs/'.format(get_home_directory()),
exist_ok=True)
with open(
'{}/rendered-configs/{}.{}'.format(
get_home_directory(), node_object[index]['name'],
template.replace('jinja2', '')) + 'conf', 'w') as file:
config = baseline.render(node=node_object[index],
with_remediation=with_remediation)
file.write(config)
file.close
return config
def push_local(args):
argument_filename = args.filename
argument_node = args.node
commands = initialize.configuration
redirect = []
authentication = True
"""
:param argument_filename: Argument accepted as template name.
:type augument_filename: str
:param argument_node: Argument accepted as regular expression.
:type augument_node: str
:param commands: Referenced to global variable commands which keeps track of all commands per node.
:type commands: list
:param redirect: A list of which method superloop will access. This variable is sent to the multithread_engine. Each element is a redirect per node.
:type alt_key_file: list
"""
node_object = process_nodes()
match_node = search_node(argument_node, node_object)
"""
:param node_object: All node(s) in the database with all attributes.
:type node_object: list
:param match_node: Nodes that matches the arguements passed in by user.
:type match_node: list
"""
if len(match_node) == 0:
print('+ No matching node(s) found in database.')
print('')
else:
node_element(match_node, node_object)
node_create(match_node, node_object)
for index in initialize.element:
redirect.append('push_cfgs')
for index in initialize.element:
config_list = []
with open('{}/{}'.format(get_home_directory(), argument_filename),
'r') as file:
init_config = file.readlines()
parse_commands(node_object[index], init_config, set_notation=True)
confirm(redirect, commands, authentication)
return None
def discover(args):
argument_node = args.node
database = process_nodes()
match_node = search_node(argument_node,database)
"""
:param node_object: All node(s) in the database with all attributes.
:type node_object: list
:param match_node: Nodes that matches the arguements passed in by user.
:type match_node: list
"""
if len(match_node) == 0:
print('+ No matching node(s) found in database.')
return None
else:
for node in match_node:
print('{}'.format(node))
index = 0
try:
for element in database:
if element['name'] == node or element['mgmt_ip4'] == node:
break
else:
index = index + 1
"""
Identified node from list.
"""
device = snmp(node)
print('')
except Exception as error:
print('SNMP query timeout. Please ensure that FQDN or IP address is reachable via SNMP.')
for attribute in database[index]:
if 'created_at' == attribute or 'created_by' == attribute:
continue
else:
database[index][attribute] = device[0][attribute]
database[index]['updated_at'] = timestamp()
database[index]['updated_by'] = '{}'.format(os.environ.get('USER'))
updated_database = yaml.dump(database,default_flow_style = False)
with open('{}/superloop_code/database/nodes.yaml'.format(get_home_directory()),'w') as file:
file.write('---\n')
file.write(updated_database)
print('+ SNMP discovery successful.')
def update(args):
argument_node = args.node
argument_attribute = args.attribute
argument_amend = args.amend
database = process_nodes()
index = 0
try:
for element in database:
if element['name'] == argument_node or element['mgmt_ip4'] == argument_node:
break
else:
index = index + 1
"""
Identified node from list.
"""
try:
if argument_attribute == 'data':
return print('+ Attribute \'data\' cannot be modified via host update.')
else:
check = str(input('Please confirm you would like to change the value from {} : {} : {} to {} : {} : {}. [y/N]: '.format(database[index]['name'],argument_attribute,database[index][argument_attribute],database[index]['name'],argument_attribute,argument_amend)))
if check[0] == 'y':
database[index][argument_attribute] = argument_amend
database[index]['updated_at'] = timestamp()
database[index]['updated_by'] = '{}'.format(os.environ.get('USER'))
updated_database = yaml.dump(database,default_flow_style = False)
try:
with open('{}/superloop_code/database/nodes.yaml'.format(get_home_directory()),'w') as file:
file.write('---\n')
file.write(updated_database)
print('+ Amendment to database was successful.')
except FileNotFoundError as error:
print("FileNotFoundError: file cannot be found")
print(error)
elif check[0] == 'N':
return False
else:
print("RuntimeError: aborted at user request")
except Exception as error:
print('+ Invalid attribute \'{}\' for \'{}\'. Please check node details via \'superloop node list {}\''.format(argument_attribute,database[index]['name'],database[index]['name']))
except IndexError as error:
print('+ Node does not exist in database.')
def render(template_list, node_object, auditcreeper, output, with_remediation):
global with_remediate
with_remediate = with_remediation
template_list_copy = template_list
if auditcreeper:
template_list = template_list_copy[0]
for index in initialize.element:
get_hardware_vendor_template_directory = get_template_directory(
node_object[index]['hardware_vendor'],
node_object[index]['opersys'], node_object[index]['type'])
print("{}".format(node_object[index]['name']))
for template in template_list:
config = process_jinja2_template(node_object, index, template,
with_remediation)
print("{}{}".format(get_hardware_vendor_template_directory,
template))
if (output):
print("{}".format(config))
with open(
'{}/rendered-configs/{}.{}'.format(
get_home_directory(), node_object[index]['name'],
template.replace('jinja2', '')) + 'conf', 'r') as file:
init_config = f.readlines()
"""
The below parse_commands() function will only get executed if
it needs to store commands in the global variable initialize.configuration
for push. push_cfgs(output = True) vs render_config(output = False) functions.
"""
if output != True:
parse_commands(node_object[index],
init_config,
set_notation=False)
print()
if auditcreeper:
template_list = get_updated_list(template_list_copy)
return None
def mediator(template_list,node_object,auditcreeper,output,with_remediation):
redirect = []
command = []
template_counter = 0
node_index = 0
AUDIT_FILTER_RE = r'\[.*\]'
template_list_original = template_list[:]
template_list_copy = template_list
authentication = True
"""
:param redirect: A list of which method superloop will access. This variable is sent to the multithread_engine. Each element is a redirect per node.
:type alt_key_file: list
:param command: A list within a list where each element represents per node of commands to execute.
:type command: list
:param template_counter: Used to keep track of the number of templates it cycles through for Juniper.
:type template_counter: int
:param node_index: Keeps track of the index in initialize.ntw_device. If remediation is not required (configs matches template), then the node is popped of initialize.ntw_device and nothing is changed on that device.
:type node_index: int
:param AUDIT_FILTER_RE: Regular expression to filter out the audit filter in every template.
:type AUDIT_FILTER_RE: str
:param template_list_original: Take a duplicate copy of template_list
:type template_list_original: list
:param template_list_copy: Memory reference to template_list
:type template_list_copy: list
"""
"""
The mediator is broken up into two sections. The first section of code will gather all rendered configs first as it's required for all hardware vendors
(Cisco, Juniper & F5). Juniper does not require backup-configs in order to be diff'ed. The diff is server (node) side processed and the results
are returned back to superloop. Cisco hardware_vendors will require backup-configs (get_config) where the diffs are processed locally.
"""
if auditcreeper:
template_list = template_list_copy[0]
for index in initialize.element:
rendered_config = []
if node_object[index]['hardware_vendor'] == 'juniper':
"""
Juniper's diff output are always in a certain stanza order.
The template_list ordered processed may very well not be in the
same order as Juniper's. In order to keep it consistent, we must
call the function get_sorted_juniper_template() and it will
return a sorted Juniper's stanza list.
"""
template_list = get_sorted_juniper_template_list(template_list)
rendered_config.append('load replace terminal')
for template in template_list:
process_jinja2_template(node_object,index,template,with_remediation)
"""
Compiling the rendered configs from template and preparing
for pushing to node.
"""
if node_object[index]['hardware_vendor'] == 'juniper':
template_counter = template_counter + 1
with open('{}/rendered-configs/{}.{}'.format(get_home_directory(),node_object[index]['name'],template.split('.')[0]) + '.conf','r') as file:
init_config = file.readlines()
for config_line in init_config:
strip_config = config_line.strip('\n')
if(strip_config == '' or strip_config == "!"):
continue
else:
rendered_config.append(strip_config)
"""
This below statement will check to see if it's the last
template for the node. It will then append 3 commands to the list.
"""
if template_counter == len(template_list):
rendered_config.append('\x04')
rendered_config.append('show | compare')
rendered_config.append('rollback 0')
"""
Uncomment the below print statement for debugging purposes
"""
#print("RENDERED CONFIG: {}".format(rendered_config))
"""
The below statement will only execute if user is auditing
against multiple templates. If only one template is being
audited, do no pop off element.
"""
if len(template_list) != 1:
template_list = get_updated_list(template_list_copy)
if node_object[index]['hardware_vendor'] == 'cisco' or node_object[index]['hardware_vendor'] == 'f5':
redirect.append('get_config')
command.append([''])
"""
Juniper devices will receive a different redirect than
Cisco. Three additional commands are appeneded
at the end, ^d, show | compare and rollback 0. All templates
matching are executed at once per device
"""
elif node_object[index]['hardware_vendor'] == 'juniper':
redirect.append('get_diff')
command.append(rendered_config)
template_counter = 0
"""
Uncomment the below print statement for debugging purposes
"""
#print('REDIRECT: {}'.format(redirect))
#print('TEMPLATE_LIST: {}'.format(template_list))
#print('COMMAND: {}'.format(command))
multithread_engine(initialize.ntw_device,redirect,command,authentication)
template_list = template_list_original
if(auditcreeper):
template_list = template_list_original[0]
for index in initialize.element:
edit_list = []
node_configs = []
diff_config = []
ntw_device_pop = True
"""
:param edit_list: Anchor points for Juniper audits based on the edit stanza.
:type edit_list: list
:param node_configs: Finalized configs used to push to device
:type node_configs: list
:param diff_config: Differential configs generated by the ~/diff-configs/*.conf file
:type diff_config: list
:param ntw_device_pop: Pop off each node once audit is complete.
:type ntw_device_pop: bool
"""
if not with_remediation:
print("Only in the device: -")
print("Only in the generated config: +")
print ("{}".format(node_object[index]['name']))
if node_object[index]['hardware_vendor'] == 'cisco' or node_object[index]['hardware_vendor'] == 'f5':
generic_audit_diff(node_object,index,template,template_list,AUDIT_FILTER_RE,output,with_remediation)
elif node_object[index]['hardware_vendor'] == 'juniper':
template_list = get_sorted_juniper_template_list(template_list)
directory = get_template_directory(node_object[index]['hardware_vendor'],node_object[index]['opersys'],node_object[index]['type'])
with open('{}/diff-configs/{}'.format(get_home_directory(),node_object[index]['name']) + '.conf','r') as file:
init_config = file.readlines()
for config_line in init_config:
strip_config = config_line.strip('\n')
diff_config.append(strip_config)
for output in diff_config:
if 'errors' in output:
error = True
break
else:
error = False
if error:
print('+ Please check error(s) in template(s)')
break
else:
juniper_mediator(node_object,template_list,diff_config,edit_list,index)
juniper_audit_diff(directory,template_list,diff_config,edit_list)
if auditcreeper:
initialize.configuration.append(node_configs)
if ntw_device_pop == True:
initialize.ntw_device.pop(node_index)
initialize.configuration.pop(node_index)
template_list = get_updated_list(template_list_original)
return None
