def as3_post(task: Task, as3_tenant: str) -> Result:
failover_status = task.run(
name="Get failover status", task=bigip_cm_failover_status
).result
if failover_status == "ACTIVE":
task.run(
name="AS3 POST",
task=atc,
atc_delay=0,
atc_method="POST",
atc_retries=3,
atc_service="AS3",
as3_tenant=as3_tenant,
atc_declaration_file=task.host["appsvcs"][as3_tenant][
"atc_declaration_file"
],
)
task.run(
name="Synchronize the devices",
task=bigip_cm_config_sync,
delay=0,
device_group=task.host["device_group"],
retries=3,
)
return Result(
host=task.host,
result="ACTIVE device, AS3 declaration successfully deployed.",
)
else:
return Result(host=task.host, result="STANDBY device, skipped.")
def render_template_apply(task: Task,
template_name: str,
template_path: Optional[str] = None) -> Result:
from jinja2 import Environment, FileSystemLoader, StrictUndefined
logger.info(f"{task.host}: Rendering Config")
if not template_path:
template_path = os.path.join(os.getcwd(),
f'templates/{task.host.platform}')
env = Environment(
loader=FileSystemLoader(template_path),
undefined=StrictUndefined,
trim_blocks=False,
)
config = task.run(task=template_file,
name='Render Device Configuration',
template=f'{template_name}.j2',
jinja_env=env,
path=template_path,
**task.host)
result = task.run(task=napalm_configure,
dry_run=False,
replace=False,
configuration=config.result)
return Result(host=task.host,
result=result.result,
failed=False,
changed=False)
def create_fake_task(output,
vendor_vars,
vrf_name,
nos,
test_obj,
effect=None):
'''Create instance of nornir.core.task.Task and nornir.core.task.Host,
assign latter to former, but mocking Host get_connection method to not
really connect to hosts and return files outputs. Can use single return
value or side effect with list of files.
Arguments:
* output - string with CLI output of some command, if we need single
output
* vendor_vars - dict with NOS CLI commands
* vrf_name - name of VRF to test upon
* nos - NOS name
* test_obj - instance of nornir.core.task.Task which we will test
* effect (defaults to None) - Mock.side_effect to provide different
outputs
Returns:
* instance of nornir.core.task.Task with mocked internals
'''
connection = Mock()
if effect is None:
connection.send_command = Mock(return_value=output)
else:
connection.send_command = Mock(side_effect=effect)
host = OvercomeHostSlots('test-host')
host['vendor_vars'] = vendor_vars
host['vrf_name'] = vrf_name
host.platform = nos
host.get_connection = Mock(return_value=connection)
fake_task = Task(test_obj)
fake_task.host = host
return fake_task
def set_loopbacks(task: Task, ):
loopbacks = task.host.data['loopbacks']
for key in loopbacks:
interface = 'interface loopback' + str(key)
# ip_addr = 'ip address ' + loopbacks[key] + ' 255.255.255.255'
ip_addr = f'ip address {loopbacks[key]} 255.255.255.255'
config_cmds = [interface, ip_addr]
task.run(task=netmiko_send_config, config_commands=config_cmds)
def run_backup( # pylint: disable=too-many-arguments
task: Task, logger, global_settings, remove_regex_dict,
replace_regex_dict) -> Result:
r"""Backup configurations to disk.
Args:
task (Task): Nornir task individual object
remove_regex_dict (dict): {'cisco_ios': ['^Building\\s+configuration.*\\n', '^Current\\s+configuration.*\\n', '^!\\s+Last\\s+configuration.*'], 'arista_eos': ['.s*']}
replace_regex_dict (dict): {'cisco_ios': [{'regex_replacement': '<redacted_config>', 'regex_search': 'username\\s+\\S+\\spassword\\s+5\\s+(\\S+)\\s+role\\s+\\S+'}]}
Returns:
result (Result): Result from Nornir task
"""
obj = task.host.data["obj"]
backup_obj = GoldenConfig.objects.filter(device=obj).first()
if not backup_obj:
backup_obj = GoldenConfig.objects.create(device=obj, )
backup_obj.backup_last_attempt_date = task.host.defaults.data["now"]
backup_obj.save()
backup_directory = get_repository_working_dir("backup", obj, logger,
global_settings)
backup_path_template_obj = render_jinja_template(
obj, logger, global_settings.backup_path_template)
backup_file = os.path.join(backup_directory, backup_path_template_obj)
if global_settings.backup_test_connectivity is not False:
task.run(
task=dispatcher,
name="TEST CONNECTIVITY",
method="check_connectivity",
obj=obj,
logger=logger,
default_drivers_mapping=get_dispatcher(),
)
running_config = task.run(
task=dispatcher,
name="SAVE BACKUP CONFIGURATION TO FILE",
method="get_config",
obj=obj,
logger=logger,
backup_file=backup_file,
remove_lines=remove_regex_dict.get(obj.platform.slug, []),
substitute_lines=replace_regex_dict.get(obj.platform.slug, []),
default_drivers_mapping=get_dispatcher(),
)[1].result["config"]
backup_obj.backup_last_success_date = task.host.defaults.data["now"]
backup_obj.backup_config = running_config
backup_obj.save()
logger.log_success(
obj, "Successfully extracted running configuration from device.")
return Result(host=task.host, result=running_config)
def set_loopbacks(task: Task,):
loopbacks = task.host.data['loopbacks']
task.run(task=netmiko_send_config, config_commands = ["interface loopback 99"])
# clock=task.run(task=netmiko_send_command, command_string="show clock")
return Result(
host=task.host,
result=f"{task.host.name} loopbacks configured to - {loopbacks}"
)
def upgrade_os(task: Task, version: str) -> Result:
# we use task get_verion to retrieve current OS running
result = task.run(task=get_version)
# if the version matches what we want to install we are done!
if result.result["full_version"] == version:
return Result(host=task.host, result="nothing to do!!!")
# otherwise we call install_os_version task to install the image
task.run(task=install_os_version, version=version)
return Result(host=task.host, changed=True, result="success!!!")
def run_backup(task: Task, logger, global_settings, backup_root_folder) -> Result:
"""Backup configurations to disk.
Args:
task (Task): Nornir task individual object
Returns:
result (Result): Result from Nornir task
"""
obj = task.host.data["obj"]
backup_obj = GoldenConfiguration.objects.filter(device=obj).first()
if not backup_obj:
backup_obj = GoldenConfiguration.objects.create(
device=obj,
)
backup_obj.backup_last_attempt_date = task.host.defaults.data["now"]
backup_obj.save()
backup_path_template_obj = check_jinja_template(obj, logger, global_settings.backup_path_template)
backup_file = os.path.join(backup_root_folder, backup_path_template_obj)
substitute_lines = get_substitute_lines(global_settings.substitute_lines)
if global_settings.backup_test_connectivity is not False:
task.run(
task=dispatcher,
name="TEST CONNECTIVITY",
method="check_connectivity",
obj=obj,
logger=logger,
)
running_config = task.run(
task=dispatcher,
name="SAVE BACKUP CONFIGURATION TO FILE",
method="get_config",
obj=obj,
logger=logger,
backup_file=backup_file,
remove_lines=global_settings.remove_lines.splitlines(),
substitute_lines=substitute_lines,
)[1].result["config"]
backup_obj.backup_last_success_date = task.host.defaults.data["now"]
backup_obj.backup_config = running_config
backup_obj.save()
logger.log_success(obj, "Successfully backed up device.")
return Result(host=task.host, result=running_config)
def run(self, task: Task, hosts: List[Host]) -> AggregatedResult:
"""
This is where the magic happens
"""
# we instantiate the aggregated result
result = AggregatedResult(task.name)
with tqdm(
total=len(hosts),
desc="progress",
) as progress:
with ThreadPoolExecutor(max_workers=self.num_workers) as pool:
futures = {
pool.submit(task.copy().start, host): host
for host in hosts
}
for future in as_completed(futures):
worker_result = future.result()
result[worker_result.host.name] = worker_result
progress.update()
if worker_result.failed:
tqdm.write(f"{worker_result.host.name}: failure")
else:
tqdm.write(f"{worker_result.host.name}: success")
return result
def napalm_configure(
task: Task,
dry_run: Optional[bool] = None,
filename: Optional[str] = None,
configuration: Optional[str] = None,
replace: bool = False,
) -> Result:
"""
Loads configuration into a network devices using napalm
Arguments:
dry_run: Whether to apply changes or not
filename: filename containing the configuration to load into the device
configuration: configuration to load into the device
replace: whether to replace or merge the configuration
Returns:
Result object with the following attributes set:
* changed (``bool``): whether the task is changing the system or not
* diff (``string``): change in the system
"""
device = task.host.get_connection("napalm", task.nornir.config)
if replace:
device.load_replace_candidate(filename=filename, config=configuration)
else:
device.load_merge_candidate(filename=filename, config=configuration)
diff = device.compare_config()
dry_run = task.is_dry_run(dry_run)
if not dry_run and diff:
device.commit_config()
else:
device.discard_config()
return Result(host=task.host, diff=diff, changed=len(diff) > 0)
def update_configuration( # pylint: disable=C0330
task: Task,
configs_directory,
config_extension="txt") -> Result:
"""
Collect running configurations on all devices
Supported Devices:
Default: Napalm (TODO)
Cisco: Netmiko
Args:
task: Task:
configs_directory:
config_extension: (Default value = "txt")
Returns:
"""
config_filename = f"{configs_directory}/{task.host.name}.{config_extension}"
new_config = None
current_md5 = None
if os.path.exists(config_filename):
current_config = Path(config_filename).read_text()
previous_md5 = hashlib.md5(current_config.encode("utf-8")).hexdigest()
try:
results = task.run(task=napalm_get,
getters=["config"],
retrieve="running")
except:
logger.debug("An exception occured while pulling the configuration",
exc_info=True)
return Result(host=task.host, failed=True)
if results.failed:
return Result(host=task.host, failed=True)
new_config = results[0].result["config"]["running"]
# Currently the configuration is going to be different everytime because there is a timestamp on it
# Will need to do some clean up
with open(config_filename, "w") as config_:
config_.write(new_config)
new_md5 = hashlib.md5(new_config.encode("utf-8")).hexdigest()
changed = False
if current_md5 and current_md5 == new_md5:
logger.debug(
f"{task.host.name} | Latest config file already present ... ")
else:
logger.info(f"{task.host.name} | Configuration file updated ")
changed = True
return Result(host=task.host, result=True, changed=changed)
def check_if_reachable(task: Task) -> Result:
"""Check if a device is reachable by doing a TCP ping it on port 22.
Will change the status of the variable `host.is_reachable` based on the results
Args:
task: Nornir Task
Returns:
Result: Nornir Result
"""
port_to_check = 22
try:
results = task.run(task=tcp_ping, ports=[port_to_check])
except: # noqa: E722 # pylint: disable=bare-except
LOGGER.debug(
"An exception occured while running the reachability test (tcp_ping)",
exc_info=True,
)
return Result(host=task.host, failed=True)
is_reachable = results[0].result[port_to_check]
if not is_reachable:
LOGGER.debug("%s | device is not reachable on port %s", task.host.name,
port_to_check)
task.host.is_reachable = False
task.host.not_reachable_reason = f"device not reachable on port {port_to_check}"
task.host.status = "fail-ip"
return Result(host=task.host, result=is_reachable)
def sftp(
task: Task, src: str, dst: str, action: str, dry_run: Optional[bool] = None
) -> Result:
"""
Transfer files from/to the device using sftp protocol
Example::
nornir.run(files.sftp,
action="put",
src="README.md",
dst="/tmp/README.md")
Arguments:
dry_run: Whether to apply changes or not
src: source file
dst: destination
action: ``put``, ``get``.
Returns:
Result object with the following attributes set:
* changed (``bool``):
* files_changed (``list``): list of files that changed
"""
dry_run = task.is_dry_run(dry_run)
actions = {"put": put, "get": get}
client = task.host.get_connection("paramiko", task.nornir.config)
scp_client = SCPClient(client.get_transport())
sftp_client = paramiko.SFTPClient.from_transport(client.get_transport())
files_changed = actions[action](task, scp_client, sftp_client, src, dst, dry_run)
return Result(
host=task.host, changed=bool(files_changed), files_changed=files_changed
)
def netmiko_run_iperf(self, task: Task) -> Result:
"""
Runs iperf between a host and several destinations.
During setup, the destinations have been set up
to act as servers.
Note: The destination is not included in the nornir result
if the iperf test fails. Therefore we cannot know which destination
was not reachable, so we must patch the destination
onto the result object to know later
which host-destination pair actually failed.
:param task: nornir task for iperf
:return: All iperf results per host
"""
destinations_per_host = [
entry["destination"] for entry in self.nuts_parameters["test_data"]
if entry["host"] == task.host.name
]
for destination in destinations_per_host:
escaped_dest = shlex.quote(destination)
result = task.run(
task=netmiko_send_command,
command_string=f"iperf3 -c {escaped_dest} --json",
)
result[0].destination = destination # type: ignore[attr-defined]
return Result(host=task.host, result=f"iperf executed for {task.host}")
def write_file(
task: Task,
filename: str,
content: str,
append: bool = False,
dry_run: Optional[bool] = None,
) -> Result:
"""
Write contents to a file (locally)
Arguments:
dry_run: Whether to apply changes or not
filename: file you want to write into
content: content you want to write
append: whether you want to replace the contents or append to it
Returns:
Result object with the following attributes set:
* changed (``bool``):
* diff (``str``): unified diff
"""
diff = _generate_diff(filename, content, append)
if not task.is_dry_run(dry_run):
mode = "a+" if append else "w+"
with open(filename, mode=mode) as f:
f.write(content)
return Result(host=task.host, diff=diff, changed=bool(diff))
def get_vlans(task: Task) -> Result:
"""Get a list of vlans from the device using netmiko and genie parser.
Args:
task (Task): Nornir Task
Returns:
Result: Nornir Result object with a dict as a result containing the vlans
The format of the result but must be similar to Vlans defined in network_importer.processors.get_vlans
"""
LOGGER.debug("Executing get_vlans for %s (%s)", task.host.name, task.host.platform)
try:
results = task.run(task=netmiko_send_command, command_string="show vlan", use_genie=True)
except NornirSubTaskError:
LOGGER.debug(
"An exception occured while pulling the vlans information",
exc_info=True,
)
return Result(host=task.host, failed=True)
if not isinstance(results[0].result, dict) or "vlans" not in results[0].result:
LOGGER.warning("%s | No vlans information returned", task.host.name)
return Result(host=task.host, result=False)
results = convert_cisco_genie_vlans(device_name=task.host.name, data=results[0].result)
return Result(host=task.host, result=results.dict())
def get_neighbors(task: Task) -> Result:
"""Get a list of neighbors from the device.
Args:
task (Task): Nornir Task
Returns:
Result: Nornir Result object with a dict as a result containing the neighbors
The format of the result but must be similar to Neighbors defined in network_importer.processors.get_neighbors
"""
LOGGER.debug("Executing get_neighbor for %s (%s)", task.host.name, task.host.platform)
if config.SETTINGS.main.import_cabling == "lldp":
command = "show lldp neighbors detail"
converter = convert_cisco_genie_lldp_neighbors_details
cmd_type = "LLDP"
elif config.SETTINGS.main.import_cabling == "cdp":
command = "show cdp neighbors detail"
converter = convert_cisco_genie_cdp_neighbors_details
cmd_type = "CDP"
else:
return Result(host=task.host, failed=True)
try:
result = task.run(task=netmiko_send_command, command_string=command, use_genie=True)
except NornirSubTaskError:
LOGGER.debug("An exception occured while pulling %s data", cmd_type, exc_info=True)
return Result(host=task.host, failed=True)
if result[0].failed:
return result
results = converter(device_name=task.host.name, data=result[0].result)
return Result(host=task.host, result=results.dict())
def run_template( # pylint: disable=too-many-arguments
task: Task, logger, global_settings, job_result, jinja_root_path,
intended_root_folder) -> Result:
"""Render Jinja Template.
Only one template is supported, so the expectation is that that template includes all other templates.
Args:
task (Task): Nornir task individual object
Returns:
result (Result): Result from Nornir task
"""
obj = task.host.data["obj"]
intended_obj = GoldenConfig.objects.filter(device=obj).first()
if not intended_obj:
intended_obj = GoldenConfig.objects.create(device=obj)
intended_obj.intended_last_attempt_date = task.host.defaults.data["now"]
intended_obj.save()
intended_path_template_obj = check_jinja_template(
obj, logger, global_settings.intended_path_template)
output_file_location = os.path.join(intended_root_folder,
intended_path_template_obj)
jinja_template = check_jinja_template(obj, logger,
global_settings.jinja_path_template)
status, device_data = graph_ql_query(job_result.request, obj,
global_settings.sot_agg_query)
if status != 200:
logger.log_failure(
obj,
f"The GraphQL query return a status of {str(status)} with error of {str(device_data)}"
)
raise NornirNautobotException()
task.host.data.update(device_data)
generated_config = task.run(
task=dispatcher,
name="GENERATE CONFIG",
method="generate_config",
obj=obj,
logger=logger,
jinja_template=jinja_template,
jinja_root_path=jinja_root_path,
output_file_location=output_file_location,
default_drivers_mapping=get_dispatcher(),
)[1].result["config"]
intended_obj.intended_last_success_date = task.host.defaults.data["now"]
intended_obj.intended_config = generated_config
intended_obj.save()
logger.log_success(obj,
"Successfully generated the intended configuration.")
return Result(host=task.host, result=generated_config)
def my_napalm_get(task: Task) -> Result:
result = task.run(
task=napalm_get,
name='Facts',
getters=['facts']
)
return Result(
host=task.host,
result=f'Task {task.name} on host {task.host} {"failed" if result.failed else "completed succesfully"}'
)
def my_netmiko_command(task: Task) -> Result:
result = task.run(
task=netmiko_send_command,
command_string = "show lldp neighbor",
use_textfsm=True
)
return Result(
host=task.host,
result=f'Task {task.name} on host {task.host} {"failed" if result.failed else "completed succesfully"}'
)
def save_show(task: Task, command: str, filename_mark: str) -> Result:
show_result = task.run(task=netmiko_send_command,
command_string=command,
enable=True,
delay_factor=5,
max_loops=6000)
task.host['show_result'] = show_result.result
isExist = os.path.exists(f"output/{command}-{filename_mark}")
if not isExist:
os.mkdir(f"output/{command}-{filename_mark}")
task.run(task=write_file,
filename=f"output/{command}-{filename_mark}/{task.host}.cfg",
content=task.host['show_result'])
return Result(host=task.host,
result=f"output/{command}-{filename_mark}/{task.host}.cfg",
severity_level=logging.WARNING)
def download_configs(task: Task, resources: set) -> None:
"""
Loads configs from device to inventory
Arguments:
resources: list of resource paths to load
Examples:
"""
config = {}
for rsc_path in resources:
r = task.run(
name=f"Get {rsc_path}",
task=nc.get_config,
source=nc.NcDatastore.running,
path=rsc_path,
)
config.update({rsc_path: r.result})
task.host["config"] = config
def get_nxos_version(task: Task) -> dict:
"""
get the NX-OS version of a device
"""
result = task.run(task=netmiko_send_command,
command_string="show version",
use_genie=True)
os = result.result["platform"]["os"]
version = result.result["platform"]["software"]["system_version"]
return {"os": os, "version": version}
def run(self, task: Task, hosts: List[Host]) -> AggregatedResult:
result = AggregatedResult(task.name)
futures = []
with ThreadPoolExecutor(self.num_workers) as pool:
for host in hosts:
future = pool.submit(task.copy().start, host)
futures.append(future)
for future in futures:
worker_result = future.result()
result[worker_result.host.name] = worker_result
return result
def greet_and_count(task: Task, number: int):
"""This function is for learning of how to group tasks
Args:
task (Task): [description]
number (int): count times
Returns:
Result: [description]
"""
task.run(
name="Greeting is the polite thing to do",
task=say,
text="hi!",
)
task.run(
name="Counting beans",
task=count,
number=number,
)
task.run(
name="We should say bye too",
task=say,
text="bye!",
)
# let's inform if we counted even or odd times
even_or_odds = "even" if number % 2 == 1 else "odd"
return Result(
host=task.host,
result=f"{task.host} counted {even_or_odds} times!",
)
def get_interface_info(task: Task, target: EUI) -> None:
"""
Get MAC addresses of all interfaces and compare to target.
If present, identify Device and Interface
"""
interfaces = task.run(task=netmiko_send_command,
command_string="show interfaces",
use_genie=True).result
for intf in interfaces:
mac_addr = EUI(interfaces[intf]["mac_address"])
if target == mac_addr:
print_info(task, intf, target)
def get_ios_xe_version(task: Task) -> dict:
"""
get the IOS version of a device
"""
result = task.run(task=netmiko_send_command,
command_string="show version",
use_genie=True)
os = result.result["version"]["os"]
version = result.result["version"]["version"]
return {"os": os, "version": version}
def run(self, task: Task, hosts: List[Host]) -> AggregatedResult:
connectors_q = queue.Queue()
work_q = queue.Queue()
result = AggregatedResult(task.name)
# enqueue hosts in connectors queue
for host in hosts:
connectors_q.put(
(task.copy(), host, {"connection_retry": 0, "task_retry": 0}, result)
)
# start connectors threads
connector_threads = []
for i in range(self.num_connectors):
t = threading.Thread(target=self.connector, args=(connectors_q, work_q))
t.start()
connector_threads.append(t)
# start worker threads
worker_threads = []
for i in range(self.num_workers):
t = threading.Thread(target=self.worker, args=(connectors_q, work_q))
t.start()
worker_threads.append(t)
# wait until all hosts completed task or timeout reached
start_time = time.time()
while True:
with LOCK:
hosts_no_result = [h.name for h in hosts if h.name not in result]
if hosts_no_result == []:
break
if time.time() - start_time > self.task_timeout:
log.error("RetryRunner task '{}', '{}' seconds wait timeout reached, hosts that did not return results '{}'".format(
task.name, self.task_timeout, hosts_no_result
)
)
break
time.sleep(0.1)
# block until all queues empty
connectors_q.join()
work_q.join()
# stop connector threads
for i in range(self.num_connectors):
connectors_q.put(None)
for t in connector_threads:
t.join()
# stop worker threads
for i in range(self.num_workers):
work_q.put(None)
for t in worker_threads:
t.join()
# delete queues and threads
del(connectors_q, work_q, connector_threads, worker_threads)
return result
def _run_parallel(self, task: Task, hosts, num_workers, **kwargs):
result = AggregatedResult(kwargs.get("name") or task.name)
pool = Pool(processes=num_workers)
result_pool = [
pool.apply_async(task.copy().start, args=(h, self)) for h in hosts
]
pool.close()
pool.join()
for rp in result_pool:
r = rp.get()
result[r.host.name] = r
return result
def napalm_config_run(task: Task,
configuration: str,
dry_run: Optional[bool] = True,
replace: Optional[bool] = False) -> Result:
result = task.run(task=napalm_configure,
dry_run=dry_run,
replace=replace,
configuration=configuration)
# TODO: Fix up output
return Result(host=task.host,
result=result.result,
failed=False,
changed=False)
