def as3_post(task: Task, as3_tenant: str) -> Result: failover_status = task.run( name="Get failover status", task=bigip_cm_failover_status ).result if failover_status == "ACTIVE": task.run( name="AS3 POST", task=atc, atc_delay=0, atc_method="POST", atc_retries=3, atc_service="AS3", as3_tenant=as3_tenant, atc_declaration_file=task.host["appsvcs"][as3_tenant][ "atc_declaration_file" ], ) task.run( name="Synchronize the devices", task=bigip_cm_config_sync, delay=0, device_group=task.host["device_group"], retries=3, ) return Result( host=task.host, result="ACTIVE device, AS3 declaration successfully deployed.", ) else: return Result(host=task.host, result="STANDBY device, skipped.")
def render_template_apply(task: Task, template_name: str, template_path: Optional[str] = None) -> Result: from jinja2 import Environment, FileSystemLoader, StrictUndefined logger.info(f"{task.host}: Rendering Config") if not template_path: template_path = os.path.join(os.getcwd(), f'templates/{task.host.platform}') env = Environment( loader=FileSystemLoader(template_path), undefined=StrictUndefined, trim_blocks=False, ) config = task.run(task=template_file, name='Render Device Configuration', template=f'{template_name}.j2', jinja_env=env, path=template_path, **task.host) result = task.run(task=napalm_configure, dry_run=False, replace=False, configuration=config.result) return Result(host=task.host, result=result.result, failed=False, changed=False)
def create_fake_task(output, vendor_vars, vrf_name, nos, test_obj, effect=None): '''Create instance of nornir.core.task.Task and nornir.core.task.Host, assign latter to former, but mocking Host get_connection method to not really connect to hosts and return files outputs. Can use single return value or side effect with list of files. Arguments: * output - string with CLI output of some command, if we need single output * vendor_vars - dict with NOS CLI commands * vrf_name - name of VRF to test upon * nos - NOS name * test_obj - instance of nornir.core.task.Task which we will test * effect (defaults to None) - Mock.side_effect to provide different outputs Returns: * instance of nornir.core.task.Task with mocked internals ''' connection = Mock() if effect is None: connection.send_command = Mock(return_value=output) else: connection.send_command = Mock(side_effect=effect) host = OvercomeHostSlots('test-host') host['vendor_vars'] = vendor_vars host['vrf_name'] = vrf_name host.platform = nos host.get_connection = Mock(return_value=connection) fake_task = Task(test_obj) fake_task.host = host return fake_task
def set_loopbacks(task: Task, ): loopbacks = task.host.data['loopbacks'] for key in loopbacks: interface = 'interface loopback' + str(key) # ip_addr = 'ip address ' + loopbacks[key] + ' 255.255.255.255' ip_addr = f'ip address {loopbacks[key]} 255.255.255.255' config_cmds = [interface, ip_addr] task.run(task=netmiko_send_config, config_commands=config_cmds)
def run_backup( # pylint: disable=too-many-arguments task: Task, logger, global_settings, remove_regex_dict, replace_regex_dict) -> Result: r"""Backup configurations to disk. Args: task (Task): Nornir task individual object remove_regex_dict (dict): {'cisco_ios': ['^Building\\s+configuration.*\\n', '^Current\\s+configuration.*\\n', '^!\\s+Last\\s+configuration.*'], 'arista_eos': ['.s*']} replace_regex_dict (dict): {'cisco_ios': [{'regex_replacement': '<redacted_config>', 'regex_search': 'username\\s+\\S+\\spassword\\s+5\\s+(\\S+)\\s+role\\s+\\S+'}]} Returns: result (Result): Result from Nornir task """ obj = task.host.data["obj"] backup_obj = GoldenConfig.objects.filter(device=obj).first() if not backup_obj: backup_obj = GoldenConfig.objects.create(device=obj, ) backup_obj.backup_last_attempt_date = task.host.defaults.data["now"] backup_obj.save() backup_directory = get_repository_working_dir("backup", obj, logger, global_settings) backup_path_template_obj = render_jinja_template( obj, logger, global_settings.backup_path_template) backup_file = os.path.join(backup_directory, backup_path_template_obj) if global_settings.backup_test_connectivity is not False: task.run( task=dispatcher, name="TEST CONNECTIVITY", method="check_connectivity", obj=obj, logger=logger, default_drivers_mapping=get_dispatcher(), ) running_config = task.run( task=dispatcher, name="SAVE BACKUP CONFIGURATION TO FILE", method="get_config", obj=obj, logger=logger, backup_file=backup_file, remove_lines=remove_regex_dict.get(obj.platform.slug, []), substitute_lines=replace_regex_dict.get(obj.platform.slug, []), default_drivers_mapping=get_dispatcher(), )[1].result["config"] backup_obj.backup_last_success_date = task.host.defaults.data["now"] backup_obj.backup_config = running_config backup_obj.save() logger.log_success( obj, "Successfully extracted running configuration from device.") return Result(host=task.host, result=running_config)
def set_loopbacks(task: Task,): loopbacks = task.host.data['loopbacks'] task.run(task=netmiko_send_config, config_commands = ["interface loopback 99"]) # clock=task.run(task=netmiko_send_command, command_string="show clock") return Result( host=task.host, result=f"{task.host.name} loopbacks configured to - {loopbacks}" )
def upgrade_os(task: Task, version: str) -> Result: # we use task get_verion to retrieve current OS running result = task.run(task=get_version) # if the version matches what we want to install we are done! if result.result["full_version"] == version: return Result(host=task.host, result="nothing to do!!!") # otherwise we call install_os_version task to install the image task.run(task=install_os_version, version=version) return Result(host=task.host, changed=True, result="success!!!")
def run_backup(task: Task, logger, global_settings, backup_root_folder) -> Result: """Backup configurations to disk. Args: task (Task): Nornir task individual object Returns: result (Result): Result from Nornir task """ obj = task.host.data["obj"] backup_obj = GoldenConfiguration.objects.filter(device=obj).first() if not backup_obj: backup_obj = GoldenConfiguration.objects.create( device=obj, ) backup_obj.backup_last_attempt_date = task.host.defaults.data["now"] backup_obj.save() backup_path_template_obj = check_jinja_template(obj, logger, global_settings.backup_path_template) backup_file = os.path.join(backup_root_folder, backup_path_template_obj) substitute_lines = get_substitute_lines(global_settings.substitute_lines) if global_settings.backup_test_connectivity is not False: task.run( task=dispatcher, name="TEST CONNECTIVITY", method="check_connectivity", obj=obj, logger=logger, ) running_config = task.run( task=dispatcher, name="SAVE BACKUP CONFIGURATION TO FILE", method="get_config", obj=obj, logger=logger, backup_file=backup_file, remove_lines=global_settings.remove_lines.splitlines(), substitute_lines=substitute_lines, )[1].result["config"] backup_obj.backup_last_success_date = task.host.defaults.data["now"] backup_obj.backup_config = running_config backup_obj.save() logger.log_success(obj, "Successfully backed up device.") return Result(host=task.host, result=running_config)
def run(self, task: Task, hosts: List[Host]) -> AggregatedResult: """ This is where the magic happens """ # we instantiate the aggregated result result = AggregatedResult(task.name) with tqdm( total=len(hosts), desc="progress", ) as progress: with ThreadPoolExecutor(max_workers=self.num_workers) as pool: futures = { pool.submit(task.copy().start, host): host for host in hosts } for future in as_completed(futures): worker_result = future.result() result[worker_result.host.name] = worker_result progress.update() if worker_result.failed: tqdm.write(f"{worker_result.host.name}: failure") else: tqdm.write(f"{worker_result.host.name}: success") return result
def napalm_configure( task: Task, dry_run: Optional[bool] = None, filename: Optional[str] = None, configuration: Optional[str] = None, replace: bool = False, ) -> Result: """ Loads configuration into a network devices using napalm Arguments: dry_run: Whether to apply changes or not filename: filename containing the configuration to load into the device configuration: configuration to load into the device replace: whether to replace or merge the configuration Returns: Result object with the following attributes set: * changed (``bool``): whether the task is changing the system or not * diff (``string``): change in the system """ device = task.host.get_connection("napalm", task.nornir.config) if replace: device.load_replace_candidate(filename=filename, config=configuration) else: device.load_merge_candidate(filename=filename, config=configuration) diff = device.compare_config() dry_run = task.is_dry_run(dry_run) if not dry_run and diff: device.commit_config() else: device.discard_config() return Result(host=task.host, diff=diff, changed=len(diff) > 0)
def update_configuration( # pylint: disable=C0330 task: Task, configs_directory, config_extension="txt") -> Result: """ Collect running configurations on all devices Supported Devices: Default: Napalm (TODO) Cisco: Netmiko Args: task: Task: configs_directory: config_extension: (Default value = "txt") Returns: """ config_filename = f"{configs_directory}/{task.host.name}.{config_extension}" new_config = None current_md5 = None if os.path.exists(config_filename): current_config = Path(config_filename).read_text() previous_md5 = hashlib.md5(current_config.encode("utf-8")).hexdigest() try: results = task.run(task=napalm_get, getters=["config"], retrieve="running") except: logger.debug("An exception occured while pulling the configuration", exc_info=True) return Result(host=task.host, failed=True) if results.failed: return Result(host=task.host, failed=True) new_config = results[0].result["config"]["running"] # Currently the configuration is going to be different everytime because there is a timestamp on it # Will need to do some clean up with open(config_filename, "w") as config_: config_.write(new_config) new_md5 = hashlib.md5(new_config.encode("utf-8")).hexdigest() changed = False if current_md5 and current_md5 == new_md5: logger.debug( f"{task.host.name} | Latest config file already present ... ") else: logger.info(f"{task.host.name} | Configuration file updated ") changed = True return Result(host=task.host, result=True, changed=changed)
def check_if_reachable(task: Task) -> Result: """Check if a device is reachable by doing a TCP ping it on port 22. Will change the status of the variable `host.is_reachable` based on the results Args: task: Nornir Task Returns: Result: Nornir Result """ port_to_check = 22 try: results = task.run(task=tcp_ping, ports=[port_to_check]) except: # noqa: E722 # pylint: disable=bare-except LOGGER.debug( "An exception occured while running the reachability test (tcp_ping)", exc_info=True, ) return Result(host=task.host, failed=True) is_reachable = results[0].result[port_to_check] if not is_reachable: LOGGER.debug("%s | device is not reachable on port %s", task.host.name, port_to_check) task.host.is_reachable = False task.host.not_reachable_reason = f"device not reachable on port {port_to_check}" task.host.status = "fail-ip" return Result(host=task.host, result=is_reachable)
def sftp( task: Task, src: str, dst: str, action: str, dry_run: Optional[bool] = None ) -> Result: """ Transfer files from/to the device using sftp protocol Example:: nornir.run(files.sftp, action="put", src="README.md", dst="/tmp/README.md") Arguments: dry_run: Whether to apply changes or not src: source file dst: destination action: ``put``, ``get``. Returns: Result object with the following attributes set: * changed (``bool``): * files_changed (``list``): list of files that changed """ dry_run = task.is_dry_run(dry_run) actions = {"put": put, "get": get} client = task.host.get_connection("paramiko", task.nornir.config) scp_client = SCPClient(client.get_transport()) sftp_client = paramiko.SFTPClient.from_transport(client.get_transport()) files_changed = actions[action](task, scp_client, sftp_client, src, dst, dry_run) return Result( host=task.host, changed=bool(files_changed), files_changed=files_changed )
def netmiko_run_iperf(self, task: Task) -> Result: """ Runs iperf between a host and several destinations. During setup, the destinations have been set up to act as servers. Note: The destination is not included in the nornir result if the iperf test fails. Therefore we cannot know which destination was not reachable, so we must patch the destination onto the result object to know later which host-destination pair actually failed. :param task: nornir task for iperf :return: All iperf results per host """ destinations_per_host = [ entry["destination"] for entry in self.nuts_parameters["test_data"] if entry["host"] == task.host.name ] for destination in destinations_per_host: escaped_dest = shlex.quote(destination) result = task.run( task=netmiko_send_command, command_string=f"iperf3 -c {escaped_dest} --json", ) result[0].destination = destination # type: ignore[attr-defined] return Result(host=task.host, result=f"iperf executed for {task.host}")
def write_file( task: Task, filename: str, content: str, append: bool = False, dry_run: Optional[bool] = None, ) -> Result: """ Write contents to a file (locally) Arguments: dry_run: Whether to apply changes or not filename: file you want to write into content: content you want to write append: whether you want to replace the contents or append to it Returns: Result object with the following attributes set: * changed (``bool``): * diff (``str``): unified diff """ diff = _generate_diff(filename, content, append) if not task.is_dry_run(dry_run): mode = "a+" if append else "w+" with open(filename, mode=mode) as f: f.write(content) return Result(host=task.host, diff=diff, changed=bool(diff))
def get_vlans(task: Task) -> Result: """Get a list of vlans from the device using netmiko and genie parser. Args: task (Task): Nornir Task Returns: Result: Nornir Result object with a dict as a result containing the vlans The format of the result but must be similar to Vlans defined in network_importer.processors.get_vlans """ LOGGER.debug("Executing get_vlans for %s (%s)", task.host.name, task.host.platform) try: results = task.run(task=netmiko_send_command, command_string="show vlan", use_genie=True) except NornirSubTaskError: LOGGER.debug( "An exception occured while pulling the vlans information", exc_info=True, ) return Result(host=task.host, failed=True) if not isinstance(results[0].result, dict) or "vlans" not in results[0].result: LOGGER.warning("%s | No vlans information returned", task.host.name) return Result(host=task.host, result=False) results = convert_cisco_genie_vlans(device_name=task.host.name, data=results[0].result) return Result(host=task.host, result=results.dict())
def get_neighbors(task: Task) -> Result: """Get a list of neighbors from the device. Args: task (Task): Nornir Task Returns: Result: Nornir Result object with a dict as a result containing the neighbors The format of the result but must be similar to Neighbors defined in network_importer.processors.get_neighbors """ LOGGER.debug("Executing get_neighbor for %s (%s)", task.host.name, task.host.platform) if config.SETTINGS.main.import_cabling == "lldp": command = "show lldp neighbors detail" converter = convert_cisco_genie_lldp_neighbors_details cmd_type = "LLDP" elif config.SETTINGS.main.import_cabling == "cdp": command = "show cdp neighbors detail" converter = convert_cisco_genie_cdp_neighbors_details cmd_type = "CDP" else: return Result(host=task.host, failed=True) try: result = task.run(task=netmiko_send_command, command_string=command, use_genie=True) except NornirSubTaskError: LOGGER.debug("An exception occured while pulling %s data", cmd_type, exc_info=True) return Result(host=task.host, failed=True) if result[0].failed: return result results = converter(device_name=task.host.name, data=result[0].result) return Result(host=task.host, result=results.dict())
def run_template( # pylint: disable=too-many-arguments task: Task, logger, global_settings, job_result, jinja_root_path, intended_root_folder) -> Result: """Render Jinja Template. Only one template is supported, so the expectation is that that template includes all other templates. Args: task (Task): Nornir task individual object Returns: result (Result): Result from Nornir task """ obj = task.host.data["obj"] intended_obj = GoldenConfig.objects.filter(device=obj).first() if not intended_obj: intended_obj = GoldenConfig.objects.create(device=obj) intended_obj.intended_last_attempt_date = task.host.defaults.data["now"] intended_obj.save() intended_path_template_obj = check_jinja_template( obj, logger, global_settings.intended_path_template) output_file_location = os.path.join(intended_root_folder, intended_path_template_obj) jinja_template = check_jinja_template(obj, logger, global_settings.jinja_path_template) status, device_data = graph_ql_query(job_result.request, obj, global_settings.sot_agg_query) if status != 200: logger.log_failure( obj, f"The GraphQL query return a status of {str(status)} with error of {str(device_data)}" ) raise NornirNautobotException() task.host.data.update(device_data) generated_config = task.run( task=dispatcher, name="GENERATE CONFIG", method="generate_config", obj=obj, logger=logger, jinja_template=jinja_template, jinja_root_path=jinja_root_path, output_file_location=output_file_location, default_drivers_mapping=get_dispatcher(), )[1].result["config"] intended_obj.intended_last_success_date = task.host.defaults.data["now"] intended_obj.intended_config = generated_config intended_obj.save() logger.log_success(obj, "Successfully generated the intended configuration.") return Result(host=task.host, result=generated_config)
def my_napalm_get(task: Task) -> Result: result = task.run( task=napalm_get, name='Facts', getters=['facts'] ) return Result( host=task.host, result=f'Task {task.name} on host {task.host} {"failed" if result.failed else "completed succesfully"}' )
def my_netmiko_command(task: Task) -> Result: result = task.run( task=netmiko_send_command, command_string = "show lldp neighbor", use_textfsm=True ) return Result( host=task.host, result=f'Task {task.name} on host {task.host} {"failed" if result.failed else "completed succesfully"}' )
def save_show(task: Task, command: str, filename_mark: str) -> Result: show_result = task.run(task=netmiko_send_command, command_string=command, enable=True, delay_factor=5, max_loops=6000) task.host['show_result'] = show_result.result isExist = os.path.exists(f"output/{command}-{filename_mark}") if not isExist: os.mkdir(f"output/{command}-{filename_mark}") task.run(task=write_file, filename=f"output/{command}-{filename_mark}/{task.host}.cfg", content=task.host['show_result']) return Result(host=task.host, result=f"output/{command}-{filename_mark}/{task.host}.cfg", severity_level=logging.WARNING)
def download_configs(task: Task, resources: set) -> None: """ Loads configs from device to inventory Arguments: resources: list of resource paths to load Examples: """ config = {} for rsc_path in resources: r = task.run( name=f"Get {rsc_path}", task=nc.get_config, source=nc.NcDatastore.running, path=rsc_path, ) config.update({rsc_path: r.result}) task.host["config"] = config
def get_nxos_version(task: Task) -> dict: """ get the NX-OS version of a device """ result = task.run(task=netmiko_send_command, command_string="show version", use_genie=True) os = result.result["platform"]["os"] version = result.result["platform"]["software"]["system_version"] return {"os": os, "version": version}
def run(self, task: Task, hosts: List[Host]) -> AggregatedResult: result = AggregatedResult(task.name) futures = [] with ThreadPoolExecutor(self.num_workers) as pool: for host in hosts: future = pool.submit(task.copy().start, host) futures.append(future) for future in futures: worker_result = future.result() result[worker_result.host.name] = worker_result return result
def greet_and_count(task: Task, number: int): """This function is for learning of how to group tasks Args: task (Task): [description] number (int): count times Returns: Result: [description] """ task.run( name="Greeting is the polite thing to do", task=say, text="hi!", ) task.run( name="Counting beans", task=count, number=number, ) task.run( name="We should say bye too", task=say, text="bye!", ) # let's inform if we counted even or odd times even_or_odds = "even" if number % 2 == 1 else "odd" return Result( host=task.host, result=f"{task.host} counted {even_or_odds} times!", )
def get_interface_info(task: Task, target: EUI) -> None: """ Get MAC addresses of all interfaces and compare to target. If present, identify Device and Interface """ interfaces = task.run(task=netmiko_send_command, command_string="show interfaces", use_genie=True).result for intf in interfaces: mac_addr = EUI(interfaces[intf]["mac_address"]) if target == mac_addr: print_info(task, intf, target)
def get_ios_xe_version(task: Task) -> dict: """ get the IOS version of a device """ result = task.run(task=netmiko_send_command, command_string="show version", use_genie=True) os = result.result["version"]["os"] version = result.result["version"]["version"] return {"os": os, "version": version}
def run(self, task: Task, hosts: List[Host]) -> AggregatedResult: connectors_q = queue.Queue() work_q = queue.Queue() result = AggregatedResult(task.name) # enqueue hosts in connectors queue for host in hosts: connectors_q.put( (task.copy(), host, {"connection_retry": 0, "task_retry": 0}, result) ) # start connectors threads connector_threads = [] for i in range(self.num_connectors): t = threading.Thread(target=self.connector, args=(connectors_q, work_q)) t.start() connector_threads.append(t) # start worker threads worker_threads = [] for i in range(self.num_workers): t = threading.Thread(target=self.worker, args=(connectors_q, work_q)) t.start() worker_threads.append(t) # wait until all hosts completed task or timeout reached start_time = time.time() while True: with LOCK: hosts_no_result = [h.name for h in hosts if h.name not in result] if hosts_no_result == []: break if time.time() - start_time > self.task_timeout: log.error("RetryRunner task '{}', '{}' seconds wait timeout reached, hosts that did not return results '{}'".format( task.name, self.task_timeout, hosts_no_result ) ) break time.sleep(0.1) # block until all queues empty connectors_q.join() work_q.join() # stop connector threads for i in range(self.num_connectors): connectors_q.put(None) for t in connector_threads: t.join() # stop worker threads for i in range(self.num_workers): work_q.put(None) for t in worker_threads: t.join() # delete queues and threads del(connectors_q, work_q, connector_threads, worker_threads) return result
def _run_parallel(self, task: Task, hosts, num_workers, **kwargs): result = AggregatedResult(kwargs.get("name") or task.name) pool = Pool(processes=num_workers) result_pool = [ pool.apply_async(task.copy().start, args=(h, self)) for h in hosts ] pool.close() pool.join() for rp in result_pool: r = rp.get() result[r.host.name] = r return result
def napalm_config_run(task: Task, configuration: str, dry_run: Optional[bool] = True, replace: Optional[bool] = False) -> Result: result = task.run(task=napalm_configure, dry_run=dry_run, replace=replace, configuration=configuration) # TODO: Fix up output return Result(host=task.host, result=result.result, failed=False, changed=False)