def get_app(): from cnaas_nms.scheduler.scheduler import Scheduler from cnaas_nms.plugins.pluginmanager import PluginManagerHandler from cnaas_nms.db.session import sqla_session from cnaas_nms.db.joblock import Joblock from cnaas_nms.db.job import Job # If running inside uwsgi, a separate "mule" will run the scheduler try: import uwsgi print("Running inside uwsgi") except (ModuleNotFoundError, ImportError): scheduler = Scheduler() scheduler.start() pmh = PluginManagerHandler() pmh.load_plugins() try: with sqla_session() as session: Joblock.clear_locks(session) except Exception as e: print("Unable to clear old locks from database at startup: {}".format(str(e))) try: with sqla_session() as session: Job.clear_jobs(session) except Exception as e: print("Unable to clear jobs with invalid states: {}".format(str(e))) return app.app
def arista_post_flight_check(task, post_waittime: int, job_id: Optional[str] = None) -> str: """ NorNir task to update device facts after a switch have been upgraded Args: task: NorNir task post_waittime: Time to wait before trying to gather facts Returns: String, describing the result """ set_thread_data(job_id) logger = get_logger() time.sleep(int(post_waittime)) logger.info( 'Post-flight check wait ({}s) complete, starting check for {}'.format( post_waittime, task.host.name)) with sqla_session() as session: if Job.check_job_abort_status(session, job_id): return "Post-flight aborted" try: res = task.run(napalm_get, getters=["facts"]) os_version = res[0].result['facts']['os_version'] with sqla_session() as session: dev: Device = session.query(Device).filter( Device.hostname == task.host.name).one() prev_os_version = dev.os_version dev.os_version = os_version if prev_os_version == os_version: logger.error( "OS version did not change, activation failed on {}". format(task.host.name)) raise Exception("OS version did not change, activation failed") else: dev.confhash = None dev.synchronized = False except Exception as e: logger.exception("Could not update OS version on device {}: {}".format( task.host.name, str(e))) return 'Post-flight failed, could not update OS version: {}'.format( str(e)) return "Post-flight, OS version updated from {} to {}.".format( prev_os_version, os_version)
def device_erase(device_id: int = None, job_id: int = None) -> NornirJobResult: with sqla_session() as session: dev: Device = session.query(Device).filter(Device.id == device_id).one_or_none() if dev: hostname = dev.hostname device_type = dev.device_type else: raise Exception('Could not find a device with ID {}'.format( device_id)) if device_type != DeviceType.ACCESS: raise Exception('Can only do factory default on access') nr = cnaas_nms.confpush.nornir_helper.cnaas_init() nr_filtered = nr.filter(name=hostname).filter(managed=True) device_list = list(nr_filtered.inventory.hosts.keys()) logger.info("Device selected: {}".format( device_list )) try: nrresult = nr_filtered.run(task=device_erase_task, hostname=hostname) print_result(nrresult) except Exception as e: logger.exception('Exception while erasing device: {}'.format( str(e))) return NornirJobResult(nrresult=nrresult) failed_hosts = list(nrresult.failed_hosts.keys()) for hostname in failed_hosts: logger.error("Failed to factory default device '{}' failed".format( hostname)) if nrresult.failed: logger.error("Factory default failed") if failed_hosts == []: with sqla_session() as session: dev: Device = session.query(Device).filter(Device.id == device_id).one_or_none() session.delete(dev) session.commit() return NornirJobResult(nrresult=nrresult)
def put(self, job_id): json_data = request.get_json() if 'action' not in json_data: return empty_result(status='error', data="Action must be specified"), 400 with sqla_session() as session: job = session.query(Job).filter(Job.id == job_id).one_or_none() if not job: return empty_result( status='error', data="No job with id {} found".format(job_id)), 400 job_status = job.status action = str(json_data['action']).upper() if action == 'ABORT': allowed_jobstates = [JobStatus.SCHEDULED, JobStatus.RUNNING] if job_status not in allowed_jobstates: return empty_result( status='error', data="Job id {} is in state {}, must be {} to abort". format(job_id, job_status, (" or ".join([x.name for x in allowed_jobstates])))), 400 abort_reason = "Aborted via API call" if 'abort_reason' in json_data and isinstance( json_data['abort_reason'], str): abort_reason = json_data['abort_reason'][:255] abort_reason += " (aborted by {})".format(get_jwt_identity()) if job_status == JobStatus.SCHEDULED: scheduler = Scheduler() scheduler.remove_scheduled_job(job_id=job_id, abort_message=abort_reason) time.sleep(2) elif job_status == JobStatus.RUNNING: with sqla_session() as session: job = session.query(Job).filter( Job.id == job_id).one_or_none() job.status = JobStatus.ABORTING with sqla_session() as session: job = session.query(Job).filter(Job.id == job_id).one_or_none() return empty_result(data={"jobs": [job.as_dict()]}) else: return empty_result(status='error', data="Unknown action: {}".format(action)), 400
def test_add_group(self): with sqla_session() as session: new_group = Groups() new_group.name = 'Foo' new_group.description = 'Bar' result = session.add(new_group) session.commit()
def get_config_hash(cls, hostname): with sqla_session() as session: instance: Device = session.query(Device).filter( Device.hostname == hostname).one_or_none() if not instance: return None return instance.confhash
def reset_access_device(self): nr = cnaas_nms.confpush.nornir_helper.cnaas_init() nr_filtered = nr.filter(name=self.testdata['init_access_new_hostname']) nr_filtered.inventory.hosts[self.testdata['init_access_new_hostname']].\ connection_options["napalm"] = ConnectionOptions(extras={"timeout": 5}) data_dir = pkg_resources.resource_filename(__name__, 'data') with open(os.path.join(data_dir, 'access_reset.j2'), 'r') as f_reset_config: print(self.testdata['init_access_new_hostname']) config = f_reset_config.read() print(config) nrresult = nr_filtered.run( task=networking.napalm_configure, name="Reset config", replace=False, configuration=config, dry_run=False # TODO: temp for testing ) print_result(nrresult) reset_interfacedb(self.testdata['init_access_new_hostname']) with sqla_session() as session: dev: Device = session.query(Device).filter( Device.hostname == self.testdata['init_access_new_hostname']).one() dev.management_ip = None dev.hostname = self.testdata['init_access_old_hostname'] dev.state = DeviceState.DISCOVERED dev.device_type = DeviceType.UNKNOWN
def test_get_device_linknets(self): hostname = self.testdata['query_neighbor_device'] with sqla_session() as session: d = session.query(Device).filter(Device.hostname == hostname).one() for linknet in d.get_linknets(session): self.assertIsInstance(linknet, Linknet) pprint.pprint(linknet.as_dict())
def delete(self, device_id): """ Delete device from ID """ json_data = request.get_json() if json_data and 'factory_default' in json_data: if isinstance(json_data['factory_default'], bool) and json_data['factory_default'] is True: scheduler = Scheduler() job_id = scheduler.add_onetime_job( 'cnaas_nms.confpush.erase:device_erase', when=1, scheduled_by=get_jwt_identity(), kwargs={'device_id': device_id}) return empty_result(data='Scheduled job {} to factory default device'.format(job_id)) else: with sqla_session() as session: dev: Device = session.query(Device).filter(Device.id == device_id).one_or_none() if not dev: return empty_result('error', "Device not found"), 404 try: session.delete(dev) session.commit() except IntegrityError as e: session.rollback() return empty_result( status='error', data="Could not remove device because existing references: {}".format(e)) except Exception as e: session.rollback() return empty_result( status='error', data="Could not remove device: {}".format(e)) return empty_result(status="success", data={"deleted_device": dev.as_dict()}), 200
def arista_pre_flight_check(task, job_id: Optional[str] = None) -> str: """ NorNir task to do some basic checks before attempting to upgrade a switch. Args: task: NorNir task Returns: String, describing the result """ set_thread_data(job_id) logger = get_logger() with sqla_session() as session: if Job.check_job_abort_status(session, job_id): return "Pre-flight aborted" flash_diskspace = 'bash timeout 5 df /mnt/flash | awk \'{print $4}\'' flash_cleanup = 'bash timeout 30 ls -t /mnt/flash/*.swi | tail -n +2 | grep -v `cut -d"/" -f2 /mnt/flash/boot-config` | xargs rm -f' # Get amount of free disk space res = task.run(napalm_cli, commands=[flash_diskspace]) if not isinstance(res, MultiResult) or len(res.result.keys()) != 1: raise Exception('Could not check free space') # Remove old firmware images if needed free_bytes = next(iter(res.result.values())).split('\n')[1] if int(free_bytes) < 2500000: logger.info('Cleaning up old firmware images on {}'.format( task.host.name)) res = task.run(napalm_cli, commands=[flash_cleanup]) else: logger.info('Enough free space ({}b), no cleanup'.format(free_bytes)) return "Pre-flight check done."
def test_find_mgmtdomain_by_ip(self): with sqla_session() as session: mgmtdomain = cnaas_nms.db.helper.find_mgmtdomain_by_ip( session, IPv4Address('10.0.6.6')) self.assertEqual( IPv4Interface(mgmtdomain.ipv4_gw).network, IPv4Network('10.0.6.0/24'))
def post(self): """ Add a device """ json_data = request.get_json() supported_platforms = ['eos', 'junos', 'ios', 'iosxr', 'nxos', 'nxos_ssh'] data = {} errors = [] data, errors = Device.validate(**json_data) if errors != []: return empty_result(status='error', data=errors), 400 with sqla_session() as session: instance: Device = session.query(Device).filter(Device.hostname == data['hostname']).one_or_none() if instance: errors.append('Device already exists') return empty_result(status='error', data=errors), 400 if 'platform' not in data or data['platform'] not in supported_platforms: errors.append("Device platform not specified or not known (must be any of: {})". format(', '.join(supported_platforms))) return empty_result(status='error', data=errors), 400 if data['device_type'] in ['DIST', 'CORE']: if 'management_ip' not in data or not data['management_ip']: data['management_ip'] = cnaas_nms.confpush.underlay.find_free_mgmt_lo_ip(session) if 'infra_ip' not in data or not data['infra_ip']: data['infra_ip'] = cnaas_nms.confpush.underlay.find_free_infra_ip(session) new_device = Device.device_create(**data) session.add(new_device) session.flush() return empty_result(status='success', data={"added_device": new_device.as_dict()}), 200
def get(self): """ Get settings """ args = request.args hostname = None device_type = None model = None if 'hostname' in args: if Device.valid_hostname(args['hostname']): hostname = args['hostname'] else: return empty_result('error', "Invalid hostname specified"), 400 with sqla_session() as session: dev: Device = session.query(Device).\ filter(Device.hostname == hostname).one_or_none() if dev: device_type = dev.device_type model = dev.model else: return empty_result('error', "Hostname not found in database"), 400 if 'device_type' in args: if DeviceType.has_name(args['device_type'].upper()): device_type = DeviceType[args['device_type'].upper()] else: return empty_result('error', "Invalid device type specified"), 400 try: settings, settings_origin = get_settings(hostname, device_type, model) except Exception as e: return empty_result('error', "Error getting settings: {}".format(str(e))), 400 return empty_result(data={'settings': settings, 'settings_origin': settings_origin})
def update_interfacedb(hostname: str, replace: bool = False, delete_all: bool = False, mlag_peer_hostname: Optional[str] = None, job_id: Optional[str] = None, scheduled_by: Optional[str] = None) -> DictJobResult: """Update interface DB with any new physical interfaces for specified device. If replace is set, any existing records in the database will get overwritten. If delete_all is set, all entries in database for this device will be removed. Returns: List of interfaces that was added to DB """ with sqla_session() as session: dev: Device = session.query(Device).filter( Device.hostname == hostname).one_or_none() if not dev: raise ValueError(f"Hostname {hostname} not found in database") if dev.state != DeviceState.MANAGED: raise ValueError(f"Hostname {hostname} is not a managed device") if dev.device_type != DeviceType.ACCESS: raise ValueError( "This function currently only supports access devices") result = update_interfacedb_worker(session, dev, replace, delete_all, mlag_peer_hostname) if result: dev.synchronized = False return DictJobResult(result={"interfaces": result})
def check_settings_collisions(unique_vlans: bool = True): """Check settings for any duplicates/collisions. This will call get_settings on all devices so make sure to not call this from get_settings. Args: unique_vlans: If enabled VLANs has to be globally unique Returns: """ mgmt_vlans: Set[int] = set() devices_dict: dict[str, dict] = {} with sqla_session() as session: mgmtdoms = session.query(Mgmtdomain).all() for mgmtdom in mgmtdoms: if mgmtdom.vlan and isinstance(mgmtdom.vlan, int): if unique_vlans and mgmtdom.vlan in mgmt_vlans: raise VlanConflictError( "Management VLAN {} used in multiple management domains" .format(mgmtdom.vlan)) mgmt_vlans.add(mgmtdom.vlan) managed_devices: List[Device] = \ session.query(Device).filter(Device.state == DeviceState.MANAGED).all() for dev in managed_devices: dev_settings, _ = get_settings(dev.hostname, dev.device_type) devices_dict[dev.hostname] = dev_settings check_vlan_collisions(devices_dict, mgmt_vlans, unique_vlans)
def get(self): """ Get job locks """ locks = [] with sqla_session() as session: for lock in session.query(Joblock).all(): locks.append(lock.as_dict()) return empty_result('success', data={'locks': locks})
def device_upgrade(download: Optional[bool] = False, activate: Optional[bool] = False, filename: Optional[bool] = None, group: Optional[str] = None, hostname: Optional[str] = None, url: Optional[str] = None, job_id: Optional[str] = None, pre_flight: Optional[bool] = False, reboot: Optional[bool] = False, scheduled_by: Optional[str] = None) -> NornirJobResult: nr = cnaas_init() if hostname: nr_filtered, dev_count, _ = inventory_selector(nr, hostname=hostname) elif group: nr_filtered, dev_count, _ = inventory_selector(nr, group=group) else: raise ValueError( "Neither hostname nor group specified for device_upgrade") device_list = list(nr_filtered.inventory.hosts.keys()) logger.info("Device(s) selected for firmware upgrade ({}): {}".format( dev_count, ", ".join(device_list))) # Make sure we only upgrade Arista access switches for device in device_list: with sqla_session() as session: dev: Device = session.query(Device).\ filter(Device.hostname == device).one_or_none() if not dev: raise Exception('Could not find device: {}'.format(device)) if dev.platform != 'eos': raise Exception( 'Invalid device platform "{}" for device: {}'.format( dev.platform, device)) # Start tasks to take care of the upgrade try: nrresult = nr_filtered.run(task=device_upgrade_task, job_id=job_id, download=download, filename=filename, url=url, pre_flight=pre_flight, reboot=reboot, activate=activate) print_result(nrresult) except Exception as e: logger.exception('Exception while upgrading devices: {}'.format( str(e))) return NornirJobResult(nrresult=nrresult) failed_hosts = list(nrresult.failed_hosts.keys()) for hostname in failed_hosts: logger.error("Firmware upgrade of device '{}' failed".format(hostname)) if nrresult.failed: logger.error("Not all devices were successfully upgraded") return NornirJobResult(nrresult=nrresult)
def sync_check_hash(task, force=False, job_id=None): """ Start the task which will compare device configuration hashes. Args: task: Nornir task force: Ignore device hash """ set_thread_data(job_id) logger = get_logger() if force is True: return with sqla_session() as session: stored_hash = Device.get_config_hash(session, task.host.name) if stored_hash is None: return task.host.open_connection("napalm", configuration=task.nornir.config) res = task.run(task=napalm_get, getters=["config"]) task.host.close_connection("napalm") running_config = dict(res.result)['config']['running'].encode() if running_config is None: raise Exception('Failed to get running configuration') hash_obj = sha256(running_config) running_hash = hash_obj.hexdigest() if stored_hash != running_hash: raise Exception('Device {} configuration is altered outside of CNaaS!'.format(task.host.name))
def pre_bounce_check(hostname: str, interfaces: List[str]): # Check1: Database state with sqla_session() as session: dev: Device = session.query(Device).filter( Device.hostname == hostname).one_or_none() if not dev: raise ValueError(f"Hostname {hostname} not found in database") if dev.device_type != DeviceType.ACCESS or dev.state != DeviceState.MANAGED: raise ValueError( f"Hostname {hostname} is not of type ACCESS or not in state MANAGED" ) db_intfs: List = session.query(Interface).filter(Interface.device == dev).\ filter(Interface.configtype == InterfaceConfigType.ACCESS_UPLINK).all() uplink_intf_names = [x.name for x in db_intfs] for interface in interfaces: if interface in uplink_intf_names: raise ValueError( "Can't bounce UPLINK port {} for device {}".format( interface, hostname)) # Check2: Current interface state on device intf_states = get_interface_states(hostname) for interface in interfaces: if interface not in intf_states.keys(): raise ValueError( "Specified interface {} not found on device {}".format( interface, hostname)) if 'is_enabled' not in intf_states[ interface] or not intf_states[interface]['is_enabled']: raise ValueError( "Specified interface {} on device {} is not enabled".format( interface, hostname))
def put(self, mgmtdomain_id): json_data = request.get_json() data = {} errors = [] if 'vlan' in json_data: try: vlan_id_int = int(json_data['vlan']) except: errors.append('Invalid VLAN received.') else: data['vlan'] = vlan_id_int if 'ipv4_gw' in json_data: try: addr = IPv4Interface(json_data['ipv4_gw']) prefix_len = int(addr.network.prefixlen) except: errors.append( 'Invalid ipv4_gw received. Must be correct IPv4 address with mask' ) else: if prefix_len <= 31 and prefix_len >= 16: data['ipv4_gw'] = str(addr) else: errors.append( "Bad prefix length for management network: {}".format( prefix_len)) with sqla_session() as session: instance = session.query(Mgmtdomain).filter( Mgmtdomain.id == mgmtdomain_id).one_or_none() if instance: #TODO: auto loop through class members and match if 'vlan' in data: instance.vlan = data['vlan'] if 'ipv4_gw' in data: instance.ipv4_gw = data['ipv4_gw']
def set_config_hash(cls, hostname, hexdigest): with sqla_session() as session: instance: Device = session.query(Device).filter( Device.hostname == hostname).one_or_none() if not instance: return 'Device not found' instance.confhash = hexdigest
def test_find_free_mgmt_ip(self): mgmtdomain_id = 1 with sqla_session() as session: mgmtdomain = session.query(Mgmtdomain).filter( Mgmtdomain.id == mgmtdomain_id).one() if mgmtdomain: print(mgmtdomain.find_free_mgmt_ip(session))
def test_get_device_neighbors(self): hostname = self.testdata['query_neighbor_device'] with sqla_session() as session: d = session.query(Device).filter(Device.hostname == hostname).one() for nei in d.get_neighbors(session): self.assertIsInstance(nei, Device) pprint.pprint(nei.as_dict())
def get(self): result = [] with sqla_session() as session: query = session.query(Linknet) for instance in query: result.append(instance.as_dict()) return result
def post(self, hostname: str): """Restore configuration to previous version""" json_data = request.get_json() apply_kwargs = {'hostname': hostname} config = None if not Device.valid_hostname(hostname): return empty_result(status='error', data=f"Invalid hostname specified"), 400 if 'job_id' in json_data: try: job_id = int(json_data['job_id']) except Exception: return empty_result('error', "job_id must be an integer"), 400 else: return empty_result('error', "job_id must be specified"), 400 with sqla_session() as session: try: prev_config_result = Job.get_previous_config(session, hostname, job_id=job_id) failed = prev_config_result['failed'] if not failed and 'config' in prev_config_result: config = prev_config_result['config'] except JobNotFoundError as e: return empty_result('error', str(e)), 404 except InvalidJobError as e: return empty_result('error', str(e)), 500 except Exception as e: return empty_result('error', "Unhandled exception: {}".format(e)), 500 if failed: return empty_result( 'error', "The specified job_id has a failed status"), 400 if not config: return empty_result('error', "No config found in this job"), 500 if 'dry_run' in json_data and isinstance(json_data['dry_run'], bool) \ and not json_data['dry_run']: apply_kwargs['dry_run'] = False else: apply_kwargs['dry_run'] = True apply_kwargs['config'] = config scheduler = Scheduler() job_id = scheduler.add_onetime_job( 'cnaas_nms.confpush.sync_devices:apply_config', when=1, scheduled_by=get_jwt_identity(), kwargs=apply_kwargs, ) res = empty_result(data=f"Scheduled job to restore {hostname}") res['job_id'] = job_id return res, 200
def get(self): result = {'linknet': []} with sqla_session() as session: query = session.query(Linknet) for instance in query: result['linknet'].append(instance.as_dict()) return empty_result(status='success', data=result)
def renew_cert_task(task, job_id: str) -> str: set_thread_data(job_id) logger = get_logger() with sqla_session() as session: dev: Device = session.query(Device). \ filter(Device.hostname == task.host.name).one_or_none() ip = dev.management_ip if not ip: raise Exception("Device {} has no management_ip".format( task.host.name)) try: generate_device_cert(task.host.name, ipv4_address=ip) except Exception as e: raise Exception( "Could not generate certificate for device {}: {}".format( task.host.name, e)) if task.host.platform == "eos": try: res = task.run(task=arista_copy_cert, job_id=job_id) except Exception as e: logger.exception('Exception while copying certificates: {}'.format( str(e))) raise e else: raise ValueError("Unsupported platform: {}".format(task.host.platform)) return "Certificate renew success for device {}".format(task.host.name)
def update_facts(hostname: str, job_id: Optional[str] = None, scheduled_by: Optional[str] = None): logger = get_logger() with sqla_session() as session: dev: Device = session.query(Device).filter( Device.hostname == hostname).one_or_none() if not dev: raise ValueError( "Device with hostname {} not found".format(hostname)) if not (dev.state == DeviceState.MANAGED or dev.state == DeviceState.UNMANAGED): raise ValueError( "Device with hostname {} is in incorrect state: {}".format( hostname, str(dev.state))) hostname = dev.hostname nr = cnaas_nms.confpush.nornir_helper.cnaas_init() nr_filtered = nr.filter(name=hostname) nrresult = nr_filtered.run(task=networking.napalm_get, getters=["facts"]) if nrresult.failed: logger.error( "Could not contact device with hostname {}".format(hostname)) return NornirJobResult(nrresult=nrresult) try: facts = nrresult[hostname][0].result['facts'] with sqla_session() as session: dev: Device = session.query(Device).filter( Device.hostname == hostname).one() dev.serial = facts['serial_number'] dev.vendor = facts['vendor'] dev.model = facts['model'] dev.os_version = facts['os_version'] logger.debug("Updating facts for device {}: {}, {}, {}, {}".format( hostname, facts['serial_number'], facts['vendor'], facts['model'], facts['os_version'])) except Exception as e: logger.exception( "Could not update device with hostname {} with new facts: {}". format(hostname, str(e))) logger.debug("Get facts nrresult for hostname {}: {}".format( hostname, nrresult)) raise e return NornirJobResult(nrresult=nrresult)
def refresh_repo(repo_type: RepoType = RepoType.TEMPLATES, scheduled_by: str = None) -> str: """Refresh the repository for repo_type Args: repo_type: Which repository to refresh Returns: String describing what was updated. Raises: cnaas_nms.db.settings.SettingsSyntaxError cnaas_nms.db.joblock.JoblockError """ # Acquire lock for devices to make sure no one refreshes the repository # while another task is building configuration for devices using repo data with sqla_session() as session: job = Job() job.start_job(function_name="refresh_repo", scheduled_by=scheduled_by) session.add(job) session.flush() job_id = job.id logger.info( "Trying to acquire lock for devices to run refresh repo: {}". format(job_id)) if not Joblock.acquire_lock(session, name='devices', job_id=job_id): raise JoblockError( "Unable to acquire lock for configuring devices") try: result = _refresh_repo_task(repo_type) job.finish_time = datetime.datetime.utcnow() job.status = JobStatus.FINISHED job.result = {"message": result, "repository": repo_type.name} try: logger.info( "Releasing lock for devices from refresh repo job: {}". format(job_id)) Joblock.release_lock(session, job_id=job_id) except Exception: logger.error( "Unable to release devices lock after refresh repo job") return result except Exception as e: logger.exception( "Exception while scheduling job for refresh repo: {}".format( str(e))) job.finish_time = datetime.datetime.utcnow() job.status = JobStatus.EXCEPTION job.result = {"error": str(e), "repository": repo_type.name} try: logger.info( "Releasing lock for devices from refresh repo job: {}". format(job_id)) Joblock.release_lock(session, job_id=job_id) except Exception: logger.error( "Unable to release devices lock after refresh repo job") raise e
def test_delete_dist_device(self): with sqla_session() as session: instance = session.query(Device).filter(Device.hostname == 'unittest').first() if instance: session.delete(instance) session.commit() else: print('Device not found: ')