def generate_only(hostname: str) -> (str, dict):
"""
Generate configuration for a device and return it as a text string.
Args:
hostname: Hostname of device generate config for
Returns:
(string with config, dict with available template variables)
"""
logger = get_logger()
nr = cnaas_init()
nr_filtered, _, _ = inventory_selector(nr, hostname=hostname)
template_vars = {}
if len(nr_filtered.inventory.hosts) != 1:
raise ValueError("Invalid hostname: {}".format(hostname))
try:
nrresult = nr_filtered.run(task=push_sync_device, generate_only=True)
if nrresult[hostname][0].failed:
raise Exception("Could not generate config for device {}: {}".format(
hostname, nrresult[hostname][0].result
))
if "template_vars" in nrresult[hostname][1].host:
template_vars = nrresult[hostname][1].host["template_vars"]
if nrresult.failed:
print_result(nrresult)
raise Exception("Failed to generate config for {}".format(hostname))
return nrresult[hostname][1].result, template_vars
except Exception as e:
logger.exception("Exception while generating config: {}".format(str(e)))
if len(nrresult[hostname]) >= 2:
return nrresult[hostname][1].result, template_vars
else:
return str(e), template_vars
def device_upgrade(download: Optional[bool] = False,
activate: Optional[bool] = False,
filename: Optional[bool] = None,
group: Optional[str] = None,
hostname: Optional[str] = None,
url: Optional[str] = None,
job_id: Optional[str] = None,
pre_flight: Optional[bool] = False,
reboot: Optional[bool] = False,
scheduled_by: Optional[str] = None) -> NornirJobResult:
nr = cnaas_init()
if hostname:
nr_filtered, dev_count, _ = inventory_selector(nr, hostname=hostname)
elif group:
nr_filtered, dev_count, _ = inventory_selector(nr, group=group)
else:
raise ValueError(
"Neither hostname nor group specified for device_upgrade")
device_list = list(nr_filtered.inventory.hosts.keys())
logger.info("Device(s) selected for firmware upgrade ({}): {}".format(
dev_count, ", ".join(device_list)))
# Make sure we only upgrade Arista access switches
for device in device_list:
with sqla_session() as session:
dev: Device = session.query(Device).\
filter(Device.hostname == device).one_or_none()
if not dev:
raise Exception('Could not find device: {}'.format(device))
if dev.platform != 'eos':
raise Exception(
'Invalid device platform "{}" for device: {}'.format(
dev.platform, device))
# Start tasks to take care of the upgrade
try:
nrresult = nr_filtered.run(task=device_upgrade_task,
job_id=job_id,
download=download,
filename=filename,
url=url,
pre_flight=pre_flight,
reboot=reboot,
activate=activate)
print_result(nrresult)
except Exception as e:
logger.exception('Exception while upgrading devices: {}'.format(
str(e)))
return NornirJobResult(nrresult=nrresult)
failed_hosts = list(nrresult.failed_hosts.keys())
for hostname in failed_hosts:
logger.error("Firmware upgrade of device '{}' failed".format(hostname))
if nrresult.failed:
logger.error("Not all devices were successfully upgraded")
return NornirJobResult(nrresult=nrresult)
def copy_cert(self):
nr = cnaas_init()
nr_filtered = nr.filter(name=self.testdata['copycert_hostname'])
nrresult = nr_filtered.run(
task=arista_copy_cert
)
if nrresult.failed:
print_result(nrresult)
self.assertFalse(nrresult.failed, "Task arista_copy_cert returned failed status")
def confcheck_devices(hostnames: List[str], job_id=None):
nr = cnaas_init()
nr_filtered, dev_count, skipped_hostnames = \
inventory_selector(nr, hostname=hostnames)
try:
nrresult = nr_filtered.run(task=sync_check_hash,
job_id=job_id)
except Exception as e:
raise e
else:
if nrresult.failed:
raise Exception('Configuration hash check failed for {}'.format(
' '.join(nrresult.failed_hosts.keys())))
def get_interface_states(hostname) -> dict:
logger = get_logger()
nr = cnaas_init()
nr_filtered = nr.filter(name=hostname).filter(managed=True)
if len(nr_filtered.inventory) != 1:
raise ValueError(f"Hostname {hostname} not found in inventory")
nrresult = nr_filtered.run(task=napalm_get, getters=["interfaces"])
if not len(nrresult) == 1:
raise Exception(f"Could not get interfaces for {hostname}: no Nornir result")
if nrresult.failed or nrresult[hostname].failed:
raise Exception("Could not get interfaces for {}, NAPALM failed: {}".format(
hostname, nrresult[hostname].exception
))
return nrresult[hostname][0].result['interfaces']
def renew_cert(hostname: Optional[str] = None,
group: Optional[str] = None,
job_id: Optional[str] = None,
scheduled_by: Optional[str] = None) -> NornirJobResult:
logger = get_logger()
nr = cnaas_init()
if hostname:
nr_filtered, dev_count, _ = inventory_selector(nr, hostname=hostname)
elif group:
nr_filtered, dev_count, _ = inventory_selector(nr, group=group)
else:
raise ValueError("Neither hostname nor group specified for renew_cert")
device_list = list(nr_filtered.inventory.hosts.keys())
logger.info("Device(s) selected for renew certificate ({}): {}".format(
dev_count, ", ".join(device_list)))
supported_platforms = ['eos']
# Make sure we only attempt supported devices
for device in device_list:
with sqla_session() as session:
dev: Device = session.query(Device). \
filter(Device.hostname == device).one_or_none()
if not dev:
raise Exception('Could not find device: {}'.format(device))
if dev.platform not in supported_platforms:
raise Exception(
'Unsupported device platform "{}" for device: {}'.format(
dev.platform, device))
try:
nrresult = nr_filtered.run(task=renew_cert_task, job_id=job_id)
except Exception as e:
logger.exception('Exception while renewing certificates: {}'.format(
str(e)))
return NornirJobResult(nrresult=nrresult)
failed_hosts = list(nrresult.failed_hosts.keys())
for hostname in failed_hosts:
logger.error(
"Certificate renew on device '{}' failed".format(hostname))
if nrresult.failed:
logger.error("Not all devices got new certificates")
return NornirJobResult(nrresult=nrresult)
def bounce_interfaces(hostname: str, interfaces: List[str]) -> bool:
"""Returns true if the device changed config down and then up.
Returns false if config did not change, and raises Exception if an
error was encountered."""
pre_bounce_check(hostname, interfaces)
nr = cnaas_init()
nr_filtered = nr.filter(name=hostname).filter(managed=True)
if len(nr_filtered.inventory) != 1:
raise ValueError(f"Hostname {hostname} not found in inventory")
nrresult = nr_filtered.run(task=bounce_task, interfaces=interfaces)
# 5 results: bounce_task, gen down config, push down config, gen up config, push up config
if not len(nrresult[hostname]) == 5:
raise Exception("Not all steps of port bounce completed")
if nrresult[hostname][2].changed and nrresult[hostname][4].changed:
return True
else:
return False
def apply_config(hostname: str,
config: str,
dry_run: bool,
job_id: Optional[int] = None,
scheduled_by: Optional[str] = None) -> NornirJobResult:
"""Apply a static configuration (from backup etc) to a device.
Args:
hostname: Specify a single host by hostname to synchronize
config: Static configuration to apply
dry_run: Set to false to actually apply config to device
job_id: Job ID number
scheduled_by: Username from JWT
Returns:
NornirJobResult
"""
logger = get_logger()
with sqla_session() as session:
dev: Device = session.query(Device).filter(
Device.hostname == hostname).one_or_none()
if not dev:
raise Exception("Device {} not found".format(hostname))
elif not (dev.state == DeviceState.MANAGED
or dev.state == DeviceState.UNMANAGED):
raise Exception("Device {} is in invalid state: {}".format(
hostname, dev.state))
if not dry_run:
dev.state = DeviceState.UNMANAGED
dev.synchronized = False
nr = cnaas_init()
nr_filtered, _, _ = inventory_selector(nr, hostname=hostname)
try:
nrresult = nr_filtered.run(task=push_static_config,
config=config,
dry_run=dry_run,
job_id=job_id)
except Exception as e:
logger.exception("Exception in apply_config: {}".format(e))
return NornirJobResult(nrresult=nrresult)
def sync_devices(hostnames: Optional[List[str]] = None, device_type: Optional[str] = None,
group: Optional[str] = None, dry_run: bool = True, force: bool = False,
auto_push: bool = False, job_id: Optional[int] = None,
scheduled_by: Optional[str] = None, resync: bool = False) -> NornirJobResult:
"""Synchronize devices to their respective templates. If no arguments
are specified then synchronize all devices that are currently out
of sync.
Args:
hostname: Specify a single host by hostname to synchronize
device_type: Specify a device type to synchronize
group: Specify a group of devices to synchronize
dry_run: Don't commit generated config to device
force: Commit config even if changes made outside CNaaS will get
overwritten
auto_push: Automatically do live-run after dry-run if change score is low
job_id: job_id provided by scheduler when adding a new job
scheduled_by: Username from JWT
resync: Re-synchronize a device even if it's marked as synced in the
database, a device selected by hostname is always re-synced
Returns:
NornirJobResult
"""
logger = get_logger()
nr = cnaas_init()
dev_count = 0
skipped_hostnames = []
if hostnames:
nr_filtered, dev_count, skipped_hostnames = \
inventory_selector(nr, hostname=hostnames)
else:
if device_type:
nr_filtered, dev_count, skipped_hostnames = \
inventory_selector(nr, resync=resync, device_type=device_type)
elif group:
nr_filtered, dev_count, skipped_hostnames = \
inventory_selector(nr, resync=resync, group=group)
else:
# all devices
nr_filtered, dev_count, skipped_hostnames = \
inventory_selector(nr, resync=resync)
if skipped_hostnames:
logger.info("Device(s) already synchronized, skipping ({}): {}".format(
len(skipped_hostnames), ", ".join(skipped_hostnames)
))
device_list = list(nr_filtered.inventory.hosts.keys())
logger.info("Device(s) selected for synchronization ({}): {}".format(
dev_count, ", ".join(device_list)
))
try:
nrresult = nr_filtered.run(task=sync_check_hash,
force=force,
job_id=job_id)
except Exception as e:
logger.exception("Exception while checking config hash: {}".format(str(e)))
raise e
else:
if nrresult.failed:
# Mark devices as unsynchronized if config hash check failed
with sqla_session() as session:
session.query(Device).filter(Device.hostname.in_(nrresult.failed_hosts.keys())).\
update({Device.synchronized: False}, synchronize_session=False)
raise Exception('Configuration hash check failed for {}'.format(
' '.join(nrresult.failed_hosts.keys())))
if not dry_run:
with sqla_session() as session:
logger.info("Trying to acquire lock for devices to run syncto job: {}".format(job_id))
if not Joblock.acquire_lock(session, name='devices', job_id=job_id):
raise JoblockError("Unable to acquire lock for configuring devices")
try:
nrresult = nr_filtered.run(task=push_sync_device, dry_run=dry_run,
job_id=job_id)
except Exception as e:
logger.exception("Exception while synchronizing devices: {}".format(str(e)))
try:
if not dry_run:
with sqla_session() as session:
logger.info("Releasing lock for devices from syncto job: {}".format(job_id))
Joblock.release_lock(session, job_id=job_id)
except Exception:
logger.error("Unable to release devices lock after syncto job")
return NornirJobResult(nrresult=nrresult)
failed_hosts = list(nrresult.failed_hosts.keys())
for hostname in failed_hosts:
logger.error("Synchronization of device '{}' failed".format(hostname))
if nrresult.failed:
logger.error("Not all devices were successfully synchronized")
total_change_score = 1
change_scores = []
changed_hosts = []
unchanged_hosts = []
# calculate change impact score
for host, results in nrresult.items():
if len(results) != 3:
logger.debug("Unable to calculate change score for failed device {}".format(host))
elif results[2].diff:
changed_hosts.append(host)
if "change_score" in results[0].host:
change_scores.append(results[0].host["change_score"])
logger.debug("Change score for host {}: {:.1f}".format(
host, results[0].host["change_score"]))
else:
unchanged_hosts.append(host)
change_scores.append(0)
logger.debug("Empty diff for host {}, 0 change score".format(
host))
nr_confighash = None
if dry_run and force:
# update config hash for devices that had an empty diff because local
# changes on a device can cause reordering of CLI commands that results
# in config hash mismatch even if the calculated diff was empty
def include_filter(host, include_list=unchanged_hosts):
if host.name in include_list:
return True
else:
return False
nr_confighash = nr_filtered.filter(filter_func=include_filter)
elif not dry_run:
# set new config hash for devices that was successfully updated
def exclude_filter(host, exclude_list=failed_hosts+unchanged_hosts):
if host.name in exclude_list:
return False
else:
return True
nr_confighash = nr_filtered.filter(filter_func=exclude_filter)
if nr_confighash:
try:
nrresult_confighash = nr_confighash.run(task=update_config_hash)
except Exception as e:
logger.exception("Exception while updating config hashes: {}".format(str(e)))
else:
if nrresult_confighash.failed:
logger.error("Unable to update some config hashes: {}".format(
list(nrresult_confighash.failed_hosts.keys())))
# set devices as synchronized if needed
with sqla_session() as session:
for hostname in changed_hosts:
if dry_run:
dev: Device = session.query(Device).filter(Device.hostname == hostname).one()
dev.synchronized = False
else:
dev: Device = session.query(Device).filter(Device.hostname == hostname).one()
dev.synchronized = True
for hostname in unchanged_hosts:
dev: Device = session.query(Device).filter(Device.hostname == hostname).one()
dev.synchronized = True
if not dry_run:
logger.info("Releasing lock for devices from syncto job: {}".format(job_id))
Joblock.release_lock(session, job_id=job_id)
if len(device_list) == 0:
total_change_score = 0
elif not change_scores or total_change_score >= 100 or failed_hosts:
total_change_score = 100
else:
# use individual max as total_change_score, range 1-100
total_change_score = max(min(int(max(change_scores) + 0.5), 100), 1)
logger.info(
"Change impact score: {:.1f} (dry_run: {}, selected devices: {}, changed devices: {})".
format(total_change_score, dry_run, len(device_list), len(changed_hosts)))
next_job_id = None
if auto_push and len(device_list) == 1 and hostnames and dry_run:
if not changed_hosts:
logger.info("None of the selected host has any changes (diff), skipping auto-push")
elif total_change_score < AUTOPUSH_MAX_SCORE:
scheduler = Scheduler()
next_job_id = scheduler.add_onetime_job(
'cnaas_nms.confpush.sync_devices:sync_devices',
when=0,
scheduled_by=scheduled_by,
kwargs={'hostnames': hostnames, 'dry_run': False, 'force': force})
logger.info(f"Auto-push scheduled live-run of commit as job id {next_job_id}")
else:
logger.info(
f"Auto-push of config to device {hostnames} failed because change score of "
f"{total_change_score} is higher than auto-push limit {AUTOPUSH_MAX_SCORE}"
)
return NornirJobResult(nrresult=nrresult, next_job_id=next_job_id, change_score=total_change_score)
def post(self):
""" Start sync of device(s) """
json_data = request.get_json()
# default args
kwargs: dict = {
'dry_run': True,
'auto_push': False,
'force': False,
'resync': False
}
if 'dry_run' in json_data and isinstance(json_data['dry_run'], bool) \
and not json_data['dry_run']:
kwargs['dry_run'] = False
if 'force' in json_data and isinstance(json_data['force'], bool):
kwargs['force'] = json_data['force']
if 'auto_push' in json_data and isinstance(json_data['auto_push'],
bool):
kwargs['auto_push'] = json_data['auto_push']
if 'resync' in json_data and isinstance(json_data['resync'], bool):
kwargs['resync'] = json_data['resync']
if 'comment' in json_data and isinstance(json_data['comment'], str):
kwargs['job_comment'] = json_data['comment']
if 'ticket_ref' in json_data and isinstance(json_data['ticket_ref'],
str):
kwargs['job_ticket_ref'] = json_data['ticket_ref']
total_count: Optional[int] = None
nr = cnaas_init()
if 'hostname' in json_data:
hostname = str(json_data['hostname'])
if not Device.valid_hostname(hostname):
return empty_result(
status='error',
data=f"Hostname '{hostname}' is not a valid hostname"), 400
_, total_count, _ = inventory_selector(nr, hostname=hostname)
if total_count != 1:
return empty_result(
status='error',
data=
f"Hostname '{hostname}' not found or is not a managed device"
), 400
kwargs['hostnames'] = [hostname]
what = hostname
elif 'device_type' in json_data:
devtype_str = str(json_data['device_type']).upper()
if DeviceType.has_name(devtype_str):
kwargs['device_type'] = devtype_str
else:
return empty_result(
status='error',
data=
f"Invalid device type '{json_data['device_type']}' specified"
), 400
what = f"{json_data['device_type']} devices"
_, total_count, _ = inventory_selector(nr,
resync=kwargs['resync'],
device_type=devtype_str)
elif 'group' in json_data:
group_name = str(json_data['group'])
if group_name not in get_groups():
return empty_result(
status='error',
data='Could not find a group with name {}'.format(
group_name))
kwargs['group'] = group_name
what = 'group {}'.format(group_name)
_, total_count, _ = inventory_selector(nr,
resync=kwargs['resync'],
group=group_name)
elif 'all' in json_data and isinstance(json_data['all'],
bool) and json_data['all']:
what = "all devices"
_, total_count, _ = inventory_selector(nr, resync=kwargs['resync'])
else:
return empty_result(
status='error',
data=f"No devices to synchronize were specified"), 400
scheduler = Scheduler()
job_id = scheduler.add_onetime_job(
'cnaas_nms.confpush.sync_devices:sync_devices',
when=1,
scheduled_by=get_jwt_identity(),
kwargs=kwargs)
res = empty_result(data=f"Scheduled job to synchronize {what}")
res['job_id'] = job_id
resp = make_response(json.dumps(res), 200)
if total_count:
resp.headers['X-Total-Count'] = total_count
resp.headers['Content-Type'] = "application/json"
return resp
def post(self):
"""Execute certificate related actions on device"""
json_data = request.get_json()
# default args
kwargs: dict = {}
if 'action' in json_data and isinstance(json_data['action'], str):
action = json_data['action'].upper()
else:
return empty_result(
status='error',
data=f"Required field 'action' was not specified"), 400
if 'comment' in json_data and isinstance(json_data['comment'], str):
kwargs['job_comment'] = json_data['comment']
if 'ticket_ref' in json_data and isinstance(json_data['ticket_ref'],
str):
kwargs['job_ticket_ref'] = json_data['ticket_ref']
total_count: Optional[int] = None
nr = cnaas_init()
if 'hostname' in json_data:
hostname = str(json_data['hostname'])
if not Device.valid_hostname(hostname):
return empty_result(
status='error',
data=f"Hostname '{hostname}' is not a valid hostname"), 400
_, total_count, _ = inventory_selector(nr, hostname=hostname)
if total_count != 1:
return empty_result(
status='error',
data=
f"Hostname '{hostname}' not found or is not a managed device"
), 400
kwargs['hostname'] = hostname
elif 'group' in json_data:
group_name = str(json_data['group'])
if group_name not in get_groups():
return empty_result(
status='error',
data='Could not find a group with name {}'.format(
group_name))
kwargs['group'] = group_name
_, total_count, _ = inventory_selector(nr, group=group_name)
else:
return empty_result(status='error',
data=f"No devices were specified"), 400
if action == 'RENEW':
scheduler = Scheduler()
job_id = scheduler.add_onetime_job(
'cnaas_nms.confpush.cert:renew_cert',
when=1,
scheduled_by=get_jwt_identity(),
kwargs=kwargs)
res = empty_result(data=f"Scheduled job to renew certificates")
res['job_id'] = job_id
resp = make_response(json.dumps(res), 200)
if total_count:
resp.headers['X-Total-Count'] = total_count
resp.headers['Content-Type'] = "application/json"
return resp
else:
return empty_result(
status='error',
data=f"Unknown action specified: {action}"), 400
def post(self):
""" Upgrade firmware on device """
json_data = request.get_json()
kwargs = dict()
seconds = 1
date_format = "%Y-%m-%d %H:%M:%S"
url = self.firmware_url()
if 'url' not in json_data and url == '':
return empty_result(status='error',
data='No external address configured for '
'HTTPD, please specify one with "url"')
if 'url' not in json_data:
kwargs['url'] = url
else:
if isinstance(json_data['url'], str):
kwargs['url'] = json_data['url']
else:
return empty_result(status='error',
data='url should be a string')
if 'activate' in json_data:
if isinstance(json_data['activate'], bool):
kwargs['activate'] = json_data['activate']
else:
return empty_result(status='error',
data='activate should be a boolean')
if 'download' in json_data:
if isinstance(json_data['download'], bool):
kwargs['download'] = json_data['download']
else:
return empty_result(status='error',
data='download should be a boolean')
if 'reboot' in json_data:
if isinstance(json_data['reboot'], bool):
kwargs['reboot'] = json_data['reboot']
else:
return empty_result(status='error',
data='reboot should be a boolean')
if 'pre_flight' in json_data:
if isinstance(json_data['pre_flight'], bool):
kwargs['pre_flight'] = json_data['pre_flight']
else:
return empty_result(status='error',
data='pre_flight should be a boolean')
if 'post_flight' in json_data:
if isinstance(json_data['post_flight'], bool):
kwargs['post_flight'] = json_data['post_flight']
else:
return empty_result(status='error',
data='post_flight should be a boolean')
if 'post_waittime' in json_data:
if isinstance(json_data['post_waittime'], int):
kwargs['post_waittime'] = json_data['post_waittime']
else:
return empty_result(status='error',
data='post_waittime should be an integer')
if 'filename' in json_data:
if isinstance(json_data['filename'], str):
kwargs['filename'] = json_data['filename']
else:
return empty_result(status='error',
data='filename should be a string')
total_count: Optional[int] = None
nr = cnaas_init()
if 'hostname' in json_data:
hostname = str(json_data['hostname'])
if not Device.valid_hostname(hostname):
return empty_result(
status='error',
data=f"Hostname '{hostname}' is not a valid hostname"
), 400
_, total_count, _ = inventory_selector(nr, hostname=hostname)
if total_count != 1:
return empty_result(
status='error',
data=f"Hostname '{hostname}' not found or is not a managed device"
), 400
kwargs['hostname'] = hostname
elif 'group' in json_data:
group_name = str(json_data['group'])
if group_name not in get_groups():
return empty_result(status='error', data='Could not find a group with name {}'.format(group_name))
kwargs['group'] = group_name
_, total_count, _ = inventory_selector(nr, group=group_name)
kwargs['group'] = group_name
else:
return empty_result(
status='error',
data=f"No devices to upgrade were specified"
), 400
if 'comment' in json_data and isinstance(json_data['comment'], str):
kwargs['job_comment'] = json_data['comment']
if 'ticket_ref' in json_data and isinstance(json_data['ticket_ref'], str):
kwargs['job_ticket_ref'] = json_data['ticket_ref']
if 'start_at' in json_data:
try:
time_start = datetime.strptime(json_data['start_at'],
date_format)
time_now = datetime.utcnow()
if time_start < time_now:
return empty_result(status='error',
data='start_at must be in the future')
time_diff = time_start - time_now
seconds = int(time_diff.total_seconds())
except Exception as e:
logger.exception(f'Exception when scheduling job: {e}')
return empty_result(status='error',
data=f'Invalid date format, should be: {date_format}')
scheduler = Scheduler()
job_id = scheduler.add_onetime_job(
'cnaas_nms.confpush.firmware:device_upgrade',
when=seconds,
scheduled_by=get_jwt_identity(),
kwargs=kwargs)
res = empty_result(data='Scheduled job to upgrade devices')
res['job_id'] = job_id
resp = make_response(json.dumps(res), 200)
if total_count:
resp.headers['X-Total-Count'] = total_count
resp.headers['Content-Type'] = "application/json"
return resp