def thread_websocket_events():
redis: StrictRedis
with redis_session() as redis:
while True:
result = redis.xread({"events": b"$"}, count=10, block=200)
for item in result:
event = parse_redis_event(item)
if not event:
continue
emit_redis_event(event)
def update_device_progress(job_id: int):
new_finished_devices = []
with redis_session() as db:
while db.llen('finished_devices_' + str(job_id)) != 0:
last_finished = db.lpop('finished_devices_' + str(job_id))
new_finished_devices.append(last_finished)
if new_finished_devices:
with sqla_session() as session:
job = session.query(Job).filter(Job.id == job_id).one_or_none()
if not job:
raise ValueError(
"Could not find Job with ID {}".format(job_id))
finished_devices = job.finished_devices + new_finished_devices
job.finished_devices = finished_devices
def add_event(message: Optional[str] = None,
event_type: str = "log",
level: str = "INFO",
update_type: Optional[str] = None,
json_data: Optional[str] = None):
with redis_session() as redis:
try:
send_data = {"type": event_type, "level": level}
if event_type == "log":
send_data['message'] = message
elif event_type == "update":
send_data['update_type'] = update_type
send_data['json'] = json_data
redis.xadd("events", send_data, maxlen=100)
except Exception as e:
print("Error in add_event: {}".format(e))
def push_sync_device(task, dry_run: bool = True, generate_only: bool = False,
job_id: Optional[str] = None,
scheduled_by: Optional[str] = None):
"""
Nornir task to generate config and push to device
Args:
task: nornir task, sent by nornir when doing .run()
dry_run: Don't commit config to device, just do compare/diff
generate_only: Only generate text config, don't try to commit or
even do dry_run compare to running config
Returns:
"""
set_thread_data(job_id)
logger = get_logger()
hostname = task.host.name
with sqla_session() as session:
dev: Device = session.query(Device).filter(Device.hostname == hostname).one()
template_vars = populate_device_vars(session, dev)
platform = dev.platform
devtype = dev.device_type
with open('/etc/cnaas-nms/repository.yml', 'r') as db_file:
repo_config = yaml.safe_load(db_file)
local_repo_path = repo_config['templates_local']
mapfile = os.path.join(local_repo_path, platform, 'mapping.yml')
if not os.path.isfile(mapfile):
raise RepoStructureException("File {} not found in template repo".format(mapfile))
with open(mapfile, 'r') as f:
mapping = yaml.safe_load(f)
template = mapping[devtype.name]['entrypoint']
logger.debug("Generate config for host: {}".format(task.host.name))
r = task.run(task=template_file,
name="Generate device config",
template=template,
jinja_env=cnaas_jinja_env,
path=f"{local_repo_path}/{task.host.platform}",
**template_vars)
# TODO: Handle template not found, variables not defined
# jinja2.exceptions.UndefinedError
task.host["config"] = r.result
task.host["template_vars"] = template_vars
if generate_only:
task.host["change_score"] = 0
else:
logger.debug("Synchronize device config for host: {} ({}:{})".format(
task.host.name, task.host.hostname, task.host.port))
task.host.open_connection("napalm", configuration=task.nornir.config)
task.run(task=napalm_configure,
name="Sync device config",
replace=True,
configuration=task.host["config"],
dry_run=dry_run
)
task.host.close_connection("napalm")
if task.results[1].diff:
config = task.results[1].host["config"]
diff = task.results[1].diff
task.host["change_score"] = calculate_score(config, diff)
else:
task.host["change_score"] = 0
if job_id:
with redis_session() as db:
db.lpush('finished_devices_' + str(job_id), task.host.name)
def device_upgrade_task(task,
job_id: str,
reboot: False,
filename: str,
url: str,
download: Optional[bool] = False,
pre_flight: Optional[bool] = False,
activate: Optional[bool] = False) -> NornirJobResult:
# If pre-flight is selected, execute the pre-flight task which
# will verify the amount of disk space and so on.
if pre_flight:
logger.info('Running pre-flight check on {}'.format(task.host.name))
try:
res = task.run(task=arista_pre_flight_check)
print_result(res)
except Exception as e:
logger.exception(
"Exception while doing pre-flight check: {}".format(str(e)))
raise Exception('Pre-flight check failed')
else:
if res.failed:
logger.exception('Pre-flight check failed for: {}'.format(
' '.join(res.failed_hosts.keys())))
raise e
# If download is true, go ahead and download the firmware
if download:
# Download the firmware from the HTTP container.
logger.info('Downloading firmware {} on {}'.format(
filename, task.host.name))
try:
res = task.run(task=arista_firmware_download,
filename=filename,
httpd_url=url)
print_result(res)
except Exception as e:
logger.exception('Exception while downloading firmware: {}'.format(
str(e)))
raise e
# If download_only is false, continue to activate the newly downloaded
# firmware and verify that it if present in the boot-config.
if activate:
logger.info('Activating firmware {} on {}'.format(
filename, task.host.name))
try:
res = task.run(task=arista_firmware_activate, filename=filename)
print_result(res)
except Exception as e:
logger.exception('Exception while activating firmware: {}'.format(
str(e)))
raise e
# Reboot the device if needed, we will then lose the connection.
if reboot:
logger.info('Rebooting {}'.format(task.host.name))
try:
res = task.run(task=arista_device_reboot)
except Exception as e:
pass
if job_id:
with redis_session() as db:
db.lpush('finished_devices_' + str(job_id), task.host.name)
def _refresh_repo_task(repo_type: RepoType = RepoType.TEMPLATES) -> str:
"""Should only be called by refresh_repo function."""
with open('/etc/cnaas-nms/repository.yml', 'r') as db_file:
repo_config = yaml.safe_load(db_file)
if repo_type == RepoType.TEMPLATES:
local_repo_path = repo_config['templates_local']
remote_repo_path = repo_config['templates_remote']
elif repo_type == RepoType.SETTINGS:
local_repo_path = repo_config['settings_local']
remote_repo_path = repo_config['settings_remote']
else:
raise ValueError("Invalid repository")
ret = ''
changed_files: Set[str] = set()
try:
local_repo = Repo(local_repo_path)
prev_commit = local_repo.commit().hexsha
diff = local_repo.remotes.origin.pull()
for item in diff:
ret += 'Commit {} by {} at {}\n'.format(
item.commit.name_rev, item.commit.committer,
item.commit.committed_datetime)
diff_files = local_repo.git.diff('{}..{}'.format(
prev_commit, item.commit.hexsha),
name_only=True).split()
changed_files.update(diff_files)
prev_commit = item.commit.hexsha
except (InvalidGitRepositoryError, NoSuchPathError) as e:
logger.info("Local repository {} not found, cloning from remote".\
format(local_repo_path))
try:
local_repo = Repo.clone_from(remote_repo_path, local_repo_path)
except NoSuchPathError as e:
raise ConfigException("Invalid remote repository {}: {}".format(
remote_repo_path, str(e)))
except GitCommandError as e:
raise ConfigException(
"Error cloning remote repository {}: {}".format(
remote_repo_path, str(e)))
ret = 'Cloned new from remote. Last commit {} by {} at {}'.format(
local_repo.head.commit.name_rev, local_repo.head.commit.committer,
local_repo.head.commit.committed_datetime)
if repo_type == RepoType.SETTINGS:
try:
logger.debug("Clearing redis-lru cache for settings")
with redis_session() as redis_db:
cache = RedisLRU(redis_db)
cache.clear_all_cache()
get_settings()
test_devtypes = [
DeviceType.ACCESS, DeviceType.DIST, DeviceType.CORE
]
for devtype in test_devtypes:
get_settings(device_type=devtype)
for hostname in os.listdir(os.path.join(local_repo_path,
'devices')):
hostname_path = os.path.join(local_repo_path, 'devices',
hostname)
if not os.path.isdir(hostname_path) or hostname.startswith(
'.'):
continue
if not Device.valid_hostname(hostname):
continue
get_settings(hostname)
check_settings_collisions()
except SettingsSyntaxError as e:
logger.exception("Error in settings repo configuration: {}".format(
str(e)))
raise e
except VlanConflictError as e:
logger.exception("VLAN conflict in repo configuration: {}".format(
str(e)))
raise e
logger.debug(
"Files changed in settings repository: {}".format(changed_files))
updated_devtypes, updated_hostnames = settings_syncstatus(
updated_settings=changed_files)
logger.debug(
"Devicestypes to be marked unsynced after repo refresh: {}".format(
', '.join([dt.name for dt in updated_devtypes])))
logger.debug(
"Devices to be marked unsynced after repo refresh: {}".format(
', '.join(updated_hostnames)))
with sqla_session() as session:
devtype: DeviceType
for devtype in updated_devtypes:
Device.set_devtype_syncstatus(session,
devtype,
syncstatus=False)
for hostname in updated_hostnames:
dev: Device = session.query(Device).\
filter(Device.hostname == hostname).one_or_none()
if dev:
dev.synchronized = False
else:
logger.warn(
"Settings updated for unknown device: {}".format(
hostname))
if repo_type == RepoType.TEMPLATES:
logger.debug(
"Files changed in template repository: {}".format(changed_files))
updated_devtypes = template_syncstatus(updated_templates=changed_files)
updated_list = [
'{}:{}'.format(platform, dt.name)
for dt, platform in updated_devtypes
]
logger.debug(
"Devicestypes to be marked unsynced after repo refresh: {}".format(
', '.join(updated_list)))
with sqla_session() as session:
devtype: DeviceType
for devtype, platform in updated_devtypes:
Device.set_devtype_syncstatus(session,
devtype,
platform,
syncstatus=False)
return ret
def device_upgrade_task(task,
job_id: str,
filename: str,
url: str,
reboot: Optional[bool] = False,
download: Optional[bool] = False,
pre_flight: Optional[bool] = False,
post_flight: Optional[bool] = False,
post_waittime: Optional[int] = 0,
activate: Optional[bool] = False) -> NornirJobResult:
# If pre-flight is selected, execute the pre-flight task which
# will verify the amount of disk space and so on.
set_thread_data(job_id)
logger = get_logger()
if pre_flight:
logger.info('Running pre-flight check on {}'.format(task.host.name))
try:
res = task.run(task=arista_pre_flight_check, job_id=job_id)
except Exception as e:
logger.exception(
"Exception while doing pre-flight check: {}".format(str(e)))
raise Exception('Pre-flight check failed')
else:
if res.failed:
logger.exception('Pre-flight check failed for: {}'.format(
' '.join(res.failed_hosts.keys())))
raise
# If download is true, go ahead and download the firmware
if download:
# Download the firmware from the HTTP container.
logger.info('Downloading firmware {} on {}'.format(
filename, task.host.name))
try:
res = task.run(task=arista_firmware_download,
filename=filename,
httpd_url=url,
job_id=job_id)
except Exception as e:
logger.exception('Exception while downloading firmware: {}'.format(
str(e)))
raise e
# If download_only is false, continue to activate the newly downloaded
# firmware and verify that it if present in the boot-config.
already_active = False
if activate:
logger.info('Activating firmware {} on {}'.format(
filename, task.host.name))
try:
res = task.run(task=arista_firmware_activate,
filename=filename,
job_id=job_id)
except NornirSubTaskError as e:
subtask_result = e.result[0]
logger.debug(
'Exception while activating firmware for {}: {}'.format(
task.host.name, subtask_result))
if subtask_result.exception:
if isinstance(subtask_result.exception,
FirmwareAlreadyActiveException):
already_active = True
logger.info(
"Firmware already active, skipping reboot and post_flight: {}"
.format(subtask_result.exception))
else:
logger.exception(
'Firmware activate subtask exception for {}: {}'.
format(task.host.name, str(subtask_result.exception)))
raise e
else:
logger.error('Activate subtask result for {}: {}'.format(
task.host.name, subtask_result.result))
raise e
except Exception as e:
logger.exception(
'Exception while activating firmware for {}: {}'.format(
task.host.name, str(e)))
raise e
# Reboot the device if needed, we will then lose the connection.
if reboot and not already_active:
logger.info('Rebooting {}'.format(task.host.name))
try:
res = task.run(task=arista_device_reboot, job_id=job_id)
except Exception as e:
pass
# If post-flight is selected, execute the post-flight task which
# will update device facts for the selected devices
if post_flight and not already_active:
logger.info(
'Running post-flight check on {}, delay start by {}s'.format(
task.host.name, post_waittime))
try:
res = task.run(task=arista_post_flight_check,
post_waittime=post_waittime,
job_id=job_id)
except Exception as e:
logger.exception('Failed to run post-flight check: {}'.format(
str(e)))
else:
if res.failed:
logger.error('Post-flight check failed for: {}'.format(
' '.join(res.failed_hosts.keys())))
if job_id:
with redis_session() as db:
db.lpush('finished_devices_' + str(job_id), task.host.name)
def push_sync_device(task,
dry_run: bool = True,
generate_only: bool = False,
job_id: Optional[str] = None,
scheduled_by: Optional[str] = None):
"""
Nornir task to generate config and push to device
Args:
task: nornir task, sent by nornir when doing .run()
dry_run: Don't commit config to device, just do compare/diff
generate_only: Only generate text config, don't try to commit or
even do dry_run compare to running config
Returns:
"""
set_thread_data(job_id)
logger = get_logger()
hostname = task.host.name
with sqla_session() as session:
dev: Device = session.query(Device).filter(
Device.hostname == hostname).one()
mgmt_ip = dev.management_ip
infra_ip = dev.infra_ip
if not mgmt_ip:
raise Exception(
"Could not find management IP for device {}".format(hostname))
devtype: DeviceType = dev.device_type
if isinstance(dev.platform, str):
platform: str = dev.platform
else:
raise ValueError("Unknown platform: {}".format(dev.platform))
settings, settings_origin = get_settings(hostname, devtype)
device_variables = {
'mgmt_ip': str(mgmt_ip),
'device_model': dev.model,
'device_os_version': dev.os_version
}
if devtype == DeviceType.ACCESS:
mgmtdomain = cnaas_nms.db.helper.find_mgmtdomain_by_ip(
session, dev.management_ip)
if not mgmtdomain:
raise Exception(
"Could not find appropriate management domain for management_ip: {}"
.format(dev.management_ip))
mgmt_gw_ipif = IPv4Interface(mgmtdomain.ipv4_gw)
access_device_variables = {
'mgmt_vlan_id':
mgmtdomain.vlan,
'mgmt_gw':
str(mgmt_gw_ipif.ip),
'mgmt_ipif':
str(
IPv4Interface('{}/{}'.format(
mgmt_ip, mgmt_gw_ipif.network.prefixlen))),
'mgmt_prefixlen':
int(mgmt_gw_ipif.network.prefixlen),
'interfaces': []
}
intfs = session.query(Interface).filter(
Interface.device == dev).all()
intf: Interface
for intf in intfs:
untagged_vlan = None
tagged_vlan_list = []
intfdata = None
if intf.data:
if 'untagged_vlan' in intf.data:
untagged_vlan = resolve_vlanid(
intf.data['untagged_vlan'], settings['vxlans'])
if 'tagged_vlan_list' in intf.data:
tagged_vlan_list = resolve_vlanid_list(
intf.data['tagged_vlan_list'], settings['vxlans'])
intfdata = dict(intf.data)
access_device_variables['interfaces'].append({
'name':
intf.name,
'ifclass':
intf.configtype.name,
'untagged_vlan':
untagged_vlan,
'tagged_vlan_list':
tagged_vlan_list,
'data':
intfdata
})
mlag_vars = get_mlag_vars(session, dev)
device_variables = {
**access_device_variables,
**device_variables,
**mlag_vars
}
elif devtype == DeviceType.DIST or devtype == DeviceType.CORE:
asn = generate_asn(infra_ip)
fabric_device_variables = {
'mgmt_ipif': str(IPv4Interface('{}/32'.format(mgmt_ip))),
'mgmt_prefixlen': 32,
'infra_ipif': str(IPv4Interface('{}/32'.format(infra_ip))),
'infra_ip': str(infra_ip),
'interfaces': [],
'bgp_ipv4_peers': [],
'bgp_evpn_peers': [],
'mgmtdomains': [],
'asn': asn
}
ifname_peer_map = dev.get_linknet_localif_mapping(session)
if 'interfaces' in settings and settings['interfaces']:
for intf in settings['interfaces']:
try:
ifindexnum: int = Interface.interface_index_num(
intf['name'])
except ValueError as e:
ifindexnum: int = 0
if 'ifclass' in intf and intf['ifclass'] == 'downlink':
data = {}
if intf['name'] in ifname_peer_map:
data['description'] = ifname_peer_map[intf['name']]
fabric_device_variables['interfaces'].append({
'name':
intf['name'],
'ifclass':
intf['ifclass'],
'indexnum':
ifindexnum,
'data':
data
})
elif 'ifclass' in intf and intf['ifclass'] == 'custom':
fabric_device_variables['interfaces'].append({
'name':
intf['name'],
'ifclass':
intf['ifclass'],
'config':
intf['config'],
'indexnum':
ifindexnum
})
for mgmtdom in cnaas_nms.db.helper.get_all_mgmtdomains(
session, hostname):
fabric_device_variables['mgmtdomains'].append({
'id':
mgmtdom.id,
'ipv4_gw':
mgmtdom.ipv4_gw,
'vlan':
mgmtdom.vlan,
'description':
mgmtdom.description,
'esi_mac':
mgmtdom.esi_mac
})
# find fabric neighbors
fabric_links = []
for neighbor_d in dev.get_neighbors(session):
if neighbor_d.device_type == DeviceType.DIST or neighbor_d.device_type == DeviceType.CORE:
# TODO: support multiple links to the same neighbor?
local_if = dev.get_neighbor_local_ifname(
session, neighbor_d)
local_ipif = dev.get_neighbor_local_ipif(
session, neighbor_d)
neighbor_ip = dev.get_neighbor_ip(session, neighbor_d)
if local_if:
fabric_device_variables['interfaces'].append({
'name':
local_if,
'ifclass':
'fabric',
'ipv4if':
local_ipif,
'peer_hostname':
neighbor_d.hostname,
'peer_infra_lo':
str(neighbor_d.infra_ip),
'peer_ip':
str(neighbor_ip),
'peer_asn':
generate_asn(neighbor_d.infra_ip)
})
fabric_device_variables['bgp_ipv4_peers'].append({
'peer_hostname':
neighbor_d.hostname,
'peer_infra_lo':
str(neighbor_d.infra_ip),
'peer_ip':
str(neighbor_ip),
'peer_asn':
generate_asn(neighbor_d.infra_ip)
})
# populate evpn peers data
for neighbor_d in get_evpn_spines(session, settings):
if neighbor_d.hostname == dev.hostname:
continue
fabric_device_variables['bgp_evpn_peers'].append({
'peer_hostname':
neighbor_d.hostname,
'peer_infra_lo':
str(neighbor_d.infra_ip),
'peer_asn':
generate_asn(neighbor_d.infra_ip)
})
device_variables = {**fabric_device_variables, **device_variables}
# Add all environment variables starting with TEMPLATE_SECRET_ to
# the list of configuration variables. The idea is to store secret
# configuration outside of the templates repository.
template_secrets = {}
for env in os.environ:
if env.startswith('TEMPLATE_SECRET_'):
template_secrets[env] = os.environ[env]
# Merge device variables with settings before sending to template rendering
# Device variables override any names from settings, for example the
# interfaces list from settings are replaced with an interface list from
# device variables that contains more information
template_vars = {**settings, **device_variables, **template_secrets}
with open('/etc/cnaas-nms/repository.yml', 'r') as db_file:
repo_config = yaml.safe_load(db_file)
local_repo_path = repo_config['templates_local']
mapfile = os.path.join(local_repo_path, platform, 'mapping.yml')
if not os.path.isfile(mapfile):
raise RepoStructureException(
"File {} not found in template repo".format(mapfile))
with open(mapfile, 'r') as f:
mapping = yaml.safe_load(f)
template = mapping[devtype.name]['entrypoint']
logger.debug("Generate config for host: {}".format(task.host.name))
r = task.run(task=text.template_file,
name="Generate device config",
template=template,
path=f"{local_repo_path}/{task.host.platform}",
**template_vars)
# TODO: Handle template not found, variables not defined
# jinja2.exceptions.UndefinedError
task.host["config"] = r.result
task.host["template_vars"] = template_vars
if generate_only:
task.host["change_score"] = 0
else:
logger.debug("Synchronize device config for host: {} ({}:{})".format(
task.host.name, task.host.hostname, task.host.port))
task.host.open_connection("napalm", configuration=task.nornir.config)
task.run(task=networking.napalm_configure,
name="Sync device config",
replace=True,
configuration=task.host["config"],
dry_run=dry_run)
task.host.close_connection("napalm")
if task.results[1].diff:
config = task.results[1].host["config"]
diff = task.results[1].diff
task.host["change_score"] = calculate_score(config, diff)
else:
task.host["change_score"] = 0
if job_id:
with redis_session() as db:
db.lpush('finished_devices_' + str(job_id), task.host.name)