def save_configuration(self, net_connect):
"""Save the device's configuration.
:param net_connect: a netmiko connection object.
:raises GenericSwitchNetmikoConfigError if saving the configuration
fails.
"""
# Junos configuration is transactional, and requires an explicit commit
# of changes in order for them to be applied. Since committing requires
# an exclusive lock on the configuration database, it can fail if
# another session has a lock. We use a retry mechanism to work around
# this.
class DBLocked(Exception):
"""Switch configuration DB is locked by another user."""
@tenacity.retry(
# Log a message after each failed attempt.
after=tenacity.after_log(LOG, logging.DEBUG),
# Reraise exceptions if our final attempt fails.
reraise=True,
# Retry on failure to commit the configuration due to the DB
# being locked by another session.
retry=(tenacity.retry_if_exception_type(DBLocked)),
# Stop after the configured timeout.
stop=tenacity.stop_after_delay(
int(self.ngs_config['ngs_commit_timeout'])),
# Wait for the configured interval between attempts.
wait=tenacity.wait_fixed(
int(self.ngs_config['ngs_commit_interval'])),
)
def commit():
try:
net_connect.commit()
except ValueError as e:
# Netmiko raises ValueError on commit failure, and appends the
# CLI output to the exception message. Raise a more specific
# exception for a locked DB, on which tenacity will retry.
DB_LOCKED_MSG = "error: configuration database locked"
if DB_LOCKED_MSG in str(e):
raise DBLocked(e)
raise
try:
commit()
except DBLocked as e:
msg = ("Reached timeout waiting for switch configuration DB lock. "
"Configuration might not be committed. Error: %s" % str(e))
LOG.error(msg)
raise exc.GenericSwitchNetmikoConfigError(
config=device_utils.sanitise_config(self.config), error=msg)
except ValueError as e:
msg = "Failed to commit configuration: %s" % e
LOG.error(msg)
raise exc.GenericSwitchNetmikoConfigError(
config=device_utils.sanitise_config(self.config), error=msg)
def check_output(self, output, operation):
"""Check the output from the device following an operation.
Drivers should implement this method to handle output from devices and
perform any checks necessary to validate that the configuration was
applied successfully.
:param output: Output from the device.
:param operation: Operation being attempted. One of 'add network',
'delete network', 'plug port', 'unplug port'.
:raises: GenericSwitchNetmikoConfigError if the driver detects that an
error has occurred.
"""
if not output:
return
for pattern in self.ERROR_MSG_PATTERNS:
if pattern.search(output):
msg = ("Found invalid configuration in device response. "
"Operation: %(operation)s. Output: %(output)s" % {
'operation': operation,
'output': output
})
raise exc.GenericSwitchNetmikoConfigError(
config=device_utils.sanitise_config(self.config),
error=msg)
def _get_connection(self):
"""Context manager providing a netmiko SSH connection object.
This function hides the complexities of gracefully handling retrying
failed connection attempts.
"""
retry_exc_types = (paramiko.SSHException, EOFError)
# Use tenacity to handle retrying.
@tenacity.retry(
# Log a message after each failed attempt.
after=tenacity.after_log(LOG, logging.DEBUG),
# Reraise exceptions if our final attempt fails.
reraise=True,
# Retry on SSH connection errors.
retry=tenacity.retry_if_exception_type(retry_exc_types),
# Stop after the configured timeout.
stop=tenacity.stop_after_delay(
int(self.ngs_config['ngs_ssh_connect_timeout'])),
# Wait for the configured interval between attempts.
wait=tenacity.wait_fixed(
int(self.ngs_config['ngs_ssh_connect_interval'])),
)
def _create_connection():
return netmiko.ConnectHandler(**self.config)
# First, create a connection.
try:
net_connect = _create_connection()
except tenacity.RetryError as e:
LOG.error("Reached maximum SSH connection attempts, not retrying")
raise exc.GenericSwitchNetmikoConnectError(
config=device_utils.sanitise_config(self.config), error=e)
except Exception as e:
LOG.error("Unexpected exception during SSH connection")
raise exc.GenericSwitchNetmikoConnectError(
config=device_utils.sanitise_config(self.config), error=e)
# Now yield the connection to the caller.
with net_connect:
yield net_connect
def send_commands_to_device(self, cmd_set):
if not cmd_set:
LOG.debug("Nothing to execute")
return
try:
with ngs_lock.PoolLock(self.locker, **self.lock_kwargs):
with self._get_connection() as net_connect:
output = self.send_config_set(net_connect, cmd_set)
# NOTE (vsaienko) always save configuration
# when configuration is applied successfully.
self.save_configuration(net_connect)
except exc.GenericSwitchException:
# Reraise without modification exceptions originating from this
# module.
raise
except Exception as e:
raise exc.GenericSwitchNetmikoConnectError(
config=device_utils.sanitise_config(self.config), error=e)
LOG.debug(output)
return output
def send_commands_to_device(self, cmd_set):
if not cmd_set:
LOG.debug("Nothing to execute")
return
try:
with ngs_lock.PoolLock(self.locker, **self.lock_kwargs):
with self._get_connection() as net_connect:
net_connect.enable()
output = net_connect.send_config_set(
config_commands=cmd_set)
# NOTE (vsaienko) always save configuration
# when configuration is applied successfully.
if self.SAVE_CONFIGURATION:
net_connect.send_command(
self._format_commands(self.SAVE_CONFIGURATION,
single=True))
except Exception as e:
raise exc.GenericSwitchNetmikoConnectError(
config=device_utils.sanitise_config(self.config), error=e)
LOG.debug(output)
return output
def save_configuration(self, net_connect):
"""Save the device's configuration.
:param net_connect: a netmiko connection object.
:raises GenericSwitchNetmikoConfigError if saving the configuration
fails.
"""
# Junos configuration is transactional, and requires an explicit commit
# of changes in order for them to be applied. Since committing requires
# an exclusive lock on the configuration database, it can fail if
# another session has a lock. We use a retry mechanism to work around
# this.
class BaseRetryable(Exception):
"""Base class for retryable exceptions."""
class DBLocked(BaseRetryable):
"""Switch configuration DB is locked by another user."""
class WarningStmtExists(BaseRetryable):
"""Attempting to add a statement that already exists."""
class WarningStmtNotExist(BaseRetryable):
"""Attempting to remove a statement that does not exist."""
@tenacity.retry(
# Log a message after each failed attempt.
after=tenacity.after_log(LOG, logging.DEBUG),
# Reraise exceptions if our final attempt fails.
reraise=True,
# Retry on certain failures.
retry=(tenacity.retry_if_exception_type(BaseRetryable)),
# Stop after the configured timeout.
stop=tenacity.stop_after_delay(
int(self.ngs_config['ngs_commit_timeout'])),
# Wait for the configured interval between attempts.
wait=tenacity.wait_fixed(
int(self.ngs_config['ngs_commit_interval'])),
)
def commit():
try:
net_connect.commit()
except ValueError as e:
# Netmiko raises ValueError on commit failure, and appends the
# CLI output to the exception message.
# Certain strings indicate a temporary failure, or a harmless
# warning. In these cases we should retry the operation. We
# don't ignore warning messages, in case there is some other
# less benign cause for the failure.
retryable_msgs = {
# Concurrent access to the switch can lead to contention
# for the configuration database lock, and potentially
# failure to commit changes with the following message.
"error: configuration database locked": DBLocked,
# Can be caused by concurrent configuration if two sessions
# attempt to remove the same statement.
"warning: statement does not exist": WarningStmtNotExist,
# Can be caused by concurrent configuration if two sessions
# attempt to add the same statement.
"warning: statement already exists": WarningStmtExists,
}
for msg in retryable_msgs:
if msg in str(e):
raise retryable_msgs[msg](e)
raise
try:
commit()
except DBLocked as e:
msg = ("Reached timeout waiting for switch configuration DB lock. "
"Configuration might not be committed. Error: %s" % str(e))
LOG.error(msg)
raise exc.GenericSwitchNetmikoConfigError(
config=device_utils.sanitise_config(self.config), error=msg)
except (WarningStmtNotExist, WarningStmtExists) as e:
msg = ("Reached timeout while attempting to apply configuration. "
"This is likely to be caused by multiple sessions "
"configuring the device concurrently. Error: %s" % str(e))
LOG.error(msg)
raise exc.GenericSwitchNetmikoConfigError(
config=device_utils.sanitise_config(self.config), error=msg)
except ValueError as e:
msg = "Failed to commit configuration: %s" % e
LOG.error(msg)
raise exc.GenericSwitchNetmikoConfigError(
config=device_utils.sanitise_config(self.config), error=msg)
def test_sanitise_config(self):
config = {'username': '******', 'password': '******'}
result = device_utils.sanitise_config(config)
expected = {'username': '******', 'password': '******'}
self.assertEqual(expected, result)
