class NetDevicesTest(unittest.TestCase):
def setUp(self):
self.nd = NetDevices()
self.nodename = self.nd.keys()[0]
self.nodeobj = self.nd.values()[0]
def testBasics(self):
"""Basic test of NetDevices functionality."""
self.assertEqual(len(self.nd), 1)
self.assertEqual(self.nodeobj.nodeName, self.nodename)
self.assertEqual(self.nodeobj.manufacturer, 'JUNIPER')
def testAclsdb(self):
"""Test acls.db handling."""
self.assert_('181j' in self.nodeobj.acls)
def testAutoacls(self):
"""Test autoacls.py handling."""
self.assert_('115j' in self.nodeobj.acls)
def testFind(self):
"""Test the find() method."""
self.assertEqual(self.nd.find(self.nodename), self.nodeobj)
nodebasename = self.nodename[:self.nodename.index('.')]
self.assertEqual(self.nd.find(nodebasename), self.nodeobj)
self.assertRaises(KeyError, lambda: self.nd.find(self.nodename[0:3]))
class NetDevicesTest(unittest.TestCase):
def setUp(self):
self.nd = NetDevices()
self.nodename = self.nd.keys()[0]
self.nodeobj = self.nd.values()[0]
def testBasics(self):
"""Basic test of NetDevices functionality."""
self.assertEqual(len(self.nd), 1)
self.assertEqual(self.nodeobj.nodeName, self.nodename)
self.assertEqual(self.nodeobj.manufacturer, 'JUNIPER')
def testAclsdb(self):
"""Test acls.db handling."""
self.assert_('181j' in self.nodeobj.acls)
def testAutoacls(self):
"""Test autoacls.py handling."""
self.assert_('115j' in self.nodeobj.acls)
def testFind(self):
"""Test the find() method."""
self.assertEqual(self.nd.find(self.nodename), self.nodeobj)
nodebasename = self.nodename[:self.nodename.index('.')]
self.assertEqual(self.nd.find(nodebasename), self.nodeobj)
self.assertRaises(KeyError, lambda: self.nd.find(self.nodename[0:3]))
class TestNetDevicesWithAcls(unittest.TestCase):
"""
Test NetDevices with ``settings.WITH_ACLs set`` to ``True``.
"""
def setUp(self):
self.nd = NetDevices()
self.nodename = self.nd.keys()[0]
self.device = self.nd.values()[0]
self.device.explicit_acls = set(['test1-abc-only'])
def test_basics(self):
"""Basic test of NetDevices functionality."""
self.assertEqual(len(self.nd), 1)
self.assertEqual(self.device.nodeName, self.nodename)
self.assertEqual(self.device.manufacturer, 'JUNIPER')
def test_aclsdb(self):
"""Test acls.db handling."""
self.assertTrue('test1-abc-only' in self.device.explicit_acls)
def test_autoacls(self):
"""Test autoacls.py handling."""
self.assertTrue('router-protect.core' in self.device.implicit_acls)
def test_find(self):
"""Test the find() method."""
self.assertEqual(self.nd.find(self.nodename), self.device)
nodebasename = self.nodename[:self.nodename.index('.')]
self.assertEqual(self.nd.find(nodebasename), self.device)
self.assertRaises(KeyError, lambda: self.nd.find(self.nodename[0:3]))
def test_all(self):
"""Test the all() method."""
expected = [self.device]
self.assertEqual(expected, self.nd.all())
def test_search(self):
"""Test the search() method."""
expected = [self.device]
self.assertEqual(expected, self.nd.search(self.nodename))
self.assertEqual(expected, self.nd.search('17', field='onCallID'))
self.assertEqual(expected, self.nd.search('juniper', field='vendor'))
def test_match(self):
"""Test the match() method."""
expected = [self.device]
self.assertEqual(expected, self.nd.match(nodename=self.nodename))
self.assertEqual(expected, self.nd.match(vendor='juniper'))
self.assertNotEqual(expected, self.nd.match(vendor='cisco'))
def tearDown(self):
NetDevices._Singleton = None
class TestNetDevicesWithAcls(unittest.TestCase):
"""
Test NetDevices with ``settings.WITH_ACLs set`` to ``True``.
"""
def setUp(self):
self.nd = NetDevices()
self.nodename = self.nd.keys()[0]
self.device = self.nd.values()[0]
self.device.explicit_acls = set(['test1-abc-only'])
def test_basics(self):
"""Basic test of NetDevices functionality."""
self.assertEqual(len(self.nd), 1)
self.assertEqual(self.device.nodeName, self.nodename)
self.assertEqual(self.device.manufacturer, 'JUNIPER')
def test_aclsdb(self):
"""Test acls.db handling."""
self.assertTrue('test1-abc-only' in self.device.explicit_acls)
def test_autoacls(self):
"""Test autoacls.py handling."""
self.assertTrue('router-protect.core' in self.device.implicit_acls)
def test_find(self):
"""Test the find() method."""
self.assertEqual(self.nd.find(self.nodename), self.device)
nodebasename = self.nodename[:self.nodename.index('.')]
self.assertEqual(self.nd.find(nodebasename), self.device)
self.assertRaises(KeyError, lambda: self.nd.find(self.nodename[0:3]))
def test_all(self):
"""Test the all() method."""
expected = [self.device]
self.assertEqual(expected, self.nd.all())
def test_search(self):
"""Test the search() method."""
expected = [self.device]
self.assertEqual(expected, self.nd.search(self.nodename))
self.assertEqual(expected, self.nd.search('17', field='onCallID'))
self.assertEqual(expected, self.nd.search('juniper', field='vendor'))
def test_match(self):
"""Test the match() method."""
expected = [self.device]
self.assertEqual(expected, self.nd.match(nodename=self.nodename))
self.assertEqual(expected, self.nd.match(vendor='juniper'))
self.assertNotEqual(expected, self.nd.match(vendor='cisco'))
def tearDown(self):
NetDevices._Singleton = None
class NetDevicesTest(unittest.TestCase):
def setUp(self):
self.nd = NetDevices(with_acls=False)
print self.nd.values()
self.nodename = self.nd.keys()[0]
self.nodeobj = self.nd.values()[0]
def testBasics(self):
"""Basic test of NetDevices functionality."""
self.assertEqual(len(self.nd), 3)
self.assertEqual(self.nodeobj.nodeName, self.nodename)
self.assertEqual(self.nodeobj.manufacturer, "JUNIPER")
def testFind(self):
"""Test the find() method."""
self.assertEqual(self.nd.find(self.nodename), self.nodeobj)
nodebasename = self.nodename[: self.nodename.index(".")]
self.assertEqual(self.nd.find(nodebasename), self.nodeobj)
self.assertRaises(KeyError, lambda: self.nd.find(self.nodename[0:3]))
class NetDevicesTest(unittest.TestCase):
def setUp(self):
self.nd = NetDevices(with_acls=False)
print self.nd.values()
self.nodename = self.nd.keys()[0]
self.nodeobj = self.nd.values()[0]
def testBasics(self):
"""Basic test of NetDevices functionality."""
self.assertEqual(len(self.nd), 3)
self.assertEqual(self.nodeobj.nodeName, self.nodename)
self.assertEqual(self.nodeobj.manufacturer, 'JUNIPER')
def testFind(self):
"""Test the find() method."""
self.assertEqual(self.nd.find(self.nodename), self.nodeobj)
nodebasename = self.nodename[:self.nodename.index('.')]
self.assertEqual(self.nd.find(nodebasename), self.nodeobj)
self.assertRaises(KeyError, lambda: self.nd.find(self.nodename[0:3]))
class TestAclsDB(unittest.TestCase):
def setUp(self):
self.nd = NetDevices()
self.acl = ACL_NAME
self.device = self.nd.find(DEVICE_NAME)
self.implicit_acls = set(['115j', 'router-protect.core'])
def test_01_add_acl_success(self):
"""Test associate ACL to device success"""
exp = 'added acl %s to %s' % (self.acl, self.device)
self.assertEqual(exp, adb.add_acl(self.device, self.acl))
def test_02_add_acl_failure(self):
"""Test associate ACL to device failure"""
exp = exceptions.ACLSetError
self.assertRaises(exp, adb.add_acl, self.device, self.acl)
def test_03_remove_acl_success(self):
"""Test remove ACL from device success"""
exp = 'removed acl %s from %s' % (self.acl, self.device)
self.assertEqual(exp, adb.remove_acl(self.device, self.acl))
def test_04_remove_acl_failure(self):
"""Test remove ACL from device failure"""
exp = exceptions.ACLSetError
self.assertRaises(exp, adb.remove_acl, self.device, self.acl)
def test_05_get_acl_dict(self):
"""Test get dict of associations"""
exp = {
'all': self.implicit_acls,
'explicit': set(),
'implicit': self.implicit_acls
}
self.assertEqual(exp, adb.get_acl_dict(self.device))
def test_06_get_acl_set_success(self):
"""Test get set of associations success"""
exp = self.implicit_acls
self.assertEqual(exp, adb.get_acl_set(self.device))
def test_07_get_acl_set_failure(self):
"""Test get set of associations failure"""
exp = exceptions.InvalidACLSet
acl_set = 'bogus'
self.assertRaises(exp, adb.get_acl_set, self.device, acl_set)
def tearDown(self):
NetDevices._Singleton = None
def test_trigger():
from twisted.internet import reactor
nd = NetDevices()
dev = nd.find('fortinet')
d = defer.Deferred()
creds = tacacsrc.get_device_password(dev.nodeName)
factory = TriggerSSHPtyClientFactory(d,
Interactor(),
creds,
display_banner=None,
init_commands=None,
device=dev)
reactor.connectTCP(dev.nodeName, 22, factory)
d.addCallback(lambda x: stop_reactor())
cli.setup_tty_for_pty(reactor.run)
class TestAclsDB(unittest.TestCase):
def setUp(self):
self.nd = NetDevices()
self.acl = ACL_NAME
self.device = self.nd.find(DEVICE_NAME)
self.implicit_acls = set(['115j', 'router-protect.core'])
def test_01_add_acl_success(self):
"""Test associate ACL to device success"""
exp = 'added acl %s to %s' % (self.acl, self.device)
self.assertEqual(exp, adb.add_acl(self.device, self.acl))
def test_02_add_acl_failure(self):
"""Test associate ACL to device failure"""
exp = exceptions.ACLSetError
self.assertRaises(exp, adb.add_acl, self.device, self.acl)
def test_03_remove_acl_success(self):
"""Test remove ACL from device success"""
exp = 'removed acl %s from %s' % (self.acl, self.device)
self.assertEqual(exp, adb.remove_acl(self.device, self.acl))
def test_04_remove_acl_failure(self):
"""Test remove ACL from device failure"""
exp = exceptions.ACLSetError
self.assertRaises(exp, adb.remove_acl, self.device, self.acl)
def test_05_get_acl_dict(self):
"""Test get dict of associations"""
exp = {'all': self.implicit_acls, 'explicit': set(),
'implicit': self.implicit_acls}
self.assertEqual(exp, adb.get_acl_dict(self.device))
def test_06_get_acl_set_success(self):
"""Test get set of associations success"""
exp = self.implicit_acls
self.assertEqual(exp, adb.get_acl_set(self.device))
def test_07_get_acl_set_failure(self):
"""Test get set of associations failure"""
exp = exceptions.InvalidACLSet
acl_set = 'bogus'
self.assertRaises(exp, adb.get_acl_set, self.device, acl_set)
def tearDown(self):
NetDevices._Singleton = None
__all__ = ['mock_redis']
# misc
from . import misc
from misc import *
__all__.extend(misc.__all__)
if __name__ == '__main__':
os.environ['NETDEVICES_SOURCE'] = 'data/netdevices.xml'
mock_redis.install()
import redis
from trigger.netdevices import NetDevices
from trigger.acl.db import AclsDB
r = redis.Redis()
a = AclsDB()
nd = NetDevices()
dev = nd.find('test1-abc')
print r.keys('*')
print a.add_acl(dev, 'bacon')
print r.keys('*')
_k = 'acls:explicit:'
key = _k + dev.nodeName
print r.smembers(key)
device_list = NetDevices()
gi = GatherInfo(devices=device_list)
gi.run()
print(gi.results)
gi = GatherInfo(devices='hsc-hmg-uu-gw')
gi.run()
print(gi.results)
from trigger.netdevices import NetDevices
from trigger.contrib.commando.plugins import gather_info
nd = NetDevices()
device_list = []
devs = [nd.find('hsc-hmg-uu-gw'), nd.find('hsc-hmg-vz-gw1')]
print devs
gi = gather_info.GatherInfo(devs)
gi.run()
print(gi.results)
# >>> import os
# >>> from trigger import tacacsrc
# >>> from trigger.conf import settings
# >>> from trigger.netdevices import NetDevices
# >>> from trigger.contrib.commando.plugins import gather_info
# >>>
# >>>
# >>> settings.NETDEVICES_SOURCE = os.path.abspath('netdevices.csv')
# >>> settings.DEFAULT_REALM = 'MyRealm'
class Commando(object):
"""
Execute commands asynchronously on multiple network devices.
This class is designed to be extended but can still be used as-is to execute
commands and return the results as-is.
At the bare minimum you must specify a list of ``devices`` to interact with.
You may optionally specify a list of ``commands`` to execute on those
devices, but doing so will execute the same commands on every device
regardless of platform.
If ``commands`` are not specified, they will be expected to be emitted by
the ``generate`` method for a given platform. Otherwise no commands will be
executed.
If you wish to customize the commands executed by device, you must define a
``to_{vendor_name}`` method containing your custom logic.
If you wish to customize what is done with command results returned from a
device, you must define a ``from_{vendor_name}`` method containing your
custom logic.
:param devices:
A list of device hostnames or `~trigger.netdevices.NetDevice` objects
:param commands:
(Optional) A list of commands to execute on the ``devices``.
:param creds:
(Optional) A 3-tuple of (username, password, realm). If only (username,
password) are provided, realm will be populated from
:setting:`DEFAULT_REALM`. If unset it will fetch from ``.tacacsrc``.
:param incremental:
(Optional) A callback that will be called with an empty sequence upon
connection and then called every time a result comes back from the
device, with the list of all results.
:param max_conns:
(Optional) The maximum number of simultaneous connections to keep open.
:param verbose:
(Optional) Whether or not to display informational messages to the
console.
:param timeout:
(Optional) Time in seconds to wait for each command executed to return a
result. Set to ``None`` to disable timeout (not recommended).
:param production_only:
(Optional) If set, includes all devices instead of excluding any devices
where ``adminStatus`` is not set to ``PRODUCTION``.
:param allow_fallback:
If set (default), allow fallback to base parse/generate methods when
they are not customized in a subclass, otherwise an exception is raised
when a method is called that has not been explicitly defined.
:param with_errors:
(Optional) Return exceptions as results instead of raising them. The
default is to always return them.
:param force_cli:
(Optional) Juniper only. If set, sends commands using CLI instead of
Junoscript.
:param with_acls:
Whether to load ACL associations (requires Redis). Defaults to whatever
is specified in settings.WITH_ACLS
:param command_interval:
(Optional) Amount of time in seconds to wait between sending commands.
:param stop_reactor:
Whether to stop the reactor loop when all results have returned.
(Default: ``True``)
"""
# Defaults to all supported vendors
vendors = settings.SUPPORTED_VENDORS
# Defaults to all supported platforms
platforms = settings.SUPPORTED_PLATFORMS
# The commands to run (defaults to [])
commands = None
# The timeout for commands to return results. We are setting this to 0
# so that if it's not overloaded in a subclass, the timeout value passed to
# the constructor will be preferred, especially if it is set to ``None``
# which Twisted uses to disable timeouts completely.
timeout = 0
# How results are stored (defaults to {})
results = None
# How parsed results are stored (defaults to {})
parsed_results = None
# How errors are stored (defaults to {})
errors = None
# Whether to stop the reactor when all results have returned.
stop_reactor = None
def __init__(self,
devices=None,
commands=None,
creds=None,
incremental=None,
max_conns=10,
verbose=False,
timeout=DEFAULT_TIMEOUT,
production_only=True,
allow_fallback=True,
with_errors=True,
force_cli=False,
with_acls=False,
command_interval=0,
stop_reactor=True):
if devices is None:
raise exceptions.ImproperlyConfigured(
'You must specify some `devices` to interact with!')
self.devices = devices
self.commands = self.commands or (commands or []
) # Always fallback to []
self.creds = creds
self.incremental = incremental
self.max_conns = max_conns
self.verbose = verbose
self.timeout = timeout if timeout != self.timeout else self.timeout
self.nd = NetDevices(production_only=production_only,
with_acls=with_acls)
self.allow_fallback = allow_fallback
self.with_errors = with_errors
self.force_cli = force_cli
self.command_interval = command_interval
self.stop_reactor = self.stop_reactor or stop_reactor
self.curr_conns = 0
self.jobs = []
# Always fallback to {} for these
self.errors = self.errors if self.errors is not None else {}
self.results = self.results if self.results is not None else {}
self.parsed_results = self.parsed_results if self.parsed_results is not None else collections.defaultdict(
dict)
#self.deferrals = []
self.supported_platforms = self._validate_platforms()
self._setup_jobs()
def _validate_platforms(self):
"""
Determine the set of supported platforms for this instance by making
sure the specified vendors/platforms for the class match up.
"""
supported_platforms = {}
for vendor in self.vendors:
if vendor in self.platforms:
types = self.platforms[vendor]
if not types:
raise exceptions.MissingPlatform(
'No platforms specified for %r' % vendor)
else:
#self.supported_platforms[vendor] = types
supported_platforms[vendor] = types
else:
raise exceptions.ImproperlyConfigured(
'Platforms for vendor %r not found. Please provide it at either the class level or using the arguments.'
% vendor)
return supported_platforms
def _decrement_connections(self, data=None):
"""
Self-explanatory. Called by _add_worker() as both callback/errback
so we can accurately refill the jobs queue, which relies on the
current connection count.
"""
self.curr_conns -= 1
return data
def _increment_connections(self, data=None):
"""Increment connection count."""
self.curr_conns += 1
return True
def _setup_jobs(self):
"""
"Maps device hostnames to `~trigger.netdevices.NetDevice` objects and
populates the job queue.
"""
for dev in self.devices:
log.msg('Adding', dev)
if self.verbose:
print 'Adding', dev
# Make sure that devices are actually in netdevices and keep going
try:
devobj = self.nd.find(str(dev))
except KeyError:
msg = 'Device not found in NetDevices: %s' % dev
log.err(msg)
if self.verbose:
print 'ERROR:', msg
# Track the errors and keep moving
self.store_error(dev, msg)
continue
# We only want to add devices for which we've enabled support in
# this class
if devobj.vendor not in self.vendors:
raise exceptions.UnsupportedVendor(
"The vendor '%s' is not specified in ``vendors``. Could not add %s to job queue. Please check the attribute in the class object."
% (devobj.vendor, devobj))
self.jobs.append(devobj)
def select_next_device(self, jobs=None):
"""
Select another device for the active queue.
Currently only returns the next device in the job queue. This is
abstracted out so that this behavior may be customized, such as for
future support for incremental callbacks.
If a device is determined to be invalid, you must return ``None``.
:param jobs:
(Optional) The jobs queue. If not set, uses ``self.jobs``.
:returns:
A `~trigger.netdevices.NetDevice` object or ``None``.
"""
if jobs is None:
jobs = self.jobs
return jobs.pop()
def _add_worker(self):
"""
Adds devices to the work queue to keep it populated with the maximum
connections as specified by ``max_conns``.
"""
while self.jobs and self.curr_conns < self.max_conns:
device = self.select_next_device()
if device is None:
log.msg('No device returned when adding worker. Moving on.')
continue
self._increment_connections()
log.msg('connections:', self.curr_conns)
log.msg('Adding work to queue...')
if self.verbose:
print 'connections:', self.curr_conns
print 'Adding work to queue...'
# Setup the async Deferred object with a timeout and error printing.
commands = self.generate(device)
async = device.execute(commands,
creds=self.creds,
incremental=self.incremental,
timeout=self.timeout,
with_errors=self.with_errors,
force_cli=self.force_cli,
command_interval=self.command_interval)
# Add the template parser callback for great justice!
async .addCallback(self.parse_template, device, commands)
# Add the parser callback for even greater justice!
async .addCallback(self.parse, device, commands)
# If parse fails, still decrement and track the error
async .addErrback(self.errback, device)
# Make sure any further uncaught errors get logged
async .addErrback(log.err)
# Here we addBoth to continue on after pass/fail, decrement the
# connections and move on.
async .addBoth(self._decrement_connections)
async .addBoth(lambda x: self._add_worker())
# Do this once we've exhausted the job queue
else:
if not self.curr_conns and self.reactor_running:
self._stop()
elif not self.jobs and not self.reactor_running:
log.msg('No work left.')
if self.verbose:
print 'No work left.'
def _lookup_method(self, device, method):
"""
Base lookup method. Looks up stuff by device manufacturer like:
from_juniper
to_foundry
and defaults to ``self.from_base`` and ``self.to_base`` methods if
customized methods not found.
:param device:
A `~trigger.netdevices.NetDevice` object
:param method:
One of 'generate', 'parse'
"""
METHOD_MAP = {
'generate': 'to_%s',
'parse': 'from_%s',
}
assert method in METHOD_MAP
desired_method = None
# Select the desired vendor name.
desired_vendor = device.vendor.name
# Workaround until we implement device drivers
if device.is_netscreen():
desired_vendor = 'netscreen'
vendor_types = self.platforms.get(desired_vendor)
method_name = METHOD_MAP[method] % desired_vendor # => 'to_cisco'
device_type = device.deviceType
if device_type in vendor_types:
if hasattr(self, method_name):
log.msg('[%s] Found %r method: %s' %
(device, method, method_name))
desired_method = method_name
else:
log.msg('[%s] Did not find %r method: %s' %
(device, method, method_name))
else:
raise exceptions.UnsupportedDeviceType(
'Device %r has an invalid type %r for vendor %r. Must be '
'one of %r.' %
(device.nodeName, device_type, desired_vendor, vendor_types))
if desired_method is None:
if self.allow_fallback:
desired_method = METHOD_MAP[method] % 'base'
log.msg('[%s] Fallback enabled. Using base method: %r' %
(device, desired_method))
else:
raise exceptions.UnsupportedVendor(
'The vendor %r had no available %s method. Please check '
'your `vendors` and `platforms` attributes in your class '
'object.' % (device.vendor.name, method))
func = getattr(self, desired_method)
return func
def generate(self, device, commands=None, extra=None):
"""
Generate commands to be run on a device. If you don't provide
``commands`` to the class constructor, this will return an empty list.
Define a 'to_{vendor_name}' method to customize the behavior for each
platform.
:param device:
NetDevice object
:type device:
`~trigger.netdevices.NetDevice`
:param commands:
(Optional) A list of commands to execute on the device. If not
specified in they will be inherited from commands passed to the
class constructor.
:type commands:
list
:param extra:
(Optional) A dictionary of extra data to send to the generate
method for the device.
"""
if commands is None:
commands = self.commands
if extra is None:
extra = {}
func = self._lookup_method(device, method='generate')
return func(device, commands, extra)
def parse_template(self, results, device, commands=None):
"""
Generator function that processes unstructured CLI data and yields either
a TextFSM based object or generic raw output.
:param results:
The unstructured "raw" CLI data from device.
:type results:
str
:param device:
NetDevice object
:type device:
`~trigger.netdevices.NetDevice`
"""
device_type = device.os
ret = []
for idx, command in enumerate(commands):
if device_type:
try:
re_table = load_cmd_template(command, dev_type=device_type)
fsm = get_textfsm_object(re_table, results[idx])
self.append_parsed_results(
device, self.map_parsed_results(command, fsm))
except:
log.msg(
"Unable to load TextFSM template, just updating with unstructured output"
)
ret.append(results[idx])
self.parsed_results = dict(self.parsed_results)
return ret
def parse(self, results, device, commands=None):
"""
Parse output from a device. Calls to ``self._lookup_method`` to find
specific ``from`` method.
Define a 'from_{vendor_name}' method to customize the behavior for each
platform.
:param results:
The results of the commands executed on the device
:type results:
list
:param device:
Device object
:type device:
`~trigger.netdevices.NetDevice`
:param commands:
(Optional) A list of commands to execute on the device. If not
specified in they will be inherited from commands passed to the
class constructor.
:type commands:
list
"""
func = self._lookup_method(device, method='parse')
return func(results, device, commands)
def errback(self, failure, device):
"""
The default errback. Overload for custom behavior but make sure it
always decrements the connections.
:param failure:
Usually a Twisted ``Failure`` instance.
:param device:
A `~trigger.netdevices.NetDevice` object
"""
failure.trap(Exception)
self.store_error(device, failure)
#self._decrement_connections(failure)
return failure
def store_error(self, device, error):
"""
A simple method for storing an error called by all default
parse/generate methods.
If you want to customize the default method for storing results,
overload this in your subclass.
:param device:
A `~trigger.netdevices.NetDevice` object
:param error:
The error to store. Anything you want really, but usually a Twisted
``Failure`` instance.
"""
devname = str(device)
self.errors[devname] = error
return True
def append_parsed_results(self, device, results):
"""
A simple method for appending results called by template parser
method.
If you want to customize the default method for storing parsed
results, overload this in your subclass.
:param device:
A `~trigger.netdevices.NetDevice` object
:param results:
The results to store. Anything you want really.
"""
devname = str(device)
log.msg("Appending results for %r: %r" % (devname, results))
self.parsed_results[devname].update(results)
return True
def store_results(self, device, results):
"""
A simple method for storing results called by all default
parse/generate methods.
If you want to customize the default method for storing results,
overload this in your subclass.
:param device:
A `~trigger.netdevices.NetDevice` object
:param results:
The results to store. Anything you want really.
"""
devname = str(device)
log.msg("Storing results for %r: %r" % (devname, results))
self.results[devname] = results
return True
def map_parsed_results(self, command=None, fsm=None):
"""Return a dict of ``{command: fsm, ...}``"""
if fsm is None:
fsm = {}
return {command: fsm}
def map_results(self, commands=None, results=None):
"""Return a dict of ``{command: result, ...}``"""
if commands is None:
commands = self.commands
if results is None:
results = []
return dict(itertools.izip_longest(commands, results))
@property
def reactor_running(self):
"""Return whether reactor event loop is running or not"""
from twisted.internet import reactor
log.msg("Reactor running? %s" % reactor.running)
return reactor.running
def _stop(self):
"""Stop the reactor event loop"""
if self.stop_reactor:
log.msg('Stop reactor enabled: stopping reactor...')
from twisted.internet import reactor
if reactor.running:
reactor.stop()
else:
log.msg('stopping reactor... except not really.')
if self.verbose:
print 'stopping reactor... except not really.'
def _start(self):
"""Start the reactor event loop"""
log.msg('starting reactor. maybe.')
if self.verbose:
print 'starting reactor. maybe.'
if self.curr_conns:
from twisted.internet import reactor
if not reactor.running:
reactor.run()
else:
msg = "Won't start reactor with no work to do!"
log.msg(msg)
if self.verbose:
print msg
def run(self):
"""
Nothing happens until you execute this to perform the actual work.
"""
self._add_worker()
self._start()
#=======================================
# Base generate (to_)/parse (from_) methods
#=======================================
def to_base(self, device, commands=None, extra=None):
commands = commands or self.commands
log.msg('Sending %r to %s' % (commands, device))
return commands
def from_base(self, results, device, commands=None):
commands = commands or self.commands
log.msg('Received %r from %s' % (results, device))
self.store_results(device, self.map_results(commands, results))
#=======================================
# Vendor-specific generate (to_)/parse (from_) methods
#=======================================
def to_juniper(self, device, commands=None, extra=None):
"""
This just creates a series of ``<command>foo</command>`` elements to
pass along to execute_junoscript()"""
commands = commands or self.commands
# If we've set force_cli, use to_base() instead
if self.force_cli:
return self.to_base(device, commands, extra)
ret = []
for command in commands:
cmd = Element('command')
cmd.text = command
ret.append(cmd)
return ret
class Commando(object):
"""
Execute commands asynchronously on multiple network devices.
This class is designed to be extended but can still be used as-is to execute
commands and return the results as-is.
At the bare minimum you must specify a list of ``devices`` to interact with.
You may optionally specify a list of ``commands`` to execute on those
devices, but doing so will execute the same commands on every device
regardless of platform.
If ``commands`` are not specified, they will be expected to be emitted by
the ``generate`` method for a given platform. Otherwise no commands will be
executed.
If you wish to customize the commands executed by device, you must define a
``to_{vendor_name}`` method containing your custom logic.
If you wish to customize what is done with command results returned from a
device, you must define a ``from_{vendor_name}`` method containing your
custom logic.
:param devices:
A list of device hostnames or `~trigger.netdevices.NetDevice` objects
:param commands:
(Optional) A list of commands to execute on the ``devices``.
:param creds:
(Optional) A 3-tuple of (username, password, realm). If only (username,
password) are provided, realm will be populated from
:setting:`DEFAULT_REALM`. If unset it will fetch from ``.tacacsrc``.
:param incremental:
(Optional) A callback that will be called with an empty sequence upon
connection and then called every time a result comes back from the
device, with the list of all results.
:param max_conns:
(Optional) The maximum number of simultaneous connections to keep open.
:param verbose:
(Optional) Whether or not to display informational messages to the
console.
:param timeout:
(Optional) Time in seconds to wait for each command executed to return a
result. Set to ``None`` to disable timeout (not recommended).
:param production_only:
(Optional) If set, includes all devices instead of excluding any devices
where ``adminStatus`` is not set to ``PRODUCTION``.
:param allow_fallback:
If set (default), allow fallback to base parse/generate methods when
they are not customized in a subclass, otherwise an exception is raised
when a method is called that has not been explicitly defined.
:param with_errors:
(Optional) Return exceptions as results instead of raising them. The
default is to always return them.
:param force_cli:
(Optional) Juniper only. If set, sends commands using CLI instead of
Junoscript.
:param with_acls:
Whether to load ACL associations (requires Redis). Defaults to whatever
is specified in settings.WITH_ACLS
:param command_interval:
(Optional) Amount of time in seconds to wait between sending commands.
"""
# Defaults to all supported vendors
vendors = settings.SUPPORTED_VENDORS
# Defaults to all supported platforms
platforms = settings.SUPPORTED_PLATFORMS
# The commands to run (defaults to [])
commands = None
# The timeout for commands to return results. We are setting this to 0
# so that if it's not overloaded in a subclass, the timeout value passed to
# the constructor will be preferred, especially if it is set to ``None``
# which Twisted uses to disable timeouts completely.
timeout = 0
# How results are stored (defaults to {})
results = None
# How errors are stored (defaults to {})
errors = None
def __init__(self, devices=None, commands=None, creds=None,
incremental=None, max_conns=10, verbose=False,
timeout=DEFAULT_TIMEOUT, production_only=True,
allow_fallback=True, with_errors=True, force_cli=False,
with_acls=False, command_interval=0):
if devices is None:
raise exceptions.ImproperlyConfigured('You must specify some `devices` to interact with!')
self.devices = devices
self.commands = self.commands or (commands or []) # Always fallback to []
self.creds = creds
self.incremental = incremental
self.max_conns = max_conns
self.verbose = verbose
self.timeout = timeout if timeout != self.timeout else self.timeout
self.nd = NetDevices(production_only=production_only, with_acls=with_acls)
self.allow_fallback = allow_fallback
self.with_errors = with_errors
self.force_cli = force_cli
self.command_interval = command_interval
self.curr_conns = 0
self.jobs = []
# Always fallback to {} for these
self.errors = self.errors if self.errors is not None else {}
self.results = self.results if self.results is not None else {}
#self.deferrals = []
self.supported_platforms = self._validate_platforms()
self._setup_jobs()
def _validate_platforms(self):
"""
Determine the set of supported platforms for this instance by making
sure the specified vendors/platforms for the class match up.
"""
supported_platforms = {}
for vendor in self.vendors:
if vendor in self.platforms:
types = self.platforms[vendor]
if not types:
raise exceptions.MissingPlatform('No platforms specified for %r' % vendor)
else:
#self.supported_platforms[vendor] = types
supported_platforms[vendor] = types
else:
raise exceptions.ImproperlyConfigured('Platforms for vendor %r not found. Please provide it at either the class level or using the arguments.' % vendor)
return supported_platforms
def _decrement_connections(self, data=None):
"""
Self-explanatory. Called by _add_worker() as both callback/errback
so we can accurately refill the jobs queue, which relies on the
current connection count.
"""
self.curr_conns -= 1
return data
def _increment_connections(self, data=None):
"""Increment connection count."""
self.curr_conns += 1
return True
def _setup_jobs(self):
"""
"Maps device hostnames to `~trigger.netdevices.NetDevice` objects and
populates the job queue.
"""
for dev in self.devices:
log.msg('Adding', dev)
if self.verbose:
print 'Adding', dev
# Make sure that devices are actually in netdevices and keep going
try:
devobj = self.nd.find(str(dev))
except KeyError:
msg = 'Device not found in NetDevices: %s' % dev
log.err(msg)
if self.verbose:
print 'ERROR:', msg
# Track the errors and keep moving
self.store_error(dev, msg)
continue
# We only want to add devices for which we've enabled support in
# this class
if devobj.vendor not in self.vendors:
raise exceptions.UnsupportedVendor("The vendor '%s' is not specified in ``vendors``. Could not add %s to job queue. Please check the attribute in the class object." % (devobj.vendor, devobj))
self.jobs.append(devobj)
def select_next_device(self, jobs=None):
"""
Select another device for the active queue.
Currently only returns the next device in the job queue. This is
abstracted out so that this behavior may be customized, such as for
future support for incremental callbacks.
If a device is determined to be invalid, you must return ``None``.
:param jobs:
(Optional) The jobs queue. If not set, uses ``self.jobs``.
:returns:
A `~trigger.netdevices.NetDevice` object or ``None``.
"""
if jobs is None:
jobs = self.jobs
return jobs.pop()
def _add_worker(self):
"""
Adds devices to the work queue to keep it populated with the maximum
connections as specified by ``max_conns``.
"""
while self.jobs and self.curr_conns < self.max_conns:
device = self.select_next_device()
if device is None:
log.msg('No device returned when adding worker. Moving on.')
continue
self._increment_connections()
log.msg('connections:', self.curr_conns)
log.msg('Adding work to queue...')
if self.verbose:
print 'connections:', self.curr_conns
print 'Adding work to queue...'
# Setup the async Deferred object with a timeout and error printing.
commands = self.generate(device)
async = device.execute(commands, creds=self.creds,
incremental=self.incremental,
timeout=self.timeout,
with_errors=self.with_errors,
force_cli=self.force_cli,
command_interval=self.command_interval)
# Add the parser callback for great justice!
async.addCallback(self.parse, device, commands)
# If parse fails, still decrement and track the error
async.addErrback(self.errback, device)
# Make sure any further uncaught errors get logged
async.addErrback(log.err)
# Here we addBoth to continue on after pass/fail, decrement the
# connections and move on.
async.addBoth(self._decrement_connections)
async.addBoth(lambda x: self._add_worker())
# Do this once we've exhausted the job queue
else:
if not self.curr_conns and self.reactor_running:
self._stop()
elif not self.jobs and not self.reactor_running:
log.msg('No work left.')
if self.verbose:
print 'No work left.'
def _lookup_method(self, device, method):
"""
Base lookup method. Looks up stuff by device manufacturer like:
from_juniper
to_foundry
and defaults to ``self.from_base`` and ``self.to_base`` methods if
customized methods not found.
:param device:
A `~trigger.netdevices.NetDevice` object
:param method:
One of 'generate', 'parse'
"""
METHOD_MAP = {
'generate': 'to_%s',
'parse': 'from_%s',
}
assert method in METHOD_MAP
desired_method = None
# Select the desired vendor name.
desired_vendor = device.vendor.name
# Workaround until we implement device drivers
if device.is_netscreen():
desired_vendor = 'netscreen'
vendor_types = self.platforms.get(desired_vendor)
method_name = METHOD_MAP[method] % desired_vendor # => 'to_cisco'
device_type = device.deviceType
if device_type in vendor_types:
if hasattr(self, method_name):
log.msg(
'[%s] Found %r method: %s' % (device, method, method_name)
)
desired_method = method_name
else:
log.msg(
'[%s] Did not find %r method: %s' % (device, method,
method_name)
)
else:
raise exceptions.UnsupportedDeviceType(
'Device %r has an invalid type %r for vendor %r. Must be '
'one of %r.' % (device.nodeName, device_type,
desired_vendor, vendor_types)
)
if desired_method is None:
if self.allow_fallback:
desired_method = METHOD_MAP[method] % 'base'
log.msg('[%s] Fallback enabled. Using base method: %r' %
(device, desired_method))
else:
raise exceptions.UnsupportedVendor(
'The vendor %r had no available %s method. Please check '
'your `vendors` and `platforms` attributes in your class '
'object.' % (device.vendor.name, method)
)
func = getattr(self, desired_method)
return func
def generate(self, device, commands=None, extra=None):
"""
Generate commands to be run on a device. If you don't provide
``commands`` to the class constructor, this will return an empty list.
Define a 'to_{vendor_name}' method to customize the behavior for each
platform.
:param device:
NetDevice object
:type device:
`~trigger.netdevices.NetDevice`
:param commands:
(Optional) A list of commands to execute on the device. If not
specified in they will be inherited from commands passed to the
class constructor.
:type commands:
list
:param extra:
(Optional) A dictionary of extra data to send to the generate
method for the device.
"""
if commands is None:
commands = self.commands
if extra is None:
extra = {}
func = self._lookup_method(device, method='generate')
return func(device, commands, extra)
def parse(self, results, device, commands=None):
"""
Parse output from a device. Calls to ``self._lookup_method`` to find
specific ``from`` method.
Define a 'from_{vendor_name}' method to customize the behavior for each
platform.
:param results:
The results of the commands executed on the device
:type results:
list
:param device:
Device object
:type device:
`~trigger.netdevices.NetDevice`
:param commands:
(Optional) A list of commands to execute on the device. If not
specified in they will be inherited from commands passed to the
class constructor.
:type commands:
list
"""
func = self._lookup_method(device, method='parse')
return func(results, device, commands)
def errback(self, failure, device):
"""
The default errback. Overload for custom behavior but make sure it
always decrements the connections.
:param failure:
Usually a Twisted ``Failure`` instance.
:param device:
A `~trigger.netdevices.NetDevice` object
"""
failure.trap(Exception)
self.store_error(device, failure)
#self._decrement_connections(failure)
return failure
def store_error(self, device, error):
"""
A simple method for storing an error called by all default
parse/generate methods.
If you want to customize the default method for storing results,
overload this in your subclass.
:param device:
A `~trigger.netdevices.NetDevice` object
:param error:
The error to store. Anything you want really, but usually a Twisted
``Failure`` instance.
"""
devname = str(device)
self.errors[devname] = error
return True
def store_results(self, device, results):
"""
A simple method for storing results called by all default
parse/generate methods.
If you want to customize the default method for storing results,
overload this in your subclass.
:param device:
A `~trigger.netdevices.NetDevice` object
:param results:
The results to store. Anything you want really.
"""
devname = str(device)
log.msg("Storing results for %r: %r" % (devname, results))
self.results[devname] = results
return True
def map_results(self, commands=None, results=None):
"""Return a dict of ``{command: result, ...}``"""
if commands is None:
commands = self.commands
if results is None:
results = []
return dict(itertools.izip_longest(commands, results))
@property
def reactor_running(self):
"""Return whether reactor event loop is running or not"""
from twisted.internet import reactor
log.msg("Reactor running? %s" % reactor.running)
return reactor.running
def _stop(self):
"""Stop the reactor event loop"""
log.msg('stopping reactor')
if self.verbose:
print 'stopping reactor'
from twisted.internet import reactor
reactor.stop()
def _start(self):
"""Start the reactor event loop"""
log.msg('starting reactor')
if self.verbose:
print 'starting reactor'
if self.curr_conns:
from twisted.internet import reactor
reactor.run()
else:
msg = "Won't start reactor with no work to do!"
log.msg(msg)
if self.verbose:
print msg
def run(self):
"""
Nothing happens until you execute this to perform the actual work.
"""
self._add_worker()
self._start()
#=======================================
# Base generate (to_)/parse (from_) methods
#=======================================
def to_base(self, device, commands=None, extra=None):
commands = commands or self.commands
log.msg('Sending %r to %s' % (commands, device))
return commands
def from_base(self, results, device, commands=None):
commands = commands or self.commands
log.msg('Received %r from %s' % (results, device))
self.store_results(device, self.map_results(commands, results))
#=======================================
# Vendor-specific generate (to_)/parse (from_) methods
#=======================================
def to_juniper(self, device, commands=None, extra=None):
"""
This just creates a series of ``<command>foo</command>`` elements to
pass along to execute_junoscript()"""
commands = commands or self.commands
# If we've set force_cli, use to_base() instead
if self.force_cli:
return self.to_base(device, commands, extra)
ret = []
for command in commands:
cmd = Element('command')
cmd.text = command
ret.append(cmd)
return ret
#Appending the content to the master report file.
report_file.seek(0)
master_report.write('\n\n*** Device: ' + each_device + ' ***\n\n')
master_report.write(report_file.read())
#Defining the list of devices to monitor. These are my Cisco 2691 / Juniper SRX100H / Arista vEOS devices. Replace with your own.
devices = ['172.16.1.2', '172.16.1.100', '172.16.1.101']
#Extracting the running config to a file, depending on the device vendor (Cisco, Juniper or Arista).
for each_device in devices:
lab = NetDevices()
device = lab.find(each_device)
#Using Trigger to check for the device vendor.
if str(device.vendor) == 'cisco':
diff_function('cisco_ios', 'cisco', 'mihai', 'python', 'show running')
#Using Trigger to check for the device vendor.
elif str(device.vendor) == 'juniper':
diff_function('juniper', 'juniper', 'mihai1', 'python1', 'show configuration')
#Using Trigger to check for the device vendor.
elif str(device.vendor) == 'arista':
diff_function('arista_eos', 'arista', 'mihai', 'python', 'show running')
else:
print '\nThis device type is not supported, sorry!\n'
import sys
from time import sleep
from twisted.internet.defer import Deferred
from zope.interface import implements
from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.web.http_headers import Headers
from twisted.internet.defer import succeed
from twisted.web.iweb import IBodyProducer
# from twisted.python import log
# log.startLogging(sys.stdout, setStdout=False)
from trigger.netdevices import NetDevices
# Create reference to upgraded switch.
nd = NetDevices()
dev = nd.find('arista-sw1.demo.local')
# Create payload body
class StringProducer(object):
implements(IBodyProducer)
def __init__(self, body):
self.body = body
self.length = len(body)
def startProducing(self, consumer):
consumer.write(self.body)
return succeed(None)
def pauseProducing(self):
reactor,
command,
"username",
"ssh.example.com",
22,
password="******"
)
factory = Factory()
d = endpoint.connect(factory)
d.addCallback(lambda protocol: protocol.finished)
# trigger
# Source: https://trigger.readthedocs.io/en/latest/examples.html#execute-commands-asynchronously-using-twisted
from trigger.netdevices import NetDevices
nd = NetDevices()
dev = nd.find('ssh.example.com')
dev.execute([command])
# parallel-ssh
# Source: https://github.com/ParallelSSH/parallel-ssh
from pssh.clients import SSHClient
client = SSHClient('ssh.example.com')
host_out = client.run_command(command)
# scrapli (can use paramiko, ssh2, asyncssh as transport)
# Source: https://github.com/carlmontanari/scrapli
from scrapli import Scrapli
conn = Scrapli(host='ssh.example.com' auth_username="******", auth_password="******")
conn.open()
class Commando(object):
"""
I run commands on devices but am not much use unless you subclass me and
configure vendor-specific parse/generate methods.
"""
def __init__(self, devices=None, max_conns=10, verbose=False, timeout=30,
production_only=True):
self.curr_connections = 0
self.reactor_running = False
self.devices = devices or []
self.verbose = verbose
self.max_conns = max_conns
self.nd = NetDevices(production_only=production_only)
self.jobs = []
self.errors = {}
self.data = {}
self.deferrals = self._setup_jobs()
self.timeout = timeout # in seconds
def _decrement_connections(self, data):
"""
Self-explanatory. Called by _add_worker() as both callback/errback
so we can accurately refill the jobs queue, which relies on the
current connection count.
"""
self.curr_connections -= 1
return True
def set_data(self, device, data):
"""
Another method for storing results. If you'd rather just change the
default method for storing results, overload this. All default
parse/generate methods call this."""
self.data[device] = data
return True
#=======================================
# Vendor-specific parse/generate methods
#=======================================
def _normalize_manufacturer(self, manufacturer):
"""Normalize the manufacturer name into a method"""
return manufacturer.replace(' ', '_').lower()
def _lookup(self, device, prefix):
"""Base lookup method."""
manuf = self._normalize_manufacturer(device.manufacturer)
try:
func = getattr(self, prefix + manuf)
except AttributeError:
return 'base prefix' + prefix (device)
#return self._base_generate_cmd(device)
return func(device)
def _parse_lookup(self, device):
"""Base parse method."""
manuf = self._normalize_manufacturer
try:
func = getattr(self, 'parse_' + manuf)
except AttributeError:
return self._base_generate_cmd(device)
return func(device)
# Yes there is probably a better way to do this in the long-run instead of
# individual parse/generate methods for each vendor, but this works for now.
def _base_parse(self, data, device):
"""
Parse output from a device. Overload this to customize this default
behavior.
"""
self.set_data(device, data)
return True
def _base_generate_cmd(self, dev=None):
"""
Generate commands to be run on a device. If you don't overload this, it
returns an empty list.
"""
return []
# TODO (jathan): Find a way to dynamically generate/call these methods
# TODO (jathan): Methods should be prefixed with their action, not vendor
# IOS (Cisco)
ios_parse = _base_parse
generate_ios_cmd = _base_generate_cmd
# Brocade
brocade_parse = _base_parse
generate_brocade_cmd = _base_generate_cmd
# Foundry
foundry_parse = _base_parse
generate_foundry_cmd = _base_generate_cmd
# Juniper (JUNOS)
junos_parse = _base_parse
generate_junos_cmd = _base_generate_cmd
# Citrix NetScaler
netscaler_parse = _base_parse
generate_netscaler_cmd = _base_generate_cmd
# Arista
arista_parse = _base_parse
generate_arista_cmd = _base_generate_cmd
# Dell
dell_parse = _base_parse
generate_dell_cmd = _base_generate_cmd
def _setup_callback(self, dev):
"""
Map execute, parse and generate callbacks to device by manufacturer.
This is ripe for optimization, especially if/when we need to add
support for multiple OS types/revisions per vendor.
:param dev: NetDevice object
Notes::
+ Arista, Brocade, Cisco, Dell, Foundry all use execute_ioslike
+ Citrix is assumed to be a NetScaler (switch)
+ Juniper is assumed to be a router/switch running JUNOS
"""
callback_map = {
'ARISTA NETWORKS':[dev, execute_ioslike,
self.generate_arista_cmd,
self.arista_parse],
'BROCADE': [dev, execute_ioslike,
self.generate_brocade_cmd,
self.brocade_parse],
'CISCO SYSTEMS': [dev, execute_ioslike,
self.generate_ios_cmd,
self.ios_parse],
'CITRIX': [dev, execute_netscaler,
self.generate_netscaler_cmd,
self.netscaler_parse],
'DELL': [dev, execute_ioslike,
self.generate_dell_cmd,
self.dell_parse],
'FOUNDRY': [dev, execute_ioslike,
self.generate_foundry_cmd,
self.foundry_parse],
'JUNIPER': [dev, execute_junoscript,
self.generate_junos_cmd,
self.junos_parse],
}
result = callback_map[dev.manufacturer]
return result
def _setup_jobs(self):
for dev in self.devices:
if self.verbose:
print 'Adding', dev
# Make sure that devices are actually in netdevices and keep going
try:
devobj = self.nd.find(str(dev))
except KeyError:
msg = 'Device not found in NetDevices: %s' % dev
if self.verbose:
print 'ERROR:', msg
# Track the errors and keep moving
self.errors[dev] = msg
continue
this_callback = self._setup_callback(devobj)
self.jobs.append(this_callback)
def run(self):
"""Nothing happens until you execute this to perform the actual work."""
self._add_worker()
self._start()
def eb(self, x):
self._decrement_connections(x)
return True
def _add_worker(self):
work = None
try:
work = self.jobs.pop()
if self.verbose:
print 'Adding work to queue...'
except (AttributeError, IndexError):
#if not self.curr_connections:
if not self.curr_connections and self.reactor_running:
self._stop()
else:
if self.verbose:
print 'No work left.'
while work:
if self.verbose:
print 'connections:', self.curr_connections
if self.curr_connections >= self.max_conns:
self.jobs.append(work)
return
self.curr_connections += 1
if self.verbose:
print 'connections:', self.curr_connections
# Unpack the job parts
#dev, execute, cmd, parser = work
dev, execute, generate, parser = work
# Setup the deferred object with a timeout and error printing.
#defer = execute(dev, cmd, timeout=self.timeout, with_errors=True)
cmds = generate(dev)
defer = execute(dev, cmds, timeout=self.timeout, with_errors=True)
# Add the callbacks for great justice!
defer.addCallback(parser, dev)
# Here we addBoth to continue on after pass/fail
defer.addBoth(self._decrement_connections)
defer.addBoth(lambda x: self._add_worker())
defer.addErrback(self.eb) # If worker add fails, still decrement
try:
work = self.jobs.pop()
except (AttributeError, IndexError):
work = None
def _stop(self):
if self.verbose:
print 'stopping reactor'
self.reactor_running = False
from twisted.internet import reactor
reactor.stop()
def _start(self):
if self.verbose:
print 'starting reactor'
self.reactor_running = True
from twisted.internet import reactor
if self.curr_connections:
reactor.run()
else:
if self.verbose:
print "Won't start reactor with no work to do!"
# names = sorted(nd)
# for name in names:
# dev = nd[name]
# if dev.deviceType not in ('ROUTER', 'SWITCH'):
# continue
# acls = sorted(dev.acls)
# print '%-39s %s' % (name, ' '.join(acls))
#
#if __name__ == '__main__':
# main()
from trigger.netdevices import NetDevices
from twisted.internet import reactor
nd = NetDevices()
dev = nd.find('foo1-abc')
def print_result(data):
"""Display results from a command"""
print 'Result:', data
def stop_reactor(data):
"""Stop the event loop"""
print 'Stopping reactor'
if reactor.running:
reactor.stop()
# Create an event chain that will execute a given list of commands on this
class Queue(object):
"""
Interacts with firewalls database to insert/remove items into the queue.
:param verbose:
Toggle verbosity
:type verbose:
Boolean
"""
def __init__(self, verbose=True):
self.nd = NetDevices()
self.verbose = verbose
self.login = get_user()
def vprint(self, msg):
"""
Print something if ``verbose`` instance variable is set.
:param msg:
The string to print
"""
if self.verbose:
print msg
def get_model(self, queue):
"""
Given a queue name, return its DB model.
:param queue:
Name of the queue whose object you want
"""
return models.MODEL_MAP.get(queue, None)
def create_task(self, queue, *args, **kwargs):
"""
Create a task in the specified queue.
:param queue:
Name of the queue whose object you want
"""
model = self.get_model(queue)
taskobj = model.create(*args, **kwargs)
def _normalize(self, arg, prefix=''):
"""
Remove ``prefix`` from ``arg``, and set "escalation" bit.
:param arg:
Arg (typically an ACL filename) to trim
:param prefix:
Prefix to trim from arg
"""
if arg.startswith(prefix):
arg = arg[len(prefix):]
escalation = False
if arg.upper().endswith(' ESCALATION'):
escalation = True
arg = arg[:-11]
return (escalation, arg)
def insert(self, acl, routers, escalation=False):
"""
Insert an ACL and associated devices into the ACL load queue.
Attempts to insert into integrated queue. If ACL test fails, then
item is inserted into manual queue.
:param acl:
ACL name
:param routers:
List of device names
:param escalation:
Whether this is an escalated task
"""
if not acl:
raise exceptions.ACLQueueError('You must specify an ACL to insert into the queue')
if not routers:
routers = []
escalation, acl = self._normalize(acl)
if routers:
for router in routers:
try:
dev = self.nd.find(router)
except KeyError:
msg = 'Could not find device %s' % router
raise exceptions.TriggerError(msg)
if acl not in dev.acls:
msg = "Could not find %s in ACL list for %s" % (acl, router)
raise exceptions.TriggerError(msg)
self.create_task(queue='integrated', acl=acl, router=router,
escalation=escalation)
self.vprint('ACL %s injected into integrated load queue for %s' %
(acl, ', '.join(dev[:dev.find('.')] for dev in routers)))
else:
self.create_task(queue='manual', q_name=acl, login=self.login)
self.vprint('"%s" injected into manual load queue' % acl)
def delete(self, acl, routers=None, escalation=False):
"""
Delete an ACL from the firewall database queue.
Attempts to delete from integrated queue. If ACL test fails
or if routers are not specified, the item is deleted from manual queue.
:param acl:
ACL name
:param routers:
List of device names. If this is ommitted, the manual queue is used.
:param escalation:
Whether this is an escalated task
"""
if not acl:
raise exceptions.ACLQueueError('You must specify an ACL to delete from the queue')
escalation, acl = self._normalize(acl)
m = self.get_model('integrated')
if routers is not None:
devs = routers
else:
self.vprint('Fetching routers from database')
result = m.select(m.router).distinct().where(
m.acl == acl, m.loaded >> None).order_by(m.router)
rows = result.tuples()
devs = [row[0] for row in rows]
if devs:
for dev in devs:
m.delete().where(m.acl == acl, m.router == dev,
m.loaded >> None).execute()
self.vprint('ACL %s cleared from integrated load queue for %s' %
(acl, ', '.join(dev[:dev.find('.')] for dev in devs)))
return True
else:
m = self.get_model('manual')
if m.delete().where(m.q_name == acl, m.done == False).execute():
self.vprint('%r cleared from manual load queue' % acl)
return True
self.vprint('%r not found in any queues' % acl)
return False
def complete(self, device, acls):
"""
Mark a device and its ACLs as complete using current timestamp.
(Integrated queue only.)
:param device:
Device names
:param acls:
List of ACL names
"""
m = self.get_model('integrated')
for acl in acls:
now = loaded=datetime.datetime.now()
m.update(loaded=now).where(m.acl == acl, m.router == device,
m.loaded >> None).execute()
self.vprint('Marked the following ACLs as complete for %s:' % device)
self.vprint(', '.join(acls))
def remove(self, acl, routers, escalation=False):
"""
Integrated queue only.
Mark an ACL and associated devices as "removed" (loaded=0). Intended
for use when performing manual actions on the load queue when
troubleshooting or addressing errors with automated loads. This leaves
the items in the database but removes them from the active queue.
:param acl:
ACL name
:param routers:
List of device names
:param escalation:
Whether this is an escalated task
"""
if not acl:
raise exceptions.ACLQueueError('You must specify an ACL to remove from the queue')
m = self.get_model('integrated')
loaded = 0
if settings.DATABASE_ENGINE == 'postgresql':
loaded = '-infinity' # See: http://bit.ly/15f0J3z
for router in routers:
m.update(loaded=loaded).where(m.acl == acl, m.router == router,
m.loaded >> None).execute()
self.vprint('Marked the following devices as removed for ACL %s: ' % acl)
self.vprint(', '.join(routers))
def list(self, queue='integrated', escalation=False, q_names=QUEUE_NAMES):
"""
List items in the specified queue, defauls to integrated queue.
:param queue:
Name of the queue to list
:param escalation:
Whether this is an escalated task
:param q_names:
(Optional) List of valid queue names
"""
if queue not in q_names:
self.vprint('Queue must be one of %s, not: %s' % (q_names, queue))
return False
m = self.get_model(queue)
if queue == 'integrated':
result = m.select(m.router, m.acl).distinct().where(
m.loaded >> None, m.escalation == escalation)
elif queue == 'manual':
result = m.select(m.q_name, m.login, m.q_ts, m.done).where(
m.done == False)
else:
raise RuntimeError('This should never happen!!')
all_data = list(result.tuples())
return all_data
class Commando(object):
"""
Execute commands asynchronously on multiple network devices.
This class is designed to be extended but can still be used as-is to execute
commands and return the results as-is.
At the bare minimum you must specify a list of ``devices`` to interact with.
You may optionally specify a list of ``commands`` to execute on those
devices, but doing so will execute the same commands on every device
regardless of platform.
If ``commands`` are not specified, they will be expected to be emitted by
the ``generate`` method for a given platform. Otherwise no commands will be
executed.
If you wish to customize the commands executed by device, you must define a
``to_{vendor_name}`` method containing your custom logic.
If you wish to customize what is done with command results returned from a
device, you must define a ``from_{vendor_name}`` method containing your
custom logic.
:param devices:
A list of device hostnames or `~trigger.netdevices.NetDevice` objects
:param commands:
(Optional) A list of commands to execute on the ``devices``.
:param incremental:
(Optional) A callback that will be called with an empty sequence upon
connection and then called every time a result comes back from the
device, with the list of all results.
:param max_conns:
(Optional) The maximum number of simultaneous connections to keep open.
:param verbose:
(Optional) Whether or not to display informational messages to the
console.
:param timeout:
(Optional) Time in seconds to wait for each command executed to return a
result
:param production_only:
(Optional) If set, includes all devices instead of excluding any devices
where ``adminStatus`` is not set to ``PRODUCTION``.
:param allow_fallback:
If set (default), allow fallback to base parse/generate methods when
they are not customized in a subclass, otherwise an exception is raised
when a method is called that has not been explicitly defined.
"""
# Defaults to all supported vendors
vendors = settings.SUPPORTED_VENDORS
# Defaults to all supported platforms
platforms = settings.SUPPORTED_PLATFORMS
# The commands to run
commands = []
# How results are stored (defaults to dict)
results = {}
def __init__(self, devices=None, commands=None, incremental=None,
max_conns=10, verbose=False, timeout=30,
production_only=True, allow_fallback=True):
if devices is None:
raise exceptions.ImproperlyConfigured('You must specify some ``devices`` to interact with!')
self.devices = devices
self.commands = self.commands or (commands or []) # Always fallback to []
self.incremental = incremental
self.max_conns = max_conns
self.verbose = verbose
self.timeout = timeout # in seconds
self.nd = NetDevices(production_only=production_only)
self.allow_fallback = allow_fallback
self.curr_conns = 0
self.jobs = []
self.errors = {}
self.results = self.results
self.deferrals = self._setup_jobs()
self.supported_platforms = self._validate_platforms()
def _validate_platforms(self):
"""
Determine the set of supported platforms for this instance by making
sure the specified vendors/platforms for the class match up.
"""
supported_platforms = {}
for vendor in self.vendors:
if vendor in self.platforms:
types = self.platforms[vendor]
if not types:
raise exceptions.MissingPlatform('No platforms specified for %r' % vendor)
else:
#self.supported_platforms[vendor] = types
supported_platforms[vendor] = types
else:
raise exceptions.ImproperlyConfigured('Platforms for vendor %r not found. Please provide it at either the class level or using the arguments.' % vendor)
return supported_platforms
def _decrement_connections(self, data=None):
"""
Self-explanatory. Called by _add_worker() as both callback/errback
so we can accurately refill the jobs queue, which relies on the
current connection count.
"""
self.curr_conns -= 1
return True
def _increment_connections(self, data=None):
"""Increment connection count."""
self.curr_conns += 1
return True
def _setup_jobs(self):
"""
"Maps device hostnames to `~trigger.netdevices.NetDevice` objects and
populates the job queue.
"""
for dev in self.devices:
if self.verbose:
print 'Adding', dev
# Make sure that devices are actually in netdevices and keep going
try:
devobj = self.nd.find(str(dev))
except KeyError:
msg = 'Device not found in NetDevices: %s' % dev
if self.verbose:
print 'ERROR:', msg
# Track the errors and keep moving
self.errors[dev] = msg
continue
# We only want to add devices for which we've enabled support in
# this class
if devobj.vendor not in self.vendors:
raise exceptions.UnsupportedVendor("The vendor '%s' is not specified in ``vendors``. Could not add %s to job queue. Please check the attribute in the class object." % (devobj.vendor, devobj))
self.jobs.append(devobj)
def select_next_device(self, jobs=None):
"""
Select another device for the active queue.
Currently only returns the next device in the job queue. This is
abstracted out so that this behavior may be customized, such as for
future support for incremental callbacks.
:param jobs:
(Optional) The jobs queue. If not set, uses ``self.jobs``.
:returns:
A `~trigger.netdevices.NetDevice` object
"""
if jobs is None:
jobs = self.jobs
return jobs.pop()
def _add_worker(self):
"""
Adds devices to the work queue to keep it populated with the maximum
connections as specified by ``max_conns``.
"""
while self.jobs and self.curr_conns < self.max_conns:
device = self.select_next_device()
self._increment_connections()
if self.verbose:
print 'connections:', self.curr_conns
print 'Adding work to queue...'
# Setup the async Deferred object with a timeout and error printing.
commands = self.generate(device)
async = device.execute(commands, incremental=self.incremental,
timeout=self.timeout, with_errors=True)
# Add the parser callback for great justice!
async.addCallback(self.parse, device)
# Here we addBoth to continue on after pass/fail
async.addBoth(self._decrement_connections)
async.addBoth(lambda x: self._add_worker())
# If worker add fails, still decrement and track the error
async.addErrback(self.errback, device)
# Do this once we've exhausted the job queue
else:
if not self.curr_conns and self.reactor_running:
self._stop()
elif not self.jobs and not self.reactor_running:
if self.verbose:
print 'No work left.'
def _lookup_method(self, device, method):
"""
Base lookup method. Looks up stuff by device manufacturer like:
from_juniper
to_foundry
:param device:
A `~trigger.netdevices.NetDevice` object
:param method:
One of 'generate', 'parse'
"""
METHOD_MAP = {
'generate': 'to_%s',
'parse': 'from_%s',
}
assert method in METHOD_MAP
desired_attr = None
for vendor, types in self.platforms.iteritems():
meth_attr = METHOD_MAP[method] % device.vendor
if device.deviceType in types:
if hasattr(self, meth_attr):
desired_attr = meth_attr
break
if desired_attr is None:
if self.allow_fallback:
desired_attr = METHOD_MAP[method] % 'base'
else:
raise exceptions.UnsupportedVendor("The vendor '%s' had no available %s method. Please check your ``vendors`` and ``platforms`` attributes in your class object." % (device.vendor, method))
func = getattr(self, desired_attr)
return func
def generate(self, device, commands=None, extra=None):
"""
Generate commands to be run on a device. If you don't provide
``commands`` to the class constructor, this will return an empty list.
Define a 'to_{vendor_name}' method to customize the behavior for each
platform.
:param device:
A `~trigger.netdevices.NetDevice` object
:param commands:
(Optional) A list of commands to execute on the device. If not
specified in they will be inherited from commands passed to the
class constructor.
:param extra:
(Optional) A dictionary of extra data to send to the generate method for the
device.
"""
if commands is None:
commands = self.commands
if extra is None:
extra = {}
func = self._lookup_method(device, method='generate')
return func(device, commands, extra)
def parse(self, results, device):
"""
Parse output from a device.
Define a 'from_{vendor_name}' method to customize the behavior for each
platform.
:param results:
The results of the commands executed on the device
:param device:
A `~trigger.netdevices.NetDevice` object
"""
func = self._lookup_method(device, method='parse')
return func(results, device)
def errback(self, failure, device):
"""
The default errback. Overload for custom behavior but make sure it
always decrements the connections.
:param failure:
Usually a Twisted ``Failure`` instance.
:param device:
A `~trigger.netdevices.NetDevice` object
"""
self.store_error(device, failure)
self._decrement_connections(failure)
return True
def store_error(self, device, error):
"""
A simple method for storing an error called by all default
parse/generate methods.
If you want to customize the default method for storing results,
overload this in your subclass.
:param device:
A `~trigger.netdevices.NetDevice` object
:param error:
The error to store. Anything you want really, but usually a Twisted
``Failure`` instance.
"""
self.errors[device.nodeName] = error
return True
def store_results(self, device, results):
"""
A simple method for storing results called by all default
parse/generate methods.
If you want to customize the default method for storing results,
overload this in your subclass.
:param device:
A `~trigger.netdevices.NetDevice` object
:param results:
The results to store. Anything you want really.
"""
log.msg("Storing results for %r: %r" % (device.nodeName, results))
self.results[device.nodeName] = results
return True
def map_results(self, commands=None, results=None):
"""Return a dict of ``{command: result, ...}``"""
if commands is None:
commands = self.commands
if results is None:
results = []
return dict(itertools.izip_longest(commands, results))
@property
def reactor_running(self):
"""Return whether reactor event loop is running or not"""
from twisted.internet import reactor
log.msg("Reactor running? %s" % reactor.running)
return reactor.running
def _stop(self):
"""Stop the reactor event loop"""
if self.verbose:
print 'stopping reactor'
from twisted.internet import reactor
reactor.stop()
def _start(self):
"""Start the reactor event loop"""
if self.verbose:
print 'starting reactor'
if self.curr_conns:
from twisted.internet import reactor
reactor.run()
else:
if self.verbose:
print "Won't start reactor with no work to do!"
def run(self):
"""
Nothing happens until you execute this to perform the actual work.
"""
self._add_worker()
self._start()
#=======================================
# Base generate (to_)/parse (from_) methods
#=======================================
def to_base(self, device, commands=None, extra=None):
commands = commands or self.commands
print 'Sending %r to %s' % (commands, device)
return commands
def from_base(self, results, device):
print 'Received %r from %s' % (results, device)
self.store_results(device, self.map_results(self.commands, results))
#=======================================
# Vendor-specific generate (to_)/parse (from_) methods
#=======================================
def to_juniper(self, device, commands=None, extra=None):
"""
This just creates a series of ``<command>foo</command>`` elements to
pass along to execute_junoscript()"""
commands = commands or self.commands
ret = []
for command in commands:
cmd = Element('command')
cmd.text = command
ret.append(cmd)
return ret
class Queue(object):
"""
Interacts with firewalls database to insert/remove items into the queue. You
may optionally suppress informational messages by passing ``verbose=False``
to the constructor.
:param verbose: Toggle verbosity
:type verbose: Boolean
"""
def __init__(self, verbose=True):
self.dbconn = self._get_firewall_db_conn()
self.cursor = self.dbconn.cursor()
self.nd = NetDevices()
self.verbose = verbose
def _get_firewall_db_conn(self):
"""Returns a MySQL db connection used for the ACL queues using database
settings found withing ``settings.py``."""
if MySQLdb is None:
raise RuntimeError("You must install ``MySQL-python`` to use the queue")
try:
return MySQLdb.connect(host=settings.DATABASE_HOST,
db=settings.DATABASE_NAME,
port=settings.DATABASE_PORT,
user=settings.DATABASE_USER,
passwd=settings.DATABASE_PASSWORD)
# catch if we can't connect, and shut down
except MySQLdb.OperationalError as e:
sys.exit("Can't connect to the database - %s (error %d)" % (e[1],e[0]))
def _normalize(self, arg):
if arg.startswith('acl.'):
arg = arg[4:]
escalation = False
if arg.upper().endswith(' ESCALATION'):
escalation = True
arg = arg[:-11]
return (escalation, arg)
def insert(self, acl, routers, escalation=False):
"""
Insert an ACL and associated devices into the ACL load queue.
Attempts to insert into integrated queue. If ACL test fails, then
item is inserted into manual queue.
"""
assert acl, 'no ACL defined'
if not routers:
routers = []
(escalation, acl) = self._normalize(acl)
if len(routers):
for router in routers:
try:
dev = self.nd.find(router)
except KeyError:
raise "Could not find %s in netdevices" % router
return
if acl not in dev.acls:
raise "Could not find %s in %s's acl list" % (acl, router)
return
self.cursor.execute('''insert into acl_queue
(acl, router, queued, escalation)
values (%s, %s, now(), %s)''',
(acl, router, escalation))
if self.verbose:
print 'ACL', acl, 'injected into integrated load queue for',
print ', '.join([dev[:dev.find('.')] for dev in routers])
else:
self.cursor.execute('''insert into queue (q_name, login) values (%s, %s)''',
(acl, os.getlogin()))
if self.verbose: print '"%s" injected into manual load queue' % acl
self.dbconn.commit()
def delete(self, acl, routers=None, escalation=False):
"""
Delete an ACL from the firewall database queue.
Attempts to delete from integrated queue. If ACL test fails, then
item is deleted from manual queue.
"""
assert acl, 'no ACL defined'
(escalation, acl) = self._normalize(acl)
if routers is not None:
devs = routers
else:
if self.verbose: print 'fetching routers from database'
self.cursor.execute('''select distinct router from acl_queue
where acl = %s and loaded is null
order by router''', (acl,))
rows = self.cursor.fetchall()
devs = [row[0] for row in rows] or []
if len(devs):
for dev in devs:
self.cursor.execute('''delete from acl_queue where acl=%s and
router=%s and loaded is null''', (acl, dev))
if self.verbose:
print 'ACL', acl, 'cleared from integrated load queue for',
print ', '.join([dev[:dev.find('.')] for dev in devs])
elif self.cursor.execute('''delete from queue
where q_name = %s and done = 0''', (acl,)):
if self.verbose: print '"%s" cleared from manual load queue' % acl
else:
if self.verbose: print '"%s" not found in manual or integrated queues' % acl
self.dbconn.commit()
def complete(self, device, acls):
"""
Integrated queue only.
Mark a device and associated ACLs as complete my updating loaded to
current timestampe. Migrated from clear_load_queue() in load_acl.
"""
for acl in acls:
self.cursor.execute("""update acl_queue set loaded=now() where
acl=%s and router=%s and loaded is null""",
(acl, device))
if self.verbose:
print 'Marked the following ACLs as complete for %s: ' % device
print ', '.join(acls)
def remove(self, acl, routers, escalation=False):
"""
Integrated queue only.
Mark an ACL and associated devices as "removed" (loaded=0). Intended
for use when performing manual actions on the load queue when
troubleshooting or addressing errors with automated loads. This leaves
the items in the database but removes them from the active queue.
"""
assert acl, 'no ACL defined'
for router in routers:
self.cursor.execute("""update acl_queue set loaded=0 where acl=%s
and router=%s and loaded is null""", (acl, router))
if self.verbose:
print 'Marked the following devices as removed for ACL %s: ' % acl
print ', '.join(routers)
def list(self, queue='integrated', escalation=False):
"""
List items in the queue, defauls to integrated queue.
Valid queue arguments are 'integrated' or 'manual'.
"""
if queue == 'integrated':
query = 'select distinct router, acl from acl_queue where loaded is null'
if escalation:
query += ' and escalation = true'
elif queue == 'manual':
query = 'select q_name, login, q_ts, done from queue where done=0'
else:
print 'Queue must be integrated or manual, you specified: %s' % queue
return False
self.cursor.execute(query)
all_data = self.cursor.fetchall()
self.dbconn.commit()
return all_data
class TestNetDevicesWithAcls(unittest.TestCase):
"""
Test NetDevices with ``settings.WITH_ACLs set`` to ``True``.
"""
def setUp(self):
self.nd = NetDevices()
self.device = self.nd[DEVICE_NAME]
self.device2 = self.nd[DEVICE2_NAME]
self.nodename = self.device.nodeName
self.device.explicit_acls = set(['test1-abc-only'])
def test_basics(self):
"""Basic test of NetDevices functionality."""
self.assertEqual(len(self.nd), 2)
self.assertEqual(self.device.nodeName, self.nodename)
self.assertEqual(self.device.manufacturer, 'JUNIPER')
def test_aclsdb(self):
"""Test acls.db handling."""
self.assertTrue('test1-abc-only' in self.device.explicit_acls)
def test_autoacls(self):
"""Test autoacls.py handling."""
self.assertTrue('router-protect.core' in self.device.implicit_acls)
def test_find(self):
"""Test the find() method."""
self.assertEqual(self.nd.find(self.nodename), self.device)
nodebasename = self.nodename[:self.nodename.index('.')]
self.assertEqual(self.nd.find(nodebasename), self.device)
self.assertRaises(KeyError, lambda: self.nd.find(self.nodename[0:3]))
def test_all(self):
"""Test the all() method."""
expected = [self.device, self.device2]
self.assertEqual(sorted(expected), sorted(self.nd.all()))
def test_search(self):
"""Test the search() method."""
expected = [self.device]
self.assertEqual([self.device], self.nd.search(self.nodename))
self.assertEqual(self.nd.all(), self.nd.search('17', field='onCallID'))
def test_match(self):
"""Test the match() method."""
self.assertEqual([self.device], self.nd.match(nodename=self.nodename))
self.assertEqual(self.nd.all(), self.nd.match(vendor='juniper'))
self.assertEqual([], self.nd.match(vendor='cisco'))
def test_multiple_filter_match(self):
"""Test that passing multiple kwargs filters properly."""
# There should be only one Juniper router.
self.assertEqual(self.nd.match(nodename='test1-abc'),
self.nd.match(vendor='juniper', devicetype='router'))
# And only one Juniper switch.
self.assertEqual(self.nd.match(nodename='test2-abc'),
self.nd.match(vendor='juniper', devicetype='switch'))
def test_match_with_null_value(self):
"""Test the match() method when attr value is ``None``."""
self.device.site = None # Zero it out!
expected = [self.device]
# None raw
self.assertEqual(expected, self.nd.match(site=None))
# "None" string
self.assertEqual(expected, self.nd.match(site='None'))
# Case-insensitive attr *and* value
self.assertEqual(expected, self.nd.match(SITE='NONE'))
def test_reload(self):
"""Test the .reload() method."""
nd = self.nd
nd.reload()
self.assertEqual(nd, self.nd)
def tearDown(self):
_reset_netdevices()
class Queue(object):
"""
Interacts with firewalls database to insert/remove items into the queue. You
may optionally suppress informational messages by passing ``verbose=False``
to the constructor.
:param verbose: Toggle verbosity
:type verbose: Boolean
"""
def __init__(self, verbose=True):
self.dbconn = self._get_firewall_db_conn()
self.cursor = self.dbconn.cursor()
self.nd = NetDevices()
self.verbose = verbose
def _get_firewall_db_conn(self):
"""Returns a MySQL db connection used for the ACL queues using database
settings found withing ``settings.py``."""
import MySQLdb
return MySQLdb.connect(host=settings.DATABASE_HOST,
db=settings.DATABASE_NAME,
port=settings.DATABASE_PORT,
user=settings.DATABASE_USER,
passwd=settings.DATABASE_PASSWORD)
def _normalize(self, arg):
if arg.startswith('acl.'):
arg = arg[4:]
escalation = False
if arg.upper().endswith(' ESCALATION'):
escalation = True
arg = arg[:-11]
return (escalation, arg)
def insert(self, acl, routers, escalation=False):
"""
Insert an ACL and associated devices into the ACL load queue.
Attempts to insert into integrated queue. If ACL test fails, then
item is inserted into manual queue.
"""
assert acl, 'no ACL defined'
if not routers:
routers = []
(escalation, acl) = self._normalize(acl)
if len(routers):
for router in routers:
try:
dev = self.nd.find(router)
except KeyError:
raise "Could not find %s in netdevices" % router
return
if acl not in dev.acls:
raise "Could not find %s in %s's acl list" % (acl, router)
return
self.cursor.execute(
'''insert into acl_queue
(acl, router, queued, escalation)
values (%s, %s, now(), %s)''',
(acl, router, escalation))
if self.verbose:
print 'ACL', acl, 'injected into integrated load queue for',
print ', '.join([dev[:dev.find('.')] for dev in routers])
else:
self.cursor.execute(
'''insert into queue (q_name, login) values (%s, %s)''',
(acl, os.getlogin()))
if self.verbose: print '"%s" injected into manual load queue' % acl
self.dbconn.commit()
def delete(self, acl, routers=None, escalation=False):
"""
Delete an ACL from the firewall database queue.
Attempts to delete from integrated queue. If ACL test fails, then
item is deleted from manual queue.
"""
assert acl, 'no ACL defined'
(escalation, acl) = self._normalize(acl)
if routers is not None:
devs = routers
else:
if self.verbose: print 'fetching routers from database'
self.cursor.execute(
'''select distinct router from acl_queue
where acl = %s and loaded is null
order by router''', (acl, ))
rows = self.cursor.fetchall()
devs = [row[0] for row in rows] or []
if len(devs):
for dev in devs:
self.cursor.execute(
'''delete from acl_queue where acl=%s and
router=%s and loaded is null''',
(acl, dev))
if self.verbose:
print 'ACL', acl, 'cleared from integrated load queue for',
print ', '.join([dev[:dev.find('.')] for dev in devs])
elif self.cursor.execute(
'''delete from queue
where q_name = %s and done = 0''', (acl, )):
if self.verbose: print '"%s" cleared from manual load queue' % acl
else:
if self.verbose:
print '"%s" not found in manual or integrated queues' % acl
self.dbconn.commit()
def complete(self, device, acls):
"""
Integrated queue only.
Mark a device and associated ACLs as complete my updating loaded to
current timestampe. Migrated from clear_load_queue() in load_acl.
"""
for acl in acls:
self.cursor.execute(
"""update acl_queue set loaded=now() where
acl=%s and router=%s and loaded is null""",
(acl, device))
if self.verbose:
print 'Marked the following ACLs as complete for %s: ' % device
print ', '.join(acls)
def remove(self, acl, routers, escalation=False):
"""
Integrated queue only.
Mark an ACL and associated devices as "removed" (loaded=0). Intended
for use when performing manual actions on the load queue when
troubleshooting or addressing errors with automated loads. This leaves
the items in the database but removes them from the active queue.
"""
assert acl, 'no ACL defined'
for router in routers:
self.cursor.execute(
"""update acl_queue set loaded=0 where acl=%s
and router=%s and loaded is null""",
(acl, router))
if self.verbose:
print 'Marked the following devices as removed for ACL %s: ' % acl
print ', '.join(routers)
def list(self, queue='integrated', escalation=False):
"""
List items in the queue, defauls to integrated queue.
Valid queue arguments are 'integrated' or 'manual'.
"""
if queue == 'integrated':
query = 'select distinct router, acl from acl_queue where loaded is null'
if escalation:
query += ' and escalation = true'
elif queue == 'manual':
query = 'select q_name, login, q_ts, done from queue where done=0'
else:
print 'Queue must be integrated or manual, you specified: %s' % queue
return False
self.cursor.execute(query)
all_data = self.cursor.fetchall()
self.dbconn.commit()
return all_data
class TestAclQueue(unittest.TestCase):
def setUp(self):
self.nd = NetDevices()
_setup_aclsdb(self.nd)
self.q = queue.Queue(verbose=False)
self.acl = ACL_NAME
self.acl_list = [self.acl]
self.device = self.nd.find(DEVICE_NAME)
self.device_name = DEVICE_NAME
self.device_list = [self.device_name]
self.user = USERNAME
#
# Integrated queue tests
#
def test_01_insert_integrated_success(self):
"""Test insert success into integrated queue"""
self.assertTrue(self.q.insert(self.acl, self.device_list) is None)
def test_02_insert_integrated_failure_device(self):
"""Test insert invalid device"""
self.assertRaises(exceptions.TriggerError, self.q.insert, self.acl, ['bogus'])
def test_03_insert_integrated_failure_acl(self):
"""Test insert devices w/ no ACL association"""
self.assertRaises(exceptions.TriggerError, self.q.insert, 'bogus',
self.device_list)
def test_04_list_integrated_success(self):
"""Test listing integrated queue"""
self.q.insert(self.acl, self.device_list)
expected = [(u'test1-abc.net.aol.com', u'foo')]
self.assertEqual(sorted(expected), sorted(self.q.list()))
def test_05_complete_integrated(self):
"""Test mark task complete"""
self.q.complete(self.device_name, self.acl_list)
expected = []
self.assertEqual(sorted(expected), sorted(self.q.list()))
def test_06_delete_integrated_with_devices(self):
"""Test delete ACL from queue providing devices"""
self.q.insert(self.acl, self.device_list)
self.assertTrue(self.q.delete(self.acl, self.device_list))
def test_07_delete_integrated_no_devices(self):
"""Test delete ACL from queue without providing devices"""
self.q.insert(self.acl, self.device_list)
self.assertTrue(self.q.delete(self.acl))
def test_08_remove_integrated_success(self):
"""Test remove (set as loaded) ACL from integrated queue"""
self.q.insert(self.acl, self.device_list)
self.q.remove(self.acl, self.device_list)
expected = []
self.assertEqual(sorted(expected), sorted(self.q.list()))
def test_10_remove_integrated_failure(self):
"""Test remove (set as loaded) failure"""
self.assertRaises(exceptions.ACLQueueError, self.q.remove, '', self.device_list)
#
# Manual queue tests
#
def test_11_insert_manual_success(self):
"""Test insert success into manual queue"""
self.assertTrue(self.q.insert('manual task', None) is None)
def test_12_list_manual_success(self):
"""Test list success of manual queue"""
self.q.insert('manual task', None)
expected = ('manual task', self.user)
result = self.q.list('manual')
actual = result[0][:2] # First tuple, items 0-1
self.assertEqual(sorted(expected), sorted(actual))
def test_13_delete_manual_success(self):
"""Test delete from manual queue"""
self.q.delete('manual task')
expected = []
self.assertEqual(sorted(expected), sorted(self.q.list('manual')))
#
# Generic tests
#
def test_14_delete_failure(self):
"""Test delete of task not in queue"""
self.assertFalse(self.q.delete('bogus'))
def test_15_list_invalid(self):
"""Test list of invalid queue name"""
self.assertFalse(self.q.list('bogus'))
# Teardown
def test_ZZ_cleanup_db(self):
"""Cleanup the temp database file"""
self.assertTrue(os.remove(db_file) is None)
def tearDown(self):
NetDevices._Singleton = None
import sys
from time import sleep
from twisted.internet.defer import Deferred
from zope.interface import implements
from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.web.http_headers import Headers
from twisted.internet.defer import succeed
from twisted.web.iweb import IBodyProducer
# from twisted.python import log
# log.startLogging(sys.stdout, setStdout=False)
from trigger.netdevices import NetDevices
# Create reference to upgraded switch.
nd = NetDevices()
dev = nd.find('arista-sw1.demo.local')
# Create payload body
class StringProducer(object):
implements(IBodyProducer)
def __init__(self, body):
self.body = body
self.length = len(body)
def startProducing(self, consumer):
consumer.write(self.body)
return succeed(None)
def pauseProducing(self):
pass
class Commando(object):
"""
Execute commands asynchronously on multiple network devices.
This class is designed to be extended but can still be used as-is to execute
commands and return the results as-is.
At the bare minimum you must specify a list of ``devices`` to interact with.
You may optionally specify a list of ``commands`` to execute on those
devices, but doing so will execute the same commands on every device
regardless of platform.
If ``commands`` are not specified, they will be expected to be emitted by
the ``generate`` method for a given platform. Otherwise no commands will be
executed.
If you wish to customize the commands executed by device, you must define a
``to_{vendor_name}`` method containing your custom logic.
If you wish to customize what is done with command results returned from a
device, you must define a ``from_{vendor_name}`` method containing your
custom logic.
:param devices:
A list of device hostnames or `~trigger.netdevices.NetDevice` objects
:param commands:
(Optional) A list of commands to execute on the ``devices``.
:param incremental:
(Optional) A callback that will be called with an empty sequence upon
connection and then called every time a result comes back from the
device, with the list of all results.
:param max_conns:
(Optional) The maximum number of simultaneous connections to keep open.
:param verbose:
(Optional) Whether or not to display informational messages to the
console.
:param timeout:
(Optional) Time in seconds to wait for each command executed to return a
result
:param production_only:
(Optional) If set, includes all devices instead of excluding any devices
where ``adminStatus`` is not set to ``PRODUCTION``.
:param allow_fallback:
If set (default), allow fallback to base parse/generate methods when
they are not customized in a subclass, otherwise an exception is raised
when a method is called that has not been explicitly defined.
"""
# Defaults to all supported vendors
vendors = settings.SUPPORTED_VENDORS
# Defaults to all supported platforms
platforms = settings.SUPPORTED_PLATFORMS
# The commands to run
commands = []
def __init__(self, devices=None, commands=None, incremental=None,
max_conns=10, verbose=False, timeout=30,
production_only=True, allow_fallback=True):
if devices is None:
raise exceptions.ImproperlyConfigured('You must specify some ``devices`` to interact with!')
self.devices = devices
self.commands = self.commands or (commands or []) # Always fallback to []
self.incremental = incremental
self.max_conns = max_conns
self.verbose = verbose
self.timeout = timeout # in seconds
self.nd = NetDevices(production_only=production_only)
self.allow_fallback = allow_fallback
self.curr_conns = 0
self.jobs = []
self.errors = {}
self.results = {}
self.deferrals = self._setup_jobs()
self.supported_platforms = self._validate_platforms()
def _validate_platforms(self):
"""
Determine the set of supported platforms for this instance by making
sure the specified vendors/platforms for the class match up.
"""
supported_platforms = {}
for vendor in self.vendors:
if vendor in self.platforms:
types = self.platforms[vendor]
if not types:
raise exceptions.MissingPlatform('No platforms specified for %r' % vendor)
else:
#self.supported_platforms[vendor] = types
supported_platforms[vendor] = types
else:
raise exceptions.ImproperlyConfigured('Platforms for vendor %r not found. Please provide it at either the class level or using the arguments.' % vendor)
return supported_platforms
def _decrement_connections(self, data=None):
"""
Self-explanatory. Called by _add_worker() as both callback/errback
so we can accurately refill the jobs queue, which relies on the
current connection count.
"""
self.curr_conns -= 1
return True
def _increment_connections(self, data=None):
"""Increment connection count."""
self.curr_conns += 1
return True
def _setup_jobs(self):
"""
"Maps device hostnames to `~trigger.netdevices.NetDevice` objects and
populates the job queue.
"""
for dev in self.devices:
if self.verbose:
print 'Adding', dev
# Make sure that devices are actually in netdevices and keep going
try:
devobj = self.nd.find(str(dev))
except KeyError:
if self.verbose:
msg = 'Device not found in NetDevices: %s' % dev
print 'ERROR:', msg
# Track the errors and keep moving
self.errors[dev] = msg
continue
# We only want to add devices for which we've enabled support in
# this class
if devobj.vendor not in self.vendors:
raise exceptions.UnsupportedVendor("The vendor '%s' is not specified in ``vendors``. Could not add %s to job queue. Please check the attribute in the class object." % (devobj.vendor, devobj))
self.jobs.append(devobj)
def select_next_device(self, jobs=None):
"""
Select another device for the active queue.
Currently only returns the next device in the job queue. This is
abstracted out so that this behavior may be customized, such as for
future support for incremental callbacks.
:param jobs:
(Optional) The jobs queue. If not set, uses ``self.jobs``.
:returns:
A `~trigger.netdevices.NetDevice` object
"""
if jobs is None:
jobs = self.jobs
return jobs.pop()
def _add_worker(self):
"""
Adds devices to the work queue to keep it populated with the maximum
connections as specified by ``max_conns``.
"""
while self.jobs and self.curr_conns < self.max_conns:
device = self.select_next_device()
self._increment_connections()
if self.verbose:
print 'connections:', self.curr_conns
print 'Adding work to queue...'
# Setup the async Deferred object with a timeout and error printing.
commands = self.generate(device)
async = device.execute(commands, incremental=self.incremental,
timeout=self.timeout, with_errors=True)
# Add the parser callback for great justice!
async.addCallback(self.parse, device)
# Here we addBoth to continue on after pass/fail
async.addBoth(self._decrement_connections)
async.addBoth(lambda x: self._add_worker())
# If worker add fails, still decrement and track the error
async.addErrback(self.errback, device)
# Do this once we've exhausted the job queue
else:
if not self.curr_conns and self.reactor_running:
self._stop()
elif not self.jobs and not self.reactor_running:
if self.verbose:
print 'No work left.'
def _lookup_method(self, device, method):
"""
Base lookup method. Looks up stuff by device manufacturer like:
from_juniper
to_foundry
:param device:
A `~trigger.netdevices.NetDevice` object
:param method:
One of 'generate', 'parse'
"""
METHOD_MAP = {
'generate': 'to_%s',
'parse': 'from_%s',
}
assert method in METHOD_MAP
desired_attr = None
for vendor, types in self.platforms.iteritems():
meth_attr = METHOD_MAP[method] % device.vendor
if device.deviceType in types:
if hasattr(self, meth_attr):
desired_attr = meth_attr
break
if desired_attr is None:
if self.allow_fallback:
desired_attr = METHOD_MAP[method] % 'base'
else:
raise exceptions.UnsupportedVendor("The vendor '%s' had no available %s method. Please check your ``vendors`` and ``platforms`` attributes in your class object." % (device.vendor, method))
func = getattr(self, desired_attr)
return func
def generate(self, device, commands=None, extra=None):
"""
Generate commands to be run on a device. If you don't provide
``commands`` to the class constructor, this will return an empty list.
Define a 'to_{vendor_name}' method to customize the behavior for each
platform.
:param device:
A `~trigger.netdevices.NetDevice` object
:param commands:
(Optional) A list of commands to execute on the device. If not
specified in they will be inherited from commands passed to the
class constructor.
:param extra:
(Optional) A dictionary of extra data to send to the generate method for the
device.
"""
if commands is None:
commands = self.commands
if extra is None:
extra = {}
func = self._lookup_method(device, method='generate')
return func(device, commands, extra)
def parse(self, results, device):
"""
Parse output from a device.
Define a 'from_{vendor_name}' method to customize the behavior for each
platform.
:param results:
The results of the commands executed on the device
:param device:
A `~trigger.netdevices.NetDevice` object
"""
func = self._lookup_method(device, method='parse')
return func(results, device)
def errback(self, failure, device):
"""
The default errback. Overload for custom behavior but make sure it
always decrements the connections.
:param failure:
Usually a Twisted ``Failure`` instance.
:param device:
A `~trigger.netdevices.NetDevice` object
"""
self.store_error(device, failure)
self._decrement_connections(failure)
return True
def store_error(self, device, error):
"""
A simple method for storing an error called by all default
parse/generate methods.
If you want to customize the default method for storing results,
overload this in your subclass.
:param device:
A `~trigger.netdevices.NetDevice` object
:param error:
The error to store. Anything you want really, but usually a Twisted
``Failure`` instance.
"""
self.errors[device.nodeName] = error
return True
def store_results(self, device, results):
"""
A simple method for storing results called by all default
parse/generate methods.
If you want to customize the default method for storing results,
overload this in your subclass.
:param device:
A `~trigger.netdevices.NetDevice` object
:param results:
The results to store. Anything you want really.
"""
self.results[device.nodeName] = results
return True
def map_results(self, commands=None, results=None):
"""Return a dict of ``{command: result, ...}``"""
if commands is None:
commands = self.commands
if results is None:
results = []
return dict(itertools.izip_longest(commands, results))
@property
def reactor_running(self):
"""Return whether reactor event loop is running or not"""
from twisted.internet import reactor
return reactor.running
def _stop(self):
"""Stop the reactor event loop"""
if self.verbose:
print 'stopping reactor'
from twisted.internet import reactor
reactor.stop()
def _start(self):
"""Start the reactor event loop"""
if self.verbose:
print 'starting reactor'
if self.curr_conns:
from twisted.internet import reactor
reactor.run()
else:
if self.verbose:
print "Won't start reactor with no work to do!"
def run(self):
"""
Nothing happens until you execute this to perform the actual work.
"""
self._add_worker()
self._start()
#=======================================
# Base generate (to_)/parse (from_) methods
#=======================================
def to_base(self, device, commands=None, extra=None):
commands = commands or self.commands
print 'Sending %r to %s' % (commands, device)
return commands
def from_base(self, results, device):
print 'Received %r from %s' % (results, device)
self.store_results(device, self.map_results(self.commands, results))
#=======================================
# Vendor-specific generate (to_)/parse (from_) methods
#=======================================
def to_juniper(self, device, commands=None, extra=None):
"""
This just creates a series of ``<command>foo</command>`` elements to
pass along to execute_junoscript()"""
commands = commands or self.commands
ret = []
for command in commands:
cmd = Element('command')
cmd.text = command
ret.append(cmd)
return ret
class TestNetDevicesWithAcls(unittest.TestCase):
"""
Test NetDevices with ``settings.WITH_ACLs set`` to ``True``.
"""
def setUp(self):
self.nd = NetDevices()
self.device = self.nd[DEVICE_NAME]
self.device2 = self.nd[DEVICE2_NAME]
self.nodename = self.device.nodeName
self.device.explicit_acls = set(['test1-abc-only'])
def test_basics(self):
"""Basic test of NetDevices functionality."""
self.assertEqual(len(self.nd), 2)
self.assertEqual(self.device.nodeName, self.nodename)
self.assertEqual(self.device.manufacturer, 'JUNIPER')
def test_aclsdb(self):
"""Test acls.db handling."""
self.assertTrue('test1-abc-only' in self.device.explicit_acls)
def test_autoacls(self):
"""Test autoacls.py handling."""
self.assertTrue('router-protect.core' in self.device.implicit_acls)
def test_find(self):
"""Test the find() method."""
self.assertEqual(self.nd.find(self.nodename), self.device)
nodebasename = self.nodename[:self.nodename.index('.')]
self.assertEqual(self.nd.find(nodebasename), self.device)
self.assertRaises(KeyError, lambda: self.nd.find(self.nodename[0:3]))
def test_all(self):
"""Test the all() method."""
expected = [self.device, self.device2]
self.assertEqual(sorted(expected), sorted(self.nd.all()))
def test_search(self):
"""Test the search() method."""
expected = [self.device]
self.assertEqual([self.device], self.nd.search(self.nodename))
self.assertEqual(self.nd.all(), self.nd.search('17', field='onCallID'))
def test_match(self):
"""Test the match() method."""
self.assertEqual([self.device], self.nd.match(nodename=self.nodename))
self.assertEqual(self.nd.all(), self.nd.match(vendor='juniper'))
self.assertEqual([], self.nd.match(vendor='cisco'))
def test_multiple_filter_match(self):
"""Test that passing multiple kwargs filters properly."""
# There should be only one Juniper router.
self.assertEqual(
self.nd.match(nodename='test1-abc'),
self.nd.match(vendor='juniper', devicetype='router')
)
# And only one Juniper switch.
self.assertEqual(
self.nd.match(nodename='test2-abc'),
self.nd.match(vendor='juniper', devicetype='switch')
)
def test_match_with_null_value(self):
"""Test the match() method when attr value is ``None``."""
self.device.site = None # Zero it out!
expected = [self.device]
# None raw
self.assertEqual(expected, self.nd.match(site=None))
# "None" string
self.assertEqual(expected, self.nd.match(site='None'))
# Case-insensitive attr *and* value
self.assertEqual(expected, self.nd.match(SITE='NONE'))
def tearDown(self):
_reset_netdevices()