示例#1
0
class NetDevicesTest(unittest.TestCase):

    def setUp(self):
        self.nd = NetDevices()
        self.nodename = self.nd.keys()[0]
        self.nodeobj = self.nd.values()[0]

    def testBasics(self):
        """Basic test of NetDevices functionality."""
        self.assertEqual(len(self.nd), 1)
        self.assertEqual(self.nodeobj.nodeName, self.nodename)
        self.assertEqual(self.nodeobj.manufacturer, 'JUNIPER')

    def testAclsdb(self):
        """Test acls.db handling."""
        self.assert_('181j' in self.nodeobj.acls)

    def testAutoacls(self):
        """Test autoacls.py handling."""
        self.assert_('115j' in self.nodeobj.acls)

    def testFind(self):
        """Test the find() method."""
        self.assertEqual(self.nd.find(self.nodename), self.nodeobj)
        nodebasename = self.nodename[:self.nodename.index('.')]
        self.assertEqual(self.nd.find(nodebasename), self.nodeobj)
        self.assertRaises(KeyError, lambda: self.nd.find(self.nodename[0:3]))
示例#2
0
文件: cmds.py 项目: simudream/trigger
    def __init__(self, devices=None, commands=None, creds=None,
                 incremental=None, max_conns=10, verbose=False,
                 timeout=DEFAULT_TIMEOUT, production_only=True,
                 allow_fallback=True, with_errors=True, force_cli=False,
                 with_acls=False, command_interval=0):
        if devices is None:
            raise exceptions.ImproperlyConfigured('You must specify some `devices` to interact with!')

        self.devices = devices
        self.commands = self.commands or (commands or []) # Always fallback to []
        self.creds = creds
        self.incremental = incremental
        self.max_conns = max_conns
        self.verbose = verbose
        self.timeout = timeout if timeout != self.timeout else self.timeout
        self.nd = NetDevices(production_only=production_only, with_acls=with_acls)
        self.allow_fallback = allow_fallback
        self.with_errors = with_errors
        self.force_cli = force_cli
        self.command_interval = command_interval
        self.curr_conns = 0
        self.jobs = []

        # Always fallback to {} for these
        self.errors = self.errors if self.errors is not None else {}
        self.results = self.results if self.results is not None else {}

        #self.deferrals = []
        self.supported_platforms = self._validate_platforms()
        self._setup_jobs()
示例#3
0
class CheckNetDevices(unittest.TestCase):
    def setUp(self):
        self.router = NetDevices()['test1-abc.net.aol.com']
        self.when = datetime(2006, 7, 24, 20, tzinfo=UTC)

    def testNetDevicesBounce(self):
        """Test integration of bounce windows with NetDevices."""
        self.assertEquals(self.router.bounce.status(self.when), 'red')

    def testAllowability(self):
        """Test allowability checks."""
        self.failIf(self.router.allowable('load-acl', self.when))
        morning = datetime(2006, 7, 25, 9, tzinfo=UTC)        # 5 am EDT
        self.assert_(self.router.allowable('load-acl', morning))
        self.assertEquals(self.router.next_ok('load-acl', self.when), morning)
        self.assertEquals(self.router.next_ok('load-acl', morning), morning)
示例#4
0
文件: tools.py 项目: ArnesSI/trigger
def get_bulk_acls():
    """
    Returns a dict of acls with an applied count over settings.AUTOLOAD_BULK_THRESH
    """
    from trigger.netdevices import NetDevices
    nd = NetDevices()
    all_acls = defaultdict(int)
    for dev in nd.all():
        for acl in dev.acls:
            all_acls[acl] += 1

    bulk_acls = {}
    for acl, count in all_acls.items():
        if count >= settings.AUTOLOAD_BULK_THRESH and acl != '':
            bulk_acls[acl] = count

    return bulk_acls
示例#5
0
 def setUp(self):
     self.nd = NetDevices()
     _setup_aclsdb(self.nd)
     self.q = queue.Queue(verbose=False)
     self.acl = ACL_NAME
     self.acl_list = [self.acl]
     self.device = self.nd.find(DEVICE_NAME)
     self.device_name = DEVICE_NAME
     self.device_list = [self.device_name]
     self.user = USERNAME
示例#6
0
class TestNetDevicesWithoutAcls(unittest.TestCase):
    """
    Test NetDevices with ``settings.WITH_ACLs`` set to ``False``.
    """
    def setUp(self):
        self.nd = NetDevices(with_acls=False)
        self.nodename = self.nd.keys()[0]
        self.device = self.nd.values()[0]

    def test_aclsdb(self):
        """Test acls.db handling."""
        self.assertFalse('test1-abc-only' in self.device.explicit_acls)

    def test_autoacls(self):
        """Test autoacls.py handling."""
        expected = set()
        self.assertEqual(expected, self.device.implicit_acls)

    def tearDown(self):
        _reset_netdevices()
示例#7
0
文件: cmds.py 项目: bwheatley/trigger
 def __init__(self, devices=None, max_conns=10, verbose=False, timeout=30,
              production_only=True):
     self.curr_connections = 0
     self.reactor_running  = False
     self.devices = devices or []
     self.verbose = verbose
     self.max_conns = max_conns
     self.nd = NetDevices(production_only=production_only)
     self.jobs = []
     self.errors = {}
     self.data = {}
     self.deferrals = self._setup_jobs()
     self.timeout = timeout # in seconds
示例#8
0
class TestAclsDB(unittest.TestCase):
    def setUp(self):
        self.nd = NetDevices()
        self.acl = ACL_NAME
        self.device = self.nd.find(DEVICE_NAME)
        self.implicit_acls = set(['115j', 'router-protect.core'])

    def test_01_add_acl_success(self):
        """Test associate ACL to device success"""
        exp = 'added acl %s to %s' % (self.acl, self.device)
        self.assertEqual(exp, adb.add_acl(self.device, self.acl))

    def test_02_add_acl_failure(self):
        """Test associate ACL to device failure"""
        exp = exceptions.ACLSetError
        self.assertRaises(exp, adb.add_acl, self.device, self.acl)

    def test_03_remove_acl_success(self):
        """Test remove ACL from device success"""
        exp = 'removed acl %s from %s' % (self.acl, self.device)
        self.assertEqual(exp, adb.remove_acl(self.device, self.acl))

    def test_04_remove_acl_failure(self):
        """Test remove ACL from device failure"""
        exp = exceptions.ACLSetError
        self.assertRaises(exp, adb.remove_acl, self.device, self.acl)

    def test_05_get_acl_dict(self):
        """Test get dict of associations"""
        exp = {'all': self.implicit_acls, 'explicit': set(),
               'implicit': self.implicit_acls}
        self.assertEqual(exp, adb.get_acl_dict(self.device))

    def test_06_get_acl_set_success(self):
        """Test get set of associations success"""
        exp = self.implicit_acls
        self.assertEqual(exp, adb.get_acl_set(self.device))

    def test_07_get_acl_set_failure(self):
        """Test get set of associations failure"""
        exp = exceptions.InvalidACLSet
        acl_set = 'bogus'
        self.assertRaises(exp, adb.get_acl_set, self.device, acl_set)

    def tearDown(self):
        NetDevices._Singleton = None
示例#9
0
    def __init__(self, devices=None, commands=None, incremental=None,
                 max_conns=10, verbose=False, timeout=30,
                 production_only=True, allow_fallback=True):
        if devices is None:
            raise exceptions.ImproperlyConfigured('You must specify some ``devices`` to interact with!')

        self.devices = devices
        self.commands = self.commands or (commands or []) # Always fallback to []
        self.incremental = incremental
        self.max_conns = max_conns
        self.verbose = verbose
        self.timeout = timeout # in seconds
        self.nd = NetDevices(production_only=production_only)
        self.allow_fallback = allow_fallback
        self.curr_conns = 0
        self.jobs = []
        self.errors = {}
        self.results = {}
        self.deferrals = self._setup_jobs()
        self.supported_platforms = self._validate_platforms()
class NetDevicesTest(unittest.TestCase):
    def setUp(self):
        self.nd = NetDevices(with_acls=False)
        print self.nd.values()
        self.nodename = self.nd.keys()[0]
        self.nodeobj = self.nd.values()[0]

    def testBasics(self):
        """Basic test of NetDevices functionality."""
        self.assertEqual(len(self.nd), 3)
        self.assertEqual(self.nodeobj.nodeName, self.nodename)
        self.assertEqual(self.nodeobj.manufacturer, "JUNIPER")

    def testFind(self):
        """Test the find() method."""
        self.assertEqual(self.nd.find(self.nodename), self.nodeobj)
        nodebasename = self.nodename[: self.nodename.index(".")]
        self.assertEqual(self.nd.find(nodebasename), self.nodeobj)
        self.assertRaises(KeyError, lambda: self.nd.find(self.nodename[0:3]))
示例#11
0
文件: cmds.py 项目: simudream/trigger
class Commando(object):
    """
    Execute commands asynchronously on multiple network devices.

    This class is designed to be extended but can still be used as-is to execute
    commands and return the results as-is.

    At the bare minimum you must specify a list of ``devices`` to interact with.
    You may optionally specify a list of ``commands`` to execute on those
    devices, but doing so will execute the same commands on every device
    regardless of platform.

    If ``commands`` are not specified, they will be expected to be emitted by
    the ``generate`` method for a given platform. Otherwise no commands will be
    executed.

    If you wish to customize the commands executed by device, you must define a
    ``to_{vendor_name}`` method containing your custom logic.

    If you wish to customize what is done with command results returned from a
    device, you must define a ``from_{vendor_name}`` method containing your
    custom logic.

    :param devices:
        A list of device hostnames or `~trigger.netdevices.NetDevice` objects

    :param commands:
        (Optional) A list of commands to execute on the ``devices``.

    :param creds:
        (Optional) A 3-tuple of (username, password, realm). If only (username,
        password) are provided, realm will be populated from
        :setting:`DEFAULT_REALM`. If unset it will fetch from ``.tacacsrc``.

    :param incremental:
        (Optional) A callback that will be called with an empty sequence upon
        connection and then called every time a result comes back from the
        device, with the list of all results.

    :param max_conns:
        (Optional) The maximum number of simultaneous connections to keep open.

    :param verbose:
        (Optional) Whether or not to display informational messages to the
        console.

    :param timeout:
        (Optional) Time in seconds to wait for each command executed to return a
        result. Set to ``None`` to disable timeout (not recommended).

    :param production_only:
        (Optional) If set, includes all devices instead of excluding any devices
        where ``adminStatus`` is not set to ``PRODUCTION``.

    :param allow_fallback:
        If set (default), allow fallback to base parse/generate methods when
        they are not customized in a subclass, otherwise an exception is raised
        when a method is called that has not been explicitly defined.

    :param with_errors:
        (Optional) Return exceptions as results instead of raising them. The
        default is to always return them.

    :param force_cli:
        (Optional) Juniper only. If set, sends commands using CLI instead of
        Junoscript.

    :param with_acls:
         Whether to load ACL associations (requires Redis). Defaults to whatever
         is specified in settings.WITH_ACLS

    :param command_interval:
         (Optional) Amount of time in seconds to wait between sending commands.
    """
    # Defaults to all supported vendors
    vendors = settings.SUPPORTED_VENDORS

    # Defaults to all supported platforms
    platforms = settings.SUPPORTED_PLATFORMS

    # The commands to run (defaults to [])
    commands = None

    # The timeout for commands to return results. We are setting this to 0
    # so that if it's not overloaded in a subclass, the timeout value passed to
    # the constructor will be preferred, especially if it is set to ``None``
    # which Twisted uses to disable timeouts completely.
    timeout = 0

    # How results are stored (defaults to {})
    results = None

    # How errors are stored (defaults to {})
    errors = None

    def __init__(self, devices=None, commands=None, creds=None,
                 incremental=None, max_conns=10, verbose=False,
                 timeout=DEFAULT_TIMEOUT, production_only=True,
                 allow_fallback=True, with_errors=True, force_cli=False,
                 with_acls=False, command_interval=0):
        if devices is None:
            raise exceptions.ImproperlyConfigured('You must specify some `devices` to interact with!')

        self.devices = devices
        self.commands = self.commands or (commands or []) # Always fallback to []
        self.creds = creds
        self.incremental = incremental
        self.max_conns = max_conns
        self.verbose = verbose
        self.timeout = timeout if timeout != self.timeout else self.timeout
        self.nd = NetDevices(production_only=production_only, with_acls=with_acls)
        self.allow_fallback = allow_fallback
        self.with_errors = with_errors
        self.force_cli = force_cli
        self.command_interval = command_interval
        self.curr_conns = 0
        self.jobs = []

        # Always fallback to {} for these
        self.errors = self.errors if self.errors is not None else {}
        self.results = self.results if self.results is not None else {}

        #self.deferrals = []
        self.supported_platforms = self._validate_platforms()
        self._setup_jobs()

    def _validate_platforms(self):
        """
        Determine the set of supported platforms for this instance by making
        sure the specified vendors/platforms for the class match up.
        """
        supported_platforms = {}
        for vendor in self.vendors:
            if vendor in self.platforms:
                types = self.platforms[vendor]
                if not types:
                    raise exceptions.MissingPlatform('No platforms specified for %r' % vendor)
                else:
                    #self.supported_platforms[vendor] = types
                    supported_platforms[vendor] = types
            else:
                raise exceptions.ImproperlyConfigured('Platforms for vendor %r not found. Please provide it at either the class level or using the arguments.' % vendor)

        return supported_platforms

    def _decrement_connections(self, data=None):
        """
        Self-explanatory. Called by _add_worker() as both callback/errback
        so we can accurately refill the jobs queue, which relies on the
        current connection count.
        """
        self.curr_conns -= 1
        return data

    def _increment_connections(self, data=None):
        """Increment connection count."""
        self.curr_conns += 1
        return True

    def _setup_jobs(self):
        """
        "Maps device hostnames to `~trigger.netdevices.NetDevice` objects and
        populates the job queue.
        """
        for dev in self.devices:
            log.msg('Adding', dev)
            if self.verbose:
                print 'Adding', dev

            # Make sure that devices are actually in netdevices and keep going
            try:
                devobj = self.nd.find(str(dev))
            except KeyError:
                msg = 'Device not found in NetDevices: %s' % dev
                log.err(msg)
                if self.verbose:
                    print 'ERROR:', msg

                # Track the errors and keep moving
                self.store_error(dev, msg)
                continue

            # We only want to add devices for which we've enabled support in
            # this class
            if devobj.vendor not in self.vendors:
                raise exceptions.UnsupportedVendor("The vendor '%s' is not specified in ``vendors``. Could not add %s to job queue. Please check the attribute in the class object." % (devobj.vendor, devobj))

            self.jobs.append(devobj)

    def select_next_device(self, jobs=None):
        """
        Select another device for the active queue.

        Currently only returns the next device in the job queue. This is
        abstracted out so that this behavior may be customized, such as for
        future support for incremental callbacks.

        If a device is determined to be invalid, you must return ``None``.

        :param jobs:
            (Optional) The jobs queue. If not set, uses ``self.jobs``.

        :returns:
            A `~trigger.netdevices.NetDevice` object or ``None``.
        """
        if jobs is None:
            jobs = self.jobs

        return jobs.pop()

    def _add_worker(self):
        """
        Adds devices to the work queue to keep it populated with the maximum
        connections as specified by ``max_conns``.
        """
        while self.jobs and self.curr_conns < self.max_conns:
            device = self.select_next_device()
            if device is None:
                log.msg('No device returned when adding worker. Moving on.')
                continue

            self._increment_connections()
            log.msg('connections:', self.curr_conns)
            log.msg('Adding work to queue...')
            if self.verbose:
                print 'connections:', self.curr_conns
                print 'Adding work to queue...'

            # Setup the async Deferred object with a timeout and error printing.
            commands = self.generate(device)
            async = device.execute(commands, creds=self.creds,
                                   incremental=self.incremental,
                                   timeout=self.timeout,
                                   with_errors=self.with_errors,
                                   force_cli=self.force_cli,
                                   command_interval=self.command_interval)

            # Add the parser callback for great justice!
            async.addCallback(self.parse, device, commands)

            # If parse fails, still decrement and track the error
            async.addErrback(self.errback, device)

            # Make sure any further uncaught errors get logged
            async.addErrback(log.err)

            # Here we addBoth to continue on after pass/fail, decrement the
            # connections and move on.
            async.addBoth(self._decrement_connections)
            async.addBoth(lambda x: self._add_worker())

        # Do this once we've exhausted the job queue
        else:
            if not self.curr_conns and self.reactor_running:
                self._stop()
            elif not self.jobs and not self.reactor_running:
                log.msg('No work left.')
                if self.verbose:
                    print 'No work left.'

    def _lookup_method(self, device, method):
        """
        Base lookup method. Looks up stuff by device manufacturer like:

            from_juniper
            to_foundry

        and defaults to ``self.from_base`` and ``self.to_base`` methods if
        customized methods not found.

        :param device:
            A `~trigger.netdevices.NetDevice` object

        :param method:
            One of 'generate', 'parse'
        """
        METHOD_MAP = {
            'generate': 'to_%s',
            'parse': 'from_%s',
        }
        assert method in METHOD_MAP

        desired_method = None

        # Select the desired vendor name.
        desired_vendor = device.vendor.name

        # Workaround until we implement device drivers
        if device.is_netscreen():
            desired_vendor = 'netscreen'

        vendor_types = self.platforms.get(desired_vendor)
        method_name = METHOD_MAP[method] % desired_vendor  # => 'to_cisco'
        device_type = device.deviceType

        if device_type in vendor_types:
            if hasattr(self, method_name):
                log.msg(
                    '[%s] Found %r method: %s' % (device, method, method_name)
                )
                desired_method = method_name
            else:
                log.msg(
                    '[%s] Did not find %r method: %s' % (device, method,
                                                         method_name)
                )
        else:
            raise exceptions.UnsupportedDeviceType(
                'Device %r has an invalid type %r for vendor %r. Must be '
                'one of %r.' % (device.nodeName, device_type,
                                desired_vendor, vendor_types)
            )

        if desired_method is None:
            if self.allow_fallback:
                desired_method = METHOD_MAP[method] % 'base'
                log.msg('[%s] Fallback enabled. Using base method: %r' %
                        (device, desired_method))
            else:
                raise exceptions.UnsupportedVendor(
                    'The vendor %r had no available %s method. Please check '
                    'your `vendors` and `platforms` attributes in your class '
                    'object.' % (device.vendor.name, method)
                )

        func = getattr(self, desired_method)
        return func

    def generate(self, device, commands=None, extra=None):
        """
        Generate commands to be run on a device. If you don't provide
        ``commands`` to the class constructor, this will return an empty list.

        Define a 'to_{vendor_name}' method to customize the behavior for each
        platform.

        :param device:
            NetDevice object
        :type device:
            `~trigger.netdevices.NetDevice`

        :param commands:
            (Optional) A list of commands to execute on the device. If not
            specified in they will be inherited from commands passed to the
            class constructor.
        :type commands:
            list

        :param extra:
            (Optional) A dictionary of extra data to send to the generate
            method for the device.
        """
        if commands is None:
            commands = self.commands
        if extra is None:
            extra = {}

        func = self._lookup_method(device, method='generate')
        return func(device, commands, extra)

    def parse(self, results, device, commands=None):
        """
        Parse output from a device. Calls to ``self._lookup_method`` to find
        specific ``from`` method.

        Define a 'from_{vendor_name}' method to customize the behavior for each
        platform.

        :param results:
            The results of the commands executed on the device
        :type results:
            list

        :param device:
            Device object
        :type device:
            `~trigger.netdevices.NetDevice`

        :param commands:
            (Optional) A list of commands to execute on the device. If not
            specified in they will be inherited from commands passed to the
            class constructor.
        :type commands:
            list
        """
        func = self._lookup_method(device, method='parse')
        return func(results, device, commands)

    def errback(self, failure, device):
        """
        The default errback. Overload for custom behavior but make sure it
        always decrements the connections.

        :param failure:
            Usually a Twisted ``Failure`` instance.

        :param device:
            A `~trigger.netdevices.NetDevice` object
        """
        failure.trap(Exception)
        self.store_error(device, failure)
        #self._decrement_connections(failure)
        return failure

    def store_error(self, device, error):
        """
        A simple method for storing an error called by all default
        parse/generate methods.

        If you want to customize the default method for storing results,
        overload this in your subclass.

        :param device:
            A `~trigger.netdevices.NetDevice` object

        :param error:
            The error to store. Anything you want really, but usually a Twisted
            ``Failure`` instance.
        """
        devname = str(device)
        self.errors[devname] = error
        return True

    def store_results(self, device, results):
        """
        A simple method for storing results called by all default
        parse/generate methods.

        If you want to customize the default method for storing results,
        overload this in your subclass.

        :param device:
            A `~trigger.netdevices.NetDevice` object

        :param results:
            The results to store. Anything you want really.
        """
        devname = str(device)
        log.msg("Storing results for %r: %r" % (devname, results))
        self.results[devname] = results
        return True

    def map_results(self, commands=None, results=None):
        """Return a dict of ``{command: result, ...}``"""
        if commands is None:
            commands = self.commands
        if results is None:
            results = []

        return dict(itertools.izip_longest(commands, results))

    @property
    def reactor_running(self):
        """Return whether reactor event loop is running or not"""
        from twisted.internet import reactor
        log.msg("Reactor running? %s" % reactor.running)
        return reactor.running

    def _stop(self):
        """Stop the reactor event loop"""
        log.msg('stopping reactor')
        if self.verbose:
            print 'stopping reactor'

        from twisted.internet import reactor
        reactor.stop()

    def _start(self):
        """Start the reactor event loop"""
        log.msg('starting reactor')
        if self.verbose:
            print 'starting reactor'

        if self.curr_conns:
            from twisted.internet import reactor
            reactor.run()
        else:
            msg = "Won't start reactor with no work to do!"
            log.msg(msg)
            if self.verbose:
                print msg

    def run(self):
        """
        Nothing happens until you execute this to perform the actual work.
        """
        self._add_worker()
        self._start()

    #=======================================
    # Base generate (to_)/parse (from_) methods
    #=======================================
    def to_base(self, device, commands=None, extra=None):
        commands = commands or self.commands
        log.msg('Sending %r to %s' % (commands, device))
        return commands

    def from_base(self, results, device, commands=None):
        commands = commands or self.commands
        log.msg('Received %r from %s' % (results, device))
        self.store_results(device, self.map_results(commands, results))

    #=======================================
    # Vendor-specific generate (to_)/parse (from_) methods
    #=======================================
    def to_juniper(self, device, commands=None, extra=None):
        """
        This just creates a series of ``<command>foo</command>`` elements to
        pass along to execute_junoscript()"""
        commands = commands or self.commands

        # If we've set force_cli, use to_base() instead
        if self.force_cli:
            return self.to_base(device, commands, extra)

        ret = []
        for command in commands:
            cmd = Element('command')
            cmd.text = command
            ret.append(cmd)

        return ret
示例#12
0
 def setUp(self):
     self.nd = NetDevices()
     self.acl = ACL_NAME
     self.device = self.nd.find(DEVICE_NAME)
     self.implicit_acls = set(['115j', 'router-protect.core'])
示例#13
0
文件: cmds.py 项目: bwheatley/trigger
class Commando(object):
    """
    I run commands on devices but am not much use unless you subclass me and
    configure vendor-specific parse/generate methods.
    """
    def __init__(self, devices=None, max_conns=10, verbose=False, timeout=30,
                 production_only=True):
        self.curr_connections = 0
        self.reactor_running  = False
        self.devices = devices or []
        self.verbose = verbose
        self.max_conns = max_conns
        self.nd = NetDevices(production_only=production_only)
        self.jobs = []
        self.errors = {}
        self.data = {}
        self.deferrals = self._setup_jobs()
        self.timeout = timeout # in seconds

    def _decrement_connections(self, data):
        """
        Self-explanatory. Called by _add_worker() as both callback/errback
        so we can accurately refill the jobs queue, which relies on the
        current connection count.
        """
        self.curr_connections -= 1
        return True

    def set_data(self, device, data):
        """
        Another method for storing results. If you'd rather just change the
        default method for storing results, overload this. All default
        parse/generate methods call this."""
        self.data[device] = data
        return True

    #=======================================
    # Vendor-specific parse/generate methods
    #=======================================

    def _normalize_manufacturer(self, manufacturer):
        """Normalize the manufacturer name into a method"""
        return manufacturer.replace(' ', '_').lower()

    def _lookup(self, device, prefix):
        """Base lookup method."""
        manuf = self._normalize_manufacturer(device.manufacturer)
        try:
            func = getattr(self, prefix + manuf)
        except AttributeError:
            return 'base prefix' + prefix  (device)
            #return self._base_generate_cmd(device)

        return func(device)

    def _parse_lookup(self, device):
        """Base parse method."""
        manuf = self._normalize_manufacturer
        try:
            func = getattr(self, 'parse_' + manuf)
        except AttributeError:
            return self._base_generate_cmd(device)

        return func(device)

    # Yes there is probably a better way to do this in the long-run instead of
    # individual parse/generate methods for each vendor, but this works for now.
    def _base_parse(self, data, device):
        """
        Parse output from a device. Overload this to customize this default
        behavior.
        """
        self.set_data(device, data)
        return True

    def _base_generate_cmd(self, dev=None):
        """
        Generate commands to be run on a device. If you don't overload this, it
        returns an empty list.
        """
        return []

    # TODO (jathan): Find a way to dynamically generate/call these methods
    # TODO (jathan): Methods should be prefixed with their action, not vendor

    # IOS (Cisco)
    ios_parse = _base_parse
    generate_ios_cmd = _base_generate_cmd

    # Brocade
    brocade_parse = _base_parse
    generate_brocade_cmd = _base_generate_cmd

    # Foundry
    foundry_parse = _base_parse
    generate_foundry_cmd = _base_generate_cmd

    # Juniper (JUNOS)
    junos_parse = _base_parse
    generate_junos_cmd = _base_generate_cmd

    # Citrix NetScaler
    netscaler_parse = _base_parse
    generate_netscaler_cmd = _base_generate_cmd

    # Arista
    arista_parse = _base_parse
    generate_arista_cmd = _base_generate_cmd

    # Dell
    dell_parse = _base_parse
    generate_dell_cmd = _base_generate_cmd

    def _setup_callback(self, dev):
        """
        Map execute, parse and generate callbacks to device by manufacturer.
        This is ripe for optimization, especially if/when we need to add
        support for multiple OS types/revisions per vendor.

        :param dev: NetDevice object

        Notes::

        + Arista, Brocade, Cisco, Dell, Foundry all use execute_ioslike
        + Citrix is assumed to be a NetScaler (switch)
        + Juniper is assumed to be a router/switch running JUNOS
        """
        callback_map = {
            'ARISTA NETWORKS':[dev, execute_ioslike,
                              self.generate_arista_cmd,
                              self.arista_parse],
            'BROCADE':       [dev, execute_ioslike,
                              self.generate_brocade_cmd,
                              self.brocade_parse],
            'CISCO SYSTEMS': [dev, execute_ioslike,
                              self.generate_ios_cmd,
                              self.ios_parse],
            'CITRIX':        [dev, execute_netscaler,
                              self.generate_netscaler_cmd,
                              self.netscaler_parse],
            'DELL':          [dev, execute_ioslike,
                              self.generate_dell_cmd,
                              self.dell_parse],
            'FOUNDRY':       [dev, execute_ioslike,
                              self.generate_foundry_cmd,
                              self.foundry_parse],
            'JUNIPER':       [dev, execute_junoscript,
                              self.generate_junos_cmd,
                              self.junos_parse],
        }
        result = callback_map[dev.manufacturer]

        return result

    def _setup_jobs(self):
        for dev in self.devices:
            if self.verbose:
                print 'Adding', dev

            # Make sure that devices are actually in netdevices and keep going
            try:
                devobj = self.nd.find(str(dev))
            except KeyError:
                msg = 'Device not found in NetDevices: %s' % dev
                if self.verbose:
                    print 'ERROR:', msg

                # Track the errors and keep moving
                self.errors[dev] = msg
                continue

            this_callback = self._setup_callback(devobj)
            self.jobs.append(this_callback)

    def run(self):
        """Nothing happens until you execute this to perform the actual work."""
        self._add_worker()
        self._start()

    def eb(self, x):
        self._decrement_connections(x)
        return True

    def _add_worker(self):
        work = None

        try:
            work = self.jobs.pop()
            if self.verbose:
                print 'Adding work to queue...'
        except (AttributeError, IndexError):
            #if not self.curr_connections:
            if not self.curr_connections and self.reactor_running:
                self._stop()
            else:
                if self.verbose:
                    print 'No work left.'

        while work:
            if self.verbose:
                print 'connections:', self.curr_connections
            if self.curr_connections >= self.max_conns:
                self.jobs.append(work)
                return

            self.curr_connections += 1
            if self.verbose:
                print 'connections:', self.curr_connections

            # Unpack the job parts
            #dev, execute, cmd, parser = work
            dev, execute, generate, parser = work

            # Setup the deferred object with a timeout and error printing.
            #defer = execute(dev, cmd, timeout=self.timeout, with_errors=True)
            cmds = generate(dev)
            defer = execute(dev, cmds, timeout=self.timeout, with_errors=True)

            # Add the callbacks for great justice!
            defer.addCallback(parser, dev)
            # Here we addBoth to continue on after pass/fail
            defer.addBoth(self._decrement_connections)
            defer.addBoth(lambda x: self._add_worker())
            defer.addErrback(self.eb) # If worker add fails, still decrement

            try:
                work = self.jobs.pop()
            except (AttributeError, IndexError):
                work = None

    def _stop(self):
        if self.verbose:
            print 'stopping reactor'
        self.reactor_running = False
        from twisted.internet import reactor
        reactor.stop()

    def _start(self):
        if self.verbose:
            print 'starting reactor'
        self.reactor_running = True
        from twisted.internet import reactor
        if self.curr_connections:
            reactor.run()
        else:
            if self.verbose:
                print "Won't start reactor with no work to do!"
示例#14
0
 def setUp(self):
     self.router = NetDevices()['iwg1-r3.router.aol.com']
     self.when = datetime(2006, 7, 24, 20, tzinfo=UTC)
示例#15
0
 def setUp(self):
     self.nd = NetDevices()
     self.nodename = self.nd.keys()[0]
     self.device = self.nd.values()[0]
     self.device.explicit_acls = set(['test1-abc-only'])
示例#16
0
 def setUp(self):
     self.router = NetDevices()['test1-abc.net.aol.com']
     self.when = datetime(2006, 7, 24, 20, tzinfo=UTC)
示例#17
0
__all__ = ['mock_redis']

# misc
from . import misc
from misc import *
__all__.extend(misc.__all__)

if __name__ == '__main__':
    os.environ['NETDEVICES_SOURCE'] = 'data/netdevices.xml'

    mock_redis.install()
    import redis
    from trigger.netdevices import NetDevices
    from trigger.acl.db import AclsDB

    r = redis.Redis()
    a = AclsDB()
    nd = NetDevices()

    dev = nd.find('test1-abc')

    print r.keys('*')
    print a.add_acl(dev, 'bacon')
    print r.keys('*')

    _k = 'acls:explicit:'
    key = _k + dev.nodeName

    print r.smembers(key)
示例#18
0
 def setUp(self):
     self.nd = NetDevices()
     self.device = self.nd[DEVICE_NAME]
     self.device2 = self.nd[DEVICE2_NAME]
     self.nodename = self.device.nodeName
     self.device.explicit_acls = set(['test1-abc-only'])
示例#19
0
class TestNetDeviceObject(unittest.TestCase):
    """
    Test NetDevice object methods.
    """
    def setUp(self):
        self.nd = NetDevices()
        self.nodename = self.nd.keys()[0]
        self.device = self.nd.values()[0]

    def test_stringify(self):
        """Test casting NetDevice to string"""
        expected = DEVICE_NAME
        self.assertEqual(expected, str(self.device))

    def test_bounce(self):
        """Test .bounce property"""
        expected = changemgmt.BounceWindow
        self.assertTrue(isinstance(self.device.bounce, expected))

    def test_shortName(self):
        """Test .shortName property"""
        expected = self.nodename.split('.', 1)[0]
        self.assertEqual(expected, self.device.shortName)

    def test_allowable(self):
        """Test allowable() method"""
        # This is already tested in test_changemgmt.py, so this is a stub.
        pass

    def test_next_ok(self):
        """Test next_ok() method"""
        # This is already tested in test_changemgmt.py, so this is a stub.
        pass

    def test_identity(self):
        """Exercise NetDevice identity tests."""
        # It's a router...
        self.assertTrue(self.device.is_router())
        # And therefore none of these other things...
        self.assertFalse(self.device.is_switch())
        self.assertFalse(self.device.is_firewall())
        self.assertFalse(self.device.is_netscaler())
        self.assertFalse(self.device.is_netscreen())
        self.assertFalse(self.device.is_ioslike())
        self.assertFalse(self.device.is_brocade_vdx())

    def test_hash_ssh(self):
        """Exercise NetDevice ssh test."""
        # TODO (jathan): Mock SSH connections so we can test actual connectivity
        # Device won't be reachable, so this should always fail
        self.assertFalse(self.device.has_ssh())
        # Since there's no SSH, no aync
        self.assertFalse(self.device.can_ssh_pty())

    def test_reachability(self):
        """Exercise NetDevice ssh test."""
        # TODO (jathan): Mock SSH connections so we can test actual connectivity
        self.assertFalse(self.device.is_reachable())

    def test_dump(self):
        """Test the dump() method."""
        with captured_output() as (out, err):
            self.device.dump()
        expected = NETDEVICE_DUMP_EXPECTED
        output = out.getvalue()
        self.assertEqual(expected, output)

    def tearDown(self):
        _reset_netdevices()
示例#20
0
class TestAclQueue(unittest.TestCase):
    def setUp(self):
        self.nd = NetDevices()
        _setup_aclsdb(self.nd)
        self.q = queue.Queue(verbose=False)
        self.acl = ACL_NAME
        self.acl_list = [self.acl]
        self.device = self.nd.find(DEVICE_NAME)
        self.device_name = DEVICE_NAME
        self.device_list = [self.device_name]
        self.user = USERNAME

    #
    # Integrated queue tests
    #

    def test_01_insert_integrated_success(self):
        """Test insert success into integrated queue"""
        self.assertTrue(self.q.insert(self.acl, self.device_list) is None)

    def test_02_insert_integrated_failure_device(self):
        """Test insert invalid device"""
        self.assertRaises(exceptions.TriggerError, self.q.insert, self.acl, ['bogus'])

    def test_03_insert_integrated_failure_acl(self):
        """Test insert devices w/ no ACL association"""
        self.assertRaises(exceptions.TriggerError, self.q.insert, 'bogus',
                          self.device_list)

    def test_04_list_integrated_success(self):
        """Test listing integrated queue"""
        self.q.insert(self.acl, self.device_list)
        expected = [(u'test1-abc.net.aol.com', u'foo')]
        self.assertEqual(sorted(expected), sorted(self.q.list()))

    def test_05_complete_integrated(self):
        """Test mark task complete"""
        self.q.complete(self.device_name, self.acl_list)
        expected = []
        self.assertEqual(sorted(expected), sorted(self.q.list()))

    def test_06_delete_integrated_with_devices(self):
        """Test delete ACL from queue providing devices"""
        self.q.insert(self.acl, self.device_list)
        self.assertTrue(self.q.delete(self.acl, self.device_list))

    def test_07_delete_integrated_no_devices(self):
        """Test delete ACL from queue without providing devices"""
        self.q.insert(self.acl, self.device_list)
        self.assertTrue(self.q.delete(self.acl))

    def test_08_remove_integrated_success(self):
        """Test remove (set as loaded) ACL from integrated queue"""
        self.q.insert(self.acl, self.device_list)
        self.q.remove(self.acl, self.device_list)
        expected = []
        self.assertEqual(sorted(expected), sorted(self.q.list()))

    def test_10_remove_integrated_failure(self):
        """Test remove (set as loaded) failure"""
        self.assertRaises(exceptions.ACLQueueError, self.q.remove, '', self.device_list)

    #
    # Manual queue tests
    #

    def test_11_insert_manual_success(self):
        """Test insert success into manual queue"""
        self.assertTrue(self.q.insert('manual task', None) is None)

    def test_12_list_manual_success(self):
        """Test list success of manual queue"""
        self.q.insert('manual task', None)
        expected = ('manual task', self.user)
        result = self.q.list('manual')
        actual = result[0][:2] # First tuple, items 0-1
        self.assertEqual(sorted(expected), sorted(actual))

    def test_13_delete_manual_success(self):
        """Test delete from manual queue"""
        self.q.delete('manual task')
        expected = []
        self.assertEqual(sorted(expected), sorted(self.q.list('manual')))

    #
    # Generic tests
    #

    def test_14_delete_failure(self):
        """Test delete of task not in queue"""
        self.assertFalse(self.q.delete('bogus'))

    def test_15_list_invalid(self):
        """Test list of invalid queue name"""
        self.assertFalse(self.q.list('bogus'))

    # Teardown

    def test_ZZ_cleanup_db(self):
        """Cleanup the temp database file"""
        self.assertTrue(os.remove(db_file) is None)

    def tearDown(self):
        NetDevices._Singleton = None
示例#21
0
class TestNetDevicesWithAcls(unittest.TestCase):
    """
    Test NetDevices with ``settings.WITH_ACLs set`` to ``True``.
    """
    def setUp(self):
        self.nd = NetDevices()
        self.nodename = self.nd.keys()[0]
        self.device = self.nd.values()[0]
        self.device.explicit_acls = set(['test1-abc-only'])

    def test_basics(self):
        """Basic test of NetDevices functionality."""
        self.assertEqual(len(self.nd), 1)
        self.assertEqual(self.device.nodeName, self.nodename)
        self.assertEqual(self.device.manufacturer, 'JUNIPER')

    def test_aclsdb(self):
        """Test acls.db handling."""
        self.assertTrue('test1-abc-only' in self.device.explicit_acls)

    def test_autoacls(self):
        """Test autoacls.py handling."""
        self.assertTrue('router-protect.core' in self.device.implicit_acls)

    def test_find(self):
        """Test the find() method."""
        self.assertEqual(self.nd.find(self.nodename), self.device)
        nodebasename = self.nodename[:self.nodename.index('.')]
        self.assertEqual(self.nd.find(nodebasename), self.device)
        self.assertRaises(KeyError, lambda: self.nd.find(self.nodename[0:3]))

    def test_all(self):
        """Test the all() method."""
        expected = [self.device]
        self.assertEqual(expected, self.nd.all())

    def test_search(self):
        """Test the search() method."""
        expected = [self.device]
        self.assertEqual(expected, self.nd.search(self.nodename))
        self.assertEqual(expected, self.nd.search('17', field='onCallID'))
        self.assertEqual(expected, self.nd.search('juniper', field='vendor'))

    def test_match(self):
        """Test the match() method."""
        expected = [self.device]
        self.assertEqual(expected, self.nd.match(nodename=self.nodename))
        self.assertEqual(expected, self.nd.match(vendor='juniper'))
        self.assertNotEqual(expected, self.nd.match(vendor='cisco'))

    def tearDown(self):
        NetDevices._Singleton = None
示例#22
0
文件: queue.py 项目: aakapoor/trigger
class Queue(object):
    """
    Interacts with firewalls database to insert/remove items into the queue.

    :param verbose:
        Toggle verbosity

    :type verbose:
        Boolean
    """
    def __init__(self, verbose=True):
        self.nd = NetDevices()
        self.verbose = verbose
        self.login = get_user()

    def vprint(self, msg):
        """
        Print something if ``verbose`` instance variable is set.

        :param msg:
            The string to print
        """
        if self.verbose:
            print msg

    def get_model(self, queue):
        """
        Given a queue name, return its DB model.

        :param queue:
            Name of the queue whose object you want
        """
        return models.MODEL_MAP.get(queue, None)

    def create_task(self, queue, *args, **kwargs):
        """
        Create a task in the specified queue.

        :param queue:
            Name of the queue whose object you want
        """
        model = self.get_model(queue)
        taskobj = model.create(*args, **kwargs)

    def _normalize(self, arg, prefix=''):
        """
        Remove ``prefix`` from ``arg``, and set "escalation" bit.

        :param arg:
            Arg (typically an ACL filename) to trim

        :param prefix:
            Prefix to trim from arg
        """
        if arg.startswith(prefix):
            arg = arg[len(prefix):]
        escalation = False
        if arg.upper().endswith(' ESCALATION'):
            escalation = True
            arg = arg[:-11]
        return (escalation, arg)

    def insert(self, acl, routers, escalation=False):
        """
        Insert an ACL and associated devices into the ACL load queue.

        Attempts to insert into integrated queue. If ACL test fails, then
        item is inserted into manual queue.

        :param acl:
            ACL name

        :param routers:
            List of device names

        :param escalation:
            Whether this is an escalated task
        """
        if not acl:
            raise exceptions.ACLQueueError('You must specify an ACL to insert into the queue')
        if not routers:
            routers = []

        escalation, acl = self._normalize(acl)
        if routers:
            for router in routers:
                try:
                    dev = self.nd.find(router)
                except KeyError:
                    msg = 'Could not find device %s' % router
                    raise exceptions.TriggerError(msg)

                if acl not in dev.acls:
                    msg = "Could not find %s in ACL list for %s" % (acl, router)
                    raise exceptions.TriggerError(msg)

                self.create_task(queue='integrated', acl=acl, router=router,
                                 escalation=escalation)

            self.vprint('ACL %s injected into integrated load queue for %s' %
                        (acl, ', '.join(dev[:dev.find('.')] for dev in routers)))

        else:
            self.create_task(queue='manual', q_name=acl, login=self.login)
            self.vprint('"%s" injected into manual load queue' % acl)

    def delete(self, acl, routers=None, escalation=False):
        """
        Delete an ACL from the firewall database queue.

        Attempts to delete from integrated queue. If ACL test fails
        or if routers are not specified, the item is deleted from manual queue.

        :param acl:
            ACL name

        :param routers:
            List of device names. If this is ommitted, the manual queue is used.

        :param escalation:
            Whether this is an escalated task
        """
        if not acl:
            raise exceptions.ACLQueueError('You must specify an ACL to delete from the queue')

        escalation, acl = self._normalize(acl)
        m = self.get_model('integrated')

        if routers is not None:
            devs = routers
        else:
            self.vprint('Fetching routers from database')
            result = m.select(m.router).distinct().where(
                              m.acl == acl, m.loaded >> None).order_by(m.router)
            rows = result.tuples()
            devs = [row[0] for row in rows]

        if devs:
            for dev in devs:
                m.delete().where(m.acl == acl, m.router == dev,
                                 m.loaded >> None).execute()

            self.vprint('ACL %s cleared from integrated load queue for %s' %
                        (acl, ', '.join(dev[:dev.find('.')] for dev in devs)))
            return True

        else:
            m = self.get_model('manual')
            if m.delete().where(m.q_name == acl, m.done == False).execute():
                self.vprint('%r cleared from manual load queue' % acl)
                return True

        self.vprint('%r not found in any queues' % acl)
        return False

    def complete(self, device, acls):
        """
        Mark a device and its ACLs as complete using current timestamp.

        (Integrated queue only.)

        :param device:
            Device names

        :param acls:
            List of ACL names
        """
        m = self.get_model('integrated')
        for acl in acls:
            now = loaded=datetime.datetime.now()
            m.update(loaded=now).where(m.acl == acl, m.router == device,
                                       m.loaded >> None).execute()

        self.vprint('Marked the following ACLs as complete for %s:' % device)
        self.vprint(', '.join(acls))

    def remove(self, acl, routers, escalation=False):
        """
        Integrated queue only.

        Mark an ACL and associated devices as "removed" (loaded=0). Intended
        for use when performing manual actions on the load queue when
        troubleshooting or addressing errors with automated loads. This leaves
        the items in the database but removes them from the active queue.

        :param acl:
            ACL name

        :param routers:
            List of device names

        :param escalation:
            Whether this is an escalated task
        """
        if not acl:
            raise exceptions.ACLQueueError('You must specify an ACL to remove from the queue')

        m = self.get_model('integrated')
        loaded = 0
        if settings.DATABASE_ENGINE == 'postgresql':
            loaded = '-infinity' # See: http://bit.ly/15f0J3z
        for router in routers:
            m.update(loaded=loaded).where(m.acl == acl, m.router == router,
                                          m.loaded >> None).execute()

        self.vprint('Marked the following devices as removed for ACL %s: ' % acl)
        self.vprint(', '.join(routers))

    def list(self, queue='integrated', escalation=False, q_names=QUEUE_NAMES):
        """
        List items in the specified queue, defauls to integrated queue.

        :param queue:
            Name of the queue to list

        :param escalation:
            Whether this is an escalated task

        :param q_names:
            (Optional) List of valid queue names
        """
        if queue not in q_names:
            self.vprint('Queue must be one of %s, not: %s' % (q_names, queue))
            return False

        m = self.get_model(queue)

        if queue == 'integrated':
            result = m.select(m.router, m.acl).distinct().where(
                              m.loaded >> None, m.escalation == escalation)
        elif queue == 'manual':
            result = m.select(m.q_name, m.login, m.q_ts, m.done).where(
                              m.done == False)
        else:
            raise RuntimeError('This should never happen!!')

        all_data = list(result.tuples())
        return all_data
import sys
from time import sleep
from twisted.internet.defer import Deferred
from zope.interface import implements
from twisted.internet import reactor
from twisted.web.client import Agent
from twisted.web.http_headers import Headers
from twisted.internet.defer import succeed
from twisted.web.iweb import IBodyProducer
# from twisted.python import log
# log.startLogging(sys.stdout, setStdout=False)
from trigger.netdevices import NetDevices

# Create reference to upgraded switch.
nd = NetDevices()
dev = nd.find('arista-sw1.demo.local')

# Create payload body
class StringProducer(object):
    implements(IBodyProducer)

    def __init__(self, body):
        self.body = body
        self.length = len(body)

    def startProducing(self, consumer):
        consumer.write(self.body)
        return succeed(None)

    def pauseProducing(self):
示例#24
0
 def __init__(self, verbose=True):
     self.dbconn = self._get_firewall_db_conn()
     self.cursor = self.dbconn.cursor()
     self.nd = NetDevices()
     self.verbose = verbose
示例#25
0
class Queue(object):
    """
    Interacts with firewalls database to insert/remove items into the queue. You
    may optionally suppress informational messages by passing ``verbose=False``
    to the constructor.

    :param verbose: Toggle verbosity
    :type verbose: Boolean
    """
    def __init__(self, verbose=True):
        self.dbconn = self._get_firewall_db_conn()
        self.cursor = self.dbconn.cursor()
        self.nd = NetDevices()
        self.verbose = verbose

    def _get_firewall_db_conn(self):
        """Returns a MySQL db connection used for the ACL queues using database
        settings found withing ``settings.py``."""
        if MySQLdb is None:
            raise RuntimeError("You must install ``MySQL-python`` to use the queue")
        try:
            return MySQLdb.connect(host=settings.DATABASE_HOST,
                                   db=settings.DATABASE_NAME,
                                   port=settings.DATABASE_PORT,
                                   user=settings.DATABASE_USER,
                                   passwd=settings.DATABASE_PASSWORD)
        # catch if we can't connect, and shut down
        except MySQLdb.OperationalError as e:
            sys.exit("Can't connect to the database - %s (error %d)" % (e[1],e[0]))

    def _normalize(self, arg):
        if arg.startswith('acl.'):
            arg = arg[4:]
        escalation = False
        if arg.upper().endswith(' ESCALATION'):
            escalation = True
            arg = arg[:-11]
        return (escalation, arg)

    def insert(self, acl, routers, escalation=False):
        """
        Insert an ACL and associated devices into the ACL load queue.  

        Attempts to insert into integrated queue.  If ACL test fails, then 
        item is inserted into manual queue.
        """
        assert acl, 'no ACL defined'
        if not routers: 
            routers = []

        (escalation, acl) = self._normalize(acl)
        if len(routers):
            for router in routers:
                try:
                    dev = self.nd.find(router)
                except KeyError:
                    raise  "Could not find %s in netdevices" % router
                    return
                if acl not in dev.acls:
                    raise "Could not find %s in %s's acl list" % (acl, router)
                    return

                self.cursor.execute('''insert into acl_queue 
                                        (acl, router, queued, escalation) 
                                        values (%s, %s, now(), %s)''', 
                                        (acl, router, escalation))

            if self.verbose:
                print 'ACL', acl, 'injected into integrated load queue for',
                print ', '.join([dev[:dev.find('.')] for dev in routers])

        else:
            self.cursor.execute('''insert into queue (q_name, login) values (%s, %s)''',
                                    (acl, os.getlogin()))
            if self.verbose: print '"%s" injected into manual load queue' % acl

        self.dbconn.commit()

    def delete(self, acl, routers=None, escalation=False):
        """
        Delete an ACL from the firewall database queue.

        Attempts to delete from integrated queue.  If ACL test fails, then 
        item is deleted from manual queue.
        """
        assert acl, 'no ACL defined' 
        (escalation, acl) = self._normalize(acl)

        if routers is not None:
            devs = routers
        else:
            if self.verbose: print 'fetching routers from database'
            self.cursor.execute('''select distinct router from acl_queue
                                where acl = %s and loaded is null
                                order by router''', (acl,))
            rows = self.cursor.fetchall()
            devs = [row[0] for row in rows] or []

        if len(devs):
            for dev in devs:
                self.cursor.execute('''delete from acl_queue where acl=%s and 
                                    router=%s and loaded is null''', (acl, dev))

            if self.verbose:
                print 'ACL', acl, 'cleared from integrated load queue for',
                print ', '.join([dev[:dev.find('.')] for dev in devs])
        elif self.cursor.execute('''delete from queue
                                where q_name = %s and done = 0''', (acl,)):
            if self.verbose: print '"%s" cleared from manual load queue' % acl
        else:
            if self.verbose: print '"%s" not found in manual or integrated queues' % acl

        self.dbconn.commit()

    def complete(self, device, acls):
        """
        Integrated queue only.

        Mark a device and associated ACLs as complete my updating loaded to
        current timestampe.  Migrated from clear_load_queue() in load_acl.
        """
        for acl in acls:
            self.cursor.execute("""update acl_queue set loaded=now() where
                                acl=%s and router=%s and loaded is null""",
                                (acl, device))
        if self.verbose: 
            print 'Marked the following ACLs as complete for %s: ' % device
            print ', '.join(acls)

    def remove(self, acl, routers, escalation=False):
        """
        Integrated queue only.

        Mark an ACL and associated devices as "removed" (loaded=0). Intended
        for use when performing manual actions on the load queue when
        troubleshooting or addressing errors with automated loads.  This leaves
        the items in the database but removes them from the active queue.
        """
        assert acl, 'no ACL defined' 
        for router in routers:
            self.cursor.execute("""update acl_queue set loaded=0 where acl=%s
                                and router=%s and loaded is null""", (acl, router))
        if self.verbose: 
            print 'Marked the following devices as removed for ACL %s: ' % acl
            print ', '.join(routers)

    def list(self, queue='integrated', escalation=False):
        """
        List items in the queue, defauls to integrated queue.

        Valid queue arguments are 'integrated' or 'manual'.
        """
        if queue == 'integrated':
            query = 'select distinct router, acl from acl_queue where loaded is null'
            if escalation:
                query += ' and escalation = true' 
        elif queue == 'manual':
            query = 'select q_name, login, q_ts, done from queue where done=0'
        else:
            print 'Queue must be integrated or manual, you specified: %s' % queue
            return False

        self.cursor.execute(query)
        all_data = self.cursor.fetchall()

        self.dbconn.commit()

        return all_data
示例#26
0
文件: queue.py 项目: aakapoor/trigger
 def __init__(self, verbose=True):
     self.nd = NetDevices()
     self.verbose = verbose
     self.login = get_user()
示例#27
0
class TestNetDevicesWithAcls(unittest.TestCase):
    """
    Test NetDevices with ``settings.WITH_ACLs set`` to ``True``.
    """
    def setUp(self):
        self.nd = NetDevices()
        self.device = self.nd[DEVICE_NAME]
        self.device2 = self.nd[DEVICE2_NAME]
        self.nodename = self.device.nodeName
        self.device.explicit_acls = set(['test1-abc-only'])

    def test_basics(self):
        """Basic test of NetDevices functionality."""
        self.assertEqual(len(self.nd), 2)
        self.assertEqual(self.device.nodeName, self.nodename)
        self.assertEqual(self.device.manufacturer, 'JUNIPER')

    def test_aclsdb(self):
        """Test acls.db handling."""
        self.assertTrue('test1-abc-only' in self.device.explicit_acls)

    def test_autoacls(self):
        """Test autoacls.py handling."""
        self.assertTrue('router-protect.core' in self.device.implicit_acls)

    def test_find(self):
        """Test the find() method."""
        self.assertEqual(self.nd.find(self.nodename), self.device)
        nodebasename = self.nodename[:self.nodename.index('.')]
        self.assertEqual(self.nd.find(nodebasename), self.device)
        self.assertRaises(KeyError, lambda: self.nd.find(self.nodename[0:3]))

    def test_all(self):
        """Test the all() method."""
        expected = [self.device, self.device2]
        self.assertEqual(sorted(expected), sorted(self.nd.all()))

    def test_search(self):
        """Test the search() method."""
        expected = [self.device]
        self.assertEqual([self.device], self.nd.search(self.nodename))
        self.assertEqual(self.nd.all(), self.nd.search('17', field='onCallID'))

    def test_match(self):
        """Test the match() method."""
        self.assertEqual([self.device], self.nd.match(nodename=self.nodename))
        self.assertEqual(self.nd.all(), self.nd.match(vendor='juniper'))
        self.assertEqual([], self.nd.match(vendor='cisco'))

    def test_multiple_filter_match(self):
        """Test that passing multiple kwargs filters properly."""
        # There should be only one Juniper router.
        self.assertEqual(
            self.nd.match(nodename='test1-abc'),
            self.nd.match(vendor='juniper', devicetype='router')
        )

        # And only one Juniper switch.
        self.assertEqual(
            self.nd.match(nodename='test2-abc'),
            self.nd.match(vendor='juniper', devicetype='switch')
        )

    def test_match_with_null_value(self):
        """Test the match() method when attr value is ``None``."""
        self.device.site = None  # Zero it out!
        expected = [self.device]

        # None raw
        self.assertEqual(expected, self.nd.match(site=None))

        # "None" string
        self.assertEqual(expected, self.nd.match(site='None'))

        # Case-insensitive attr *and* value
        self.assertEqual(expected, self.nd.match(SITE='NONE'))

    def tearDown(self):
        _reset_netdevices()
示例#28
0
 def setUp(self):
     self.nd = NetDevices()
     self.nodename = self.nd.keys()[0]
     self.device = self.nd.values()[0]
示例#29
0
class Commando(object):
    """
    Execute commands asynchronously on multiple network devices.

    This class is designed to be extended but can still be used as-is to execute
    commands and return the results as-is.

    At the bare minimum you must specify a list of ``devices`` to interact with.
    You may optionally specify a list of ``commands`` to execute on those
    devices, but doing so will execute the same commands on every device
    regardless of platform.

    If ``commands`` are not specified, they will be expected to be emitted by
    the ``generate`` method for a given platform. Otherwise no commands will be
    executed.

    If you wish to customize the commands executed by device, you must define a
    ``to_{vendor_name}`` method containing your custom logic.

    If you wish to customize what is done with command results returned from a
    device, you must define a ``from_{vendor_name}`` method containing your
    custom logic.

    :param devices:
        A list of device hostnames or `~trigger.netdevices.NetDevice` objects

    :param commands:
        (Optional) A list of commands to execute on the ``devices``.

    :param incremental:
        (Optional) A callback that will be called with an empty sequence upon
        connection and then called every time a result comes back from the
        device, with the list of all results.

    :param max_conns:
        (Optional) The maximum number of simultaneous connections to keep open.

    :param verbose:
        (Optional) Whether or not to display informational messages to the
        console.

    :param timeout:
        (Optional) Time in seconds to wait for each command executed to return a
        result

    :param production_only:
        (Optional) If set, includes all devices instead of excluding any devices
        where ``adminStatus`` is not set to ``PRODUCTION``.

    :param allow_fallback:
        If set (default), allow fallback to base parse/generate methods when
        they are not customized in a subclass, otherwise an exception is raised
        when a method is called that has not been explicitly defined.
    """
    # Defaults to all supported vendors
    vendors = settings.SUPPORTED_VENDORS

    # Defaults to all supported platforms
    platforms = settings.SUPPORTED_PLATFORMS

    # The commands to run
    commands = []

    def __init__(self, devices=None, commands=None, incremental=None,
                 max_conns=10, verbose=False, timeout=30,
                 production_only=True, allow_fallback=True):
        if devices is None:
            raise exceptions.ImproperlyConfigured('You must specify some ``devices`` to interact with!')

        self.devices = devices
        self.commands = self.commands or (commands or []) # Always fallback to []
        self.incremental = incremental
        self.max_conns = max_conns
        self.verbose = verbose
        self.timeout = timeout # in seconds
        self.nd = NetDevices(production_only=production_only)
        self.allow_fallback = allow_fallback
        self.curr_conns = 0
        self.jobs = []
        self.errors = {}
        self.results = {}
        self.deferrals = self._setup_jobs()
        self.supported_platforms = self._validate_platforms()

    def _validate_platforms(self):
        """
        Determine the set of supported platforms for this instance by making
        sure the specified vendors/platforms for the class match up.
        """
        supported_platforms = {}
        for vendor in self.vendors:
            if vendor in self.platforms:
                types = self.platforms[vendor]
                if not types:
                    raise exceptions.MissingPlatform('No platforms specified for %r' % vendor)
                else:
                    #self.supported_platforms[vendor] = types
                    supported_platforms[vendor] = types
            else:
                raise exceptions.ImproperlyConfigured('Platforms for vendor %r not found. Please provide it at either the class level or using the arguments.' % vendor)

        return supported_platforms

    def _decrement_connections(self, data=None):
        """
        Self-explanatory. Called by _add_worker() as both callback/errback
        so we can accurately refill the jobs queue, which relies on the
        current connection count.
        """
        self.curr_conns -= 1
        return True

    def _increment_connections(self, data=None):
        """Increment connection count."""
        self.curr_conns += 1
        return True

    def _setup_jobs(self):
        """
        "Maps device hostnames to `~trigger.netdevices.NetDevice` objects and
        populates the job queue.
        """
        for dev in self.devices:
            if self.verbose:
                print 'Adding', dev

            # Make sure that devices are actually in netdevices and keep going
            try:
                devobj = self.nd.find(str(dev))
            except KeyError:
                if self.verbose:
                    msg = 'Device not found in NetDevices: %s' % dev
                    print 'ERROR:', msg

                # Track the errors and keep moving
                self.errors[dev] = msg
                continue

            # We only want to add devices for which we've enabled support in
            # this class
            if devobj.vendor not in self.vendors:
                raise exceptions.UnsupportedVendor("The vendor '%s' is not specified in ``vendors``. Could not add %s to job queue. Please check the attribute in the class object." % (devobj.vendor, devobj))

            self.jobs.append(devobj)

    def select_next_device(self, jobs=None):
        """
        Select another device for the active queue.

        Currently only returns the next device in the job queue. This is
        abstracted out so that this behavior may be customized, such as for
        future support for incremental callbacks.

        :param jobs:
            (Optional) The jobs queue. If not set, uses ``self.jobs``.

        :returns:
            A `~trigger.netdevices.NetDevice` object
        """
        if jobs is None:
            jobs = self.jobs

        return jobs.pop()

    def _add_worker(self):
        """
        Adds devices to the work queue to keep it populated with the maximum
        connections as specified by ``max_conns``.
        """
        while self.jobs and self.curr_conns < self.max_conns:
            device = self.select_next_device()

            self._increment_connections()
            if self.verbose:
                print 'connections:', self.curr_conns
                print 'Adding work to queue...'

            # Setup the async Deferred object with a timeout and error printing.
            commands = self.generate(device)
            async = device.execute(commands, incremental=self.incremental,
                                   timeout=self.timeout, with_errors=True)

            # Add the parser callback for great justice!
            async.addCallback(self.parse, device)

            # Here we addBoth to continue on after pass/fail
            async.addBoth(self._decrement_connections)
            async.addBoth(lambda x: self._add_worker())

            # If worker add fails, still decrement and track the error
            async.addErrback(self.errback, device)

        # Do this once we've exhausted the job queue
        else:
            if not self.curr_conns and self.reactor_running:
                self._stop()
            elif not self.jobs and not self.reactor_running:
                if self.verbose:
                    print 'No work left.'

    def _lookup_method(self, device, method):
        """
        Base lookup method. Looks up stuff by device manufacturer like:

            from_juniper
            to_foundry

        :param device:
            A `~trigger.netdevices.NetDevice` object

        :param method:
            One of 'generate', 'parse'
        """
        METHOD_MAP = {
            'generate': 'to_%s',
            'parse': 'from_%s',
        }
        assert method in METHOD_MAP

        desired_attr = None

        for vendor, types in self.platforms.iteritems():
            meth_attr = METHOD_MAP[method] % device.vendor
            if device.deviceType in types:
                if hasattr(self, meth_attr):
                    desired_attr = meth_attr
                    break

        if desired_attr is None:
            if self.allow_fallback:
                desired_attr = METHOD_MAP[method] % 'base'
            else:
                raise exceptions.UnsupportedVendor("The vendor '%s' had no available %s method. Please check your ``vendors`` and ``platforms`` attributes in your class object." % (device.vendor, method))

        func = getattr(self, desired_attr)
        return func

    def generate(self, device, commands=None, extra=None):
        """
        Generate commands to be run on a device. If you don't provide
        ``commands`` to the class constructor, this will return an empty list.

        Define a 'to_{vendor_name}' method to customize the behavior for each
        platform.

        :param device:
            A `~trigger.netdevices.NetDevice` object

        :param commands:
            (Optional) A list of commands to execute on the device. If not
            specified in they will be inherited from commands passed to the
            class constructor.

        :param extra:
            (Optional) A dictionary of extra data to send to the generate method for the
            device.
        """
        if commands is None:
            commands = self.commands
        if extra is None:
            extra = {}

        func = self._lookup_method(device, method='generate')
        return func(device, commands, extra)

    def parse(self, results, device):
        """
        Parse output from a device.

        Define a 'from_{vendor_name}' method to customize the behavior for each
        platform.

        :param results:
            The results of the commands executed on the device

        :param device:
            A `~trigger.netdevices.NetDevice` object
        """
        func = self._lookup_method(device, method='parse')
        return func(results, device)

    def errback(self, failure, device):
        """
        The default errback. Overload for custom behavior but make sure it
        always decrements the connections.

        :param failure:
            Usually a Twisted ``Failure`` instance.

        :param device:
            A `~trigger.netdevices.NetDevice` object
        """
        self.store_error(device, failure)
        self._decrement_connections(failure)
        return True

    def store_error(self, device, error):
        """
        A simple method for storing an error called by all default
        parse/generate methods.

        If you want to customize the default method for storing results,
        overload this in your subclass.

        :param device:
            A `~trigger.netdevices.NetDevice` object

        :param error:
            The error to store. Anything you want really, but usually a Twisted
            ``Failure`` instance.
        """
        self.errors[device.nodeName] = error
        return True

    def store_results(self, device, results):
        """
        A simple method for storing results called by all default
        parse/generate methods.

        If you want to customize the default method for storing results,
        overload this in your subclass.

        :param device:
            A `~trigger.netdevices.NetDevice` object

        :param results:
            The results to store. Anything you want really.
        """
        self.results[device.nodeName] = results
        return True

    def map_results(self, commands=None, results=None):
        """Return a dict of ``{command: result, ...}``"""
        if commands is None:
            commands = self.commands
        if results is None:
            results = []

        return dict(itertools.izip_longest(commands, results))

    @property
    def reactor_running(self):
        """Return whether reactor event loop is running or not"""
        from twisted.internet import reactor
        return reactor.running

    def _stop(self):
        """Stop the reactor event loop"""
        if self.verbose:
            print 'stopping reactor'

        from twisted.internet import reactor
        reactor.stop()

    def _start(self):
        """Start the reactor event loop"""
        if self.verbose:
            print 'starting reactor'

        if self.curr_conns:
            from twisted.internet import reactor
            reactor.run()
        else:
            if self.verbose:
                print "Won't start reactor with no work to do!"

    def run(self):
        """
        Nothing happens until you execute this to perform the actual work.
        """
        self._add_worker()
        self._start()

    #=======================================
    # Base generate (to_)/parse (from_) methods
    #=======================================

    def to_base(self, device, commands=None, extra=None):
        commands = commands or self.commands
        print 'Sending %r to %s' % (commands, device)
        return commands

    def from_base(self, results, device):
        print 'Received %r from %s' % (results, device)
        self.store_results(device, self.map_results(self.commands, results))

    #=======================================
    # Vendor-specific generate (to_)/parse (from_) methods
    #=======================================

    def to_juniper(self, device, commands=None, extra=None):
        """
        This just creates a series of ``<command>foo</command>`` elements to
        pass along to execute_junoscript()"""
        commands = commands or self.commands
        ret = []
        for command in commands:
            cmd = Element('command')
            cmd.text = command
            ret.append(cmd)

        return ret
示例#30
0
 def setUp(self):
     self.nd = NetDevices(with_acls=False)
     self.nodename = self.nd.keys()[0]
     self.device = self.nd.values()[0]