def discover_all(interactive=False, update_existing=False, outputs=None): """Runs discovery on all networks defined in the database.""" sanity_check() if outputs: stdout, stdout_verbose, stderr = outputs else: stdout = output.get(interactive) nets = Network.objects.filter( environment__isnull=False, environment__queue__isnull=False, ) for net in nets: if interactive: discover_network( net.network, interactive=True, update_existing=True, ) else: queue = django_rq.get_queue() queue.enqueue( discover_network, net.network, update_existing=update_existing, ) stdout()
def discover_network(network, plugin_name='ping', requirements=None, interactive=False, update_existing=False, outputs=None): """Runs discovery for a single `network`. The argument may be an IPv[46]Network instance, a Network instance or a string holding a network address or a network name defined in the database. If `interactive` is False all output is omitted and discovery is done asynchronously by pushing tasks to Rabbit. If `update_existing` is True, only existing IPs from the specified network are updated. """ sanity_check() if outputs: stdout, stdout_verbose, stderr = outputs else: stdout = output.get(interactive) dbnet = None if isinstance(network, (IPv4Network, IPv6Network)): net = network try: dbnet = Network.objects.get(address=str(network)) except Network.DoesNotExist: pass elif isinstance(network, Network): net = network.network dbnet = network else: try: network = Network.objects.get(address=network) except Network.DoesNotExist: network = Network.objects.get(name=network) # if raises DoesNotExist here then so be it, user passed # a non-existent network. net = network.network dbnet = network if not dbnet or not dbnet.environment or not dbnet.environment.queue: # Only do discover on networks that have a queue defined. stdout( "Skipping network {} -- no queue defined for this network " "environment.".format(net), ) return queue_name = dbnet.environment.queue.name stdout("Scanning network {} started.".format(net)) if update_existing: ip_address_queryset = IPAddress.objects.filter(number__gt=int(net.ip), number__lt=int( net.broadcast)) hosts = (i.address for i in ip_address_queryset) else: hosts = net.iterhosts() for host in hosts: discover_address(host, requirements, interactive, queue_name) if interactive: stdout() else: stdout('Scanning network {} finished.'.format(net))
def _run_plugin(context, chain, plugin_name, requirements, interactive, clear_down, done_requirements, outputs=None): if outputs: stdout, stdout_verbose, stderr = outputs else: stdout = output.get(interactive) stdout_verbose = output.get(interactive, verbose=True) stderr = output.get(interactive, err=True) message = "[{}] {}... ".format(plugin_name, context.get('ip', '')) pad = output.WIDTH - len(message) stdout(message, end='') try: new_context = {} is_up, message, new_context = plugin.run(chain, plugin_name, **context) if is_up: requirements.add(plugin_name) except plugin.Restart as e: stdout('needs to be restarted: {}'.format(unicode(e)), end='\n') raise except Exception: stdout('', end='\r') stderr("Exception in plugin '{}' for '{}': {}".format( plugin_name, context.get('ip', 'none'), traceback.format_exc()), end='\n') else: message = message or '' if clear_down and not is_up: end = '\r' else: end = '\n' pad -= len(message) message = message + (" " * pad) if is_up: stdout(message, end=end) else: stdout_verbose(message, end=end) finally: done_requirements.add(plugin_name) context.update(new_context)
def dummy_task(interactive=False, index=None): stdout = output.get(interactive) if index: if not index % 25: raise LookupError( "You called {} and it failed on purpose.".format(index), ) stdout("Ping {}.".format(index)) else: stdout("Ping.")
def _run_plugin(context, chain, plugin_name, requirements, interactive, clear_down, done_requirements, outputs=None): if outputs: stdout, stdout_verbose, stderr = outputs else: stdout = output.get(interactive) stdout_verbose = output.get(interactive, verbose=True) stderr = output.get(interactive, err=True) message = "[{}] {}... ".format(plugin_name, context.get('ip', '')) pad = output.WIDTH - len(message) stdout(message, end='') try: new_context = {} is_up, message, new_context = plugin.run(chain, plugin_name, **context) if is_up: requirements.add(plugin_name) except plugin.Restart as e: stdout('needs to be restarted: {}'.format(unicode(e)), end='\n') raise except Exception: stdout('', end='\r') stderr("{}\nException in plugin '{}' for '{}'.".format( traceback.format_exc()), plugin_name, context.get('ip', 'none'), end='\n') else: message = message or '' if clear_down and not is_up: end = '\r' else: end = '\n' pad -= len(message) message = message + (" " * pad) if is_up: stdout(message, end=end) else: stdout_verbose(message, end=end) finally: done_requirements.add(plugin_name) context.update(new_context) context['successful_plugins'] = ', '.join(sorted(requirements))
def discover_network(network, plugin_name='ping', requirements=None, interactive=False, update_existing=False, outputs=None): """Runs discovery for a single `network`. The argument may be an IPv[46]Network instance, a Network instance or a string holding a network address or a network name defined in the database. If `interactive` is False all output is omitted and discovery is done asynchronously by pushing tasks to Rabbit. If `update_existing` is True, only existing IPs from the specified network are updated.""" sanity_check() if outputs: stdout, stdout_verbose, stderr = outputs else: stdout = output.get(interactive) dbnet = None if isinstance(network, (IPv4Network, IPv6Network)): net = network try: dbnet = Network.objects.get(address=str(network)) except Network.DoesNotExist: pass elif isinstance(network, Network): net = network.network dbnet = network else: try: network = Network.objects.get(address=network) except Network.DoesNotExist: network = Network.objects.get(name=network) # if raises DoesNotExist here then so be it, user passed # a non-existent network. net = network.network dbnet = network stdout("Scanning network {} started.".format(net)) if update_existing: ip_address_queryset = IPAddress.objects.filter( number__gt=int(net.ip), number__lt=int(net.broadcast)) hosts = (i.address for i in ip_address_queryset) else: hosts = net.iterhosts() for index, host in enumerate(hosts): context = {'ip': host} if dbnet: context['queue'] = dbnet.queue if interactive: discover_single(context, plugin_name=plugin_name, requirements=requirements, interactive=True) else: discover_single.delay(context, plugin_name=plugin_name, requirements=requirements, clear_down=False) if interactive: stdout() else: stdout('Scanning network {} finished.'.format(net))
def discover_network(network, plugin_name='ping', requirements=None, interactive=False, update_existing=False, outputs=None): """Runs discovery for a single `network`. The argument may be an IPv[46]Network instance, a Network instance or a string holding a network address or a network name defined in the database. If `interactive` is False all output is omitted and discovery is done asynchronously by pushing tasks to Rabbit. If `update_existing` is True, only existing IPs from the specified network are updated.""" sanity_check() if outputs: stdout, stdout_verbose, stderr = outputs else: stdout = output.get(interactive) dbnet = None if isinstance(network, (IPv4Network, IPv6Network)): net = network try: dbnet = Network.objects.get(address=str(network)) except Network.DoesNotExist: pass elif isinstance(network, Network): net = network.network dbnet = network else: try: network = Network.objects.get(address=network) except Network.DoesNotExist: network = Network.objects.get(name=network) # if raises DoesNotExist here then so be it, user passed # a non-existent network. net = network.network dbnet = network stdout("Scanning network {} started.".format(net)) if update_existing: ip_address_queryset = IPAddress.objects.filter( number__gt=int(net.ip), number__lt=int(net.broadcast)) hosts = (i.address for i in ip_address_queryset) else: hosts = net.iterhosts() for index, host in enumerate(hosts): context = {'ip': host} if dbnet: context['queue'] = dbnet.queue.name if interactive: discover_single(context, plugin_name=plugin_name, requirements=requirements, interactive=True) else: discover_single.delay(context, plugin_name=plugin_name, requirements=requirements, clear_down=False) if interactive: stdout() else: stdout('Scanning network {} finished.'.format(net))
def discover_network(network, plugin_name='ping', requirements=None, interactive=False, update_existing=False, outputs=None): """Runs discovery for a single `network`. The argument may be an IPv[46]Network instance, a Network instance or a string holding a network address or a network name defined in the database. If `interactive` is False all output is omitted and discovery is done asynchronously by pushing tasks to Rabbit. If `update_existing` is True, only existing IPs from the specified network are updated. """ sanity_check() if outputs: stdout, stdout_verbose, stderr = outputs else: stdout = output.get(interactive) dbnet = None if isinstance(network, (IPv4Network, IPv6Network)): net = network try: dbnet = Network.objects.get(address=str(network)) except Network.DoesNotExist: pass elif isinstance(network, Network): net = network.network dbnet = network else: try: network = Network.objects.get(address=network) except Network.DoesNotExist: network = Network.objects.get(name=network) # if raises DoesNotExist here then so be it, user passed # a non-existent network. net = network.network dbnet = network if not dbnet or not dbnet.environment or not dbnet.environment.queue: # Only do discover on networks that have a queue defined. stdout( "Skipping network {} -- no queue defined for this network " "environment.".format(net), ) return queue_name = dbnet.environment.queue.name stdout("Scanning network {} started.".format(net)) if update_existing: ip_address_queryset = IPAddress.objects.filter( number__gt=int(net.ip), number__lt=int(net.broadcast)) hosts = (i.address for i in ip_address_queryset) else: hosts = net.iterhosts() for host in hosts: discover_address(host, requirements, interactive, queue_name) if interactive: stdout() else: stdout('Scanning network {} finished.'.format(net))
def _run_plugin(context, chain, plugin_name, requirements, interactive, done_requirements, outputs=None): if outputs: stdout, stdout_verbose, stderr = outputs else: stdout = output.get(interactive) stderr = output.get(interactive, err=True) message = "[{}] {}... ".format(plugin_name, _get_uid(context)) stdout(message, end='') new_context = {} try: is_up, message, new_context = plugin.run(chain, plugin_name, **context) except plugin.Restart as e: stdout('needs to be restarted: {}'.format(unicode(e))) raise except Exception: stdout('', end='\r') stderr( "{}\nException in plugin '{}' for '{}'.".format( traceback.format_exc(), plugin_name, _get_uid(context), ), end='\n', ) raise else: if message: stdout(message, verbose=not is_up) if is_up: requirements.add(plugin_name) context['successful_plugins'] = ', '.join(sorted(requirements)) context.update(new_context) finally: done_requirements.add(plugin_name)
def run_plugin(context, chains, plugin_name, requirements=None, interactive=False, done_requirements=None, restarts=MAX_RESTARTS, outputs=None): """Synchronously runs a plugin named `plugin_name` from the first of the specified `chains` using a given `context`. Automatically advances the chain scheduling the next plugin to be run. When no plugins are left in the current chain, advances to the next in the list. If `interactive` is True, returns output on stdout and runs the next plugin synchronously.""" if requirements is None: requirements = set() if done_requirements is None: done_requirements = set() restarted = False if isinstance(chains, basestring): raise NotImplementedError("API changed.") chain = chains[0] try: _run_plugin(context, chain, plugin_name, requirements, interactive, done_requirements, outputs) except plugin.Restart as e: if restarts > 0: jitter = random.randint(30, 90) after = timedelta(seconds=jitter) run = _select_run_method(context, interactive, run_plugin, after) run(context, plugin_name, requirements, interactive, done_requirements, restarts=restarts - 1) restarted = True else: if outputs: stdout, stdout_verbose, stderr = outputs else: stderr = output.get(interactive, err=True) stderr( "Exceeded allowed number of restarts in plugin '{}' for " "'{}': {}".format(plugin_name, _get_uid(context), unicode(e)), end='\n', ) finally: if not restarted: run_next_plugin(context, chains, requirements, interactive, done_requirements, outputs)
def _run_plugin(context, chain, plugin_name, requirements, interactive, clear_down, done_requirements, outputs=None): if outputs: stdout, stdout_verbose, stderr = outputs else: stdout = output.get(interactive) stdout_verbose = output.get(interactive, verbose=True) stderr = output.get(interactive, err=True) message = "[{}] {}... ".format(plugin_name, context["ip"]) pad = output.WIDTH - len(message) stdout(message, end="") try: new_context = {} is_up, message, new_context = plugin.run(chain, plugin_name, **context) if is_up: requirements.add(plugin_name) except plugin.Restart as e: stdout("needs to be restarted: {}".format(unicode(e)), end="\n") raise except Exception: stdout("", end="\r") stderr( "Exception in plugin '{}' for '{}': {}".format(plugin_name, context["ip"], traceback.format_exc()), end="\n" ) else: message = message or "" if clear_down and not is_up: end = "\r" else: end = "\n" pad -= len(message) message = message + (" " * pad) if is_up: stdout(message, end=end) else: stdout_verbose(message, end=end) finally: done_requirements.add(plugin_name) context.update(new_context)
def discover_all(interactive=False, update_existing=False, outputs=None): """Runs discovery on all networks defined in the database.""" sanity_check() if outputs: stdout, stdout_verbose, stderr = outputs else: stdout = output.get(interactive) for net in Network.objects.exclude(queue=None).exclude(queue=""): if interactive: discover_network(net.network, interactive=True, update_existing=True) else: discover_network.delay(net.network, update_existing=update_existing) stdout()
def discover_all(interactive=False, update_existing=False, outputs=None): """Runs discovery on all networks defined in the database.""" sanity_check() if outputs: stdout, stdout_verbose, stderr = outputs else: stdout = output.get(interactive) for net in Network.objects.exclude(queue=None).exclude(queue=''): if interactive: discover_network(net.network, interactive=True, update_existing=True) else: discover_network.delay(net.network, update_existing=update_existing) stdout()
def discover_single( context, plugin_name="ping", requirements=None, interactive=False, clear_down=True, done_requirements=None, restarts=MAX_RESTARTS, outputs=None, ): """ Runs discovery on a single `ip`. If `interactive` is True, returns output on stdout. In that case, when `clear_down` is True (the default), lines listing addresses of hosts which are down are cleared from output. """ if requirements is None: requirements = set() if done_requirements is None: done_requirements = set() restarted = False try: _run_plugin( context, "discovery", plugin_name, requirements, interactive, clear_down, done_requirements, outputs ) except plugin.Restart as e: if restarts > 0: discover = discover_single if not interactive: discover = discover.delay discover( context, plugin_name, requirements, interactive, clear_down, done_requirements, restarts=restarts - 1 ) restarted = True else: if outputs: stdout, stdout_verbose, stderr = outputs else: stderr = output.get(interactive, err=True) stderr( "Exceeded allowed number of restarts in plugin '{}' for " "'{}': {}".format(plugin_name, context["ip"], unicode(e)), end="\n", ) if not restarted: run_next_plugin(context, requirements, interactive, clear_down, done_requirements, outputs)
def discover_single(context, plugin_name='ping', requirements=None, interactive=False, clear_down=True, done_requirements=None, restarts=MAX_RESTARTS, outputs=None): """ Runs discovery on a single `ip`. If `interactive` is True, returns output on stdout. In that case, when `clear_down` is True (the default), lines listing addresses of hosts which are down are cleared from output. """ if requirements is None: requirements = set() if done_requirements is None: done_requirements = set() restarted = False try: _run_plugin(context, 'discovery', plugin_name, requirements, interactive, clear_down, done_requirements, outputs) except plugin.Restart as e: if restarts > 0: discover = discover_single if not interactive: discover = discover.delay discover(context, plugin_name, requirements, interactive, clear_down, done_requirements, restarts=restarts - 1) restarted = True else: if outputs: stdout, stdout_verbose, stderr = outputs else: stderr = output.get(interactive, err=True) stderr("Exceeded allowed number of restarts in plugin '{}' for " "'{}': {}".format(plugin_name, context['ip'], unicode(e)), end='\n') if not restarted: run_next_plugin(context, requirements, interactive, clear_down, done_requirements, outputs)
def dummy_task(interactive=False, index=None): stdout = output.get(interactive) if index: stdout("Ping {}.".format(index)) else: stdout("Ping.")