def discover_all(interactive=False, update_existing=False, outputs=None):
"""Runs discovery on all networks defined in the database."""
sanity_check()
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stdout = output.get(interactive)
nets = Network.objects.filter(
environment__isnull=False,
environment__queue__isnull=False,
)
for net in nets:
if interactive:
discover_network(
net.network,
interactive=True,
update_existing=True,
)
else:
queue = django_rq.get_queue()
queue.enqueue(
discover_network,
net.network,
update_existing=update_existing,
)
stdout()
def discover_all(interactive=False, update_existing=False, outputs=None):
"""Runs discovery on all networks defined in the database."""
sanity_check()
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stdout = output.get(interactive)
nets = Network.objects.filter(
environment__isnull=False,
environment__queue__isnull=False,
)
for net in nets:
if interactive:
discover_network(
net.network,
interactive=True,
update_existing=True,
)
else:
queue = django_rq.get_queue()
queue.enqueue(
discover_network,
net.network,
update_existing=update_existing,
)
stdout()
def discover_network(network,
plugin_name='ping',
requirements=None,
interactive=False,
update_existing=False,
outputs=None):
"""Runs discovery for a single `network`. The argument may be
an IPv[46]Network instance, a Network instance or a string
holding a network address or a network name defined in the database.
If `interactive` is False all output is omitted and discovery is done
asynchronously by pushing tasks to Rabbit.
If `update_existing` is True, only existing IPs from the specified
network are updated.
"""
sanity_check()
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stdout = output.get(interactive)
dbnet = None
if isinstance(network, (IPv4Network, IPv6Network)):
net = network
try:
dbnet = Network.objects.get(address=str(network))
except Network.DoesNotExist:
pass
elif isinstance(network, Network):
net = network.network
dbnet = network
else:
try:
network = Network.objects.get(address=network)
except Network.DoesNotExist:
network = Network.objects.get(name=network)
# if raises DoesNotExist here then so be it, user passed
# a non-existent network.
net = network.network
dbnet = network
if not dbnet or not dbnet.environment or not dbnet.environment.queue:
# Only do discover on networks that have a queue defined.
stdout(
"Skipping network {} -- no queue defined for this network "
"environment.".format(net), )
return
queue_name = dbnet.environment.queue.name
stdout("Scanning network {} started.".format(net))
if update_existing:
ip_address_queryset = IPAddress.objects.filter(number__gt=int(net.ip),
number__lt=int(
net.broadcast))
hosts = (i.address for i in ip_address_queryset)
else:
hosts = net.iterhosts()
for host in hosts:
discover_address(host, requirements, interactive, queue_name)
if interactive:
stdout()
else:
stdout('Scanning network {} finished.'.format(net))
def _run_plugin(context,
chain,
plugin_name,
requirements,
interactive,
clear_down,
done_requirements,
outputs=None):
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stdout = output.get(interactive)
stdout_verbose = output.get(interactive, verbose=True)
stderr = output.get(interactive, err=True)
message = "[{}] {}... ".format(plugin_name, context.get('ip', ''))
pad = output.WIDTH - len(message)
stdout(message, end='')
try:
new_context = {}
is_up, message, new_context = plugin.run(chain, plugin_name, **context)
if is_up:
requirements.add(plugin_name)
except plugin.Restart as e:
stdout('needs to be restarted: {}'.format(unicode(e)), end='\n')
raise
except Exception:
stdout('', end='\r')
stderr("Exception in plugin '{}' for '{}': {}".format(
plugin_name, context.get('ip', 'none'), traceback.format_exc()),
end='\n')
else:
message = message or ''
if clear_down and not is_up:
end = '\r'
else:
end = '\n'
pad -= len(message)
message = message + (" " * pad)
if is_up:
stdout(message, end=end)
else:
stdout_verbose(message, end=end)
finally:
done_requirements.add(plugin_name)
context.update(new_context)
def dummy_task(interactive=False, index=None):
stdout = output.get(interactive)
if index:
if not index % 25:
raise LookupError(
"You called {} and it failed on purpose.".format(index), )
stdout("Ping {}.".format(index))
else:
stdout("Ping.")
def _run_plugin(context, chain, plugin_name, requirements, interactive,
clear_down, done_requirements, outputs=None):
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stdout = output.get(interactive)
stdout_verbose = output.get(interactive, verbose=True)
stderr = output.get(interactive, err=True)
message = "[{}] {}... ".format(plugin_name, context.get('ip', ''))
pad = output.WIDTH - len(message)
stdout(message, end='')
try:
new_context = {}
is_up, message, new_context = plugin.run(chain, plugin_name,
**context)
if is_up:
requirements.add(plugin_name)
except plugin.Restart as e:
stdout('needs to be restarted: {}'.format(unicode(e)), end='\n')
raise
except Exception:
stdout('', end='\r')
stderr("{}\nException in plugin '{}' for '{}'.".format(
traceback.format_exc()),
plugin_name,
context.get('ip', 'none'),
end='\n')
else:
message = message or ''
if clear_down and not is_up:
end = '\r'
else:
end = '\n'
pad -= len(message)
message = message + (" " * pad)
if is_up:
stdout(message, end=end)
else:
stdout_verbose(message, end=end)
finally:
done_requirements.add(plugin_name)
context.update(new_context)
context['successful_plugins'] = ', '.join(sorted(requirements))
def dummy_task(interactive=False, index=None):
stdout = output.get(interactive)
if index:
if not index % 25:
raise LookupError(
"You called {} and it failed on purpose.".format(index),
)
stdout("Ping {}.".format(index))
else:
stdout("Ping.")
def discover_network(network, plugin_name='ping', requirements=None,
interactive=False, update_existing=False, outputs=None):
"""Runs discovery for a single `network`. The argument may be
an IPv[46]Network instance, a Network instance or a string
holding a network address or a network name defined in the database.
If `interactive` is False all output is omitted and discovery is done
asynchronously by pushing tasks to Rabbit.
If `update_existing` is True, only existing IPs from the specified
network are updated."""
sanity_check()
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stdout = output.get(interactive)
dbnet = None
if isinstance(network, (IPv4Network, IPv6Network)):
net = network
try:
dbnet = Network.objects.get(address=str(network))
except Network.DoesNotExist:
pass
elif isinstance(network, Network):
net = network.network
dbnet = network
else:
try:
network = Network.objects.get(address=network)
except Network.DoesNotExist:
network = Network.objects.get(name=network)
# if raises DoesNotExist here then so be it, user passed
# a non-existent network.
net = network.network
dbnet = network
stdout("Scanning network {} started.".format(net))
if update_existing:
ip_address_queryset = IPAddress.objects.filter(
number__gt=int(net.ip), number__lt=int(net.broadcast))
hosts = (i.address for i in ip_address_queryset)
else:
hosts = net.iterhosts()
for index, host in enumerate(hosts):
context = {'ip': host}
if dbnet:
context['queue'] = dbnet.queue
if interactive:
discover_single(context, plugin_name=plugin_name,
requirements=requirements, interactive=True)
else:
discover_single.delay(context, plugin_name=plugin_name,
requirements=requirements, clear_down=False)
if interactive:
stdout()
else:
stdout('Scanning network {} finished.'.format(net))
def discover_network(network, plugin_name='ping', requirements=None,
interactive=False, update_existing=False, outputs=None):
"""Runs discovery for a single `network`. The argument may be
an IPv[46]Network instance, a Network instance or a string
holding a network address or a network name defined in the database.
If `interactive` is False all output is omitted and discovery is done
asynchronously by pushing tasks to Rabbit.
If `update_existing` is True, only existing IPs from the specified
network are updated."""
sanity_check()
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stdout = output.get(interactive)
dbnet = None
if isinstance(network, (IPv4Network, IPv6Network)):
net = network
try:
dbnet = Network.objects.get(address=str(network))
except Network.DoesNotExist:
pass
elif isinstance(network, Network):
net = network.network
dbnet = network
else:
try:
network = Network.objects.get(address=network)
except Network.DoesNotExist:
network = Network.objects.get(name=network)
# if raises DoesNotExist here then so be it, user passed
# a non-existent network.
net = network.network
dbnet = network
stdout("Scanning network {} started.".format(net))
if update_existing:
ip_address_queryset = IPAddress.objects.filter(
number__gt=int(net.ip), number__lt=int(net.broadcast))
hosts = (i.address for i in ip_address_queryset)
else:
hosts = net.iterhosts()
for index, host in enumerate(hosts):
context = {'ip': host}
if dbnet:
context['queue'] = dbnet.queue.name
if interactive:
discover_single(context, plugin_name=plugin_name,
requirements=requirements, interactive=True)
else:
discover_single.delay(context, plugin_name=plugin_name,
requirements=requirements, clear_down=False)
if interactive:
stdout()
else:
stdout('Scanning network {} finished.'.format(net))
def discover_network(network, plugin_name='ping', requirements=None,
interactive=False, update_existing=False, outputs=None):
"""Runs discovery for a single `network`. The argument may be
an IPv[46]Network instance, a Network instance or a string
holding a network address or a network name defined in the database.
If `interactive` is False all output is omitted and discovery is done
asynchronously by pushing tasks to Rabbit.
If `update_existing` is True, only existing IPs from the specified
network are updated.
"""
sanity_check()
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stdout = output.get(interactive)
dbnet = None
if isinstance(network, (IPv4Network, IPv6Network)):
net = network
try:
dbnet = Network.objects.get(address=str(network))
except Network.DoesNotExist:
pass
elif isinstance(network, Network):
net = network.network
dbnet = network
else:
try:
network = Network.objects.get(address=network)
except Network.DoesNotExist:
network = Network.objects.get(name=network)
# if raises DoesNotExist here then so be it, user passed
# a non-existent network.
net = network.network
dbnet = network
if not dbnet or not dbnet.environment or not dbnet.environment.queue:
# Only do discover on networks that have a queue defined.
stdout(
"Skipping network {} -- no queue defined for this network "
"environment.".format(net),
)
return
queue_name = dbnet.environment.queue.name
stdout("Scanning network {} started.".format(net))
if update_existing:
ip_address_queryset = IPAddress.objects.filter(
number__gt=int(net.ip), number__lt=int(net.broadcast))
hosts = (i.address for i in ip_address_queryset)
else:
hosts = net.iterhosts()
for host in hosts:
discover_address(host, requirements, interactive, queue_name)
if interactive:
stdout()
else:
stdout('Scanning network {} finished.'.format(net))
def _run_plugin(context,
chain,
plugin_name,
requirements,
interactive,
done_requirements,
outputs=None):
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stdout = output.get(interactive)
stderr = output.get(interactive, err=True)
message = "[{}] {}... ".format(plugin_name, _get_uid(context))
stdout(message, end='')
new_context = {}
try:
is_up, message, new_context = plugin.run(chain, plugin_name, **context)
except plugin.Restart as e:
stdout('needs to be restarted: {}'.format(unicode(e)))
raise
except Exception:
stdout('', end='\r')
stderr(
"{}\nException in plugin '{}' for '{}'.".format(
traceback.format_exc(),
plugin_name,
_get_uid(context),
),
end='\n',
)
raise
else:
if message:
stdout(message, verbose=not is_up)
if is_up:
requirements.add(plugin_name)
context['successful_plugins'] = ', '.join(sorted(requirements))
context.update(new_context)
finally:
done_requirements.add(plugin_name)
def run_plugin(context,
chains,
plugin_name,
requirements=None,
interactive=False,
done_requirements=None,
restarts=MAX_RESTARTS,
outputs=None):
"""Synchronously runs a plugin named `plugin_name` from the first of the
specified `chains` using a given `context`. Automatically advances the
chain scheduling the next plugin to be run. When no plugins are left in the
current chain, advances to the next in the list.
If `interactive` is True, returns output on stdout and runs the next plugin
synchronously."""
if requirements is None:
requirements = set()
if done_requirements is None:
done_requirements = set()
restarted = False
if isinstance(chains, basestring):
raise NotImplementedError("API changed.")
chain = chains[0]
try:
_run_plugin(context, chain, plugin_name, requirements, interactive,
done_requirements, outputs)
except plugin.Restart as e:
if restarts > 0:
jitter = random.randint(30, 90)
after = timedelta(seconds=jitter)
run = _select_run_method(context, interactive, run_plugin, after)
run(context,
plugin_name,
requirements,
interactive,
done_requirements,
restarts=restarts - 1)
restarted = True
else:
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stderr = output.get(interactive, err=True)
stderr(
"Exceeded allowed number of restarts in plugin '{}' for "
"'{}': {}".format(plugin_name, _get_uid(context), unicode(e)),
end='\n',
)
finally:
if not restarted:
run_next_plugin(context, chains, requirements, interactive,
done_requirements, outputs)
def _run_plugin(context, chain, plugin_name, requirements, interactive, clear_down, done_requirements, outputs=None):
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stdout = output.get(interactive)
stdout_verbose = output.get(interactive, verbose=True)
stderr = output.get(interactive, err=True)
message = "[{}] {}... ".format(plugin_name, context["ip"])
pad = output.WIDTH - len(message)
stdout(message, end="")
try:
new_context = {}
is_up, message, new_context = plugin.run(chain, plugin_name, **context)
if is_up:
requirements.add(plugin_name)
except plugin.Restart as e:
stdout("needs to be restarted: {}".format(unicode(e)), end="\n")
raise
except Exception:
stdout("", end="\r")
stderr(
"Exception in plugin '{}' for '{}': {}".format(plugin_name, context["ip"], traceback.format_exc()), end="\n"
)
else:
message = message or ""
if clear_down and not is_up:
end = "\r"
else:
end = "\n"
pad -= len(message)
message = message + (" " * pad)
if is_up:
stdout(message, end=end)
else:
stdout_verbose(message, end=end)
finally:
done_requirements.add(plugin_name)
context.update(new_context)
def discover_all(interactive=False, update_existing=False, outputs=None):
"""Runs discovery on all networks defined in the database."""
sanity_check()
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stdout = output.get(interactive)
for net in Network.objects.exclude(queue=None).exclude(queue=""):
if interactive:
discover_network(net.network, interactive=True, update_existing=True)
else:
discover_network.delay(net.network, update_existing=update_existing)
stdout()
def _run_plugin(context, chain, plugin_name, requirements, interactive,
done_requirements, outputs=None):
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stdout = output.get(interactive)
stderr = output.get(interactive, err=True)
message = "[{}] {}... ".format(plugin_name, _get_uid(context))
stdout(message, end='')
new_context = {}
try:
is_up, message, new_context = plugin.run(chain, plugin_name,
**context)
except plugin.Restart as e:
stdout('needs to be restarted: {}'.format(unicode(e)))
raise
except Exception:
stdout('', end='\r')
stderr(
"{}\nException in plugin '{}' for '{}'.".format(
traceback.format_exc(),
plugin_name,
_get_uid(context),
),
end='\n',
)
raise
else:
if message:
stdout(message, verbose=not is_up)
if is_up:
requirements.add(plugin_name)
context['successful_plugins'] = ', '.join(sorted(requirements))
context.update(new_context)
finally:
done_requirements.add(plugin_name)
def discover_all(interactive=False, update_existing=False, outputs=None):
"""Runs discovery on all networks defined in the database."""
sanity_check()
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stdout = output.get(interactive)
for net in Network.objects.exclude(queue=None).exclude(queue=''):
if interactive:
discover_network(net.network, interactive=True,
update_existing=True)
else:
discover_network.delay(net.network,
update_existing=update_existing)
stdout()
def discover_single(
context,
plugin_name="ping",
requirements=None,
interactive=False,
clear_down=True,
done_requirements=None,
restarts=MAX_RESTARTS,
outputs=None,
):
"""
Runs discovery on a single `ip`. If `interactive` is True, returns
output on stdout. In that case, when `clear_down` is True (the default),
lines listing addresses of hosts which are down are cleared from output.
"""
if requirements is None:
requirements = set()
if done_requirements is None:
done_requirements = set()
restarted = False
try:
_run_plugin(
context, "discovery", plugin_name, requirements, interactive, clear_down, done_requirements, outputs
)
except plugin.Restart as e:
if restarts > 0:
discover = discover_single
if not interactive:
discover = discover.delay
discover(
context, plugin_name, requirements, interactive, clear_down, done_requirements, restarts=restarts - 1
)
restarted = True
else:
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stderr = output.get(interactive, err=True)
stderr(
"Exceeded allowed number of restarts in plugin '{}' for "
"'{}': {}".format(plugin_name, context["ip"], unicode(e)),
end="\n",
)
if not restarted:
run_next_plugin(context, requirements, interactive, clear_down, done_requirements, outputs)
def discover_single(context,
plugin_name='ping',
requirements=None,
interactive=False,
clear_down=True,
done_requirements=None,
restarts=MAX_RESTARTS,
outputs=None):
"""
Runs discovery on a single `ip`. If `interactive` is True, returns
output on stdout. In that case, when `clear_down` is True (the default),
lines listing addresses of hosts which are down are cleared from output.
"""
if requirements is None:
requirements = set()
if done_requirements is None:
done_requirements = set()
restarted = False
try:
_run_plugin(context, 'discovery', plugin_name, requirements,
interactive, clear_down, done_requirements, outputs)
except plugin.Restart as e:
if restarts > 0:
discover = discover_single
if not interactive:
discover = discover.delay
discover(context,
plugin_name,
requirements,
interactive,
clear_down,
done_requirements,
restarts=restarts - 1)
restarted = True
else:
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stderr = output.get(interactive, err=True)
stderr("Exceeded allowed number of restarts in plugin '{}' for "
"'{}': {}".format(plugin_name, context['ip'], unicode(e)),
end='\n')
if not restarted:
run_next_plugin(context, requirements, interactive, clear_down,
done_requirements, outputs)
def run_plugin(context, chains, plugin_name,
requirements=None, interactive=False, done_requirements=None,
restarts=MAX_RESTARTS, outputs=None):
"""Synchronously runs a plugin named `plugin_name` from the first of the
specified `chains` using a given `context`. Automatically advances the
chain scheduling the next plugin to be run. When no plugins are left in the
current chain, advances to the next in the list.
If `interactive` is True, returns output on stdout and runs the next plugin
synchronously."""
if requirements is None:
requirements = set()
if done_requirements is None:
done_requirements = set()
restarted = False
if isinstance(chains, basestring):
raise NotImplementedError("API changed.")
chain = chains[0]
try:
_run_plugin(context, chain, plugin_name, requirements, interactive,
done_requirements, outputs)
except plugin.Restart as e:
if restarts > 0:
jitter = random.randint(30, 90)
after = timedelta(seconds=jitter)
run = _select_run_method(context, interactive, run_plugin, after)
run(context, plugin_name, requirements, interactive,
done_requirements, restarts=restarts - 1)
restarted = True
else:
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stderr = output.get(interactive, err=True)
stderr(
"Exceeded allowed number of restarts in plugin '{}' for "
"'{}': {}".format(plugin_name, _get_uid(context), unicode(e)),
end='\n',
)
finally:
if not restarted:
run_next_plugin(context, chains, requirements, interactive,
done_requirements, outputs)
def dummy_task(interactive=False, index=None):
stdout = output.get(interactive)
if index:
stdout("Ping {}.".format(index))
else:
stdout("Ping.")
def dummy_task(interactive=False, index=None):
stdout = output.get(interactive)
if index:
stdout("Ping {}.".format(index))
else:
stdout("Ping.")
