def top(self, nodes):
results = cmdresult.CmdResult()
for (node, error, vals) in self.get_top_output(nodes):
top_info = {
"name": node.name,
"type": node.type,
"host": node.host,
"pid": None,
"proc": None,
"vsize": None,
"rss": None,
"cpu": None,
"cmd": None,
"error": None
}
if error:
top_info["error"] = error
results.set_node_data(node, False, {"procs": [top_info]})
continue
proclist = []
for d in vals:
top_info2 = top_info.copy()
top_info2.update(d)
proclist.append(top_info2)
results.set_node_data(node, True, {"procs": proclist})
return results
def update(self, nodes):
results = cmdresult.CmdResult()
running = self._isrunning(nodes)
zone = self.config.zoneid
if not zone:
zone = "NOZONE"
cmds = []
for (node, isrunning) in running:
if isrunning:
env = _make_env_params(node)
env += " BRO_DNS_FAKE=1"
args = " ".join(_make_bro_params(node, False))
cmds += [(node.name,
os.path.join(self.config.scriptsdir, "update") +
" %s %s %s/tcp %s" % (util.format_bro_addr(
node.addr), zone, node.getPort(), args), env,
None)]
self.ui.info("updating %s ..." % node.name)
res = execute.run_localcmds(cmds)
for (tag, success, output) in res:
node = self.config.nodes(tag=tag)[0]
if not success:
self.ui.info("failed to update %s: %s" % (tag, output[0]))
results.set_node_fail(node)
else:
self.ui.info("%s: %s" % (tag, output[0]))
results.set_node_success(node)
return results
def deploy(self):
if not self.plugins.cmdPre("deploy"):
results = cmdresult.CmdResult(ok=False)
return results
if self.config.is_cfg_changed():
self.ui.info("Reloading broctl configuration ...")
self.reload_cfg()
self.ui.info("checking configurations ...")
results = self.check(check_node_types=True)
if not results.ok:
for (node, success, output) in results.get_node_output():
if not success:
self.ui.info("%s scripts failed." % node)
self.ui.info(output)
return results
self.ui.info("installing ...")
results = self.install()
if not results.ok:
return results
self.ui.info("stopping ...")
results = self.stop()
if not results.ok:
return results
self.ui.info("starting ...")
results = self.start()
self.plugins.cmdPost("deploy")
return results
def process(self, trace, bro_options, bro_scripts):
results = cmdresult.CmdResult()
if not os.path.isfile(trace):
self.ui.error("trace file not found: %s" % trace)
results.ok = False
return results
standalone = (self.config.standalone == "1")
if standalone:
tag = "standalone"
else:
tag = "workers"
node = self.config.nodes(tag=tag)[0]
cwd = os.path.join(self.config.tmpdir, "testing")
if os.path.isdir(cwd):
try:
shutil.rmtree(cwd)
except OSError as err:
self.ui.error("cannot remove directory: %s" % err)
results.ok = False
return results
try:
os.makedirs(cwd)
except OSError as err:
self.ui.error("cannot create directory: %s" % err)
results.ok = False
return results
env = _make_env_params(node)
bro_args = " ".join(bro_options + _make_bro_params(node, False))
bro_args += " broctl/process-trace"
if bro_scripts:
bro_args += " " + " ".join(bro_scripts)
cmd = os.path.join(
self.config.scriptsdir,
"run-bro-on-trace") + " %s %s %s %s" % (0, cwd, trace, bro_args)
self.ui.info(cmd)
(success, output) = execute.run_localcmd(cmd,
env,
donotcaptureoutput=True)
if not success:
results.ok = False
for line in output:
self.ui.info(line)
self.ui.info("\n### Bro output in %s" % cwd)
return results
def install(self, local=False):
if self.plugins.cmdPre("install"):
results = self.controller.install(local)
else:
results = cmdresult.CmdResult(ok=False)
self.plugins.cmdPost("install")
return results
def process(self, trace, options, scripts):
if self.plugins.cmdPre("process", trace, options, scripts):
results = self.controller.process(trace, options, scripts)
else:
results = cmdresult.CmdResult(ok=False)
self.plugins.cmdPost("process", trace, options, scripts, results.ok)
return results
def execute_cmd(self, nodes, cmd):
results = cmdresult.CmdResult()
for node, success, out in self.executor.run_shell_cmds([
(n, cmd) for n in nodes
]):
results.set_node_output(node, success, out)
return results
def df(self, nodes):
results = cmdresult.CmdResult()
DiskInfo = namedtuple("DiskInfo",
("fs", "total", "used", "available", "percent"))
dirs = ("logdir", "bindir", "helperdir", "cfgdir", "spooldir",
"policydir", "libdir", "tmpdir", "staticdir", "scriptsdir")
df = {}
for node in nodes:
df[node.name] = {}
cmds = []
for node in nodes:
for key in dirs:
if key == "logdir" and node.type not in ("manager",
"standalone"):
# Don't need this on the workers/proxies.
continue
path = self.config.config[key]
cmds += [(node, "df", [path])]
res = self.executor.run_helper(cmds)
for (node, success, output) in res:
if success:
if not output:
df[node.name]["FAIL"] = "no output from df helper"
continue
fields = output[0].split()
fs = fields[0]
# Ignore NFS mounted volumes.
if ":" in fs:
continue
total = float(fields[1])
used = float(fields[2])
avail = float(fields[3])
perc = used * 100.0 / (used + avail)
df[node.name][fs] = DiskInfo(fs, total, used, avail, perc)
else:
if output:
msg = output[0]
else:
msg = "unknown failure"
df[node.name]["FAIL"] = msg
for node in nodes:
success = "FAIL" not in df[node.name]
results.set_node_data(node, success, df[node.name])
return results
def peerstatus(self, nodes):
results = cmdresult.CmdResult()
for (node, success, args) in self._query_peerstatus(nodes):
if success:
out = args[0]
else:
out = args
results.set_node_output(node, success, out)
return results
def netstats(self, nodes):
results = cmdresult.CmdResult()
for (node, success, args) in self._query_netstats(nodes):
if success:
out = args[0].strip()
else:
out = args
results.set_node_output(node, success, out)
return results
def get_config(self):
results = cmdresult.CmdResult()
if self.plugins.cmdPre("config"):
results.keyval = self.config.options()
else:
results.ok = False
self.plugins.cmdPost("config")
return results
def _check_config(self, nodes, installed, list_scripts):
results = cmdresult.CmdResult()
nodetmpdirs = [(node,
os.path.join(self.config.tmpdir,
"check-config-%s" % node.name))
for node in nodes]
nodes = []
for (node, cwd) in nodetmpdirs:
if os.path.isdir(cwd):
try:
shutil.rmtree(cwd)
except OSError as err:
self.ui.error("cannot remove directory: %s" % err)
results.ok = False
return results
try:
os.makedirs(cwd)
except OSError as err:
self.ui.error("cannot create temporary directory: %s" % err)
results.ok = False
return results
nodes += [(node, cwd)]
cmds = []
for (node, cwd) in nodes:
env = _make_env_params(node)
installed_policies = installed and "1" or "0"
print_scripts = list_scripts and "1" or "0"
install.make_layout(cwd, self.ui, True)
if not install.make_local_networks(cwd, self.ui, True):
results.ok = False
return results
install.make_broctl_config_policy(cwd, self.ui, True)
cmd = os.path.join(
self.config.scriptsdir, "check-config") + " %s %s %s %s" % (
installed_policies, print_scripts, cwd, " ".join(
_make_bro_params(node, False)))
cmd += " broctl/check"
cmds += [((node, cwd), cmd, env, None)]
for ((node, cwd), success, output) in execute.run_localcmds(cmds):
results.set_node_output(node, success, output)
shutil.rmtree(cwd)
return results
def capstats(self, nodes, interval):
results = cmdresult.CmdResult()
if self.config.capstatspath:
for (node, netif, success,
vals) in self.get_capstats_output(nodes, interval):
if not success:
vals = {"output": vals}
results.set_node_data(node, success, vals)
return results
def execute(self, cmd):
nodes = self.node_args(get_hosts=True)
if self.plugins.cmdPre("exec", cmd):
results = self.controller.execute_cmd(nodes, cmd)
else:
results = cmdresult.CmdResult(ok=False)
self.plugins.cmdPost("exec", cmd)
return results
def nodes(self):
results = cmdresult.CmdResult()
if self.plugins.cmdPre("nodes"):
for n in self.config.nodes():
results.set_node_data(n, True, n.to_dict())
else:
results.ok = False
self.plugins.cmdPost("nodes")
return results
def runCustomCommand(self, cmd, args, cmdout):
"""Runs a custom command *cmd* with string *args* as argument. Returns
a CmdResult object which contains the command results."""
try:
myplugin, usage, descr = self._cmds[cmd]
except LookupError:
return cmdresult.CmdResult(ok=False, unknowncmd=True)
prefix = myplugin.prefix()
if cmd.startswith("%s." % prefix):
cmd = cmd[len(prefix)+1:]
return myplugin.cmd_custom(cmd, args, cmdout)
def print_id(self, nodes, id):
running = self._isrunning(nodes)
eventlist = []
for (node, isrunning) in running:
if isrunning:
eventlist += [(node, "Control::id_value_request", [id],
"Control::id_value_response")]
results = cmdresult.CmdResult()
for (node, success, args) in events.send_events_parallel(eventlist):
results.set_node_output(node, success, args)
return results
def diag(self, nodes):
results = cmdresult.CmdResult()
crashdiag = os.path.join(self.config.scriptsdir, "crash-diag")
cmds = [(node, crashdiag, [node.cwd()]) for node in nodes]
for (node, success, output) in self.executor.run_cmds(cmds):
if not success:
errmsgs = ["error running crash-diag for %s" % node.name]
errmsgs += output
results.set_node_output(node, False, errmsgs)
continue
results.set_node_output(node, True, output)
return results
def cleanup(self, nodes, cleantmp=False):
# Given a set of node names "orig" and command results "res", add
# all node names to "orig" that have a failed result in "res".
def addfailed(orig, res):
for (node, status, output) in res:
# if status is Fail, then add the node name
if not status:
orig.add(node.name)
return orig
results = cmdresult.CmdResult()
result = self._isrunning(nodes)
running = [node for (node, on) in result if on]
notrunning = [node for (node, on) in result if not on]
for node in running:
self.ui.info(
" %s is still running, not cleaning work directory" % node)
results1 = self.executor.rmdirs([(n, n.cwd()) for n in notrunning])
results2 = self.executor.mkdirs([(n, n.cwd()) for n in notrunning])
failed = set()
failed = addfailed(failed, results1)
failed = addfailed(failed, results2)
for node in notrunning:
node.clearCrashed()
if cleantmp:
self.ui.info("cleaning %s ..." % self.config.tmpdir)
results3 = self.executor.rmdirs([(n, self.config.tmpdir)
for n in running + notrunning])
results4 = self.executor.mkdirs([(n, self.config.tmpdir)
for n in running + notrunning])
failed = addfailed(failed, results3)
failed = addfailed(failed, results4)
for node in nodes:
if node.name in failed:
results.set_node_fail(node)
else:
results.set_node_success(node)
return results
def cleanup(self, nodes, cleantmp=False):
def addfailed(orig, res):
for (n, status) in res:
if not status:
orig.add(n.name)
return orig
results = cmdresult.CmdResult()
result = self._isrunning(nodes)
running = [node for (node, on) in result if on]
notrunning = [node for (node, on) in result if not on]
for node in running:
self.ui.info(
" %s is still running, not cleaning work directory" % node)
results1 = self.executor.rmdirs([(n, n.cwd()) for n in notrunning])
results2 = self.executor.mkdirs([(n, n.cwd()) for n in notrunning])
failed = set()
failed = addfailed(failed, results1)
failed = addfailed(failed, results2)
for node in notrunning:
node.clearCrashed()
if cleantmp:
results3 = self.executor.rmdirs([(n, self.config.tmpdir)
for n in running + notrunning])
results4 = self.executor.mkdirs([(n, self.config.tmpdir)
for n in running + notrunning])
failed = addfailed(failed, results3)
failed = addfailed(failed, results4)
for node in nodes:
if node.name in failed:
results.set_node_fail(node)
else:
results.set_node_success(node)
return results
def stop(self, nodes):
results = cmdresult.CmdResult()
manager = []
proxies = []
workers = []
for n in nodes:
n.setExpectRunning(False)
if n.type == "worker":
workers += [n]
elif n.type == "proxy":
proxies += [n]
else:
manager += [n]
# Stop nodes. Do it in the order workers, proxies, manager
# (the reverse of "start").
if workers:
self._stop_nodes(workers, results)
if not results.ok:
for n in (proxies + manager):
results.set_node_fail(n)
return results
if proxies:
self._stop_nodes(proxies, results)
if not results.ok:
for n in manager:
results.set_node_fail(n)
return results
if manager:
self._stop_nodes(manager, results)
return results
def start(self, nodes):
results = cmdresult.CmdResult()
manager = []
proxies = []
workers = []
for n in nodes:
n.setExpectRunning(True)
if n.type == "worker":
workers += [n]
elif n.type == "proxy":
proxies += [n]
else:
manager += [n]
# Start nodes. Do it in the order manager, proxies, workers.
if manager:
self._start_nodes(manager, results)
if not results.ok:
for n in (proxies + workers):
results.set_node_fail(n)
return results
if proxies:
self._start_nodes(proxies, results)
if not results.ok:
for n in workers:
results.set_node_fail(n)
return results
if workers:
self._start_nodes(workers, results)
return results
def install(self, local_only):
results = cmdresult.CmdResult()
try:
self.config.record_bro_version()
except config.ConfigurationError as err:
self.ui.error("%s" % err)
results.ok = False
return results
manager = self.config.manager()
# Delete previously installed policy files to not mix things up.
policies = [
self.config.policydirsiteinstall,
self.config.policydirsiteinstallauto
]
for dirpath in policies:
if os.path.isdir(dirpath):
self.ui.info("removing old policies in %s ..." % dirpath)
try:
shutil.rmtree(dirpath)
except OSError as err:
self.ui.error("failed to remove directory: %s" % err)
results.ok = False
return results
self.ui.info("creating policy directories ...")
for dirpath in policies:
try:
os.makedirs(dirpath)
except OSError as err:
self.ui.error("failed to create directory: %s" % err)
results.ok = False
return results
# Install local site policy.
if self.config.sitepolicypath:
self.ui.info("installing site policies ...")
dst = self.config.policydirsiteinstall
for dir in self.config.sitepolicypath.split(":"):
dirpath = self.config.subst(dir)
for pathname in glob.glob(os.path.join(dirpath, "*")):
if not execute.install(pathname, dst, self.ui):
results.ok = False
return results
install.make_layout(self.config.policydirsiteinstallauto, self.ui)
self.ui.info("generating local-networks.bro ...")
if not install.make_local_networks(
self.config.policydirsiteinstallauto, self.ui):
results.ok = False
return results
self.ui.info("generating broctl-config.bro ...")
install.make_broctl_config_policy(self.config.policydirsiteinstallauto,
self.ui)
current = self.config.subst(os.path.join(self.config.logdir,
"current"))
try:
util.force_symlink(manager.cwd(), current)
except (IOError, OSError) as err:
results.ok = False
self.ui.error("failed to update symlink '%s': %s" % (current, err))
return results
self.ui.info("generating broctl-config.sh ...")
if not install.make_broctl_config_sh(self.ui):
results.ok = False
return results
if local_only:
return results
# Make sure we install each remote host only once.
nodes = self.config.hosts(nolocal=True)
# If there are no remote hosts, then we're done.
if not nodes:
# Save current configuration state.
self.config.update_cfg_hash()
return results
# Sync to clients.
self.ui.info("updating nodes ...")
dirs = []
if not self.config.havenfs:
# Non-NFS, need to explicitly synchronize.
syncs = install.get_syncs()
else:
# NFS. We only need to take care of the spool/log directories.
# We need this only on the manager.
dirs.append((manager, self.config.logdir))
syncs = install.get_nfssyncs()
createdirs = [
self.config.subst(dir) for (dir, mirror) in syncs if not mirror
]
for n in nodes:
for dir in createdirs:
dirs.append((n, dir))
for (node, success, output) in self.executor.mkdirs(dirs):
if not success:
self.ui.error("cannot create a directory on node %s" %
node.name)
self.ui.error("\n".join(output))
results.ok = False
return results
paths = [self.config.subst(dir) for (dir, mirror) in syncs if mirror]
if not execute.sync(nodes, paths, self.ui):
results.ok = False
return results
# Save current configuration state.
self.config.update_cfg_hash()
return results
def status(self, nodes):
results = cmdresult.CmdResult()
showall = self.config.statuscmdshowall
if showall:
self.ui.info("Getting process status ...")
nodestatus = self._isrunning(nodes)
running = []
cmds = []
for (node, isrunning) in nodestatus:
if isrunning:
running += [node]
cmds += [
(node, "first-line",
["%s/.startup" % node.cwd(),
"%s/.status" % node.cwd()])
]
startups = {}
statuses = {}
for (n, success, output) in self.executor.run_helper(cmds):
startups[n.name] = (success and output[0]) and util.fmttime(
output[0]) or "???"
statuses[n.name] = (success and output[1]
) and output[1].split()[0].lower() or "???"
if showall:
self.ui.info("Getting peer status ...")
peers = {}
nodes = [n for n in running if statuses[n.name] == "running"]
for (node, success, args) in self._query_peerstatus(nodes):
if success:
peers[node.name] = []
for f in args[0].split():
keyval = f.split("=")
if len(keyval) > 1:
(key, val) = keyval
if key == "peer" and val != "":
peers[node.name] += [val]
for (node, isrunning) in nodestatus:
node_info = {
"name": node.name,
"type": node.type,
"host": node.host,
"status": "stopped",
"pid": None,
"started": None,
}
if showall:
node_info["peers"] = None
if isrunning:
node_info["status"] = statuses[node.name]
elif node.hasCrashed():
node_info["status"] = "crashed"
if isrunning:
node_info["pid"] = node.getPID()
if showall:
if node.name in peers:
node_info["peers"] = len(peers[node.name])
else:
node_info["peers"] = "???"
node_info["started"] = startups[node.name]
results.set_node_data(node, True, node_info)
return results
