def top(self, nodes): results = cmdresult.CmdResult() for (node, error, vals) in self.get_top_output(nodes): top_info = { "name": node.name, "type": node.type, "host": node.host, "pid": None, "proc": None, "vsize": None, "rss": None, "cpu": None, "cmd": None, "error": None } if error: top_info["error"] = error results.set_node_data(node, False, {"procs": [top_info]}) continue proclist = [] for d in vals: top_info2 = top_info.copy() top_info2.update(d) proclist.append(top_info2) results.set_node_data(node, True, {"procs": proclist}) return results
def update(self, nodes): results = cmdresult.CmdResult() running = self._isrunning(nodes) zone = self.config.zoneid if not zone: zone = "NOZONE" cmds = [] for (node, isrunning) in running: if isrunning: env = _make_env_params(node) env += " BRO_DNS_FAKE=1" args = " ".join(_make_bro_params(node, False)) cmds += [(node.name, os.path.join(self.config.scriptsdir, "update") + " %s %s %s/tcp %s" % (util.format_bro_addr( node.addr), zone, node.getPort(), args), env, None)] self.ui.info("updating %s ..." % node.name) res = execute.run_localcmds(cmds) for (tag, success, output) in res: node = self.config.nodes(tag=tag)[0] if not success: self.ui.info("failed to update %s: %s" % (tag, output[0])) results.set_node_fail(node) else: self.ui.info("%s: %s" % (tag, output[0])) results.set_node_success(node) return results
def deploy(self): if not self.plugins.cmdPre("deploy"): results = cmdresult.CmdResult(ok=False) return results if self.config.is_cfg_changed(): self.ui.info("Reloading broctl configuration ...") self.reload_cfg() self.ui.info("checking configurations ...") results = self.check(check_node_types=True) if not results.ok: for (node, success, output) in results.get_node_output(): if not success: self.ui.info("%s scripts failed." % node) self.ui.info(output) return results self.ui.info("installing ...") results = self.install() if not results.ok: return results self.ui.info("stopping ...") results = self.stop() if not results.ok: return results self.ui.info("starting ...") results = self.start() self.plugins.cmdPost("deploy") return results
def process(self, trace, bro_options, bro_scripts): results = cmdresult.CmdResult() if not os.path.isfile(trace): self.ui.error("trace file not found: %s" % trace) results.ok = False return results standalone = (self.config.standalone == "1") if standalone: tag = "standalone" else: tag = "workers" node = self.config.nodes(tag=tag)[0] cwd = os.path.join(self.config.tmpdir, "testing") if os.path.isdir(cwd): try: shutil.rmtree(cwd) except OSError as err: self.ui.error("cannot remove directory: %s" % err) results.ok = False return results try: os.makedirs(cwd) except OSError as err: self.ui.error("cannot create directory: %s" % err) results.ok = False return results env = _make_env_params(node) bro_args = " ".join(bro_options + _make_bro_params(node, False)) bro_args += " broctl/process-trace" if bro_scripts: bro_args += " " + " ".join(bro_scripts) cmd = os.path.join( self.config.scriptsdir, "run-bro-on-trace") + " %s %s %s %s" % (0, cwd, trace, bro_args) self.ui.info(cmd) (success, output) = execute.run_localcmd(cmd, env, donotcaptureoutput=True) if not success: results.ok = False for line in output: self.ui.info(line) self.ui.info("\n### Bro output in %s" % cwd) return results
def install(self, local=False): if self.plugins.cmdPre("install"): results = self.controller.install(local) else: results = cmdresult.CmdResult(ok=False) self.plugins.cmdPost("install") return results
def process(self, trace, options, scripts): if self.plugins.cmdPre("process", trace, options, scripts): results = self.controller.process(trace, options, scripts) else: results = cmdresult.CmdResult(ok=False) self.plugins.cmdPost("process", trace, options, scripts, results.ok) return results
def execute_cmd(self, nodes, cmd): results = cmdresult.CmdResult() for node, success, out in self.executor.run_shell_cmds([ (n, cmd) for n in nodes ]): results.set_node_output(node, success, out) return results
def df(self, nodes): results = cmdresult.CmdResult() DiskInfo = namedtuple("DiskInfo", ("fs", "total", "used", "available", "percent")) dirs = ("logdir", "bindir", "helperdir", "cfgdir", "spooldir", "policydir", "libdir", "tmpdir", "staticdir", "scriptsdir") df = {} for node in nodes: df[node.name] = {} cmds = [] for node in nodes: for key in dirs: if key == "logdir" and node.type not in ("manager", "standalone"): # Don't need this on the workers/proxies. continue path = self.config.config[key] cmds += [(node, "df", [path])] res = self.executor.run_helper(cmds) for (node, success, output) in res: if success: if not output: df[node.name]["FAIL"] = "no output from df helper" continue fields = output[0].split() fs = fields[0] # Ignore NFS mounted volumes. if ":" in fs: continue total = float(fields[1]) used = float(fields[2]) avail = float(fields[3]) perc = used * 100.0 / (used + avail) df[node.name][fs] = DiskInfo(fs, total, used, avail, perc) else: if output: msg = output[0] else: msg = "unknown failure" df[node.name]["FAIL"] = msg for node in nodes: success = "FAIL" not in df[node.name] results.set_node_data(node, success, df[node.name]) return results
def peerstatus(self, nodes): results = cmdresult.CmdResult() for (node, success, args) in self._query_peerstatus(nodes): if success: out = args[0] else: out = args results.set_node_output(node, success, out) return results
def netstats(self, nodes): results = cmdresult.CmdResult() for (node, success, args) in self._query_netstats(nodes): if success: out = args[0].strip() else: out = args results.set_node_output(node, success, out) return results
def get_config(self): results = cmdresult.CmdResult() if self.plugins.cmdPre("config"): results.keyval = self.config.options() else: results.ok = False self.plugins.cmdPost("config") return results
def _check_config(self, nodes, installed, list_scripts): results = cmdresult.CmdResult() nodetmpdirs = [(node, os.path.join(self.config.tmpdir, "check-config-%s" % node.name)) for node in nodes] nodes = [] for (node, cwd) in nodetmpdirs: if os.path.isdir(cwd): try: shutil.rmtree(cwd) except OSError as err: self.ui.error("cannot remove directory: %s" % err) results.ok = False return results try: os.makedirs(cwd) except OSError as err: self.ui.error("cannot create temporary directory: %s" % err) results.ok = False return results nodes += [(node, cwd)] cmds = [] for (node, cwd) in nodes: env = _make_env_params(node) installed_policies = installed and "1" or "0" print_scripts = list_scripts and "1" or "0" install.make_layout(cwd, self.ui, True) if not install.make_local_networks(cwd, self.ui, True): results.ok = False return results install.make_broctl_config_policy(cwd, self.ui, True) cmd = os.path.join( self.config.scriptsdir, "check-config") + " %s %s %s %s" % ( installed_policies, print_scripts, cwd, " ".join( _make_bro_params(node, False))) cmd += " broctl/check" cmds += [((node, cwd), cmd, env, None)] for ((node, cwd), success, output) in execute.run_localcmds(cmds): results.set_node_output(node, success, output) shutil.rmtree(cwd) return results
def capstats(self, nodes, interval): results = cmdresult.CmdResult() if self.config.capstatspath: for (node, netif, success, vals) in self.get_capstats_output(nodes, interval): if not success: vals = {"output": vals} results.set_node_data(node, success, vals) return results
def execute(self, cmd): nodes = self.node_args(get_hosts=True) if self.plugins.cmdPre("exec", cmd): results = self.controller.execute_cmd(nodes, cmd) else: results = cmdresult.CmdResult(ok=False) self.plugins.cmdPost("exec", cmd) return results
def nodes(self): results = cmdresult.CmdResult() if self.plugins.cmdPre("nodes"): for n in self.config.nodes(): results.set_node_data(n, True, n.to_dict()) else: results.ok = False self.plugins.cmdPost("nodes") return results
def runCustomCommand(self, cmd, args, cmdout): """Runs a custom command *cmd* with string *args* as argument. Returns a CmdResult object which contains the command results.""" try: myplugin, usage, descr = self._cmds[cmd] except LookupError: return cmdresult.CmdResult(ok=False, unknowncmd=True) prefix = myplugin.prefix() if cmd.startswith("%s." % prefix): cmd = cmd[len(prefix)+1:] return myplugin.cmd_custom(cmd, args, cmdout)
def print_id(self, nodes, id): running = self._isrunning(nodes) eventlist = [] for (node, isrunning) in running: if isrunning: eventlist += [(node, "Control::id_value_request", [id], "Control::id_value_response")] results = cmdresult.CmdResult() for (node, success, args) in events.send_events_parallel(eventlist): results.set_node_output(node, success, args) return results
def diag(self, nodes): results = cmdresult.CmdResult() crashdiag = os.path.join(self.config.scriptsdir, "crash-diag") cmds = [(node, crashdiag, [node.cwd()]) for node in nodes] for (node, success, output) in self.executor.run_cmds(cmds): if not success: errmsgs = ["error running crash-diag for %s" % node.name] errmsgs += output results.set_node_output(node, False, errmsgs) continue results.set_node_output(node, True, output) return results
def cleanup(self, nodes, cleantmp=False): # Given a set of node names "orig" and command results "res", add # all node names to "orig" that have a failed result in "res". def addfailed(orig, res): for (node, status, output) in res: # if status is Fail, then add the node name if not status: orig.add(node.name) return orig results = cmdresult.CmdResult() result = self._isrunning(nodes) running = [node for (node, on) in result if on] notrunning = [node for (node, on) in result if not on] for node in running: self.ui.info( " %s is still running, not cleaning work directory" % node) results1 = self.executor.rmdirs([(n, n.cwd()) for n in notrunning]) results2 = self.executor.mkdirs([(n, n.cwd()) for n in notrunning]) failed = set() failed = addfailed(failed, results1) failed = addfailed(failed, results2) for node in notrunning: node.clearCrashed() if cleantmp: self.ui.info("cleaning %s ..." % self.config.tmpdir) results3 = self.executor.rmdirs([(n, self.config.tmpdir) for n in running + notrunning]) results4 = self.executor.mkdirs([(n, self.config.tmpdir) for n in running + notrunning]) failed = addfailed(failed, results3) failed = addfailed(failed, results4) for node in nodes: if node.name in failed: results.set_node_fail(node) else: results.set_node_success(node) return results
def cleanup(self, nodes, cleantmp=False): def addfailed(orig, res): for (n, status) in res: if not status: orig.add(n.name) return orig results = cmdresult.CmdResult() result = self._isrunning(nodes) running = [node for (node, on) in result if on] notrunning = [node for (node, on) in result if not on] for node in running: self.ui.info( " %s is still running, not cleaning work directory" % node) results1 = self.executor.rmdirs([(n, n.cwd()) for n in notrunning]) results2 = self.executor.mkdirs([(n, n.cwd()) for n in notrunning]) failed = set() failed = addfailed(failed, results1) failed = addfailed(failed, results2) for node in notrunning: node.clearCrashed() if cleantmp: results3 = self.executor.rmdirs([(n, self.config.tmpdir) for n in running + notrunning]) results4 = self.executor.mkdirs([(n, self.config.tmpdir) for n in running + notrunning]) failed = addfailed(failed, results3) failed = addfailed(failed, results4) for node in nodes: if node.name in failed: results.set_node_fail(node) else: results.set_node_success(node) return results
def stop(self, nodes): results = cmdresult.CmdResult() manager = [] proxies = [] workers = [] for n in nodes: n.setExpectRunning(False) if n.type == "worker": workers += [n] elif n.type == "proxy": proxies += [n] else: manager += [n] # Stop nodes. Do it in the order workers, proxies, manager # (the reverse of "start"). if workers: self._stop_nodes(workers, results) if not results.ok: for n in (proxies + manager): results.set_node_fail(n) return results if proxies: self._stop_nodes(proxies, results) if not results.ok: for n in manager: results.set_node_fail(n) return results if manager: self._stop_nodes(manager, results) return results
def start(self, nodes): results = cmdresult.CmdResult() manager = [] proxies = [] workers = [] for n in nodes: n.setExpectRunning(True) if n.type == "worker": workers += [n] elif n.type == "proxy": proxies += [n] else: manager += [n] # Start nodes. Do it in the order manager, proxies, workers. if manager: self._start_nodes(manager, results) if not results.ok: for n in (proxies + workers): results.set_node_fail(n) return results if proxies: self._start_nodes(proxies, results) if not results.ok: for n in workers: results.set_node_fail(n) return results if workers: self._start_nodes(workers, results) return results
def install(self, local_only): results = cmdresult.CmdResult() try: self.config.record_bro_version() except config.ConfigurationError as err: self.ui.error("%s" % err) results.ok = False return results manager = self.config.manager() # Delete previously installed policy files to not mix things up. policies = [ self.config.policydirsiteinstall, self.config.policydirsiteinstallauto ] for dirpath in policies: if os.path.isdir(dirpath): self.ui.info("removing old policies in %s ..." % dirpath) try: shutil.rmtree(dirpath) except OSError as err: self.ui.error("failed to remove directory: %s" % err) results.ok = False return results self.ui.info("creating policy directories ...") for dirpath in policies: try: os.makedirs(dirpath) except OSError as err: self.ui.error("failed to create directory: %s" % err) results.ok = False return results # Install local site policy. if self.config.sitepolicypath: self.ui.info("installing site policies ...") dst = self.config.policydirsiteinstall for dir in self.config.sitepolicypath.split(":"): dirpath = self.config.subst(dir) for pathname in glob.glob(os.path.join(dirpath, "*")): if not execute.install(pathname, dst, self.ui): results.ok = False return results install.make_layout(self.config.policydirsiteinstallauto, self.ui) self.ui.info("generating local-networks.bro ...") if not install.make_local_networks( self.config.policydirsiteinstallauto, self.ui): results.ok = False return results self.ui.info("generating broctl-config.bro ...") install.make_broctl_config_policy(self.config.policydirsiteinstallauto, self.ui) current = self.config.subst(os.path.join(self.config.logdir, "current")) try: util.force_symlink(manager.cwd(), current) except (IOError, OSError) as err: results.ok = False self.ui.error("failed to update symlink '%s': %s" % (current, err)) return results self.ui.info("generating broctl-config.sh ...") if not install.make_broctl_config_sh(self.ui): results.ok = False return results if local_only: return results # Make sure we install each remote host only once. nodes = self.config.hosts(nolocal=True) # If there are no remote hosts, then we're done. if not nodes: # Save current configuration state. self.config.update_cfg_hash() return results # Sync to clients. self.ui.info("updating nodes ...") dirs = [] if not self.config.havenfs: # Non-NFS, need to explicitly synchronize. syncs = install.get_syncs() else: # NFS. We only need to take care of the spool/log directories. # We need this only on the manager. dirs.append((manager, self.config.logdir)) syncs = install.get_nfssyncs() createdirs = [ self.config.subst(dir) for (dir, mirror) in syncs if not mirror ] for n in nodes: for dir in createdirs: dirs.append((n, dir)) for (node, success, output) in self.executor.mkdirs(dirs): if not success: self.ui.error("cannot create a directory on node %s" % node.name) self.ui.error("\n".join(output)) results.ok = False return results paths = [self.config.subst(dir) for (dir, mirror) in syncs if mirror] if not execute.sync(nodes, paths, self.ui): results.ok = False return results # Save current configuration state. self.config.update_cfg_hash() return results
def status(self, nodes): results = cmdresult.CmdResult() showall = self.config.statuscmdshowall if showall: self.ui.info("Getting process status ...") nodestatus = self._isrunning(nodes) running = [] cmds = [] for (node, isrunning) in nodestatus: if isrunning: running += [node] cmds += [ (node, "first-line", ["%s/.startup" % node.cwd(), "%s/.status" % node.cwd()]) ] startups = {} statuses = {} for (n, success, output) in self.executor.run_helper(cmds): startups[n.name] = (success and output[0]) and util.fmttime( output[0]) or "???" statuses[n.name] = (success and output[1] ) and output[1].split()[0].lower() or "???" if showall: self.ui.info("Getting peer status ...") peers = {} nodes = [n for n in running if statuses[n.name] == "running"] for (node, success, args) in self._query_peerstatus(nodes): if success: peers[node.name] = [] for f in args[0].split(): keyval = f.split("=") if len(keyval) > 1: (key, val) = keyval if key == "peer" and val != "": peers[node.name] += [val] for (node, isrunning) in nodestatus: node_info = { "name": node.name, "type": node.type, "host": node.host, "status": "stopped", "pid": None, "started": None, } if showall: node_info["peers"] = None if isrunning: node_info["status"] = statuses[node.name] elif node.hasCrashed(): node_info["status"] = "crashed" if isrunning: node_info["pid"] = node.getPID() if showall: if node.name in peers: node_info["peers"] = len(peers[node.name]) else: node_info["peers"] = "???" node_info["started"] = startups[node.name] results.set_node_data(node, True, node_info) return results