def _getProfLogs():
cmds = []
for node in config.Config.hosts():
if execute.isLocal(node):
continue
if not execute.isAlive(node.addr):
continue
cmd = os.path.join(config.Config.scriptsdir, "get-prof-log") + " %s %s %s/prof.log" % (node.name, node.host, node.cwd())
cmds += [(node, cmd, [], None)]
for (node, success, output) in execute.runLocalCmdsParallel(cmds):
if not success:
util.output("cannot get prof.log from %s" % node.name)
def _checkHosts():
for node in config.Config.hosts():
if execute.isLocal(node):
continue
tag = "alive-%s" % node.host.lower()
alive = execute.isAlive(node.addr) and "1" or "0"
if tag in config.Config.state:
previous = config.Config.state[tag]
if alive != previous:
plugin.Registry.hostStatusChanged(node.host, alive == "1")
util.output("host %s %s" % (node.host, alive == "1" and "up" or "down"))
config.Config._setState(tag, alive)
util.error("%s: more than one node defined in stand-alone setup" % file)
manageronlocalhost = False
for n in self.nodelist.values():
if not n.name:
util.error("node configured without a name")
if not n.host:
util.error("no host given for node %s" % n.name)
if not n.type:
util.error("no type given for node %s" % n.name)
if n.type == "manager":
if not execute.isLocal(n):
util.error("script must be run on manager node")
if ( n.addr == "127.0.0.1" or n.addr == "::1" ) and n.type != "standalone":
manageronlocalhost = True
# If manager is on localhost, then all other nodes must be on localhost
if manageronlocalhost:
for n in self.nodelist.values():
if n.type != "manager" and n.type != "standalone":
if n.addr != "127.0.0.1" and n.addr != "::1":
util.error("cannot use localhost/127.0.0.1/::1 for manager host in nodes configuration")
# Parses broctl.cfg or broctl.dat and returns a dictionary of all entries.
def _readConfig(self, file, allowstate = False):
config = {}
class Configuration:
def __init__(self, config, basedir, version):
global Config
Config = self
self.config = {}
self.state = {}
# Read broctl.cfg.
self.config = self._readConfig(config)
# Set defaults for options we get passed in.
self._setOption("brobase", basedir)
self._setOption("version", version)
# Initialize options.
for opt in options.options:
if not opt.dontinit:
self._setOption(opt.name.lower(), opt.default)
# Set defaults for options we derive dynamically.
self._setOption("mailto", "%s" % os.getenv("USER"))
self._setOption("mailfrom",
"Big Brother <bro@%s>" % socket.gethostname())
self._setOption("home", os.getenv("HOME"))
self._setOption("mailalarmsto", self.config["mailto"])
# Determine operating system.
(success, output) = execute.captureCmd("uname")
if not success:
util.error("cannot run uname")
self._setOption("os", output[0].lower().strip())
# Find the time command (should be a GNU time for best results).
(success, output) = execute.captureCmd("which time")
self._setOption("time", output[0].lower().strip())
def initPostPlugins(self):
plugin.Registry.addNodeKeys()
# Read node.cfg and broctl.dat.
self._readNodes()
self.readState()
# Now that the nodes have been read in, set the standalone config option.
standalone = "0"
for node in self.nodes("all"):
if node.type == "standalone":
standalone = "1"
self._setOption("standalone", standalone)
# Make sure cron flag is cleared.
self.config["cron"] = "0"
# Provides access to the configuration options via the dereference operator.
# Lookups the attribute in broctl.cfg first, then in the dynamic variables from broctl.dat.
# Defaults to empty string for unknown options.
def __getattr__(self, attr):
if attr in self.config:
return self.config[attr]
if attr in self.state:
return self.state[attr]
return ""
# Returns True if attribute is defined.
def hasAttr(self, attr):
if attr in self.config:
return True
if attr in self.state:
return True
return False
# Returns a list of all broctl.cfg entries.
# Includes dynamic variables if dynamic is true.
def options(self, dynamic=True):
if dynamic:
return self.config.items() + self.state.items()
else:
return self.config.items()
# Returns a list of Nodes.
# - If tag is "global" or "all", all Nodes are returned if "expand_all" is true.
# If "expand_all" is false, returns an empty list in this case.
# - If tag is "proxies" or "proxy", all proxy Nodes are returned.
# - If tag is "workers" or "worker", all worker Nodes are returned.
# - If tag is "manager", the manager Node is returned.
def nodes(self, tag=None, expand_all=True):
nodes = []
type = None
if tag == "cluster" or tag == "all":
if not expand_all:
return []
tag = None
elif tag == "proxies":
type = "proxy"
elif tag == "workers":
type = "worker"
elif tag in self.config:
type = tag
for n in self.nodelist.values():
if type:
if type == n.type:
nodes += [n]
elif tag == n.name or not tag:
nodes += [n]
nodes.sort(key=lambda n: (n.type, n.name))
if not nodes and tag == "manager":
nodes = self.nodes("standalone")
return nodes
# Returns the manager Node.
def manager(self):
n = self.nodes("manager")
if n:
return n[0]
n = self.nodes("standalone")
if n:
return n[0]
return None
# Returns a list of nodes which is a subset of the result a similar call to
# nodes() would yield but within which each host appears only once.
def hosts(self, tag=None):
hosts = {}
for node in self.nodes(tag):
if not node.host in hosts:
hosts[node.host] = node
return hosts.values()
# Replace all occurences of "${option}", with option being either
# broctl.cfg option or a dynamic variable, with the corresponding value.
# Defaults to replacement with the empty string for unknown options.
def subst(self, str):
while True:
m = re.search(r"(\$\{([A-Za-z]+)(:([^}]+))?\})", str)
if not m:
return str
key = m.group(2).lower()
if self.hasAttr(key):
value = self.__getattr__(key)
else:
value = m.group(4)
if not value:
value = ""
str = str[0:m.start(1)] + value + str[m.end(1):]
# Parse node.cfg.
def _readNodes(self):
self.nodelist = {}
config = ConfigParser.SafeConfigParser()
if not config.read(self.nodecfg):
util.error("cannot read '%s'" % self.nodecfg)
manager = False
proxy = False
worker = False
standalone = False
file = self.nodecfg
counts = {}
for sec in config.sections():
node = node_mod.Node(sec)
self.nodelist[sec] = node
for (key, val) in config.items(sec):
key = key.replace(".", "_")
if not key in node_mod.Node._keys:
util.warn("%s: unknown key '%s' in section '%s'" %
(file, key, sec))
continue
if key == "type":
if val == "manager":
if manager:
util.error("only one manager can be defined")
manager = True
elif val == "proxy":
proxy = True
elif val == "worker":
worker = True
elif val == "standalone":
standalone = True
else:
util.error("%s: unknown type '%s' in section '%s'" %
(file, val, sec))
node.__dict__[key] = val
try:
addrinfo = socket.getaddrinfo(node.host, None, 0, 0,
socket.SOL_TCP)
if len(addrinfo) == 0:
util.error(
"%s: no addresses resolved in section '%s' for host %s"
% (file, sec, node.host))
addr_str = addrinfo[0][4][0]
# zone_id is handled manually, so strip it if it's there
node.addr = addr_str.split('%')[0]
except AttributeError:
util.error("%s: no host given in section '%s'" % (file, sec))
except socket.gaierror, e:
util.error("%s: unknown host '%s' in section '%s' [%s]" %
(file, node.host, sec, e.args[1]))
# Each node gets a number unique across its type.
type = self.nodelist[sec].type
try:
counts[type] += 1
except KeyError:
counts[type] = 1
node.count = counts[type]
if node.lb_procs:
try:
numprocs = int(node.lb_procs)
if numprocs < 1:
util.error(
"%s: value of lb_procs must be at least 1 in section '%s'"
% (file, sec))
except ValueError:
util.error(
"%s: value of lb_procs must be an integer in section '%s'"
% (file, sec))
if not node.lb_method:
util.error(
"%s: no load balancing method given in section '%s'" %
(file, sec))
if node.lb_method not in ("pf_ring", "myricom", "interfaces"):
util.error(
"%s: unknown load balancing method given in section '%s'"
% (file, sec))
if node.lb_method == "interfaces":
if not node.lb_interfaces:
util.error(
"%s: no list of interfaces given in section '%s'" %
(file, sec))
# get list of interfaces to use, and assign one to each node
netifs = node.lb_interfaces.split(",")
if len(netifs) != int(node.lb_procs):
util.error(
"%s: number of interfaces does not match value of lb_procs in section '%s'"
% (file, sec))
node.interface = netifs.pop().strip()
# node names will have a numerical suffix
node.name = "%s-1" % sec
for num in xrange(2, int(node.lb_procs) + 1):
newnode = copy.deepcopy(node)
# only the node name and count need to be changed
newname = "%s-%d" % (sec, num)
newnode.name = newname
self.nodelist[newname] = newnode
counts[type] += 1
newnode.count = counts[type]
if newnode.lb_method == "interfaces":
newnode.interface = netifs.pop().strip()
if self.nodelist:
if not standalone:
if not manager:
util.error("%s: no manager defined" % file)
if not proxy:
util.error("%s: no proxy defined" % file)
else:
if len(self.nodelist) > 1:
util.error(
"%s: more than one node defined in stand-alone setup" %
file)
for n in self.nodelist.values():
if not n.name:
util.error("node configured without a name")
if not n.host:
util.error("no host given for node %s" % n.name)
if not n.type:
util.error("no type given for node %s" % n.name)
if n.type == "manager":
if not execute.isLocal(n):
util.error("script must be run on manager node")
if (n.addr == "127.0.0.1"
or n.addr == "::1") and n.type != "standalone":
util.error(
"cannot use localhost/127.0.0.1/::1 for manager host in nodes configuration"
)
class Configuration:
def __init__(self, config, basedir, distdir, version, standalone):
global Config
global Installing
Config = self
if "BROCTL_INSTALL" in os.environ:
Installing = True
global BroBase
BroBase = basedir
if "MAKE_DESTDIR" in os.environ:
global MakeDestDir
MakeDestDir = os.environ["MAKE_DESTDIR"]
self.config = {}
self.state = {}
# Read broctl.cfg.
self.config = self._readConfig(os.path.join(basedir, config))
# Set defaults for options we get passed in.
self._setOption("brobase", basedir)
self._setOption("distdir", distdir)
self._setOption("version", version)
self._setOption("standalone", standalone and "1" or "0")
# Initialize options.
for opt in options.options:
if not opt.dontinit:
self._setOption(opt.name.lower(), opt.default)
# Set defaults for options we derive dynamically.
self._setOption("mailto", "%s" % os.getenv("USER"))
self._setOption("mailfrom", "Big Brother <bro@%s>" % socket.gethostname())
self._setOption("home", os.getenv("HOME"))
self._setOption("mailalarmsto", self.config["mailto"])
# Determine operating system.
(success, output) = execute.captureCmd("uname")
if not success:
util.error("cannot run uname")
self._setOption("os", output[0].lower().strip())
# Find the time command (should be a GNU time for best results).
(success, output) = execute.captureCmd("which time")
self._setOption("time", output[0].lower().strip())
# Read nodes.cfg and broctl.dat.
self._readNodes()
self.readState()
# Setup the kinds of analyses which we support.
self._analysis = Analysis(self.analysiscfg)
# Make sure cron flag is cleared.
self.config["cron"] = "0"
# Provides access to the configuration options via the dereference operator.
# Lookups the attribute in broctl.cfg first, then in the dynamic variables from broctl.dat.
# Defaults to empty string for unknown options.
def __getattr__(self, attr):
if attr in self.config:
return self.config[attr]
if attr in self.state:
return self.state[attr]
return ""
# Returns True if attribute is defined.
def hasAttr(self, attr):
if attr in self.config:
return True
if attr in self.state:
return True
return False
# Returns a list of all broctl.cfg entries.
# Includes dynamic variables if dynamic is true.
def options(self, dynamic=True):
if dynamic:
return self.config.items() + self.state.items()
else:
return self.config.items()
# Returns a list of Nodes.
# - If tag is "global" or "all", all Nodes are returned if "expand_all" is true.
# If "expand_all" is false, returns an empty list in this case.
# - If tag is "proxies" or "proxy", all proxy Nodes are returned.
# - If tag is "workers" or "worker", all worker Nodes are returned.
# - If tag is "manager", the manager Node is returned.
def nodes(self, tag=None, expand_all=True):
nodes = []
type = None
if tag == "cluster" or tag == "all":
if not expand_all:
return []
tag = None
if tag == "proxies":
tag = "proxy"
if tag == "workers":
tag = "worker"
if ("scripts-%s" % tag) in self.config:
type = tag
for n in self.nodelist.values():
if type:
if type == n.type:
nodes += [n]
elif tag == n.tag or not tag:
nodes += [n]
nodes.sort(key=lambda n: (n.type, n.tag))
if not nodes and tag == "manager":
nodes = self.nodes("standalone")
return nodes
# Returns the manager Node.
def manager(self):
n = self.nodes("manager")
if n:
return n[0]
n = self.nodes("standalone")
if n:
return n[0]
return None
# Returns a list of nodes which is a subset of the result a similar call to
# nodes() would yield but within which each host appears only once.
def hosts(self, tag = None):
hosts = {}
for node in self.nodes(tag):
if not node.host in hosts:
hosts[node.host] = node
return hosts.values()
# Replace all occurences of "${option}", with option being either
# broctl.cfg option or a dynamic variable, with the corresponding value.
# Defaults to replacement with the empty string for unknown options.
def subst(self, str, make_dest=True):
while True:
m = re.search(r"(\$\{([A-Za-z]+)(:([^}]+))?\})", str)
if not m:
# This is a hack to support make's DESTDIR: if the env variable
# MAKE_DESTDIR is set, and the string we return starts with our
# installation prefix, we prepend the var's content. This it not
# totally perfect but should do the trick.
if Installing and MakeDestDir and MakeDestDir != "":
if make_dest and str.startswith(BroBase):
str = MakeDestDir + str
if not make_dest:
str = str.replace(MakeDestDir, "")
return str
key = m.group(2).lower()
if self.hasAttr(key):
value = self.__getattr__(key)
else:
value = m.group(4)
if not value:
value = ""
str = str[0:m.start(1)] + value + str[m.end(1):]
# Returns instance of class Analysis.
def analysis(self):
return self._analysis
# Parse nodes.cfg.
def _readNodes(self):
self.nodelist = {}
config = ConfigParser.SafeConfigParser()
if not config.read(self.nodecfg) and not Installing:
util.error("cannot read '%s'" % self.nodecfg)
manager = False
proxy = False
standalone = False
file = self.nodecfg
counts = {}
for sec in config.sections():
node = Node(sec)
self.nodelist[sec] = node
for (key, val) in config.items(sec):
if not key in Node._tags:
util.warn("%s: unknown key '%s' in section '%s'" % (file, key, sec))
continue
if key == "type":
# We determine which types are valid by checking for having an
# option specifying which scripts to use for it.
cfg = "scripts-%s" % val
if not cfg in self.config:
util.error("%s: unknown type '%s' in section '%s'" % (file, val, sec))
self.nodelist[sec].scripts = self.config[cfg].split()
if val == "manager":
if manager:
util.error("only one manager can be defined")
manager = True
if val == "proxy":
proxy = True
if val == "standalone":
standalone = True
node.__dict__[key] = val
try:
node.addr = socket.gethostbyname(node.host)
except AttributeError:
util.error("%s: no host given in section '%s'" % (file, sec))
except socket.gaierror, e:
util.error("%s: unknown host '%s' in section '%s' [%s]" % (file, node.host, sec, e.args[1]))
# Each node gets a number unique across its type.
type = self.nodelist[sec].type
try:
counts[type] += 1
except KeyError:
counts[type] = 1
node.count = counts[type]
if self.nodelist:
if not standalone:
if not manager:
util.error("%s: no manager defined" % file)
if not proxy:
util.error("%s: no proxy defined" % file)
else:
if len(self.nodelist) > 1:
util.error("%s: more than one node defined in stand-alone setup" % file)
for n in self.nodelist.values():
if n.type == "manager":
if not execute.isLocal(n):
util.error("script must be run on manager node")
if n.addr == "127.0.0.1" and n.type != "standalone":
util.error("cannot use localhost/127.0.0.1 for manager host in nodes configuration")
except (IOError, OSError), e:
util.error("failed to update current log symlink")
generateDynamicVariableScript()
if local_only:
return not hadError
# Sync to clients.
util.output("updating nodes ... ", False)
nodes = []
# Make sure we install each remote host only once.
for n in config.Config.hosts():
if execute.isLocal(n):
continue
if not execute.isAlive(n.addr):
hadError = True
continue
nodes += [n]
if config.Config.havenfs != "1":
# Non-NFS, need to explicitly synchronize.
dirs = []
for dir in [config.Config.subst(dir) for (dir, mirror) in Syncs if not mirror]:
dirs += [(n, dir) for n in nodes]
for (node, success) in execute.mkdirs(dirs):