def _read_config(self, fname):
type_converters = {"bool": self._to_bool, "int": int, "string": str}
config = {}
opt_names = set()
for opt in options.options:
# Convert key to lowercase because keys are stored in lowercase.
key = opt.name.lower()
opt_names.add(key)
with open(fname, "r") as f:
for line in f:
line = line.strip()
if not line or line.startswith("#"):
continue
args = line.split("=", 1)
if len(args) != 2:
raise ConfigurationError("broctl config syntax error: %s" %
line)
key, val = args
# Option names are not case-sensitive.
key = key.strip().lower()
# Warn about unrecognized options, but we can't check plugin
# options here because no plugins have been loaded yet.
if "." not in key and key not in opt_names:
self.ui.warn("ignoring unrecognized broctl option: %s" %
key)
continue
# if the key already exists, just overwrite with new value
config[key] = val.strip()
# Convert option values to correct data type
for opt in options.options:
# Convert key to lowercase because keys are stored in lowercase.
key = opt.name.lower()
if key in config:
try:
config[key] = type_converters[opt.type](config[key])
except ValueError:
raise ConfigurationError(
"broctl option '%s' has invalid value '%s' for type %s"
% (key, config[key], opt.type))
return config
def initPostPlugins(self):
# Read node.cfg
self.nodestore = self._read_nodes()
# If "env_vars" was specified in broctl.cfg, then apply to all nodes.
varlist = self.config.get("env_vars")
if varlist:
try:
global_env_vars = self._get_env_var_dict(varlist)
except ConfigurationError as err:
raise ConfigurationError("broctl config: %s" % err)
for node in self.nodes("all"):
for (key, val) in global_env_vars.items():
# Values from node.cfg take precedence over broctl.cfg
node.env_vars.setdefault(key, val)
# Check state store for any running nodes that are no longer in the
# current node config.
self._warn_dangling_bro()
# Set the standalone config option.
standalone = len(self.nodestore) == 1
self.init_option("standalone", standalone)
def _get_bro_version(self):
from BroControl import execute
bro = self.config["bro"]
if not os.path.lexists(bro):
raise ConfigurationError("cannot find Bro binary: %s" % bro)
version = ""
success, output = execute.run_localcmd("%s -v" % bro)
if success and output:
version = output.splitlines()[-1]
else:
msg = " with no output"
if output:
msg = " with output:\n%s" % output
raise RuntimeEnvironmentError('running "bro -v" failed%s' % msg)
match = re.search(".* version ([^ ]*).*$", version)
if not match:
raise RuntimeEnvironmentError(
'cannot determine Bro version ("bro -v" output: %s)' %
version.strip())
version = match.group(1)
# If bro is built with the "--enable-debug" configure option, then it
# appends "-debug" to the version string.
if version.endswith("-debug"):
version = version[:-6]
return version
def _read_config(self, fname):
type_converters = {"bool": self._to_bool, "int": int, "string": str}
config = {}
with open(fname, "r") as f:
for line in f:
line = line.strip()
if not line or line.startswith("#"):
continue
args = line.split("=", 1)
if len(args) != 2:
raise ConfigurationError("broctl config syntax error: %s" %
line)
key, val = args
# Option names are not case-sensitive.
key = key.strip().lower()
# if the key already exists, just overwrite with new value
config[key] = val.strip()
# Convert option values to correct data type
for opt in options.options:
# Convert key to lowercase because keys are stored in lowercase.
key = opt.name.lower()
if key in config:
try:
config[key] = type_converters[opt.type](config[key])
except ValueError:
raise ConfigurationError(
"broctl option '%s' has invalid value '%s' for type %s"
% (key, config[key], opt.type))
# If someone uses a deprecated option in broctl.cfg, then convert it
# to the new option, but only if the new option is not specified in
# the config file.
if "sitepolicystandalone" in config:
self.ui.warn(
"the SitePolicyStandalone option is deprecated (use SitePolicyScripts instead)."
)
if "sitepolicyscripts" not in config:
config["sitepolicyscripts"] = config["sitepolicystandalone"]
return config
def _check_nodestore(self, nodestore):
if not nodestore:
raise ConfigurationError("no nodes found in node config")
standalone = False
manager = False
proxy = False
manageronlocalhost = False
# Note: this is a subset of localaddrs
localhostaddrs = "127.0.0.1", "::1"
for n in nodestore.values():
if n.type == "manager":
if manager:
raise ConfigurationError(
"only one manager can be defined in node config")
manager = True
if n.addr in localhostaddrs:
manageronlocalhost = True
if n.addr not in self.localaddrs:
raise ConfigurationError(
"must run broctl on same machine as the manager node. The manager node has IP address %s and this machine has IP addresses: %s"
% (n.addr, ", ".join(self.localaddrs)))
elif n.type == "proxy":
proxy = True
elif n.type == "standalone":
standalone = True
if n.addr not in self.localaddrs:
raise ConfigurationError(
"must run broctl on same machine as the standalone node. The standalone node has IP address %s and this machine has IP addresses: %s"
% (n.addr, ", ".join(self.localaddrs)))
if standalone:
if len(nodestore) > 1:
raise ConfigurationError(
"more than one node defined in standalone node config")
else:
if not manager:
raise ConfigurationError("no manager defined in node config")
elif not proxy:
raise ConfigurationError("no proxy defined in node config")
# If manager is on localhost, then all other nodes must be on localhost
if manageronlocalhost:
for n in nodestore.values():
if n.type != "manager" and n.addr not in localhostaddrs:
raise ConfigurationError(
"all nodes must use localhost/127.0.0.1/::1 when manager uses it"
)
def _get_env_var_dict(self, text):
env_vars = {}
if text:
for keyval in text.split(","):
try:
key, val = keyval.split("=", 1)
except ValueError:
raise ConfigurationError(
"missing '=' in env_vars option value: %s" % keyval)
key = key.strip()
if not key:
raise ConfigurationError(
"env_vars option value must contain at least one environment variable name: %s"
% keyval)
env_vars[key] = val.strip()
return env_vars
def _read_nodes(self):
config = py3bro.configparser.SafeConfigParser()
fname = self.nodecfg
try:
if not config.read(fname):
raise ConfigurationError("cannot read node config file: %s" %
fname)
except py3bro.configparser.MissingSectionHeaderError as err:
raise ConfigurationError(err)
nodestore = NodeStore()
counts = {}
for sec in config.sections():
node = node_mod.Node(self, sec)
# Note that the keys are converted to lowercase by configparser.
for (key, val) in config.items(sec):
key = key.replace(".", "_")
if key not in node_mod.Node._keys:
self.ui.warn(
"ignoring unrecognized node config option '%s' given for node '%s'"
% (key, sec))
continue
node.__dict__[key] = val
# Perform a sanity check on the node, and update nodestore.
self._check_node(node, nodestore, counts)
# Perform a sanity check on the nodestore (make sure we have a valid
# cluster config, etc.).
self._check_nodestore(nodestore.nodestore)
return nodestore.nodestore
def add_node(self, node):
# Add a node to the nodestore, but first check for duplicate node
# names. This check is not case-sensitive, because names are stored
# lowercase in the state db, and some filesystems are not
# case-sensitive (working dir name is node name).
# Duplicate node names can occur either because the user defined two
# nodes that differ only by case (e.g. "Worker-1" and "worker-1"), or
# if a user defines a node name that conflicts with an auto-generated
# one (e.g. "worker-1" with lb_procs=2 and "worker-1-2").
namelower = node.name.lower()
if namelower in self.nodenameslower:
matchname = ""
for nn in self.nodestore:
if nn.lower() == namelower:
matchname = nn
break
raise ConfigurationError('node name "%s" is a duplicate of "%s"' %
(node.name, matchname))
self.nodestore[node.name] = node
self.nodenameslower.append(namelower)
def _check_options(self):
# Option names must be valid bash variable names because we will
# write them to broctl-config.sh (note that broctl will convert "."
# to "_" when it writes to broctl-config.sh).
allowedchars = re.compile("^[a-z0-9_.]+$")
nostartdigit = re.compile("^[^0-9]")
for key, value in self.config.items():
if re.match(allowedchars, key) is None:
raise ConfigurationError(
'broctl option name "%s" contains invalid characters (allowed characters: a-z, 0-9, ., and _)'
% key)
if re.match(nostartdigit, key) is None:
raise ConfigurationError(
'broctl option name "%s" cannot start with a number' % key)
# No broctl option ever requires the entire value to be wrapped in
# quotes, and since doing so can cause problems, we don't allow it.
if isinstance(value, str):
if (value.startswith('"') and value.endswith('"')
or value.startswith("'") and value.endswith("'")):
raise ConfigurationError(
'value of broctl option "%s" cannot be wrapped in quotes'
% key)
dirs = ("brobase", "logdir", "spooldir", "cfgdir", "broscriptdir",
"bindir", "libdirinternal", "plugindir", "scriptsdir")
files = ("makearchivename", )
for d in dirs:
v = self.config[d]
if not os.path.isdir(v):
raise ConfigurationError(
'broctl option "%s" directory not found: %s' % (d, v))
for f in files:
v = self.config[f]
if not os.path.isfile(v):
raise ConfigurationError(
'broctl option "%s" file not found: %s' % (f, v))
# Verify that logs don't expire more quickly than the rotation interval
logexpireseconds = 60 * self.config["logexpireminutes"]
if 0 < logexpireseconds < self.config["logrotationinterval"]:
raise ConfigurationError(
"Log expire interval cannot be shorter than the log rotation interval"
)
def _get_interval_minutes(self, optname):
# Conversion table for time units to minutes.
units = {"day": 24 * 60, "hr": 60, "min": 1}
ss = self.config[optname]
try:
# If no time unit, assume it's days (for backward compatibility).
v = int(ss) * units["day"]
return v
except ValueError:
pass
# Time interval is a non-negative integer followed by an optional
# space, followed by a time unit.
mm = re.match("([0-9]+) ?(day|hr|min)s?$", ss)
if mm is None:
raise ConfigurationError(
'value of broctl option "%s" is invalid (value must be integer followed by a time unit "day", "hr", or "min"): %s'
% (optname, ss))
v = int(mm.group(1))
v *= units[mm.group(2)]
return v
def _check_node(self, node, nodestore, counts):
if not node.type:
raise ConfigurationError("no type given for node %s" % node.name)
if node.type not in ("logger", "manager", "proxy", "worker",
"standalone"):
raise ConfigurationError(
"unknown node type '%s' given for node '%s'" %
(node.type, node.name))
if not node.host:
raise ConfigurationError("no host given for node '%s'" % node.name)
try:
addrinfo = socket.getaddrinfo(node.host, None, 0, 0,
socket.SOL_TCP)
except socket.gaierror as e:
raise ConfigurationError(
"hostname lookup failed for '%s' in node config [%s]" %
(node.host, e.args[1]))
addrs = [addr[4][0] for addr in addrinfo]
# By default, just use the first IP addr in the list.
addr_str = addrs[0]
# Choose the first IPv4 addr (if any) in the list.
for ip in addrs:
if ":" not in ip:
addr_str = ip
break
# zone_id is handled manually, so strip it if it's there
node.addr = addr_str.split("%")[0]
# Convert env_vars from a string to a dictionary.
try:
node.env_vars = self._get_env_var_dict(node.env_vars)
except ConfigurationError as err:
raise ConfigurationError("node '%s' config: %s" % (node.name, err))
# Each node gets a number unique across its type.
try:
counts[node.type] += 1
except KeyError:
counts[node.type] = 1
node.count = counts[node.type]
numprocs = 0
if node.lb_procs:
if node.type != "worker":
raise ConfigurationError(
"node '%s' config: load balancing node config options are only for worker nodes"
% node.name)
try:
numprocs = int(node.lb_procs)
except ValueError:
raise ConfigurationError(
"number of load-balanced processes must be an integer for node '%s'"
% node.name)
if numprocs < 1:
raise ConfigurationError(
"number of load-balanced processes must be greater than zero for node '%s'"
% node.name)
elif node.lb_method:
raise ConfigurationError(
"number of load-balanced processes not specified for node '%s'"
% node.name)
try:
pin_cpus = self._get_pin_cpu_list(node.pin_cpus, numprocs)
except ValueError:
raise ConfigurationError(
"pin cpus list must contain only non-negative integers for node '%s'"
% node.name)
if pin_cpus:
node.pin_cpus = pin_cpus[0]
if node.lb_procs:
if not node.lb_method:
raise ConfigurationError(
"no load balancing method given for node '%s'" % node.name)
if node.lb_method not in ("pf_ring", "myricom", "custom",
"interfaces"):
raise ConfigurationError(
"unknown load balancing method '%s' given for node '%s'" %
(node.lb_method, node.name))
if node.lb_method == "interfaces":
if not node.lb_interfaces:
raise ConfigurationError(
"list of load-balanced interfaces not specified for node '%s'"
% node.name)
# get list of interfaces to use, and assign one to each node
netifs = node.lb_interfaces.split(",")
if len(netifs) != numprocs:
raise ConfigurationError(
"number of load-balanced interfaces is not same as number of load-balanced processes for node '%s'"
% node.name)
node.interface = netifs.pop().strip()
origname = node.name
# node names will have a numerical suffix
node.name = "%s-1" % node.name
for num in range(2, numprocs + 1):
newnode = node.copy()
# Update the node attrs that need to be changed
newname = "%s-%d" % (origname, num)
newnode.name = newname
counts[node.type] += 1
newnode.count = counts[node.type]
if pin_cpus:
newnode.pin_cpus = pin_cpus[num - 1]
if newnode.lb_method == "interfaces":
newnode.interface = netifs.pop().strip()
nodestore.add_node(newnode)
nodestore.add_node(node)
