def _read_config(self, fname): type_converters = {"bool": self._to_bool, "int": int, "string": str} config = {} opt_names = set() for opt in options.options: # Convert key to lowercase because keys are stored in lowercase. key = opt.name.lower() opt_names.add(key) with open(fname, "r") as f: for line in f: line = line.strip() if not line or line.startswith("#"): continue args = line.split("=", 1) if len(args) != 2: raise ConfigurationError("broctl config syntax error: %s" % line) key, val = args # Option names are not case-sensitive. key = key.strip().lower() # Warn about unrecognized options, but we can't check plugin # options here because no plugins have been loaded yet. if "." not in key and key not in opt_names: self.ui.warn("ignoring unrecognized broctl option: %s" % key) continue # if the key already exists, just overwrite with new value config[key] = val.strip() # Convert option values to correct data type for opt in options.options: # Convert key to lowercase because keys are stored in lowercase. key = opt.name.lower() if key in config: try: config[key] = type_converters[opt.type](config[key]) except ValueError: raise ConfigurationError( "broctl option '%s' has invalid value '%s' for type %s" % (key, config[key], opt.type)) return config
def initPostPlugins(self): # Read node.cfg self.nodestore = self._read_nodes() # If "env_vars" was specified in broctl.cfg, then apply to all nodes. varlist = self.config.get("env_vars") if varlist: try: global_env_vars = self._get_env_var_dict(varlist) except ConfigurationError as err: raise ConfigurationError("broctl config: %s" % err) for node in self.nodes("all"): for (key, val) in global_env_vars.items(): # Values from node.cfg take precedence over broctl.cfg node.env_vars.setdefault(key, val) # Check state store for any running nodes that are no longer in the # current node config. self._warn_dangling_bro() # Set the standalone config option. standalone = len(self.nodestore) == 1 self.init_option("standalone", standalone)
def _get_bro_version(self): from BroControl import execute bro = self.config["bro"] if not os.path.lexists(bro): raise ConfigurationError("cannot find Bro binary: %s" % bro) version = "" success, output = execute.run_localcmd("%s -v" % bro) if success and output: version = output.splitlines()[-1] else: msg = " with no output" if output: msg = " with output:\n%s" % output raise RuntimeEnvironmentError('running "bro -v" failed%s' % msg) match = re.search(".* version ([^ ]*).*$", version) if not match: raise RuntimeEnvironmentError( 'cannot determine Bro version ("bro -v" output: %s)' % version.strip()) version = match.group(1) # If bro is built with the "--enable-debug" configure option, then it # appends "-debug" to the version string. if version.endswith("-debug"): version = version[:-6] return version
def _read_config(self, fname): type_converters = {"bool": self._to_bool, "int": int, "string": str} config = {} with open(fname, "r") as f: for line in f: line = line.strip() if not line or line.startswith("#"): continue args = line.split("=", 1) if len(args) != 2: raise ConfigurationError("broctl config syntax error: %s" % line) key, val = args # Option names are not case-sensitive. key = key.strip().lower() # if the key already exists, just overwrite with new value config[key] = val.strip() # Convert option values to correct data type for opt in options.options: # Convert key to lowercase because keys are stored in lowercase. key = opt.name.lower() if key in config: try: config[key] = type_converters[opt.type](config[key]) except ValueError: raise ConfigurationError( "broctl option '%s' has invalid value '%s' for type %s" % (key, config[key], opt.type)) # If someone uses a deprecated option in broctl.cfg, then convert it # to the new option, but only if the new option is not specified in # the config file. if "sitepolicystandalone" in config: self.ui.warn( "the SitePolicyStandalone option is deprecated (use SitePolicyScripts instead)." ) if "sitepolicyscripts" not in config: config["sitepolicyscripts"] = config["sitepolicystandalone"] return config
def _check_nodestore(self, nodestore): if not nodestore: raise ConfigurationError("no nodes found in node config") standalone = False manager = False proxy = False manageronlocalhost = False # Note: this is a subset of localaddrs localhostaddrs = "127.0.0.1", "::1" for n in nodestore.values(): if n.type == "manager": if manager: raise ConfigurationError( "only one manager can be defined in node config") manager = True if n.addr in localhostaddrs: manageronlocalhost = True if n.addr not in self.localaddrs: raise ConfigurationError( "must run broctl on same machine as the manager node. The manager node has IP address %s and this machine has IP addresses: %s" % (n.addr, ", ".join(self.localaddrs))) elif n.type == "proxy": proxy = True elif n.type == "standalone": standalone = True if n.addr not in self.localaddrs: raise ConfigurationError( "must run broctl on same machine as the standalone node. The standalone node has IP address %s and this machine has IP addresses: %s" % (n.addr, ", ".join(self.localaddrs))) if standalone: if len(nodestore) > 1: raise ConfigurationError( "more than one node defined in standalone node config") else: if not manager: raise ConfigurationError("no manager defined in node config") elif not proxy: raise ConfigurationError("no proxy defined in node config") # If manager is on localhost, then all other nodes must be on localhost if manageronlocalhost: for n in nodestore.values(): if n.type != "manager" and n.addr not in localhostaddrs: raise ConfigurationError( "all nodes must use localhost/127.0.0.1/::1 when manager uses it" )
def _get_env_var_dict(self, text): env_vars = {} if text: for keyval in text.split(","): try: key, val = keyval.split("=", 1) except ValueError: raise ConfigurationError( "missing '=' in env_vars option value: %s" % keyval) key = key.strip() if not key: raise ConfigurationError( "env_vars option value must contain at least one environment variable name: %s" % keyval) env_vars[key] = val.strip() return env_vars
def _read_nodes(self): config = py3bro.configparser.SafeConfigParser() fname = self.nodecfg try: if not config.read(fname): raise ConfigurationError("cannot read node config file: %s" % fname) except py3bro.configparser.MissingSectionHeaderError as err: raise ConfigurationError(err) nodestore = NodeStore() counts = {} for sec in config.sections(): node = node_mod.Node(self, sec) # Note that the keys are converted to lowercase by configparser. for (key, val) in config.items(sec): key = key.replace(".", "_") if key not in node_mod.Node._keys: self.ui.warn( "ignoring unrecognized node config option '%s' given for node '%s'" % (key, sec)) continue node.__dict__[key] = val # Perform a sanity check on the node, and update nodestore. self._check_node(node, nodestore, counts) # Perform a sanity check on the nodestore (make sure we have a valid # cluster config, etc.). self._check_nodestore(nodestore.nodestore) return nodestore.nodestore
def add_node(self, node): # Add a node to the nodestore, but first check for duplicate node # names. This check is not case-sensitive, because names are stored # lowercase in the state db, and some filesystems are not # case-sensitive (working dir name is node name). # Duplicate node names can occur either because the user defined two # nodes that differ only by case (e.g. "Worker-1" and "worker-1"), or # if a user defines a node name that conflicts with an auto-generated # one (e.g. "worker-1" with lb_procs=2 and "worker-1-2"). namelower = node.name.lower() if namelower in self.nodenameslower: matchname = "" for nn in self.nodestore: if nn.lower() == namelower: matchname = nn break raise ConfigurationError('node name "%s" is a duplicate of "%s"' % (node.name, matchname)) self.nodestore[node.name] = node self.nodenameslower.append(namelower)
def _check_options(self): # Option names must be valid bash variable names because we will # write them to broctl-config.sh (note that broctl will convert "." # to "_" when it writes to broctl-config.sh). allowedchars = re.compile("^[a-z0-9_.]+$") nostartdigit = re.compile("^[^0-9]") for key, value in self.config.items(): if re.match(allowedchars, key) is None: raise ConfigurationError( 'broctl option name "%s" contains invalid characters (allowed characters: a-z, 0-9, ., and _)' % key) if re.match(nostartdigit, key) is None: raise ConfigurationError( 'broctl option name "%s" cannot start with a number' % key) # No broctl option ever requires the entire value to be wrapped in # quotes, and since doing so can cause problems, we don't allow it. if isinstance(value, str): if (value.startswith('"') and value.endswith('"') or value.startswith("'") and value.endswith("'")): raise ConfigurationError( 'value of broctl option "%s" cannot be wrapped in quotes' % key) dirs = ("brobase", "logdir", "spooldir", "cfgdir", "broscriptdir", "bindir", "libdirinternal", "plugindir", "scriptsdir") files = ("makearchivename", ) for d in dirs: v = self.config[d] if not os.path.isdir(v): raise ConfigurationError( 'broctl option "%s" directory not found: %s' % (d, v)) for f in files: v = self.config[f] if not os.path.isfile(v): raise ConfigurationError( 'broctl option "%s" file not found: %s' % (f, v)) # Verify that logs don't expire more quickly than the rotation interval logexpireseconds = 60 * self.config["logexpireminutes"] if 0 < logexpireseconds < self.config["logrotationinterval"]: raise ConfigurationError( "Log expire interval cannot be shorter than the log rotation interval" )
def _get_interval_minutes(self, optname): # Conversion table for time units to minutes. units = {"day": 24 * 60, "hr": 60, "min": 1} ss = self.config[optname] try: # If no time unit, assume it's days (for backward compatibility). v = int(ss) * units["day"] return v except ValueError: pass # Time interval is a non-negative integer followed by an optional # space, followed by a time unit. mm = re.match("([0-9]+) ?(day|hr|min)s?$", ss) if mm is None: raise ConfigurationError( 'value of broctl option "%s" is invalid (value must be integer followed by a time unit "day", "hr", or "min"): %s' % (optname, ss)) v = int(mm.group(1)) v *= units[mm.group(2)] return v
def _check_node(self, node, nodestore, counts): if not node.type: raise ConfigurationError("no type given for node %s" % node.name) if node.type not in ("logger", "manager", "proxy", "worker", "standalone"): raise ConfigurationError( "unknown node type '%s' given for node '%s'" % (node.type, node.name)) if not node.host: raise ConfigurationError("no host given for node '%s'" % node.name) try: addrinfo = socket.getaddrinfo(node.host, None, 0, 0, socket.SOL_TCP) except socket.gaierror as e: raise ConfigurationError( "hostname lookup failed for '%s' in node config [%s]" % (node.host, e.args[1])) addrs = [addr[4][0] for addr in addrinfo] # By default, just use the first IP addr in the list. addr_str = addrs[0] # Choose the first IPv4 addr (if any) in the list. for ip in addrs: if ":" not in ip: addr_str = ip break # zone_id is handled manually, so strip it if it's there node.addr = addr_str.split("%")[0] # Convert env_vars from a string to a dictionary. try: node.env_vars = self._get_env_var_dict(node.env_vars) except ConfigurationError as err: raise ConfigurationError("node '%s' config: %s" % (node.name, err)) # Each node gets a number unique across its type. try: counts[node.type] += 1 except KeyError: counts[node.type] = 1 node.count = counts[node.type] numprocs = 0 if node.lb_procs: if node.type != "worker": raise ConfigurationError( "node '%s' config: load balancing node config options are only for worker nodes" % node.name) try: numprocs = int(node.lb_procs) except ValueError: raise ConfigurationError( "number of load-balanced processes must be an integer for node '%s'" % node.name) if numprocs < 1: raise ConfigurationError( "number of load-balanced processes must be greater than zero for node '%s'" % node.name) elif node.lb_method: raise ConfigurationError( "number of load-balanced processes not specified for node '%s'" % node.name) try: pin_cpus = self._get_pin_cpu_list(node.pin_cpus, numprocs) except ValueError: raise ConfigurationError( "pin cpus list must contain only non-negative integers for node '%s'" % node.name) if pin_cpus: node.pin_cpus = pin_cpus[0] if node.lb_procs: if not node.lb_method: raise ConfigurationError( "no load balancing method given for node '%s'" % node.name) if node.lb_method not in ("pf_ring", "myricom", "custom", "interfaces"): raise ConfigurationError( "unknown load balancing method '%s' given for node '%s'" % (node.lb_method, node.name)) if node.lb_method == "interfaces": if not node.lb_interfaces: raise ConfigurationError( "list of load-balanced interfaces not specified for node '%s'" % node.name) # get list of interfaces to use, and assign one to each node netifs = node.lb_interfaces.split(",") if len(netifs) != numprocs: raise ConfigurationError( "number of load-balanced interfaces is not same as number of load-balanced processes for node '%s'" % node.name) node.interface = netifs.pop().strip() origname = node.name # node names will have a numerical suffix node.name = "%s-1" % node.name for num in range(2, numprocs + 1): newnode = node.copy() # Update the node attrs that need to be changed newname = "%s-%d" % (origname, num) newnode.name = newname counts[node.type] += 1 newnode.count = counts[node.type] if pin_cpus: newnode.pin_cpus = pin_cpus[num - 1] if newnode.lb_method == "interfaces": newnode.interface = netifs.pop().strip() nodestore.add_node(newnode) nodestore.add_node(node)