async def start(self):
""" Start up the job queue including resource initialisation. """
awaitables = []
for worker in self.workers:
awaitables.append(await worker.start())
if self.cluster_duplicate_handler:
awaitables.append(await self.cluster_duplicate_handler.start())
# create a single ruleset engine for all workers, instantiates all the
# rules based on the ruleset configuration, may start up long-lived
# analyzer instances which are shared as well, is otherwise stateless
# to allow concurrent use by multiple worker
try:
awaitables.extend(await self.ruleset_engine.start())
except (KeyError, ValueError, PeekabooConfigException) as error:
self.shut_down()
await self.close_down()
raise PeekabooConfigException('Ruleset configuration error: %s' %
error)
except PeekabooRulesetConfigError as error:
self.shut_down()
await self.close_down()
raise PeekabooConfigException(error)
return awaitables
def __init__(self, config_file):
super().__init__(interpolation=ListInterpolation())
# interpolation calls our add_list_value() from read_file() and read()
# which needs these defined
self.lists = {}
self.relists = {}
try:
self.read_file(open(config_file))
except IOError as ioerror:
raise PeekabooConfigException(
'Configuration file "%s" can not be opened for reading: %s' %
(config_file, ioerror))
except configparser.Error as cperror:
raise PeekabooConfigException(
'Configuration file "%s" can not be parsed: %s' %
(config_file, cperror))
# main configuration file may contain options in global section that
# configure drop-file read behaviour
drop_dir_template = self.get('global',
'drop_dir_template',
fallback='{config_path}.d')
drop_file_glob = self.get('global',
'drop_file_glob',
fallback='[0-9][0-9]-*.conf')
config_dir = os.path.dirname(config_file)
config_basename = os.path.basename(config_file)
drop_dir = drop_dir_template.format(config_path=config_file,
config_dir=config_dir,
config_file=config_basename)
drop_files = sorted(glob.glob(os.path.join(drop_dir, drop_file_glob)))
try:
read_files = self.read(drop_files)
except configparser.Error as cperror:
raise PeekabooConfigException(
f'Configuration drop file can not be parsed: {cperror}'
) from cperror
missing_files = set(drop_files) - set(read_files)
if missing_files:
raise PeekabooConfigException(
'Some configuration drop files could not be read: '
f'{missing_files}')
def get_log_level(self,
section,
option,
raw=False,
vars=None,
fallback=None):
""" Get the log level from the configuration file and parse the string
into a logging loglevel such as logging.CRITICAL. Raises config
exception if the log level is unknown. Options identical to get(). """
levels = {
'CRITICAL': logging.CRITICAL,
'ERROR': logging.ERROR,
'WARNING': logging.WARNING,
'INFO': logging.INFO,
'DEBUG': logging.DEBUG
}
level = self.get(section, option, raw=raw, vars=vars, fallback=None)
if level is None:
return fallback
if level not in levels:
raise PeekabooConfigException('Unknown log level %s' % level)
return levels[level]
def __init__(self, config_file):
super().__init__()
try:
self.read_file(open(config_file))
except IOError as ioerror:
raise PeekabooConfigException(
'Configuration file "%s" can not be opened for reading: %s' %
(config_file, ioerror))
except configparser.Error as cperror:
raise PeekabooConfigException(
'Configuration file "%s" can not be parsed: %s' %
(config_file, cperror))
self.lists = {}
self.relists = {}
def __init__(self, config_file):
# super() does not work here because ConfigParser uses old-style
# classes in python 2
configparser.ConfigParser.__init__(self)
try:
self.read_file(open(config_file))
except IOError as ioerror:
raise PeekabooConfigException(
'Configuration file "%s" can not be opened for reading: %s' %
(config_file, ioerror))
except configparser.Error as cperror:
raise PeekabooConfigException(
'Configuration file "%s" can not be parsed: %s' %
(config_file, cperror))
self.lists = {}
self.relists = {}
def getlist(self, section, option, raw=False, vars=None, fallback=None):
""" Special getter where multiple options in the config file
distinguished by a .<no> suffix form a list. Matches the signature for
configparser getters. """
# cache results because the following is somewhat inefficient
if section not in self.lists:
self.lists[section] = {}
if option in self.lists[section]:
return self.lists[section][option]
if section not in self:
self.lists[section][option] = fallback
return fallback
# Go over all options in this section we want to allow "holes" in
# the lists, i.e setting.1, setting.2 but no setting.3 followed by
# setting.4. We use here that ConfigParser retains option order from
# the file.
value = []
for setting in self[section]:
if not setting.startswith(option):
continue
# Parse 'setting' into (key) and 'setting.subscript' into
# (key, subscript) and use it to determine if this setting is a
# list. Note how we do not use the subscript at all here.
name_parts = setting.split('.')
key = name_parts[0]
is_list = len(name_parts) > 1
if key != option:
continue
if not is_list:
raise PeekabooConfigException(
'Option %s in section %s is supposed to be a list '
'but given as individual setting' % (setting, section))
# Potential further checks:
# - There are no duplicate settings with ConfigParser. The last
# one always wins.
value.append(self[section].get(setting, raw=raw, vars=vars))
# it's not gonna get any better on the next call, so cache even the
# default
if not value:
value = fallback
self.lists[section][option] = value
return value
def getlist(self, section, option, raw=False, vars=None, fallback=None):
""" Special getter where multiple options in the config file
distinguished by a .<no> suffix form a list. Matches the signature for
configparser getters. """
if section in self and option in self[section]:
raise PeekabooConfigException(
f'Option {option} in section {section} is supposed to be a '
'list but given as individual setting')
if section in self.lists and option in self.lists[section]:
return self.lists[section][option]
return fallback
def check_sections(self, known_sections):
""" Check a list of known section names against this configuration
@param known_sections: names of known sections
@type known_sections: list(string)
@returns: None
@raises PeekabooConfigException: if any unknown sections are found in
the configuration.
"""
section_diff = set(self.sections()) - set(known_sections)
if section_diff:
raise PeekabooConfigException(
'Unknown section(s) found in config: %s' %
', '.join(section_diff))
def getoctal(self, section, option, raw=False, vars=None, fallback=None):
""" Get an integer in octal notation. Raises config
exception if the format is wrong. Options identical to get(). """
value = self.get(section, option, raw=raw, vars=vars, fallback=None)
if value is None:
return fallback
try:
octal = int(value, 8)
except ValueError:
raise PeekabooConfigException(
'Invalid value for octal option %s in section %s: %s' %
(option, section, value))
return octal
def getrelist(self,
section,
option,
raw=False,
vars=None,
fallback=None,
flags=0):
""" Special getter for lists of regular expressions. Returns the
compiled expression objects in a list ready for matching and searching.
"""
if section not in self.relists:
self.relists[section] = {}
if option in self.relists[section]:
return self.relists[section][option]
if section not in self:
self.relists[section][option] = fallback
return fallback
strlist = self[section].getlist(option,
raw=raw,
vars=vars,
fallback=fallback)
if strlist is None:
self.relists[section][option] = None
return None
compiled_res = []
for regex in strlist:
try:
compiled_res.append(re.compile(regex, flags))
except (ValueError, TypeError) as error:
raise PeekabooConfigException(
'Failed to compile regular expression "%s" (section %s, '
'option %s): %s' % (re, section, option, error))
# it's not gonna get any better on the next call, so cache even the
# default
if not compiled_res:
compiled_res = fallback
self.relists[section][option] = compiled_res
return compiled_res
def gettlp(self, section, option, raw=False, vars=None, fallback=None):
levels = {
'red': tlp.RED,
'3': tlp.RED,
'amber': tlp.AMBER,
'2': tlp.AMBER,
'green': tlp.GREEN,
'1': tlp.GREEN,
'white': tlp.WHITE,
'0': tlp.WHITE,
}
level = self.get(section, option, raw=raw, vars=vars, fallback=None)
if level is None:
return fallback
level = level.lower()
if level not in levels:
raise PeekabooConfigException('Unknown tlp level %s' % level)
return levels[level]
def check_section_options(self, section, known_options):
""" Check a config section for unknown options.
@param section: name of section to check
@type section: string
@param known_options: list of names of known options to check against
@type known_options: list(string)
@returns: None
@raises PeekabooConfigException: if any unknown options are found. """
try:
section_options = map(
# account for option.1 list syntax
lambda x: x.split('.')[0],
self.options(section))
except configparser.NoSectionError:
# a non-existant section can have no non-allowed options :)
return
option_diff = set(section_options) - set(known_options)
if option_diff:
raise PeekabooConfigException(
'Unknown config option(s) found in section %s: %s' %
(section, ', '.join(option_diff)))