def __create_toolbox(self):
"""
Create the toolbox
:return: None
"""
self.toolbox = Toolbox(self, self.services.list_services(multi=True))
for section in self.config_parsers[TOOLBOX_CONF_FILE].sections():
newtool = self.__create_tool(section)
if newtool is not None:
if not self.toolbox.add_tool(newtool):
logger.warning('[{filename}{ext} | Section "{section}"] Unable to add tool "{tool}" into the toolbox'.format(
filename=TOOLBOX_CONF_FILE, ext=CONF_EXT, section=section, tool=newtool.name))
def __init__(self, settings_dir, toolbox_dir, output):
"""
Initialize Settings object
@Args settings_dir: directory where config files are stored
toolbox_dir: directory where the toolbox is stored
output: Instance of CLIOutput
"""
self.settings_dir = settings_dir
self.toolbox_dir = toolbox_dir
self.output = output
# config_parsers: dict of config_parsers indexed by conf_filename
self.config_parsers = {}
# general_settings: 2 dimensions dict - [service_name][option_name]
self.general_settings = {}
self.toolbox = Toolbox(self)
# Check directory and presence of *.conf files
if not FileUtils.is_dir(settings_dir):
self.output.printError(
'Configuration directory ({0}) does not exist'.format(
settings_dir))
raise ValueError
files = FileUtils.list_directory(settings_dir)
for f in files:
if not FileUtils.check_extension(f, CONF_EXT):
files.remove(f)
if not files:
self.output.printError(
'Configuration directory ({0}) does not store any *.conf file'.
format(settings_dir))
raise ValueError
# Parse config files
self.parseAllConfFiles(files)
def __init__(self, settings_dir, toolbox_dir, output):
"""
Constructor of Settings object
@Args settings_dir: directory where config files are stored
toolbox_dir: directory where the toolbox is stored
output: Instance of CLIOutput
"""
self.settings_dir = settings_dir
self.toolbox_dir = toolbox_dir
self.output = output
# config_parsers: dict of config_parsers indexed by service_name
self.config_parsers = {}
# general_settings: 2 dimensions dict - [service_name][setting]
self.general_settings = {}
self.toolbox = Toolbox(self)
# Check directory and presence of *.conf files
if not FileUtils.is_dir(settings_dir):
self.output.printError(
'Configuration directory ({0}) does not exist'.format(
settings_dir))
sys.exit(0)
files = FileUtils.list_directory(settings_dir)
for f in files:
if not FileUtils.check_extension(f, CONF_EXT):
files.remove(f)
if not files:
self.output.printError(
'Configuration directory ({0}) does not store any *.conf file'.
format(settings_dir))
sys.exit(0)
# Parse config files
# i.e. extract tools categories and optional/specific settings for each service
self.parseConfFiles(files)
class Settings(object):
def __init__(self, settings_dir, toolbox_dir, output):
"""
Constructor of Settings object
@Args settings_dir: directory where config files are stored
toolbox_dir: directory where the toolbox is stored
output: Instance of CLIOutput
"""
self.settings_dir = settings_dir
self.toolbox_dir = toolbox_dir
self.output = output
# config_parsers: dict of config_parsers indexed by service_name
self.config_parsers = {}
# general_settings: 2 dimensions dict - [service_name][setting]
self.general_settings = {}
self.toolbox = Toolbox(self)
# Check directory and presence of *.conf files
if not FileUtils.is_dir(settings_dir):
self.output.printError(
'Configuration directory ({0}) does not exist'.format(
settings_dir))
sys.exit(0)
files = FileUtils.list_directory(settings_dir)
for f in files:
if not FileUtils.check_extension(f, CONF_EXT):
files.remove(f)
if not files:
self.output.printError(
'Configuration directory ({0}) does not store any *.conf file'.
format(settings_dir))
sys.exit(0)
# Parse config files
# i.e. extract tools categories and optional/specific settings for each service
self.parseConfFiles(files)
def parseConfFiles(self, files):
"""
Parse all *.conf files into the config directory
@Args files: list of config files to parse
@Returns None
"""
# Process *.conf files
for f in files:
self.output.printInfo(
'Parsing configuration file "{0}" ...'.format(f))
full_path = FileUtils.concat_path(self.settings_dir, f)
service_name = f[:f.rfind(CONF_EXT)].lower().strip()
self.config_parsers[service_name] = DefaultConfigParser()
self.config_parsers[service_name].read(full_path)
#config_parser = DefaultConfigParser()
#config_parser.read(full_path)
# Add the entry into general settings for the service
self.general_settings[service_name] = {}
# General settings - [general] in .conf file
self.general_settings[service_name]['tools_categories'] = [
e.lower()
for e in self.config_parsers[service_name].safeGetList(
'general', 'tools_categories', ',', [])
]
# General settings - Optional/Specific settings (depends on the targeted servicee)
if service_name in SPECIFIC_TOOL_OPTIONS.keys():
for option in SPECIFIC_TOOL_OPTIONS[service_name]:
if SPECIFIC_TOOL_OPTIONS[service_name][option]:
setting_name = SPECIFIC_TOOL_OPTIONS[service_name][
option]
self.general_settings[service_name][setting_name] = \
[ e.lower() for e in self.config_parsers[service_name].safeGetList('general', setting_name, ',', []) ]
# Check general settings for the current service
self.checkGeneralSettings(service_name)
# Add service as new toolbox section
self.toolbox.addService(service_name)
# Add tools in current config file into the toolbox, under the correct service section
for section in self.config_parsers[service_name].sections():
if section.startswith('tool_'):
newtool = self.createToolFromConfiguration(
section, service_name)
if newtool:
if not self.toolbox.addTool(newtool, service_name):
self.output.printWarning(
'Unable to add tool "{0}" into the toolbox'.
format(newtool.name))
else:
#self.output.printSettings('Tool "{0}" added into the toolbox (category "{1}")'.format(newtool.name,
# newtool.category))
pass
def checkGeneralSettings(self, service_name):
"""
Check [general] section
@Args service_name: service related to config file to check
@Returns Boolean indicating status
"""
if service_name not in self.general_settings.keys():
return False
# General settings - [general] in .conf file
if not self.general_settings[service_name]['tools_categories']:
self.output.printError(
'[{0}{1}] General settings error: Incorrect "tools_categories"'
.format(service_name, CONF_EXT))
sys.exit(0)
# General settings - Optional/Specific settings (depends on the targeted servicee)
if service_name in SPECIFIC_TOOL_OPTIONS.keys():
for option in SPECIFIC_TOOL_OPTIONS[service_name]:
if SPECIFIC_TOOL_OPTIONS[service_name][option]:
setting_name = SPECIFIC_TOOL_OPTIONS[service_name][option]
if not self.general_settings[service_name][setting_name]:
self.output.printWarning('[{0}{1}] General settings warning: No "{2}" setting for service {3}. The tool option "{4}" will not be taken into account'.format( \
service_name, CONF_EXT, setting_name, service_name, option))
return True
def createToolFromConfiguration(self, section, service_name):
"""
Create tool object from a [tool_****] entry into the settings file
@Args section: section from config file corresponding to a tool
service_name: service targeted by the tool
@Returns instance of Tool object if everything is ok, False otherwise
"""
if service_name not in self.general_settings.keys():
return False
# First, check for the presence of all needed option for the tool
options = self.config_parsers[service_name].options(section)
for o in MANDATORY_TOOL_OPTIONS:
if o not in options:
self.output.printWarning('[{0}{1}] Section "{2}" > missing mandatory option "{3}", skipped'.format( \
service_name, CONF_EXT, section, o))
return False
# Parse general+mandatory info
try:
name = self.config_parsers[service_name].safeGet(
section, 'name', '', None).strip()
category = self.config_parsers[service_name].safeGet(
section, 'category', '', None).strip().lower()
description = self.config_parsers[service_name].safeGet(
section, 'description', '', None).strip()
raw_command = self.config_parsers[service_name].safeGet(
section, 'command', '', None).strip()
except:
self.output.printWarning('[{0}{1}] Section "{2}" > syntax error with mandatory options'.format( \
service_name, CONF_EXT, section))
#traceback.print_exc()
return False
# Check general+mandatory info
if not name:
self.output.printWarning(
'[{0}{1}] Section "{2}" > option "name" is empty, section skipped'
.format(service_name, CONF_EXT, section))
return False
if not category:
self.output.printWarning(
'[{0}{1}] Section "{2}" > option "category" is empty, section skipped'
.format(service_name, CONF_EXT, section))
return False
if category not in self.general_settings[service_name][
'tools_categories']:
self.output.printWarning(
'[{0}{1}] Section "{2}" > option "category" ("{3}") not in "tools_categories", section skipped'
.format(service_name, CONF_EXT, section, category))
return False
if not raw_command:
self.output.printWarning(
'[{0}{1}] Section "{2}" > option "command" is empty, section skipped'
.format(service_name, CONF_EXT, section))
return False
# Parse general+optional info
try:
install = self.config_parsers[service_name].safeGet(
section, 'install', '', None).strip()
update = self.config_parsers[service_name].safeGet(
section, 'update', '', None).strip()
last_update = self.config_parsers[service_name].safeGet(
section, 'last_update', '', None).strip()
installed = self.config_parsers[service_name].safeGetBoolean(
section, 'installed', True)
except:
pass
# Parse specific info (depends on targeted service)
# opt_specific is a dictionary: "option" => (type, value)
opt_specific = dict()
if service_name in SPECIFIC_TOOL_OPTIONS.keys():
for option in SPECIFIC_TOOL_OPTIONS[service_name]:
# Boolean options (default False)
if SPECIFIC_TOOL_OPTIONS[service_name][option] == '':
opt_specific[option] = (
bool, self.config_parsers[service_name].safeGetBoolean(
section, option + '_specific', False))
# List-type options
else:
value_list = [
e.lower()
for e in self.config_parsers[service_name].safeGetList(
section, option + '_specific', ',', [])
]
if value_list:
for e in value_list:
if e.lower(
) not in self.general_settings[service_name][
SPECIFIC_TOOL_OPTIONS[service_name]
[option]]:
value_list.remove(e)
self.output.printWarning(
'[{0}{1}] Section "{2}" > option "{3}" contains invalid entry ("{4}")'
.format(service_name, CONF_EXT, section,
option, e))
opt_specific[option] = (list, value_list)
# Create the Tool object from parsed info
tool = Tool(
service_name,
section,
self.toolbox_dir,
# General+Mandatory tool options
name,
category,
description,
raw_command,
# General+Optional tool options
install,
update,
last_update,
installed,
# Specific tool options
opt_specific)
return tool
def changeInstalledOption(self, service_name, tool_section_name, value):
"""
Change the status of option "installed" for a given tool
@Args service_name: service targeted by the tool
tool_section_name: Tool section name as it appears in config file
value: 'True' if tool installed, 'False' otherwise
@Returns Boolean indicating operation status
"""
if value not in ('True', 'False'):
return False
if self.config_parsers[service_name].safeSet(tool_section_name,
'installed', value):
return self.saveSettings(service_name)
return False
def changeLastUpdateOption(self, service_name, tool_section_name):
"""
Update the value of the option "last_update" with the current date-time
@Args service_name: service targeted by the tool
tool_section_name: Tool section name as it appears in config file
@Returns Boolean indicating operation status
"""
current_datetime = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
if self.config_parsers[service_name].safeSet(tool_section_name,
'last_update',
current_datetime):
return self.saveSettings(service_name)
return False
def saveSettings(self, service_name):
"""
Save settings into config file.
Make sure changes are thus taken into account.
@Args service_name: service targeted by the tool
@Returns Boolean indicating operation status
"""
try:
config_file = FileUtils.concat_path(self.settings_dir,
service_name + '.conf')
with open(config_file, 'w') as handle:
self.config_parsers[service_name].write(handle)
# Re-read to take change into account
self.config_parsers[service_name].read(
config_file) # warning: takes filename as param
return True
except:
traceback.print_exc()
return False
class Settings:
"""
Class used to parse all settings files:
- toolbox.conf : File storing configurations about all tools
- <service_name>.conf : Each supported service has a corresponding .conf file
- _install_status.conf : Install status for each tool & last update if installed
- attack_profiles.conf : Attack profiles
Settings class is instanciated when starting Jok3r.
"""
def __init__(self):
"""
Start the parsing of settings files and create the Settings object.
:raises SettingsException: Exception raised if any error is encountered while
parsing files (syntax error, missing mandatory file...)
"""
self.config_parsers = dict(
) # Dict of DefaultConfigParser indexed by filename
self.toolbox = None # Receives Toolbox object
self.services = None # Receives ServicesConfig object
self.attack_profiles = None # Receives AttackProfiles object
# Check directory
if not FileUtils.is_dir(SETTINGS_DIR):
raise SettingsException('Configuration directory ({dir}) does not ' \
'exist'.format(dir=SETTINGS_DIR))
# Check presence of *.conf files
files = FileUtils.list_directory(SETTINGS_DIR)
for f in files:
if not FileUtils.check_extension(f, CONF_EXT):
files.remove(f)
if not files:
raise SettingsException('Configuration directory ({dir}) does not ' \
'store any *.conf file'.format(dir=SETTINGS_DIR))
if TOOLBOX_CONF_FILE + CONF_EXT not in files:
raise SettingsException('Missing mandatory {toolbox}{ext} settings ' \
'file in directory "{dir}"'.format(
toolbox=TOOLBOX_CONF_FILE,
ext=CONF_EXT,
dir=SETTINGS_DIR))
if ATTACK_PROFILES_CONF_FILE + CONF_EXT not in files:
raise SettingsException('Missing mandatory {profiles}{ext} settings ' \
'file in directory "{dir}"'.format(
profiles=ATTACK_PROFILES_CONF_FILE,
ext=CONF_EXT,
dir=SETTINGS_DIR))
# Create _install_status.conf file if necessary
if INSTALL_STATUS_CONF_FILE + CONF_EXT not in files:
open(SETTINGS_DIR + '/' + INSTALL_STATUS_CONF_FILE + CONF_EXT,
'a').close()
logger.info('{status}{ext} settings file created in directory ' \
'"{dir}"'.format(status=INSTALL_STATUS_CONF_FILE,
ext=CONF_EXT,
dir=SETTINGS_DIR))
files.append(INSTALL_STATUS_CONF_FILE + CONF_EXT)
# Parse configuration files and create objects from them
self.__parse_all_conf_files(files)
self.__create_toolbox()
self.__create_all_services_config_and_checks()
self.__create_attack_profiles()
#------------------------------------------------------------------------------------
# Config files reading
def __parse_all_conf_files(self, files):
"""
Parse all configuration files into the settings directory.
Initialize ServicesConfig object with list of supported services.
:param list files: List of files in settings directory
"""
services = list()
for f in files:
name = FileUtils.remove_ext(f).lower().strip()
if name not in (INSTALL_STATUS_CONF_FILE, TOOLBOX_CONF_FILE,
ATTACK_PROFILES_CONF_FILE):
services.append(name)
full_path = FileUtils.concat_path(SETTINGS_DIR, f)
self.config_parsers[name] = DefaultConfigParser()
# Utf8 to avoid encoding issues
self.config_parsers[name].read(full_path, 'utf8')
services.append('multi') # Add support for special "multi" service
self.services = ServicesConfig(services)
#------------------------------------------------------------------------------------
# Toolbox config file parsing
def __create_toolbox(self):
"""Create the toolbox and update self.toolbox."""
self.toolbox = Toolbox(self, self.services.list_services(multi=True))
for section in self.config_parsers[TOOLBOX_CONF_FILE].sections():
newtool = self.__create_tool(section)
if newtool is not None:
if not self.toolbox.add_tool(newtool):
logger.warning('[{filename}{ext} | Section "{section}"] Unable ' \
'to add tool "{tool}" into the toolbox'.format(
filename=TOOLBOX_CONF_FILE,
ext=CONF_EXT,
section=section,
tool=newtool.name))
def __create_tool(self, section):
"""
Create a Tool object.
:param str section: Section name corresponding to the tool in toolbox.conf
:return: The newly created tool
:rtype: Tool
"""
tool_config = defaultdict(str)
if not self.__parse_tool_options(section, tool_config):
return None
if not self.__parse_tool_install_status(tool_config):
return None
return Tool(tool_config['name'], tool_config['description'],
tool_config['target_service'], tool_config['installed'],
tool_config['last_update'], tool_config['install'],
tool_config['update'], tool_config['check_command'])
def __parse_tool_options(self, section, tool_config):
"""
Check and parse options from a given tool section.
:param str section: Section name corresponding to the tool in toolbox.conf
:param defaultdict(str) tool_config: Tool configuration updated in this method
:return: Status of parsing
:rtype: bool
"""
log_prefix = '[{filename}{ext} | Section "{section}"]'.format(
filename=TOOLBOX_CONF_FILE, ext=CONF_EXT, section=section)
# Check presence of mandatory options
optparsed = self.config_parsers[TOOLBOX_CONF_FILE].options(section)
for opt in TOOL_OPTIONS[MANDATORY]:
if opt not in optparsed:
logger.warning('{prefix} Missing mandatory option "{option}", ' \
'tool is skipped'.format(prefix=log_prefix, option=opt))
return False
# Loop over options
for opt in optparsed:
# Check for unsupported options
if opt not in TOOL_OPTIONS[MANDATORY] + TOOL_OPTIONS[OPTIONAL]:
logger.warning('{prefix} Option "{option}" is not supported, ' \
'it will be ignored'.format(prefix=log_prefix, option=opt))
continue
# Add value
val = self.config_parsers[TOOLBOX_CONF_FILE].safe_get(
section, opt, '', None)
if opt == 'name':
tool_config[opt] = StringUtils.clean(
val, allowed_specials=['-', '_'])
elif opt == 'description':
tool_config[opt] = val
elif opt == 'target_service':
tool_config[opt] = val.lower()
if tool_config[opt] not in self.services.list_services(
multi=True):
logger.warning('{prefix} Service specified in "target_service" is ' \
'not supported, tool is skipped'.format(prefix=log_prefix))
return False
elif opt == 'install':
tool_config[opt] = Command(cmdtype=CmdType.INSTALL,
cmdline=val)
elif opt == 'update':
tool_config[opt] = Command(cmdtype=CmdType.UPDATE, cmdline=val)
elif opt == 'check_command':
tool_config[opt] = Command(cmdtype=CmdType.CHECK, cmdline=val)
# Check for empty mandatory option
if opt in TOOL_OPTIONS[MANDATORY] and not tool_config[opt]:
logger.warning('{prefix} Mandatory option "{option}" is empty, tool ' \
'is skipped'.format(prefix=log_prefix, option=opt))
return False
return True
def __parse_tool_install_status(self, tool_config):
"""
Retrieve install status of a given tool and update tool configuration
accordingly.
By default: not installed, no last update date
Must be called after self.__parse_tool_options()
:param defaultdict(str) tool_config: Tool configuration updated in this method
:return: Status of parsing
:rtype: bool
"""
is_installed = self.config_parsers[INSTALL_STATUS_CONF_FILE].safe_get(
tool_config['target_service'], tool_config['name'], 'false',
None).lower().strip()
if is_installed == 'false':
tool_config['installed'], tool_config['last_update'] = False, ''
elif is_installed == 'true':
tool_config['installed'], tool_config['last_update'] = True, ''
else:
tool_config['installed'], tool_config[
'last_update'] = True, is_installed
return True
#------------------------------------------------------------------------------------
# Services configurations and checks parsing
def __create_all_services_config_and_checks(self):
"""Parse each <service_name>.conf file"""
for f in self.config_parsers:
if f in (INSTALL_STATUS_CONF_FILE, TOOLBOX_CONF_FILE,
ATTACK_PROFILES_CONF_FILE):
continue
self.__parse_service_checks_config_file(f)
def __parse_service_checks_config_file(self, service):
"""
Parse a service checks configuration file <service_name>.conf, in order to
create a ServiceChecks object and to update ServicesConfig object with
service information (default port, protocol, supported specific options,
supported products, authentication type for HTTP).
:param str service: Service name
"""
service_config = defaultdict(str)
categories = self.__parse_section_config(service, service_config)
self.__parse_section_specific_options(service, service_config)
self.__parse_section_supported_list_options(service, service_config)
self.__parse_section_products(service, service_config)
# Add the service configuration from settings
self.services.add_service(service, service_config['default_port'],
service_config['protocol'],
service_config['specific_options'],
service_config['supported_list_options'],
service_config['products'],
service_config['auth_types'],
ServiceChecks(service, categories))
# Add the various checks for the service into the ServiceChecks object
self.__parse_all_checks_sections(service)
#------------------------------------------------------------------------------------
# Services configurations parsing
def __parse_section_config(self, service, service_config):
"""
Parse section [config] in <service_name>.conf, retrieve basic info about service
(default port/protocol) and retrieve list of supported categories of checks for
this service.
:param str service: Service name
:param defaultdict(str) service_config: Information about the service, updated
into this method
:return: List of categories of checks
:rtype: list(str)
:raises SettingsException: Exception raised if any unrecoverable error is
encountered while parsing the section
"""
log_prefix = '[{filename}{ext} | Section "config"]'.format(
filename=service, ext=CONF_EXT)
# Check presence of mandatory options in [config]
optparsed = self.config_parsers[service].options('config')
for opt in SERVICE_CHECKS_CONFIG_OPTIONS[MANDATORY]:
if opt not in optparsed:
raise SettingsException('{prefix} Missing mandatory option "{option}"' \
', check the file'.format(prefix=log_prefix, option=opt))
# Get port number
default_port = self.config_parsers[service].safe_get_int(
'config', 'default_port', None, None)
if default_port is None or default_port < 0 or default_port > 65535:
raise SettingsException('{prefix} Invalid value for option "default_port",' \
' must be in the range [0-65535]'.format(prefix=log_prefix))
# Get protocol
protocol = self.config_parsers[service].safe_get_lower(
'config', 'protocol', 'tcp', ['tcp', 'udp'])
# Get categories of checks as a list, clean each element
categories = list(
map(
lambda x: StringUtils.clean(x.lower(),
allowed_specials=('-', '_')),
self.config_parsers[service].safe_get_list(
'config', 'categories', ',', [])))
if not categories:
raise SettingsException('{prefix} Option "categories" must have at least '\
'one category'.format(prefix=log_prefix))
# Get authentication type (for HTTP) as a list, clean each element
if 'auth_types' in optparsed:
auth_types = list(
map(
lambda x: StringUtils.clean(x.lower(),
allowed_specials=('-', '_')),
self.config_parsers[service].safe_get_list(
'config', 'auth_types', ',', [])))
else:
auth_types = None
# Update service configuration with parsed information
service_config['default_port'] = default_port
service_config['protocol'] = protocol
service_config['auth_types'] = auth_types
return categories
def __parse_section_specific_options(self, service, service_config):
"""
Parse section [specific_options] in <service_name>.conf and update service
configuration with supported specific options for the service and their
respective types.
:param str service: Service name
:param defaultdict(str) service_config: Information about the service, updated
into this method
:return: None
:raises SettingsException: Exception raised if any unrecoverable error is
encountered while parsing the section
"""
# Case when no [specific_options] can be found
try:
optparsed = self.config_parsers[service].options(
'specific_options')
except configparser.NoSectionError:
service_config['specific_options'] = dict()
return
specific_options = dict()
# Loop over supported specific options
for opt in optparsed:
# Get option type
option_type = self.config_parsers[service].safe_get_lower(
'specific_options', opt, None, None)
# Handle case when default value is specified (for boolean)
if option_type.count(':') == 1:
option_type, default_value = option_type.split(':')
opt_clean = StringUtils.clean(opt.lower(),
allowed_specials=('-', '_'))
if option_type == 'boolean':
specific_options[opt_clean] = OptionType.BOOLEAN
elif option_type == 'list':
specific_options[opt_clean] = OptionType.LIST
elif option_type == 'var':
specific_options[opt_clean] = OptionType.VAR
else:
raise SettingsException('[{filename}{ext} | Section ' \
'"specific_options"] Specific option named "{option}" has ' \
'an invalid type. Supported types are: boolean, list, var'.format(
filename = service, ext=CONF_EXT, option=opt))
# Update service configuration with specific options names and types
service_config['specific_options'] = specific_options
def __parse_section_supported_list_options(self, service, service_config):
"""
Parse section [supported_list_options] in <service_name>.conf and update service
configuration with supported values for specific options of type list.
Must be called after self.__parse_section_config() and
self.__parse_section_specific_options().
:param defaultdict(str) service_config: Information about the service, updated
into this method
:return: None
:raises SettingsException: Exception raised if any unrecoverable error is
encountered while parsing the section
"""
# Get names of specific options of type list
options_list = list(
filter(
lambda x: service_config['specific_options'][x] == OptionType.
LIST, service_config['specific_options'].keys()))
if not options_list:
return
elif not self.config_parsers[service].has_section(
'supported_list_options'):
raise SettingsException('[{filename}{ext}] Missing section ' \
'[supported_list_options] to store supported values for specific ' \
'options of type "list"'.format(filename=service, ext=CONF_EXT))
log_prefix = '[{filename}{ext} | Section "supported_list_options"]'.format(
filename=service, ext=CONF_EXT)
supported_list_options = dict()
optparsed = self.config_parsers[service].options(
'supported_list_options')
# Loop over specific options of type list
for opt in options_list:
# If missing option
if 'supported_' + opt not in optparsed:
raise SettingsException('{prefix} No option "supported_{option}" ' \
'is defined'.format(prefix=log_prefix, option=opt))
# Values are put in lowercase, no spaces, no special chars (except -, _)
values = list(
map(
lambda x: StringUtils.clean(x.lower(),
allowed_specials=('-', '_')),
self.config_parsers[service].safe_get_list(
'supported_list_options', 'supported_' + opt, ',',
[])))
if not values:
raise SettingsException('{prefix} Option "supported_{option}" is ' \
'empty'.format(prefix=log_prefix, option=opt))
supported_list_options[opt] = values
# Update service configuration with lists of supported values
service_config['supported_list_options'] = supported_list_options
def __parse_section_products(self, service, service_config):
"""
Parse section [products] in <service_name>.conf and retrieve supported values
for each product type.
:param str service: Service name
:param dict service_config: Service configuration, updated into this method
:return: None
:raises SettingsException: Exception raised if unconsistent values detected
"""
# First, check if config file has a [products] section
if not self.config_parsers[service].has_section('products'):
service_config['products'] = dict()
return
log_prefix = '[{filename}{ext} | Section "products"]'.format(
filename=service, ext=CONF_EXT)
products = dict()
optparsed = self.config_parsers[service].options('products')
# Loop over product types in [products]
for product_type in optparsed:
# Clean the product type
product_type = StringUtils.clean(product_type.lower(),
allowed_specials=('-', '_'))
# Get supported product names as a list.
# Only some special chars allowed, spaces allowed
# '/' is used to separate vendor name (optional) and product name
product_names = self.config_parsers[service].safe_get_list(
'products', product_type, ',', [])
product_names = list(
map(
lambda x: StringUtils.clean(
x, allowed_specials=('-', '_', '.', '/', '\\', ' ')),
product_names))
if not product_names:
raise SettingsException(
'{prefix} Option "{option}" is empty'.format(
prefix=log_prefix, option=opt))
products[product_type] = product_names
# Update service configuration with supported products
service_config['products'] = products
return
#------------------------------------------------------------------------------------
# Services security checks parsing
def __parse_all_checks_sections(self, service):
"""
Parse all the [check_(.+)] sections of a given service checks settings file
<service_name>.conf.
:param str service: Service name
"""
for section in self.config_parsers[service].sections():
# Check section begins with "check_"
if section.startswith(PREFIX_SECTION_CHECK):
check_config = defaultdict(str)
# Parse section
if not self.__parse_check_section(service, section,
check_config):
continue
# Create new Check object and add it to services configuration
newcheck = Check(
check_config['name'],
check_config['category'],
check_config['description'],
check_config['tool'],
check_config['commands'],
)
self.services[service]['checks'].add_check(newcheck)
def __parse_check_section(self, service, section, check_config):
"""
Check and parse options from a given check section.
:param str service: Service name
:param str section: Section corresponding to the check to parse
:param defaultdict(str) check_config: Check configuration, updated into this
method
:return: Status of parsing
:rtype: bool
"""
log_prefix = '[{filename}{ext} | Section "{section}"]'.format(
filename=service, ext=CONF_EXT, section=section)
# Check presence of mandatory options in [check_<name>] section
optparsed = self.config_parsers[service].options(section)
for opt in CHECK_OPTIONS[MANDATORY]:
if opt not in optparsed:
logger.warning('{prefix} Missing mandatory option "{option}", the ' \
' check is ignored'.format(prefix=log_prefix, option=opt))
return False
# Loop over options
for opt in optparsed:
# Check for unsupported options
if opt not in CHECK_OPTIONS[MANDATORY]+CHECK_OPTIONS[OPTIONAL] \
and not opt.startswith('command_') and not opt.startswith('context_'):
logger.warning('{prefix} Option "{option}" is not supported, the ' \
'check is ignored'.format(prefix=log_prefix, option=opt))
continue
# Add value
val = self.config_parsers[service].safe_get(section, opt, '', None)
if opt == 'name':
check_config[opt] = StringUtils.clean(val,
allowed_specials=('_',
'-'))
elif opt == 'category':
cat = StringUtils.clean(val, allowed_specials=('_', '-'))
check_config[opt] = cat.lower()
if check_config[opt] not in self.services[service][
'checks'].categories:
logger.warning('{prefix} Category "{category}" is not supported, ' \
'the check is ignored'.format(prefix=log_prefix, category=val))
return False
elif opt == 'tool':
tool = self.toolbox.get_tool(val)
if tool is None:
logger.warning('{prefix} The tool "{tool}" does not exist, the ' \
'check is ignored'.format(prefix=log_prefix, tool=val))
return False
check_config[opt] = tool
else:
check_config[opt] = val
# Check for empty mandatory option
if opt in CHECK_OPTIONS[MANDATORY] and not check_config[opt]:
logger.warning('{prefix} Mandatory option "{option}" is empty, the ' \
'check is ignored'.format(prefix=log_prefix, option=opt))
return False
# Parse commands along with optional context requirements
commands = self.__parse_commands(service, section)
if not commands:
return False
check_config['commands'] = commands
return True
def __parse_commands(self, service, section):
"""
Parse commands for a given tool and create Commands object.
Each command is defined in configuration file by:
- command_<command_number>
- context_<command_number> (optional)
:param str service: Service name
:param str section: Section name of the check
:return: Created Command objects
:rtype: list(Command)
"""
log_prefix = '[{filename}{ext} | Section "{section}"]'.format(
filename=service, ext=CONF_EXT, section=section)
commands = list()
# Get command lines
cmdlines = self.config_parsers[service].safe_get_multi(section,
'command',
default=None)
i = 0
# Loop over command lines
for cmd in cmdlines:
# Parse context requirements and create ContextRequirements object
context = self.config_parsers[service].safe_get(section,
'context_' +
str(i + 1),
default=None)
context_requirements = self.__parse_context_requirements(
service, section, i + 1, context)
if context_requirements is None:
logger.warning('{prefix} Context requirements are invalid, the check ' \
'is ignored'.format(prefix=log_prefix))
return None
# Create the Command object
command = Command(cmdtype=CmdType.RUN,
cmdline=cmdlines[i],
context_requirements=context_requirements,
services_config=self.services)
commands.append(command)
i += 1
if not commands:
logger.warning('{prefix} No command is specified, the check is ' \
'ignored'.format(prefix=log_prefix))
return commands
def __parse_context_requirements(self, service, section, num_context,
context_str):
"""
Convert the value of a "context_<command_number>" option into a valid
Python dictionary, and initialize a fresh ContextRequirements object from it.
:param str service: Service name
:param str section: Section name [check_<name>]
:param int num_context: Number in option name, ie: context_<num_context>
:param str context_str: Context string extracted from settings file
:return: Context if parsing is ok, None otherwise
:rtype: Context|None
"""
# When no context is defined in settings, it means there is no requirement
if not context_str:
return ContextRequirements(specific_options=None,
products=None,
os=None,
auth_status=None,
raw='<empty>')
log_prefix = '[{filename}{ext} | Section "{section}"] "context_{i}":'.format(
filename=service, ext=CONF_EXT, section=section, i=num_context)
# Keep raw context string for debugging
context_str_raw = context_str
# Retrieve value as dict
# Note: Make sure to replace special constants
context_str = context_str.replace('NO_AUTH', str(NO_AUTH))\
.replace('USER_ONLY', str(USER_ONLY))\
.replace('POST_AUTH', str(POST_AUTH))
try:
context = ast.literal_eval(context_str)
except Exception as e:
logger.warning('{prefix} Parsing error. Valid dictionary syntax is ' \
'probably not respected: { \'key\': \'value\', ... }'.format(
prefix=log_prefix))
return None
# Check validity of context requirements
req_specific_options = dict()
req_products = dict()
for cond, val in context.items():
# Auth status
if cond == 'auth_status':
if val not in (NO_AUTH, USER_ONLY, POST_AUTH, None):
logger.warning('{prefix} Invalid value for "auth_status" ' \
'context requirement. Supported values are: NO_AUTH, ' \
'USER_ONLY, POST_AUTH, None'.format(prefix=log_prefix))
return None
# Auth type (for HTTP)
elif cond == 'auth_type':
if service != 'http':
logger.warning('{prefix} "auth_type" context requirement is only ' \
'supported for service HTTP'.format(prefix=log_prefix))
return None
elif context[cond] not in self.services[service]['auth_types']:
logger.warning('{prefix} "auth_type" context requirement does not ' \
'have a valid value, check info --list-http-auth'.format(
prefix=log_prefix))
return None
# OS
elif cond == 'os':
if not val:
logger.warning('{prefix} "os" context requirement is specified' \
'but no value is provided'.format(prefix=log_prefix))
return None
# Specific option
elif self.services.is_specific_option_name_supported(
cond, service):
if val is None:
continue
type_ = self.services.get_specific_option_type(cond, service)
# For specific option of type "list"
# Value can be either of type: None, str, list
# Possible values:
# - None: no restriction on the specific option value (default),
# - str: restriction on a given value,
# - list: restriction on several possible values
# - 'undefined': specific option must not be defined
if type_ == OptionType.LIST:
if val == 'undefined':
req_specific_options[cond] = ['undefined']
else:
if isinstance(val, str):
val = [val]
val = list(map(lambda x: x.lower(), val))
sup_vals = self.services[service][
'supported_list_options'][cond]
for e in val:
if e not in sup_vals:
logger.warning('{prefix} Context requirement ' \
'"{option}" contains an invalid element ' \
'("{element}")'.format(prefix=log_prefix,
option=cond, element=e))
return None
req_specific_options[cond] = val
# For specific option of type "boolean" or "var"
# Context requirement must be boolean or None
elif type_ in (OptionType.BOOLEAN, OptionType.VAR):
if not isinstance(val, bool):
logger.warning('{prefix} Context requirement "{option}" must ' \
'have a boolean value (True/False) or None'.format(
prefix=log_prefix, option=cond))
return None
req_specific_options[cond] = val
# Product
elif self.services.is_product_type_supported(cond, service):
# Possible values:
# - None: no restriction on the product name (default),
# - str: restriction on a given product name,
# - list: restriction on several possible product names,
# - 'undefined': product must not be defined
# - 'any': product must be defined (any value)
# - 'any|version_known': product+version must be defined
#
# In context requirements, product name can also embed requirement on
# product version by appending "|version_requirements" to product name
if val in ('undefined', 'any', 'any|version_known'):
req_products[cond] = [val]
else:
if isinstance(val, str):
val = [val]
for e in val:
# Check if [vendor/]product_name is in the list of supported
# product names (ignore version requirements if present)
product_name = e[:e.index('|')] if '|' in e else e
# Handle the case where inversion is used with prefix "!"
if len(product_name) > 0 and product_name[0] == '!':
product_name = product_name[1:]
if product_name.lower() not in list(
map(lambda x: x.lower(),
self.services[service]['products'][cond])):
logger.warning('{prefix} Context requirement "{option}" ' \
'contains an invalid product ("{product}")'.format(
prefix=log_prefix,
option=cond,
product=product_name))
req_products[cond] = val
# Not supported
else:
logger.warning('{prefix} Context requirement "{option}" is not ' \
'supported for service {service}'.format(
prefix=log_prefix, option=cond, service=service))
return None
return ContextRequirements(specific_options=req_specific_options,
products=req_products,
os=context.get('os'),
auth_status=context.get('auth_status'),
auth_type=context.get('auth_type'),
raw=context_str_raw)
#------------------------------------------------------------------------------------
# Attack profiles parsing
def __create_attack_profiles(self):
"""Create Attack Profiles and update self.attack_profiles."""
self.attack_profiles = AttackProfiles()
for section in self.config_parsers[ATTACK_PROFILES_CONF_FILE].sections(
):
newprofile = self.__create_attack_profile(section)
if newprofile is not None:
if not self.attack_profiles.add(newprofile):
logger.warning('[{filename}{ext} | Section "{section}"] Unable ' \
'to add attack profile "{profile}" (duplicate)'.format(
filename=ATTACK_PROFILES_CONF_FILE,
ext=CONF_EXT,
section=section,
profile=newprofile.name))
def __create_attack_profile(self, section):
"""
Create an AttackProfile object.
:param str section: Section name corresponding to the profile in
attack_profiles.conf
:return: The newly created Attack Profile, or None in case of problem
:rtype: AttackProfile|None
"""
log_prefix = '[{filename}{ext} | Section "{section}"]'.format(
filename=ATTACK_PROFILES_CONF_FILE, ext=CONF_EXT, section=section)
description = ''
checks = defaultdict(list)
optparsed = self.config_parsers[ATTACK_PROFILES_CONF_FILE].options(
section)
# Loop over options
for opt in optparsed:
if opt == 'description':
description = self.config_parsers[
ATTACK_PROFILES_CONF_FILE].safe_get(
section, opt, '', None)
# List of checks (ordered) for a service (name of the option must
# correspond to the name of the service)
elif self.services.is_service_supported(opt, multi=False):
list_checks = self.config_parsers[ATTACK_PROFILES_CONF_FILE]\
.safe_get_list(section, opt, ',', [])
if not list_checks:
logger.warning('{prefix} List of checks for {service} is empty, ' \
'the attack profile is skipped'.format(
prefix=log_prefix, service=opt))
return None
# Check existence of checks
for c in list_checks:
if not self.services.get_service_checks(
opt).is_existing_check(c):
logger.warning('{prefix} The check "{check}" does not exist ' \
'for service {service}, the attack profile is ' \
'skipped'.format(prefix=log_prefix, check=c, service=opt))
return None
# Add list of checks in dictionnary for the corresponding service
checks[opt] = list_checks
# Unsupported option
else:
logger.warning('{prefix} The option "{option}" is not supported ' \
'for attack profile configuration'.format(
prefix=log_prefix, option=opt))
return None
if not description:
logger.warning('{prefix} A description must be given for the attack ' \
'profile'.format(prefix=log_prefix))
return None
return AttackProfile(section, description, checks)
#------------------------------------------------------------------------------------
# Install status configuration modification
def change_installed_status(self, target_service, tool_name,
install_status):
"""
Change the install status for a given tool.
Change is made into the INSTALL_STATUS_CONF_FILE
If tool installed, put the current datetime.
:param str target_service: Name of service targeted by the tool
:param str tool_name: Name of the tool
:param bool install_status: New install status to set
:return: Status of change
:rtype: bool
"""
if install_status:
value = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
else:
value = 'False'
parser = self.config_parsers[INSTALL_STATUS_CONF_FILE]
# Create the section [service] if needed
if target_service not in parser.sections():
parser.add_section(target_service)
# Add/Update the install status
if not parser.safe_set(target_service, tool_name, value):
raise SettingsException('Unable to change install status value for the ' \
'tool {tool}'.format(tool=tool_name))
# Save change permanently into the file
return self.save(INSTALL_STATUS_CONF_FILE)
def save(self, conf_filename):
"""
Save change permanently into the file.
:param str conf_filename: Settings filename without extension
:return: Status of saving
:rtype: bool
"""
try:
config_file = FileUtils.concat_path(SETTINGS_DIR,
conf_filename + CONF_EXT)
with open(config_file, 'w') as handle:
self.config_parsers[conf_filename].write(handle)
# Re-read to take change into account
# Warning: read() takes filename as param
self.config_parsers[conf_filename].read(config_file, 'utf8')
return True
except:
logger.error('Error occured when saving changes in settings file ' \
'named "{filename}"'.format(filename=conf_filename))
traceback.print_exc()
return False
class Settings(object):
def __init__(self, settings_dir, toolbox_dir, output):
"""
Initialize Settings object
@Args settings_dir: directory where config files are stored
toolbox_dir: directory where the toolbox is stored
output: Instance of CLIOutput
"""
self.settings_dir = settings_dir
self.toolbox_dir = toolbox_dir
self.output = output
# config_parsers: dict of config_parsers indexed by conf_filename
self.config_parsers = {}
# general_settings: 2 dimensions dict - [service_name][option_name]
self.general_settings = {}
self.toolbox = Toolbox(self)
# Check directory and presence of *.conf files
if not FileUtils.is_dir(settings_dir):
self.output.printError(
'Configuration directory ({0}) does not exist'.format(
settings_dir))
raise ValueError
files = FileUtils.list_directory(settings_dir)
for f in files:
if not FileUtils.check_extension(f, CONF_EXT):
files.remove(f)
if not files:
self.output.printError(
'Configuration directory ({0}) does not store any *.conf file'.
format(settings_dir))
raise ValueError
# Parse config files
self.parseAllConfFiles(files)
def parseAllConfFiles(self, files):
"""
Parse all *.conf files into the config directory
@Args files: list of config files to parse
@Returns None
"""
# ----
# Parse INSTALL_STATUS_CONF_FILE
if INSTALL_STATUS_CONF_FILE + CONF_EXT not in files:
self.output.printError(
'Install status file ({0}/{1}.{2}) is missing'.format(
SETTINGS_DIR, INSTALL_STATUS_CONF_FILE, CONF_EXT))
sys.exit(0)
self.config_parsers[INSTALL_STATUS_CONF_FILE] = DefaultConfigParser()
self.config_parsers[INSTALL_STATUS_CONF_FILE].read(
FileUtils.concat_path(self.settings_dir,
INSTALL_STATUS_CONF_FILE + CONF_EXT))
files.remove(INSTALL_STATUS_CONF_FILE + CONF_EXT)
# ----
# Parse MULTI_SERVICES_CONF_FILE
support_multi_services_tools = MULTI_SERVICES_CONF_FILE + CONF_EXT in files
self.parseToolsConfFile(MULTI_SERVICES_CONF_FILE + CONF_EXT)
files.remove(MULTI_SERVICES_CONF_FILE + CONF_EXT)
# ----
# Parse services *.conf files
for f in files:
self.parseToolsConfFile(f)
def parseToolsConfFile(self, file):
"""
Parse a given settings file
"""
#self.output.printInfo('Parsing configuration file "{0}" ...'.format(file))
full_path = FileUtils.concat_path(self.settings_dir, file)
conf_filename = FileUtils.remove_ext(file).lower().strip()
self.config_parsers[conf_filename] = DefaultConfigParser()
self.config_parsers[conf_filename].read(full_path)
# Add the entry into general settings for the service
self.general_settings[conf_filename] = {}
if conf_filename == MULTI_SERVICES_CONF_FILE:
self.general_settings[conf_filename]['tools_categories'] = ['all']
else:
# General settings - [general] in .conf file
tools_cats = self.config_parsers[conf_filename].safeGetList(
'general', 'tools_categories', ',', [])
self.general_settings[conf_filename]['tools_categories'] = [
StringUtils.cleanSpecialChars(e).lower() for e in tools_cats
]
# General settings - Optional/Specific settings (depends on the targeted service)
if conf_filename in SPECIFIC_TOOL_OPTIONS.keys():
for option in SPECIFIC_TOOL_OPTIONS[conf_filename]:
setting_name = SPECIFIC_TOOL_OPTIONS[conf_filename][option]
if setting_name:
self.general_settings[conf_filename][setting_name] = \
[ e.lower() for e in self.config_parsers[conf_filename].safeGetList('general', setting_name, ',', []) ]
# Check general settings for the current service
self.checkGeneralSettings(conf_filename)
# Add service as new toolbox section
self.toolbox.addService(conf_filename)
# Add tools in current config file into the toolbox, under the correct service section
for section in self.config_parsers[conf_filename].sections():
if section.startswith(PREFIX_TOOL_SECTIONNAME):
if conf_filename != MULTI_SERVICES_CONF_FILE:
newtool = self.createToolFromConfiguration(
section, conf_filename, tooltype=ToolType.STANDARD)
else:
newtool = self.createToolFromConfiguration(
section,
conf_filename,
tooltype=ToolType.MULTI_SERVICES)
elif section.startswith(PREFIX_TOOL_USEMULTI_SECTIONNAME):
newtool = self.createToolFromConfiguration(
section, conf_filename, tooltype=ToolType.USE_MULTI)
else:
continue
if newtool:
if not self.toolbox.addTool(newtool, conf_filename):
self.output.printWarning(
'Unable to add tool "{0}" into the toolbox'.format(
newtool.name))
else:
#self.output.printSettings('Tool "{0}" added into the toolbox (category "{1}")'.format(newtool.name,
# newtool.category))
pass
def checkGeneralSettings(self, service_name):
"""
Check [general] section in settings files
@Args service_name: service related to config file to check
@Returns Boolean indicating status
"""
if service_name not in self.general_settings.keys():
return False
# General settings - [general] in .conf file
if not self.general_settings[service_name]['tools_categories']:
self.output.printError(
'[{0}{1}] General settings error: Incorrect "tools_categories"'
.format(service_name, CONF_EXT))
sys.exit(0)
# General settings - Optional/Specific settings (depends on the targeted servicee)
if service_name in SPECIFIC_TOOL_OPTIONS.keys():
for option in SPECIFIC_TOOL_OPTIONS[service_name]:
if SPECIFIC_TOOL_OPTIONS[service_name][option]:
setting_name = SPECIFIC_TOOL_OPTIONS[service_name][option]
if not self.general_settings[service_name][setting_name]:
self.output.printWarning('[{0}{1}] General settings warning: No "{2}" setting for service {3}. The tool option "{4}" will not be taken into account'.format( \
service_name, CONF_EXT, setting_name, service_name, option))
return True
def createToolFromConfiguration(self,
section,
service_name,
tooltype=ToolType.STANDARD):
"""
Create tool object from a tool entry (section) into the settings file
Note: Must be called after initializing parser for INSTALL_STATUS_CONF_FILE
@Args section: Section from config file corresponding to a tool
service_name: Service targeted by the tool
tooltype: ToolType
@Returns instance of Tool object if everything is ok, False otherwise
"""
if service_name not in self.general_settings.keys():
return False
# Parse general options
options_general = self.parseToolGeneralOptions(section, service_name,
tooltype)
if not options_general:
return False
# Parse specific info (depends on targeted service)
options_specific = self.parseToolSpecificOptions(
section, service_name, tooltype)
# Create the Tool object from parsed info
tool = Tool(
service_name,
self.toolbox_dir,
tooltype,
# General tool options
options_general['name'],
options_general['tool_ref_name'],
options_general['category'],
options_general['description'],
options_general['command'],
options_general['install'],
options_general['update'],
options_general['installed'],
options_general['last_update'],
# Specific tool options
options_specific)
return tool
def parseToolGeneralOptions(self,
section,
service_name,
tooltype=ToolType.STANDARD):
"""
Parse the general options inside a tool section in settings file.
General options include:
- Mandatory options (depends on the tooltype), defined inside Constants.py
- Optional options: install, update
- Install status: extracted from INSTALL_STATUS_CONF_FILE
@Args section: Section from config file corresponding to a tool
service_name: Service targeted by the tool
tooltype: ToolType
@Returns If success: Dictionary options_general
If error: None
"""
options_general = {
'name': '',
'tool_ref_name': '',
'category': '',
'description': '',
'command': '',
'install': '',
'update': '',
'installed': False,
'last_update': ''
}
# ----
# Check presence of mandatory options
for o in MANDATORY_TOOL_OPTIONS[tooltype]:
if o not in self.config_parsers[service_name].options(section):
self.output.printWarning(
'[{0}{1}] Section "{2}" > missing mandatory option "{3}", skipped'
.format(service_name, CONF_EXT, section, o))
return None
# ----
# Parse mandatory options
try:
for o in MANDATORY_TOOL_OPTIONS[tooltype]:
options_general[o] = self.config_parsers[service_name].safeGet(
section, o, '', None).strip()
if o == 'name' or o == 'tool_ref_name':
options_general[o] = StringUtils.cleanSpecialChars(
options_general[o])
if o == 'category':
options_general[o] = StringUtils.cleanSpecialChars(
options_general[o]).lower()
except:
self.output.printWarning(
'[{0}{1}] Section "{2}" > syntax error with mandatory options'.
format(service_name, CONF_EXT, section))
return None
if tooltype == ToolType.MULTI_SERVICES:
options_general['category'] = 'all'
# ----
# Check mandatory options
for o in MANDATORY_TOOL_OPTIONS[tooltype]:
if not options_general[o]:
self.output.printWarning(
'[{0}{1}] Section "{2}" > option "{3}" is empty, section skipped'
.format(service_name, CONF_EXT, section, o))
return None
if options_general['category'] not in self.general_settings[
service_name]['tools_categories']:
self.output.printWarning(
'[{0}{1}] Section "{2}" > option "category" ("{3}") not in "tools_categories", section skipped'
.format(service_name, CONF_EXT, section, category))
return None
# ----
# Parse general+optional options
try:
options_general['install'] = self.config_parsers[
service_name].safeGet(section, 'install', '', None).strip()
options_general['update'] = self.config_parsers[
service_name].safeGet(section, 'update', '', None).strip()
except:
pass
# ----
# Retrieve install status
# By default: not installed, no last update date
# If the tool entry is actually a reference to a multi-services tool, extract the install status
# from [multi] section inside INSTALL_STATUS_CONF_FILE
if tooltype == ToolType.USE_MULTI:
tool_installed = self.config_parsers[
INSTALL_STATUS_CONF_FILE].safeGet(
MULTI_SERVICES_CONF_FILE, options_general['tool_ref_name'],
'false', None).lower().strip()
else:
tool_installed = self.config_parsers[
INSTALL_STATUS_CONF_FILE].safeGet(service_name,
options_general['name'],
'false',
None).lower().strip()
if tool_installed == 'false':
options_general['installed'] = False
options_general['last_update'] = ''
elif tool_installed == 'true':
options_general['installed'] = True
options_general['last_update'] = ''
else:
options_general['installed'] = True
options_general['last_update'] = tool_installed
return options_general
def parseToolSpecificOptions(self,
section,
service_name,
tooltype=ToolType.STANDARD):
"""
Parse the specific options inside a tool section in settings file.
@Args section: Section from config file corresponding to a tool
service_name: Service targeted by the tool
tooltype: ToolType
@Returns Dictionary options_specific
"""
# opt_specific is a dictionary: "option" => (type, value)
options_specific = dict()
if service_name in SPECIFIC_TOOL_OPTIONS.keys():
for option in SPECIFIC_TOOL_OPTIONS[service_name]:
# Boolean options (default False)
if SPECIFIC_TOOL_OPTIONS[service_name][option] == '':
options_specific[option] = (
bool, self.config_parsers[service_name].safeGetBoolean(
section, option + '_specific', False))
# List-type options
else:
value_list = [
e.lower()
for e in self.config_parsers[service_name].safeGetList(
section, option + '_specific', ',', [])
]
if value_list:
for e in value_list:
if e.lower(
) not in self.general_settings[service_name][
SPECIFIC_TOOL_OPTIONS[service_name]
[option]]:
value_list.remove(e)
self.output.printWarning(
'[{0}{1}] Section "{2}" > option "{3}" contains invalid entry ("{4}")'
.format(service_name, CONF_EXT, section,
option, e))
options_specific[option] = (list, value_list)
return options_specific
def changeInstalledOption(self, service_name, tool_name, install_status):
"""
Change the install status for a given tool.
Change is made into the INSTALL_STATUS_CONF_FILE
@Args service_name: service targeted by the tool
tool_name: Tool name as it appears in config file
install_status: Boolean
@Returns Boolean indicating operation status
"""
# If value == True: tool installed, put the current datetime
if install_status:
value = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
else:
value = 'False'
if not self.config_parsers[INSTALL_STATUS_CONF_FILE].safeSet(
service_name, tool_name, value):
raise Exception
# If "MULTI_SERVICES" tool, change the install status of all the references
if service_name == MULTI_SERVICES_CONF_FILE:
for tool in self.toolbox.searchInToolboxToolsReferencing(
tool_name):
if not self.config_parsers[INSTALL_STATUS_CONF_FILE].safeSet(
tool.service_name, tool.name, value):
print tool.service_name
print tool.name
print value
raise Exception
return self.saveSettings(INSTALL_STATUS_CONF_FILE)
def saveSettings(self, conf_filename):
"""
Save settings into config file.
Make sure changes are thus taken into account.
@Args conf_filename: configuration filename (without extension)
@Returns Boolean indicating operation status
"""
try:
config_file = FileUtils.concat_path(self.settings_dir,
conf_filename + CONF_EXT)
with open(config_file, 'w') as handle:
self.config_parsers[conf_filename].write(handle)
# Re-read to take change into account
self.config_parsers[conf_filename].read(
config_file) # warning: takes filename as param
return True
except:
traceback.print_exc()
return False
class Settings:
"""
Class used for parsing settings files:
- toolbox.conf : File storing configuration about all tools
- <service_name>.conf : Each supported service has a corresponding .conf file
- _install_status.conf : Store install status for each tool & last update if installed
"""
def __init__(self):
"""
:raises SettingsException:
"""
self.config_parsers = dict() # dict of DefaultConfigParser indexed by filename
self.toolbox = None # Receives Toolbox object
self.services = None # Receives ServicesConfig object
# Check directory and presence of *.conf files
if not FileUtils.is_dir(SETTINGS_DIR):
raise SettingsException('Configuration directory ({dir}) does not exist'.format(dir=SETTINGS_DIR))
files = FileUtils.list_directory(SETTINGS_DIR)
for f in files:
if not FileUtils.check_extension(f, CONF_EXT):
files.remove(f)
if not files:
raise SettingsException('Configuration directory ({dir}) does not store any *.conf file'.format(
dir=SETTINGS_DIR))
if TOOLBOX_CONF_FILE+CONF_EXT not in files:
raise SettingsException('Missing mandatory {toolbox}{ext} settings file in directory "{dir}"'.format(
toolbox=TOOLBOX_CONF_FILE, ext=CONF_EXT, dir=SETTINGS_DIR))
if INSTALL_STATUS_CONF_FILE+CONF_EXT not in files:
open(SETTINGS_DIR+'/'+INSTALL_STATUS_CONF_FILE+CONF_EXT, 'a').close()
logger.info('{status}{ext} settings file created in directory "{dir}"'.format(
status=INSTALL_STATUS_CONF_FILE, ext=CONF_EXT, dir=SETTINGS_DIR))
files.append(INSTALL_STATUS_CONF_FILE+CONF_EXT)
# Parse settings files, add tools inside toolbox and create scan configs
self.__parse_all_conf_files(files)
self.__create_toolbox()
self.__create_all_services_checks()
def __parse_all_conf_files(self, files):
"""
Parse all *.conf files into the settings directory.
Initialize ServicesConfig object with list of supported services
:param files: List of files in settings directory
:return: None
"""
list_services = list()
for f in files:
name = FileUtils.remove_ext(f).lower().strip()
if name not in (INSTALL_STATUS_CONF_FILE, TOOLBOX_CONF_FILE):
list_services.append(name)
full_path = FileUtils.concat_path(SETTINGS_DIR, f)
self.config_parsers[name] = DefaultConfigParser()
self.config_parsers[name].read(full_path, 'utf8') # utf8 to avoid encoding issues
list_services.append('multi') # Add support for special "multi" service
self.services = ServicesConfig(list_services)
def __create_toolbox(self):
"""
Create the toolbox
:return: None
"""
self.toolbox = Toolbox(self, self.services.list_services(multi=True))
for section in self.config_parsers[TOOLBOX_CONF_FILE].sections():
newtool = self.__create_tool(section)
if newtool is not None:
if not self.toolbox.add_tool(newtool):
logger.warning('[{filename}{ext} | Section "{section}"] Unable to add tool "{tool}" into the toolbox'.format(
filename=TOOLBOX_CONF_FILE, ext=CONF_EXT, section=section, tool=newtool.name))
def __create_tool(self, section):
"""
Create a Tool object
:param section: Tool section into the toolbox settings file
:return: The created Tool instance
"""
tool_config = defaultdict(str)
if not self.__parse_tool_options(section, tool_config): return None
if not self.__parse_tool_install_status(tool_config): return None
return Tool(
tool_config['name_clean'],
tool_config['name'],
tool_config['description'],
tool_config['target_service'],
tool_config['installed'],
tool_config['last_update'],
tool_config['install'],
tool_config['update'],
tool_config['check_command']
)
def __parse_tool_options(self, section, tool_config):
"""
Check and parse options from a given tool section
:param section: Tool section into the toolbox settings file
:param tool_config: A defaultdict(str) storing tool config which is updated into this method
:return: Boolean indicating status
"""
log_prefix = '[{filename}{ext} | Section "{section}"]'.format(
filename=TOOLBOX_CONF_FILE, ext=CONF_EXT, section=section)
optparsed = self.config_parsers[TOOLBOX_CONF_FILE].options(section)
for opt in TOOL_OPTIONS[MANDATORY]:
if opt not in optparsed:
logger.warning('{prefix} Missing mandatory option "{option}", tool is skipped'.format(
prefix=log_prefix, option=opt))
return False
tool_config['name_clean'] = section
for opt in optparsed:
if opt not in TOOL_OPTIONS[MANDATORY]+TOOL_OPTIONS[OPTIONAL]:
logger.warning('{prefix} Option "{option}" is not supported, it will be ignored'.format(
prefix=log_prefix, option=opt))
continue
if opt in TOOL_OPTIONS[MANDATORY]:
val = self.config_parsers[TOOLBOX_CONF_FILE].safe_get(section, opt, '', None)
if opt == 'name':
tool_config[opt]=StringUtils.clean(val, allowed_specials=['-', '_'])
elif opt == 'description':
tool_config[opt] = val
elif opt == 'target_service':
tool_config[opt] = val.lower()
if tool_config[opt] not in self.services.list_services(multi=True):
logger.warning('{prefix} Service specified in "target_service" is not supported, ' \
'tool is skipped'.format(prefix=log_prefix))
return False
if not tool_config[opt]:
logger.warning('{prefix} Mandatory option "{option}" is empty, tool is skipped'.format(
prefix=log_prefix, option=opt))
return False
elif opt == 'install':
tool_config[opt] = Command(cmdtype = CMD_INSTALL,
cmdline = self.config_parsers[TOOLBOX_CONF_FILE].safe_get(section, opt, '', None))
elif opt == 'update':
tool_config[opt] = Command(cmdtype = CMD_UPDATE,
cmdline = self.config_parsers[TOOLBOX_CONF_FILE].safe_get(section, opt, '', None))
elif opt == 'check_command':
tool_config[opt] = Command(cmdtype = CMD_CHECK,
cmdline = self.config_parsers[TOOLBOX_CONF_FILE].safe_get(section, opt, '', None))
return True
def __parse_tool_install_status(self, tool_config):
"""
Retrieve install status of a given tool.
By default: not installed, no last update date
Must be called after self.__parse_tool_options()
:param tool_config: A defaultdict(str) storing tool config which is updated into this method
:return: Boolean
"""
tool_installed = self.config_parsers[INSTALL_STATUS_CONF_FILE].safe_get(
tool_config['target_service'], tool_config['name_clean'], 'false', None).lower().strip()
if tool_installed == 'false' : tool_config['installed'], tool_config['last_update'] = False , ''
elif tool_installed == 'true' : tool_config['installed'], tool_config['last_update'] = True , ''
else : tool_config['installed'], tool_config['last_update'] = True , tool_installed
return True
def __create_all_services_checks(self):
"""
Parse each <service_name>.conf file and create a ServiceChecks object for each one.
A ServiceChecks object stores all checks for a given service.
:return: None
"""
for f in self.config_parsers:
if f in (TOOLBOX_CONF_FILE, INSTALL_STATUS_CONF_FILE):
continue
self.__parse_service_checks_config_file(f)
def __parse_service_checks_config_file(self, service):
"""
Parse a service checks configuration file <service_name>.conf, in order to:
- Update service info (default port, protocol, specific options, supported values for
options of list type) accordingly.
- Create a ServiceChecks object from parsing of various check sections
:param service: Service name
:return: None
"""
service_config = defaultdict(str)
categories = self.__parse_section_config(service, service_config)
self.__parse_section_specific_options(service, service_config)
self.__parse_section_supported_list_options(service, service_config)
# Add the service configuration from settings
self.services.add_service(
service,
service_config['default_port'],
service_config['protocol'],
service_config['specific_options'],
service_config['supported_list_options'],
service_config['auth_types'],
ServiceChecks(service, categories)
)
# Add the various for the service into the ServiceChecks object
self.__parse_all_checks_sections(service)
def __parse_section_config(self, service, service_config):
"""
Parse section [config] in <service_name>.conf, retrieve basic info about service
(default port/protocol) and retrieve list of categories.
:param service: Service name
:param service_config: Dict storing info about service, updated into this method
:return: List of categories of checks
:raises SettingsException:
"""
log_prefix = '[{filename}{ext} | Section "config"]'.format(filename=service, ext=CONF_EXT)
optparsed = self.config_parsers[service].options('config')
for opt in SERVICE_CHECKS_CONFIG_OPTIONS[MANDATORY]:
if opt not in optparsed:
raise SettingsException('{prefix} Missing mandatory option "{option}", check the file'.format(
prefix=log_prefix, option=opt))
default_port = self.config_parsers[service].safe_get_int('config', 'default_port', None, None)
protocol = self.config_parsers[service].safe_get_lower('config', 'protocol', 'tcp', ['tcp', 'udp'])
categories = list(map(lambda x: StringUtils.clean(x.lower(), allowed_specials=('-', '_')),
self.config_parsers[service].safe_get_list('config', 'categories', ',', [])))
auth_types = list(map(lambda x: StringUtils.clean(x.lower(), allowed_specials=('-', '_')),
self.config_parsers[service].safe_get_list('config', 'auth_types', ',', []))) \
if 'auth_types' in optparsed else None
if default_port is None or default_port < 0 or default_port > 65535:
raise SettingsException('{prefix} Invalid value for option "default_port", must be in the range ' \
'[0-65535]'.format(prefix=log_prefix))
if not categories:
raise SettingsException('{prefix} Option "categories" must have at least one category'.format(
prefix=log_prefix))
service_config['default_port'] = default_port
service_config['protocol'] = protocol
service_config['auth_types'] = auth_types
return categories
def __parse_section_specific_options(self, service, service_config):
"""
Parse section [specific_options] in <service_name>.conf and update service config
:param service: Service name
:param service_config: Dict storing info about service, updated into this method
:return: None
:raises SettingsException:
"""
try:
optparsed = self.config_parsers[service].options('specific_options')
except configparser.NoSectionError:
service_config['specific_options'] = dict()
return
specific_options = dict()
for opt in optparsed:
option_type = self.config_parsers[service].safe_get_lower('specific_options', opt, None, None)
if option_type.count(':') == 1:
option_type, default_value = option_type.split(':')
opt_clean = StringUtils.clean(opt.lower(), allowed_specials=('-', '_'))
if option_type == 'boolean' : specific_options[opt_clean] = OptionType.BOOLEAN
elif option_type == 'list' : specific_options[opt_clean] = OptionType.LIST
elif option_type == 'var' : specific_options[opt_clean] = OptionType.VAR
else:
raise SettingsException('[{filename}{ext} | Section "specific_options"] Specific option named "{option}" has ' \
'an invalid type. Supported types are: boolean, list, var'.format(
filename = service, ext=CONF_EXT, option=opt))
service_config['specific_options'] = specific_options
def __parse_section_supported_list_options(self, service, service_config):
"""
Parse section [supported_list_options] in <service_name>.conf and retrieve
supported values for specific options of type list.
Must be called after self.__parse_section_config() and self.__parse_section_specific_options()
:param service: Service name
:param service_config: Dict storing info about service, updated into this method
:return: None
:raises SettingsException:
"""
options_list = list(filter(lambda x: service_config['specific_options'][x] == OptionType.LIST,
service_config['specific_options'].keys()))
if not options_list:
return dict()
elif not self.config_parsers[service].has_section('supported_list_options'):
raise SettingsException('[{filename}{ext}] Missing section [supported_list_options] to store supported ' \
'values for specific options of type list'.format(filename=service, ext=CONF_EXT))
log_prefix = '[{filename}{ext} | Section "supported_list_options"]'.format(filename=service, ext=CONF_EXT)
supported_list_options = dict()
optparsed = self.config_parsers[service].options('supported_list_options')
for opt in options_list:
if 'supported_'+opt not in optparsed:
raise SettingsException('{prefix} No option "supported_{option}" is defined'.format(
prefix=log_prefix, option=opt))
values = list(map(lambda x: StringUtils.clean(x.lower(), allowed_specials=('-', '_')),
self.config_parsers[service].safe_get_list('supported_list_options',
'supported_'+opt, ',', [])))
if not values:
raise SettingsException('{prefix} Option "supported_{option}" is empty'.format(
prefix=log_prefix, option=opt))
supported_list_options[opt] = values
service_config['supported_list_options'] = supported_list_options
def __parse_all_checks_sections(self, service):
"""
Parse all the [check_(.+)] sections of a given service checks settings file
:param service: Service name
:return: None
"""
for section in self.config_parsers[service].sections():
if section.startswith(PREFIX_SECTION_CHECK):
check_config = defaultdict(str)
if not self.__parse_check_section(service, section, check_config):
continue
newcheck = Check(
check_config['name'],
check_config['category'],
check_config['description'],
check_config['tool'],
check_config['commands'],
check_config['postrun']
)
self.services[service]['checks'].add_check(newcheck)
def __parse_check_section(self, service, section, check_config):
"""
Check and parse options from a given check section
:param service: Service name
:param section: Tool section into the toolbox settings file
:param check_config: A defaultdict(str) storing check config which is updated into this method
:return: Boolean indicating status
"""
log_prefix = '[{filename}{ext} | Section "{section}"]'.format(
filename=service, ext=CONF_EXT, section=section)
optparsed = self.config_parsers[service].options(section)
for opt in CHECK_OPTIONS[MANDATORY]:
if opt not in optparsed:
logger.warning('{prefix} Missing mandatory option "{option}", the check ' \
'is ignored'.format(prefix=log_prefix, option=opt))
return False
for opt in optparsed:
if opt not in CHECK_OPTIONS[MANDATORY]+CHECK_OPTIONS[OPTIONAL] and \
not opt.startswith('command_') and not opt.startswith('context_'):
logger.warning('{prefix} Option "{option}" is not supported, the check is ' \
'ignored'.format(prefix=log_prefix, option=opt))
continue
value = self.config_parsers[service].safe_get(section, opt, '', None)
if opt in CHECK_OPTIONS[MANDATORY]:
if opt == 'name':
check_config[opt] = StringUtils.clean(value, allowed_specials=('_','-'))
elif opt == 'category':
check_config[opt] = StringUtils.clean(value, allowed_specials=('_','-')).lower()
if check_config[opt] not in self.services[service]['checks'].categories:
logger.warning('{prefix} Category "{category}" is not supported, the check is ' \
'ignored'.format(prefix=log_prefix, category=value))
return False
elif opt == 'tool':
tool = self.toolbox.get_tool(value)
if tool is None:
logger.warning('{prefix} The tool "{tool}" does not exist, the check is ' \
'ignored'.format(prefix=log_prefix, tool=value))
return False
check_config[opt] = tool
else:
check_config[opt] = value
if not check_config[opt]:
logger.warning('{prefix} Mandatory option "{option}" is empty, the check is ' \
'ignored'.format(prefix=log_prefix, option=opt))
return False
elif opt == 'postrun':
check_config[opt] = value
commands = self.__parse_commands(service, section)
if not commands: return False
check_config['commands'] = commands
return True
def __parse_commands(self, service, section):
"""
Create Commands object for a given tool.
Each command is defined in settings file by:
- command_<command_number>
- context_<command_number> (optional)
:param service: Service name
:param section: Section name [check_(.+)]
:return: List of Command instances
"""
log_prefix = '[{filename}{ext} | Section "{section}"]'.format(
filename=service, ext=CONF_EXT, section=section)
commands = list()
cmdlines = self.config_parsers[service].safe_get_multi(section, 'command', default=None)
i = 0
for cmd in cmdlines:
context = self.config_parsers[service].safe_get(section, 'context_'+str(i+1), default=None)
context = self.__parse_context(service, section, i, context)
if context is None:
logger.warning('{prefix} Context is invalid, the check is ignored'.format(prefix=log_prefix))
return None
commands.append(Command(cmdtype=CMD_RUN,
cmdline=cmdlines[i],
context=context,
services_config=self.services))
i += 1
if not commands:
logger.warning('{prefix} No command is specified, the check is ignored'.format(prefix=log_prefix))
return commands
def __parse_context(self, service, section, num_context, context_str):
"""
Convert the value of a "context_<command_number>" option into a valid python dict
:param service: Service name
:param section: Section name [check_(.+)]
:param num_context: Number in option name, ie: context_<num_context>
:param context_str: Context string extracted from settings file
:return: Context object if parsing is ok, None otherwise
"""
if not context_str: return Context(None)
# Retrieve value as dict
context_str = context_str.replace('NO_AUTH', str(NO_AUTH))\
.replace('USER_ONLY', str(USER_ONLY))\
.replace('POST_AUTH', str(POST_AUTH))
log_prefix = '[{filename}{ext} | Section "{section}"] "context_{i}":'.format(
filename=service, ext=CONF_EXT, section=section, i=num_context)
try:
context = ast.literal_eval(context_str)
except Exception as e:
logger.warning('{prefix} Parsing error. Valid dictionary syntax is probably not respected: ' \
'{ \'key\': \'value\', ... }'.format(prefix=log_prefix))
return None
# Check validity of values according to service name
for opt,val in context.items():
if opt == 'auth_status':
if val not in (NO_AUTH, USER_ONLY, POST_AUTH, None):
logger.warning('{prefix} Invalid value for "auth_status" context-option. Supported values are: ' \
'NO_AUTH, USER_ONLY, POST_AUTH, None'.format(prefix=log_prefix))
return None
elif opt == 'auth_type':
if service != 'http':
logger.warning('{prefix} "auth_type" context-option is only supported for service HTTP'.format(
prefix=log_prefix))
return None
elif context[opt] not in self.services[service]['auth_types']:
logger.warning('{prefix} "auth_type" context-option does not have a valid value, ' \
'check --list-http-auth'.format(prefix=log_prefix))
return None
else:
if not self.services.is_specific_option_name_supported(opt, service):
logger.warning('{prefix} Context-option "{option}" is not supported for service {service}'.format(
prefix=log_prefix, option=opt, service=service))
return None
if self.services.get_specific_option_type(opt, service) == OptionType.LIST:
if val is not None:
if val == 'undefined':
context[opt] = ['undefined']
else:
if isinstance(val, str):
val = [val]
val = list(map(lambda x: x.lower(), val))
for e in val:
if e not in self.services[service]['supported_list_options'][opt]:
logger.warning('{prefix} Context-option "{option}" contains an invalid element ' \
'("{element}")'.format(prefix=log_prefix, option=opt, element=e))
context[opt] = val
else:
if val is not None and not isinstance(val, bool):
logger.warning('{prefix} Context-option "{option}" must have a boolean value (True/False) ' \
'or None'.format(prefix=log_prefix, option=opt))
return None
return Context(context)
def change_installed_status(self, target_service, tool_name, install_status):
"""
Change the install status for a given tool.
Change is made into the INSTALL_STATUS_CONF_FILE
If tool installed, put the current datetime
:param target_service: Name of service targeted by the tool
:param tool_name: Tool name (Attention: must be the clean name !)
:param install_status: New install status to set
:return: Boolean indicating change status
"""
#
value = datetime.now().strftime('%Y-%m-%d %H:%M:%S') if install_status else 'False'
parser = self.config_parsers[INSTALL_STATUS_CONF_FILE]
# Create the section [service] if needed
if target_service not in parser.sections():
parser.add_section(target_service)
if not parser.safe_set(target_service, tool_name, value):
raise SettingsException('Unable to change install status value for the tool {tool}'.format(
tool=tool_name))
return self.save(INSTALL_STATUS_CONF_FILE)
def save(self, conf_filename):
"""
Save change permanently into the file
:param conf_filename: Settings filename without extension
:return: Boolean indicating status
"""
try:
config_file = FileUtils.concat_path(SETTINGS_DIR, conf_filename+CONF_EXT)
with open(config_file, 'w') as handle:
self.config_parsers[conf_filename].write(handle)
# Re-read to take change into account
self.config_parsers[conf_filename].read(config_file, 'utf8') # warning: takes filename as param
return True
except:
logger.error('Error occured when saving changes in settings file named "{filename}"'.format(
filename=conf_filename))
traceback.print_exc()
return False