def __parse_section_config(self, service, service_config): """ Parse section [config] in <service_name>.conf, retrieve basic info about service (default port/protocol) and retrieve list of supported categories of checks for this service. :param str service: Service name :param defaultdict(str) service_config: Information about the service, updated into this method :return: List of categories of checks :rtype: list(str) :raises SettingsException: Exception raised if any unrecoverable error is encountered while parsing the section """ log_prefix = '[{filename}{ext} | Section "config"]'.format( filename=service, ext=CONF_EXT) # Check presence of mandatory options in [config] optparsed = self.config_parsers[service].options('config') for opt in SERVICE_CHECKS_CONFIG_OPTIONS[MANDATORY]: if opt not in optparsed: raise SettingsException('{prefix} Missing mandatory option "{option}"' \ ', check the file'.format(prefix=log_prefix, option=opt)) # Get port number default_port = self.config_parsers[service].safe_get_int( 'config', 'default_port', None, None) if default_port is None or default_port < 0 or default_port > 65535: raise SettingsException('{prefix} Invalid value for option "default_port",' \ ' must be in the range [0-65535]'.format(prefix=log_prefix)) # Get protocol protocol = self.config_parsers[service].safe_get_lower( 'config', 'protocol', 'tcp', ['tcp', 'udp']) # Get categories of checks as a list, clean each element categories = list(map(lambda x: StringUtils.clean( x.lower(), allowed_specials=('-', '_')), self.config_parsers[service].safe_get_list('config', 'categories', ',', []))) if not categories: raise SettingsException('{prefix} Option "categories" must have at least '\ 'one category'.format(prefix=log_prefix)) # Get authentication type (for HTTP) as a list, clean each element if 'auth_types' in optparsed: auth_types = list(map(lambda x: StringUtils.clean( x.lower(), allowed_specials=('-', '_')), self.config_parsers[service].safe_get_list( 'config', 'auth_types', ',', []))) else: auth_types = None # Update service configuration with parsed information service_config['default_port'] = default_port service_config['protocol'] = protocol service_config['auth_types'] = auth_types return categories
def __parse_section_products(self, service, service_config): """ Parse section [products] in <service_name>.conf and retrieve supported values for each product type. :param str service: Service name :param dict service_config: Service configuration, updated into this method :return: None :raises SettingsException: Exception raised if unconsistent values detected """ # First, check if config file has a [products] section if not self.config_parsers[service].has_section('products'): service_config['products'] = dict() return log_prefix = '[{filename}{ext} | Section "products"]'.format( filename=service, ext=CONF_EXT) products = dict() optparsed = self.config_parsers[service].options('products') # Loop over product types in [products] for product_type in optparsed: # Clean the product type product_type = StringUtils.clean(product_type.lower(), allowed_specials=('-', '_')) # Get supported product names as a list. # Only some special chars allowed, spaces allowed # '/' is used to separate vendor name (optional) and product name product_names = self.config_parsers[service].safe_get_list( 'products', product_type, ',', []) product_names = list( map( lambda x: StringUtils.clean( x, allowed_specials=('-', '_', '.', '/', '\\', ' ')), product_names)) if not product_names: raise SettingsException( '{prefix} Option "{option}" is empty'.format( prefix=log_prefix, option=opt)) products[product_type] = product_names # Update service configuration with supported products service_config['products'] = products return
def __parse_section_supported_list_options(self, service, service_config): """ Parse section [supported_list_options] in <service_name>.conf and update service configuration with supported values for specific options of type list. Must be called after self.__parse_section_config() and self.__parse_section_specific_options(). :param defaultdict(str) service_config: Information about the service, updated into this method :return: None :raises SettingsException: Exception raised if any unrecoverable error is encountered while parsing the section """ # Get names of specific options of type list options_list = list( filter( lambda x: service_config['specific_options'][x] == OptionType. LIST, service_config['specific_options'].keys())) if not options_list: return elif not self.config_parsers[service].has_section( 'supported_list_options'): raise SettingsException('[{filename}{ext}] Missing section ' \ '[supported_list_options] to store supported values for specific ' \ 'options of type "list"'.format(filename=service, ext=CONF_EXT)) log_prefix = '[{filename}{ext} | Section "supported_list_options"]'.format( filename=service, ext=CONF_EXT) supported_list_options = dict() optparsed = self.config_parsers[service].options( 'supported_list_options') # Loop over specific options of type list for opt in options_list: # If missing option if 'supported_' + opt not in optparsed: raise SettingsException('{prefix} No option "supported_{option}" ' \ 'is defined'.format(prefix=log_prefix, option=opt)) # Values are put in lowercase, no spaces, no special chars (except -, _) values = list( map( lambda x: StringUtils.clean(x.lower(), allowed_specials=('-', '_')), self.config_parsers[service].safe_get_list( 'supported_list_options', 'supported_' + opt, ',', []))) if not values: raise SettingsException('{prefix} Option "supported_{option}" is ' \ 'empty'.format(prefix=log_prefix, option=opt)) supported_list_options[opt] = values # Update service configuration with lists of supported values service_config['supported_list_options'] = supported_list_options
def __parse_section_specific_options(self, service, service_config): """ Parse section [specific_options] in <service_name>.conf and update service config :param service: Service name :param service_config: Dict storing info about service, updated into this method :return: None :raises SettingsException: """ try: optparsed = self.config_parsers[service].options('specific_options') except configparser.NoSectionError: service_config['specific_options'] = dict() return specific_options = dict() for opt in optparsed: option_type = self.config_parsers[service].safe_get_lower('specific_options', opt, None, None) if option_type.count(':') == 1: option_type, default_value = option_type.split(':') opt_clean = StringUtils.clean(opt.lower(), allowed_specials=('-', '_')) if option_type == 'boolean' : specific_options[opt_clean] = OptionType.BOOLEAN elif option_type == 'list' : specific_options[opt_clean] = OptionType.LIST elif option_type == 'var' : specific_options[opt_clean] = OptionType.VAR else: raise SettingsException('[{filename}{ext} | Section "specific_options"] Specific option named "{option}" has ' \ 'an invalid type. Supported types are: boolean, list, var'.format( filename = service, ext=CONF_EXT, option=opt)) service_config['specific_options'] = specific_options
def __parse_tool_options(self, section, tool_config): """ Check and parse options from a given tool section :param section: Tool section into the toolbox settings file :param tool_config: A defaultdict(str) storing tool config which is updated into this method :return: Boolean indicating status """ log_prefix = '[{filename}{ext} | Section "{section}"]'.format( filename=TOOLBOX_CONF_FILE, ext=CONF_EXT, section=section) optparsed = self.config_parsers[TOOLBOX_CONF_FILE].options(section) for opt in TOOL_OPTIONS[MANDATORY]: if opt not in optparsed: logger.warning('{prefix} Missing mandatory option "{option}", tool is skipped'.format( prefix=log_prefix, option=opt)) return False tool_config['name_clean'] = section for opt in optparsed: if opt not in TOOL_OPTIONS[MANDATORY]+TOOL_OPTIONS[OPTIONAL]: logger.warning('{prefix} Option "{option}" is not supported, it will be ignored'.format( prefix=log_prefix, option=opt)) continue if opt in TOOL_OPTIONS[MANDATORY]: val = self.config_parsers[TOOLBOX_CONF_FILE].safe_get(section, opt, '', None) if opt == 'name': tool_config[opt]=StringUtils.clean(val, allowed_specials=['-', '_']) elif opt == 'description': tool_config[opt] = val elif opt == 'target_service': tool_config[opt] = val.lower() if tool_config[opt] not in self.services.list_services(multi=True): logger.warning('{prefix} Service specified in "target_service" is not supported, ' \ 'tool is skipped'.format(prefix=log_prefix)) return False if not tool_config[opt]: logger.warning('{prefix} Mandatory option "{option}" is empty, tool is skipped'.format( prefix=log_prefix, option=opt)) return False elif opt == 'install': tool_config[opt] = Command(cmdtype = CMD_INSTALL, cmdline = self.config_parsers[TOOLBOX_CONF_FILE].safe_get(section, opt, '', None)) elif opt == 'update': tool_config[opt] = Command(cmdtype = CMD_UPDATE, cmdline = self.config_parsers[TOOLBOX_CONF_FILE].safe_get(section, opt, '', None)) elif opt == 'check_command': tool_config[opt] = Command(cmdtype = CMD_CHECK, cmdline = self.config_parsers[TOOLBOX_CONF_FILE].safe_get(section, opt, '', None)) return True
def __parse_section_specific_options(self, service, service_config): """ Parse section [specific_options] in <service_name>.conf and update service configuration with supported specific options for the service and their respective types. :param str service: Service name :param defaultdict(str) service_config: Information about the service, updated into this method :return: None :raises SettingsException: Exception raised if any unrecoverable error is encountered while parsing the section """ # Case when no [specific_options] can be found try: optparsed = self.config_parsers[service].options( 'specific_options') except configparser.NoSectionError: service_config['specific_options'] = dict() return specific_options = dict() # Loop over supported specific options for opt in optparsed: # Get option type option_type = self.config_parsers[service].safe_get_lower( 'specific_options', opt, None, None) # Handle case when default value is specified (for boolean) if option_type.count(':') == 1: option_type, default_value = option_type.split(':') opt_clean = StringUtils.clean(opt.lower(), allowed_specials=('-', '_')) if option_type == 'boolean': specific_options[opt_clean] = OptionType.BOOLEAN elif option_type == 'list': specific_options[opt_clean] = OptionType.LIST elif option_type == 'var': specific_options[opt_clean] = OptionType.VAR else: raise SettingsException('[{filename}{ext} | Section ' \ '"specific_options"] Specific option named "{option}" has ' \ 'an invalid type. Supported types are: boolean, list, var'.format( filename = service, ext=CONF_EXT, option=opt)) # Update service configuration with specific options names and types service_config['specific_options'] = specific_options
def __parse_section_config(self, service, service_config): """ Parse section [config] in <service_name>.conf, retrieve basic info about service (default port/protocol) and retrieve list of categories. :param service: Service name :param service_config: Dict storing info about service, updated into this method :return: List of categories of checks :raises SettingsException: """ log_prefix = '[{filename}{ext} | Section "config"]'.format(filename=service, ext=CONF_EXT) optparsed = self.config_parsers[service].options('config') for opt in SERVICE_CHECKS_CONFIG_OPTIONS[MANDATORY]: if opt not in optparsed: raise SettingsException('{prefix} Missing mandatory option "{option}", check the file'.format( prefix=log_prefix, option=opt)) default_port = self.config_parsers[service].safe_get_int('config', 'default_port', None, None) protocol = self.config_parsers[service].safe_get_lower('config', 'protocol', 'tcp', ['tcp', 'udp']) categories = list(map(lambda x: StringUtils.clean(x.lower(), allowed_specials=('-', '_')), self.config_parsers[service].safe_get_list('config', 'categories', ',', []))) auth_types = list(map(lambda x: StringUtils.clean(x.lower(), allowed_specials=('-', '_')), self.config_parsers[service].safe_get_list('config', 'auth_types', ',', []))) \ if 'auth_types' in optparsed else None if default_port is None or default_port < 0 or default_port > 65535: raise SettingsException('{prefix} Invalid value for option "default_port", must be in the range ' \ '[0-65535]'.format(prefix=log_prefix)) if not categories: raise SettingsException('{prefix} Option "categories" must have at least one category'.format( prefix=log_prefix)) service_config['default_port'] = default_port service_config['protocol'] = protocol service_config['auth_types'] = auth_types return categories
def __parse_section_supported_list_options(self, service, service_config): """ Parse section [supported_list_options] in <service_name>.conf and retrieve supported values for specific options of type list. Must be called after self.__parse_section_config() and self.__parse_section_specific_options() :param service: Service name :param service_config: Dict storing info about service, updated into this method :return: None :raises SettingsException: """ options_list = list(filter(lambda x: service_config['specific_options'][x] == OptionType.LIST, service_config['specific_options'].keys())) if not options_list: return dict() elif not self.config_parsers[service].has_section('supported_list_options'): raise SettingsException('[{filename}{ext}] Missing section [supported_list_options] to store supported ' \ 'values for specific options of type list'.format(filename=service, ext=CONF_EXT)) log_prefix = '[{filename}{ext} | Section "supported_list_options"]'.format(filename=service, ext=CONF_EXT) supported_list_options = dict() optparsed = self.config_parsers[service].options('supported_list_options') for opt in options_list: if 'supported_'+opt not in optparsed: raise SettingsException('{prefix} No option "supported_{option}" is defined'.format( prefix=log_prefix, option=opt)) values = list(map(lambda x: StringUtils.clean(x.lower(), allowed_specials=('-', '_')), self.config_parsers[service].safe_get_list('supported_list_options', 'supported_'+opt, ',', []))) if not values: raise SettingsException('{prefix} Option "supported_{option}" is empty'.format( prefix=log_prefix, option=opt)) supported_list_options[opt] = values service_config['supported_list_options'] = supported_list_options
def run(self): # Create report directory dirname = '{mission}-{datetime}'.format( mission=StringUtils.clean(self.mission.replace(' ', '_'), allowed_specials=('_', '-')), datetime=datetime.datetime.now().strftime('%Y%m%d%H%M%S')) self.output_path = self.output_path + '/' + dirname if not FileUtils.create_directory(self.output_path): logger.error('Unable to create report directory: "{path}"'.format( path=self.output_path)) return False # Retrieve all services in selected mission req = ServicesRequester(self.sqlsession) req.select_mission(self.mission) services = req.get_results() # Generate screenshots processor = ScreenshotsProcessor(self.mission, self.sqlsession) processor.run() screens_dir = self.output_path + '/screenshots' if not FileUtils.create_directory(screens_dir): logger.warning( 'Unable to create screenshots directory: "{path}"'.format( path=screens_dir)) else: for service in services: if service.name == 'http' and service.screenshot is not None \ and service.screenshot.status == ScreenStatus.OK: img_name = 'scren-{ip}-{port}-{id}'.format( ip=str(service.host.ip), port=service.port, id=service.id) path = screens_dir + '/' + img_name ImageUtils.save_image(service.screenshot.image, path + '.png') ImageUtils.save_image(service.screenshot.thumbnail, path + '.thumb.png') # Create index.html html = self.__generate_index() if FileUtils.write(self.output_path + '/index.html', html): logger.info('index.html file generated') else: logger.error('An error occured while generating index.html') return False # Create results-<service>.html (1 for each service) for service in services: # Useless to create page when no check has been run for the service if len(service.results) == 0: continue html = self.__generate_results_page(service) # Create a unique name for the service HTML file filename = 'results-{ip}-{port}-{service}-{id}.html'.format( ip=str(service.host.ip), port=service.port, service=service.name, id=service.id) if FileUtils.write(self.output_path + '/' + filename, html): logger.info( '{filename} file generated'.format(filename=filename)) else: logger.error( 'An error occured while generating {filename}'.format( filename=filename)) return False logger.success('HTML Report written with success in: {path}'.format( path=self.output_path)) logger.info('Important: If running from Docker container, make sure to run ' \ '"xhost +" on the host before') if Output.prompt_confirm('Would you like to open the report now ?', default=True): webbrowser.open(self.output_path + '/index.html') return True
def __parse_check_section(self, service, section, check_config): """ Check and parse options from a given check section. :param str service: Service name :param str section: Section corresponding to the check to parse :param defaultdict(str) check_config: Check configuration, updated into this method :return: Status of parsing :rtype: bool """ log_prefix = '[{filename}{ext} | Section "{section}"]'.format( filename=service, ext=CONF_EXT, section=section) # Check presence of mandatory options in [check_<name>] section optparsed = self.config_parsers[service].options(section) for opt in CHECK_OPTIONS[MANDATORY]: if opt not in optparsed: logger.warning('{prefix} Missing mandatory option "{option}", the ' \ ' check is ignored'.format(prefix=log_prefix, option=opt)) return False # Loop over options for opt in optparsed: # Check for unsupported options if opt not in CHECK_OPTIONS[MANDATORY]+CHECK_OPTIONS[OPTIONAL] \ and not opt.startswith('command_') and not opt.startswith('context_'): logger.warning('{prefix} Option "{option}" is not supported, the ' \ 'check is ignored'.format(prefix=log_prefix, option=opt)) continue # Add value val = self.config_parsers[service].safe_get(section, opt, '', None) if opt == 'name': check_config[opt] = StringUtils.clean(val, allowed_specials=('_', '-')) elif opt == 'category': cat = StringUtils.clean(val, allowed_specials=('_', '-')) check_config[opt] = cat.lower() if check_config[opt] not in self.services[service][ 'checks'].categories: logger.warning('{prefix} Category "{category}" is not supported, ' \ 'the check is ignored'.format(prefix=log_prefix, category=val)) return False elif opt == 'tool': tool = self.toolbox.get_tool(val) if tool is None: logger.warning('{prefix} The tool "{tool}" does not exist, the ' \ 'check is ignored'.format(prefix=log_prefix, tool=val)) return False check_config[opt] = tool else: check_config[opt] = val # Check for empty mandatory option if opt in CHECK_OPTIONS[MANDATORY] and not check_config[opt]: logger.warning('{prefix} Mandatory option "{option}" is empty, the ' \ 'check is ignored'.format(prefix=log_prefix, option=opt)) return False # Parse commands along with optional context requirements commands = self.__parse_commands(service, section) if not commands: return False check_config['commands'] = commands return True
def __parse_tool_options(self, section, tool_config): """ Check and parse options from a given tool section. :param str section: Section name corresponding to the tool in toolbox.conf :param defaultdict(str) tool_config: Tool configuration updated in this method :return: Status of parsing :rtype: bool """ log_prefix = '[{filename}{ext} | Section "{section}"]'.format( filename=TOOLBOX_CONF_FILE, ext=CONF_EXT, section=section) # Check presence of mandatory options optparsed = self.config_parsers[TOOLBOX_CONF_FILE].options(section) for opt in TOOL_OPTIONS[MANDATORY]: if opt not in optparsed: logger.warning('{prefix} Missing mandatory option "{option}", ' \ 'tool is skipped'.format(prefix=log_prefix, option=opt)) return False # Loop over options for opt in optparsed: # Check for unsupported options if opt not in TOOL_OPTIONS[MANDATORY] + TOOL_OPTIONS[OPTIONAL]: logger.warning('{prefix} Option "{option}" is not supported, ' \ 'it will be ignored'.format(prefix=log_prefix, option=opt)) continue # Add value val = self.config_parsers[TOOLBOX_CONF_FILE].safe_get( section, opt, '', None) if opt == 'name': tool_config[opt] = StringUtils.clean( val, allowed_specials=['-', '_']) elif opt == 'description': tool_config[opt] = val elif opt == 'target_service': tool_config[opt] = val.lower() if tool_config[opt] not in self.services.list_services( multi=True): logger.warning('{prefix} Service specified in "target_service" is ' \ 'not supported, tool is skipped'.format(prefix=log_prefix)) return False elif opt == 'install': tool_config[opt] = Command(cmdtype=CmdType.INSTALL, cmdline=val) elif opt == 'update': tool_config[opt] = Command(cmdtype=CmdType.UPDATE, cmdline=val) elif opt == 'check_command': tool_config[opt] = Command(cmdtype=CmdType.CHECK, cmdline=val) # Check for empty mandatory option if opt in TOOL_OPTIONS[MANDATORY] and not tool_config[opt]: logger.warning('{prefix} Mandatory option "{option}" is empty, tool ' \ 'is skipped'.format(prefix=log_prefix, option=opt)) return False return True
def __parse_tool_options(self, section, tool_config): """ Check and parse options from a given tool section. :param str section: Section name corresponding to the tool in toolbox.conf :param defaultdict(str) tool_config: Tool configuration updated in this method :return: Status of parsing :rtype: bool """ log_prefix = '[{filename}{ext} | Section "{section}"]'.format( filename=TOOLBOX_CONF_FILE, ext=CONF_EXT, section=section) # Check presence of mandatory options optparsed = self.config_parsers[TOOLBOX_CONF_FILE].options(section) for opt in TOOL_OPTIONS[MANDATORY]: if opt not in optparsed: logger.warning('{prefix} Missing mandatory option "{option}", ' \ 'tool is skipped'.format(prefix=log_prefix, option=opt)) return False # Loop over options for opt in optparsed: # Check for unsupported options if opt not in TOOL_OPTIONS[MANDATORY] + TOOL_OPTIONS[OPTIONAL]: logger.warning('{prefix} Option "{option}" is not supported, ' \ 'it will be ignored'.format(prefix=log_prefix, option=opt)) continue # Add value val = self.config_parsers[TOOLBOX_CONF_FILE].safe_get( section, opt, '', None) if opt == 'name': tool_config[opt] = StringUtils.clean( val, allowed_specials=['-', '_']) elif opt == 'description': tool_config[opt] = val elif opt == 'target_service': tool_config[opt] = val.lower() if tool_config[opt] not in self.services.list_services( multi=True): logger.warning('{prefix} Service specified in "target_service" is ' \ 'not supported, tool is skipped'.format(prefix=log_prefix)) return False elif opt == 'virtualenv': tool_config[opt] = val.lower() # For Python, format must be "python<version>" if tool_config[opt].startswith('python'): m = re.match('python(?P<version>[0-9](\.[0-9])*)', tool_config[opt]) if not m: logger.warning('{prefix} Invalid Python virtualenv, must be: ' \ 'virtualenv = python<version>. Tool is skipped'.format( prefix=log_prefix)) return False # For Ruby, make sure to have a format like "ruby-<version>" # Format "ruby<version>" is accepted and turned into "ruby-<version>" if tool_config[opt].startswith('ruby'): m1 = re.match('ruby(?P<version>[0-9](\.[0-9])*)', tool_config[opt]) m2 = re.match('ruby-(?P<version>[0-9](\.[0-9])*)', tool_config[opt]) if m1: tool_config[opt] = 'ruby-{version}'.format( version=m.group('version')) elif not m2: logger.warning('{prefix} Invalid Ruby virtualenv, must be: ' \ 'virtualenv = ruby-<version>. Tool is skipped'.format( prefix=log_prefix)) return False elif opt == 'install': tool_config[opt] = Command(cmdtype=CmdType.INSTALL, cmdline=val) elif opt == 'update': tool_config[opt] = Command(cmdtype=CmdType.UPDATE, cmdline=val) elif opt == 'check_command': tool_config[opt] = Command(cmdtype=CmdType.CHECK, cmdline=val) # Check for empty mandatory option if opt in TOOL_OPTIONS[MANDATORY] and not tool_config[opt]: logger.warning('{prefix} Mandatory option "{option}" is empty, tool ' \ 'is skipped'.format(prefix=log_prefix, option=opt)) return False return True
def __parse_check_section(self, service, section, check_config): """ Check and parse options from a given check section :param service: Service name :param section: Tool section into the toolbox settings file :param check_config: A defaultdict(str) storing check config which is updated into this method :return: Boolean indicating status """ log_prefix = '[{filename}{ext} | Section "{section}"]'.format( filename=service, ext=CONF_EXT, section=section) optparsed = self.config_parsers[service].options(section) for opt in CHECK_OPTIONS[MANDATORY]: if opt not in optparsed: logger.warning('{prefix} Missing mandatory option "{option}", the check ' \ 'is ignored'.format(prefix=log_prefix, option=opt)) return False for opt in optparsed: if opt not in CHECK_OPTIONS[MANDATORY]+CHECK_OPTIONS[OPTIONAL] and \ not opt.startswith('command_') and not opt.startswith('context_'): logger.warning('{prefix} Option "{option}" is not supported, the check is ' \ 'ignored'.format(prefix=log_prefix, option=opt)) continue value = self.config_parsers[service].safe_get(section, opt, '', None) if opt in CHECK_OPTIONS[MANDATORY]: if opt == 'name': check_config[opt] = StringUtils.clean(value, allowed_specials=('_','-')) elif opt == 'category': check_config[opt] = StringUtils.clean(value, allowed_specials=('_','-')).lower() if check_config[opt] not in self.services[service]['checks'].categories: logger.warning('{prefix} Category "{category}" is not supported, the check is ' \ 'ignored'.format(prefix=log_prefix, category=value)) return False elif opt == 'tool': tool = self.toolbox.get_tool(value) if tool is None: logger.warning('{prefix} The tool "{tool}" does not exist, the check is ' \ 'ignored'.format(prefix=log_prefix, tool=value)) return False check_config[opt] = tool else: check_config[opt] = value if not check_config[opt]: logger.warning('{prefix} Mandatory option "{option}" is empty, the check is ' \ 'ignored'.format(prefix=log_prefix, option=opt)) return False elif opt == 'postrun': check_config[opt] = value commands = self.__parse_commands(service, section) if not commands: return False check_config['commands'] = commands return True