def db_load_fuzzer_extensions(): entries = [] file = cfg_get_fuzzer_ext_list(cfg_start_get()) if file != u'': if not core_utilities.check_file_path(file): raise core_error.Config_Error( u'Invalid configuration value for \'fuzzer_ext_list\' parameter' ) else: fileHandle = codecs.open(file, 'r', FILE_ENCODING) for line in fileHandle.readlines(): if line[0] != FUZZ_DATABASE_COMMENT and len(line.split()) == 1: # Note: len(u'\r\n') = len(u'\n') = 0, thus empty lines are implicitely not considered. entries.append( line.replace(u'\r', u'').replace(u'\n', u'')) else: continue fileHandle.close() return entries
def ids_load_proxy_list(): field_separator = u'[::]' entries = [] file = cfg_get_ids_rp_list(cfg_start_get()) if not core_utilities.check_file_path(file): raise core_error.Config_Error( u'Invalid configuration value for \'ids_rp_list\' parameter') else: fileHandle = codecs.open(file, 'r', FILE_ENCODING) for line in fileHandle.readlines(): if line.strip()[0] == u'#': continue else: params = [p.strip() for p in line.split(field_separator)] if len(params) == 4: if core_utilities.check_host( params[0]) and core_utilities.check_port_string( params[1] ) and core_utilities.check_generic_string( params[2] ) and core_utilities.check_generic_string( params[3]): entries.append(params) else: continue else: continue fileHandle.close() return entries
def db_custom_update(tests, banners): # load config parameters # custom_db_dir = cfg_get_custom_db_dir(cfg_start_get()) if not core_utilities.check_dir_path(custom_db_dir): raise core_error.Config_Error( u'Invalid configuration value for \'custom_db_dir\' parameter') else: fileHandle = codecs.open(custom_db_dir + u'/' + CUSTOM_DB_BANNERS, 'w', CUSTOM_DB_ENCODING) fileHandle.write(banners) fileHandle.close() fileHandle = codecs.open(custom_db_dir + u'/' + CUSTOM_DB_TESTS, 'w', CUSTOM_DB_ENCODING) fileHandle.write(tests) fileHandle.close()
def db_nikto_update(db_tests, db_variables): # load config parameters # nikto_db_dir = cfg_get_nikto_db_dir(cfg_start_get()) if not core_utilities.check_dir_path(nikto_db_dir): raise core_error.Config_Error( u'Invalid configuration value for \'nikto_db_dir\' parameter') else: fileHandle = codecs.open(nikto_db_dir + u'/' + NIKTO_DB_VARIABLES, 'w', NIKTO_DB_ENCODING) fileHandle.write(db_variables) fileHandle.close() fileHandle = codecs.open(nikto_db_dir + u'/' + NIKTO_DB_TESTS, 'w', NIKTO_DB_ENCODING) fileHandle.write(db_tests) fileHandle.close()
def db_load_known_banners(): banners = {} field_separator = u'[::]' custom_db_dir = cfg_get_custom_db_dir(cfg_start_get()) if not core_utilities.check_dir_path(custom_db_dir) or not os.path.exists( custom_db_dir + u'/' + CUSTOM_DB_BANNERS): raise core_error.Config_Error( u'Invalid configuration value for \'custom_db_dir\' parameter') else: fileHandle = codecs.open(custom_db_dir + u'/' + CUSTOM_DB_BANNERS, 'r', CUSTOM_DB_ENCODING) lines = fileHandle.readlines() fileHandle.close() for line in lines: line = line.strip() # skip comments or blank lines # if len(line) == 0 or line[0] == u'#': continue else: entry_fields = [e.strip() for e in line.split(field_separator)] if len(entry_fields) != 2: continue else: try: banners[re.compile(entry_fields[0], re.IGNORECASE)] = entry_fields[1] except re.error: print 'error' continue return banners
def db_load_nikto_tests(): nikto_variables = {} nikto_tests = [] # load config parameters # nikto_db_dir = cfg_get_nikto_db_dir(cfg_start_get()) if not core_utilities.check_dir_path(nikto_db_dir): raise core_error.Config_Error( u'Invalid configuration value for \'nikto_db_dir\' parameter') elif not os.path.exists(nikto_db_dir + u'/' + NIKTO_DB_VARIABLES): raise core_error.ConfigError(u'Inexistent file: ' + nikto_db_dir + u'/' + NIKTO_DB_VARIABLES) elif not os.path.exists(nikto_db_dir + u'/' + NIKTO_DB_TESTS): raise core_error.Config_Error(u'Inexistent file: ' + nikto_db_dir + u'/' + NIKTO_DB_TESTS) else: fileHandle = codecs.open(nikto_db_dir + u'/' + NIKTO_DB_VARIABLES, 'r', NIKTO_DB_ENCODING) lines = fileHandle.readlines() fileHandle.close() for line in lines: line = line.strip() if len(line) > 0 and line[0] == u'@': variable, sep, values_str = line.partition(u'=') values = values_str.split() nikto_variables[variable] = values else: continue fileHandle = codecs.open(nikto_db_dir + u'/' + NIKTO_DB_TESTS, 'r', NIKTO_DB_ENCODING) lines = fileHandle.readlines() fileHandle.close() for line in lines: line = line.strip() # skip comments or blank lines # if len(line) == 0 or line[0] == u'#': continue else: # replace the JUNK(X) entries in Nikto database by actual junk junk_matches = NIKTO_JUNK_REGEXP.findall(line) if len(junk_matches) > 0: for match in junk_matches: line = line.replace( match[0], core_utilities.random_string(int(match[1]))) else: pass # the code below 'extends' the test by replacing the nikto_variables by their possible values. # It can support an arbitrary number of variables per line. A single test line containing # X variables of Y possible values each will be 'extended' to produce Y^X test lines. # # @VAR1 = /value_11/ /value_21/ # @VAR2 = /value_12/ /value_22/ # LINE = "000000","0","x","@VAR1no_mather_what@VAR2","GET","200","","","","","example","","" # # ==> "000000","0","x","/value_11/no_mather_what/value_12/","GET","200","","","","","example","","" # ==> "000000","0","x","/value_11/no_mather_what/value_22/","GET","200","","","","","example","","" # ==> "000000","0","x","/value_21/no_mather_what/value_12/","GET","200","","","","","example","","" # ==> "000000","0","x","/value_21/no_mather_what/value_22/","GET","200","","","","","example","","" entries = [line] extended_entries = [] for variable in nikto_variables.keys(): for entry in entries: if variable in entry: begin, var, end = entry.partition(variable) extended_entries.extend([ begin + rpl + end for rpl in nikto_variables[variable] ]) else: extended_entries.append(entry) entries = extended_entries extended_entries = [] # once the line has been extended (if necessary) as multiple lines, # all these lines are parsed to extract the various fields. for entry in entries: entry = entry[1:-1] # clean... remove 1st and last '"' entry_fields = entry.split(u'","') if len(entry_fields) != 13: continue else: # ## NIKTO: db_tests ############################################################################################################################ # 0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 , 11 , 12 # "Test-ID","OSVDB-ID","Tuning Type","URI","HTTP Method","Match","Match And","Match Or","Fail","Fail Or","Summary","HTTP Data","Headers" # # ## WEBSHAG test ############################################################################################################################### # # OSVDB | SERVER | METHOD | PATH | HEADER | DATA | MATCH_CODE | MATCH | MATCH_AND | MATCH_OR | FAIL_CODE | FAIL | FAIL_OR | DESCRIPTION | TRIGGER test = {} test[u'OSVDB'] = entry_fields[1] test[u'SERVER'] = u'' test[u'METHOD'] = entry_fields[4] test[u'PATH'] = entry_fields[3] test[u'HEADER'] = entry_fields[12] test[u'DATA'] = entry_fields[11] # only one of the fields is allowed to have a code... if HTTP_CODE_REGEXP.match(entry_fields[5]): test[u'MATCH_CODE'] = entry_fields[5] test[u'MATCH'] = u'' test[u'MATCH_AND'] = entry_fields[6] test[u'MATCH_OR'] = entry_fields[7] elif HTTP_CODE_REGEXP.match(entry_fields[6]): test[u'MATCH_CODE'] = entry_fields[6] test[u'MATCH'] = entry_fields[5] test[u'MATCH_AND'] = u'' test[u'MATCH_OR'] = entry_fields[7] elif HTTP_CODE_REGEXP.match(entry_fields[7]): test[u'MATCH_CODE'] = entry_fields[7] test[u'MATCH'] = entry_fields[5] test[u'MATCH_AND'] = entry_fields[6] test[u'MATCH_OR'] = u'' else: test[u'MATCH_CODE'] = u'' test[u'MATCH'] = entry_fields[5] test[u'MATCH_AND'] = entry_fields[6] test[u'MATCH_OR'] = entry_fields[7] if HTTP_CODE_REGEXP.match(entry_fields[8]): test[u'FAIL_CODE'] = entry_fields[8] test[u'FAIL'] = u'' test[u'FAIL_OR'] = entry_fields[9] elif HTTP_CODE_REGEXP.match(entry_fields[9]): test[u'FAIL_CODE'] = entry_fields[9] test[u'FAIL'] = entry_fields[8] test[u'FAIL_OR'] = u'' else: test[u'FAIL_CODE'] = u'' test[u'FAIL'] = entry_fields[8] test[u'FAIL_OR'] = entry_fields[9] test[u'DESCRIPTION'] = entry_fields[10] test[u'TRIGGER'] = u'' nikto_tests.append(test) return nikto_tests
def cfg_get_use_robots(configParser): try: return configParser.getboolean(CFG_SECTION_SPIDER, u'use_robots') except ValueError: raise core_error.Config_Error( 'Invalid configuration value for \'use_robots\' parameter')
def cfg_get_spider_threads(configParser): try: return configParser.getint(CFG_SECTION_SPIDER, u'spider_threads') except ValueError: raise core_error.Config_Error( 'Invalid configuration value for \'spider_threads\' parameter')
def cfg_get_use_db_custom(configParser): try: return configParser.getboolean(CFG_SECTION_URLSCAN, u'use_db_custom') except ValueError: raise core_error.Config_Error( 'Invalid configuration value for \'use_db_custom\' parameter')
def cfg_get_nmap(configParser): try: return configParser.getboolean(CFG_SECTION_PORTSCAN, u'nmap') except ValueError: raise core_error.Config_Error( 'Invalid configuration value for \'nmap\' parameter')
def cfg_get_ssl(configParser): try: return configParser.getboolean(CFG_SECTION_HTTP, u'ssl') except ValueError: raise core_error.Config_Error( 'Invalid configuration value for \'ssl\' parameter')
def cfg_get_ids_pause_time(configParser): try: return configParser.getint(CFG_SECTION_HTTP, u'ids_pause_time') except ValueError: raise core_error.Config_Error( 'Invalid configuration value for \'ids_pause_time\' parameter')
def cfg_get_socket_timeout(configParser): try: return configParser.getint(CFG_SECTION_HTTP, u'socket_timeout') except ValueError: raise core_error.Config_Error( 'Invalid configuration value for \'socket_timeout\' parameter')
def cfg_get_proxy_port(configParser): try: return configParser.getint(CFG_SECTION_HTTP, u'proxy_port') except ValueError: raise core_error.Config_Error( 'Invalid configuration value for \'proxy_port\' parameter')
def db_load_custom_tests(triggers=[], host=None, load_generic=True): empty_field = u'__empty__' host_field = u'__host__' field_separator = u'[::]' custom_tests = [] custom_db_dir = cfg_get_custom_db_dir(cfg_start_get()) if not core_utilities.check_dir_path(custom_db_dir) or not os.path.exists( custom_db_dir + u'/' + CUSTOM_DB_TESTS): raise core_error.Config_Error( u'Invalid configuration value for \'custom_db_dir\' parameter') else: # ## custom_test.db #################################################################################################################################################### # 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 # id[::]server[::]method[::]path[::]header[::]data[::]match_code[::]indicator[::]indicator_and[::]indicator_or[::]fail_code[::]fail[::]fail_or[::]trigger[::]description # # ## WEBSHAG test ############################################################################################################################### # # OSVDB | SERVER | METHOD | PATH | HEADER | DATA | MATCH_CODE | MATCH | MATCH_AND | MATCH_OR | FAIL_CODE | FAIL | FAIL_OR | DESCRIPTION | TRIGGER fileHandle = codecs.open(custom_db_dir + u'/' + CUSTOM_DB_TESTS, 'r', CUSTOM_DB_ENCODING) lines = fileHandle.readlines() fileHandle.close() for line in lines: line = line.strip() # skip comments or blank lines # if len(line) == 0 or line[0] == u'#': continue else: # maybe stripping is not a good idea... entry_fields = [e.strip() for e in line.split(field_separator)] if len(entry_fields) != 15: continue else: if (entry_fields[1] == empty_field and not load_generic): continue if (entry_fields[1] != empty_field) and ( entry_fields[1].lower() not in [t.lower() for t in triggers]): continue else: for i in range(0, len(entry_fields)): if entry_fields[i] == empty_field: entry_fields[i] = u'' elif host_field in entry_fields[i]: if host != None: entry_fields[i] = entry_fields[i].replace( host_field, host) else: entry_fields[i] = entry_fields[i].replace( host_field, u'') else: pass test = {} test[u'OSVDB'] = u'' test[u'SERVER'] = entry_fields[1] test[u'METHOD'] = entry_fields[2] test[u'PATH'] = entry_fields[3] test[u'HEADER'] = entry_fields[4] test[u'DATA'] = entry_fields[5] test[u'MATCH_CODE'] = entry_fields[6] test[u'MATCH'] = entry_fields[7] test[u'MATCH_AND'] = entry_fields[8] test[u'MATCH_OR'] = entry_fields[9] test[u'FAIL_CODE'] = entry_fields[10] test[u'FAIL'] = entry_fields[11] test[u'FAIL_OR'] = entry_fields[12] test[u'DESCRIPTION'] = entry_fields[14] test[u'TRIGGER'] = entry_fields[13] custom_tests.append(test) return custom_tests
def __init__(self, host, port, override_disable_ssl=False): self.__host = host self.__port = port # Loading and checking the configuration settings configParser = core_file.cfg_start_get() self.__user_agent = core_file.cfg_get_user_agent(configParser) if not core_utilities.check_ascii_string(self.__user_agent): raise core_error.Config_Error( u'Invalid configuration value for \'user_agent\' parameter') self.__socket_timeout = core_file.cfg_get_socket_timeout(configParser) self.__ssl = core_file.cfg_get_ssl(configParser) self.__ids = core_file.cfg_get_ids(configParser) self.__ids_rp = core_file.cfg_get_ids_rp(configParser) if self.__ids and not core_utilities.check_boolean(self.__ids_rp): raise core_error.Config_Error( u'Invalid configuration value for \'ids_rp\' parameter') self.__ids_rp_list = core_file.cfg_get_ids_rp_list(configParser) if self.__ids and self.__ids_rp and not core_utilities.check_file_path( self.__ids_rp_list): raise core_error.Config_Error( u'Invalid configuration value for \'ids_rp_list\' parameter') self.__ids_pause = core_file.cfg_get_ids_pause(configParser) self.__ids_pause_time = core_file.cfg_get_ids_pause_time(configParser) self.__proxy = core_file.cfg_get_proxy(configParser) self.__proxy_host = core_file.cfg_get_proxy_host(configParser) if self.__proxy and not core_utilities.check_host(self.__proxy_host): raise core_error.Config_Error( u'Invalid configuration value for \'proxy_host\' parameter') self.__proxy_port = core_file.cfg_get_proxy_port(configParser) self.__proxy_auth = core_file.cfg_get_proxy_auth(configParser) if self.__proxy and not core_utilities.check_boolean( self.__proxy_auth): raise core_error.Config_Error( u'Invalid configuration value for \'proxy_auth\' parameter') self.__proxy_username = core_file.cfg_get_proxy_username(configParser) if self.__proxy and self.__proxy_auth and not core_utilities.check_generic_string( self.__proxy_username): raise core_error.Config_Error( u'Invalid configuration value for \'proxy_username\' parameter' ) self.__proxy_password = core_file.cfg_get_proxy_password(configParser) if self.__proxy and self.__proxy_auth and not core_utilities.check_generic_string( self.__proxy_password): raise core_error.Config_Error( u'Invalid configuration value for \'proxy_password\' parameter' ) self.__auth = core_file.cfg_get_auth(configParser) self.__auth_username = core_file.cfg_get_auth_username(configParser) if self.__auth and not core_utilities.check_generic_string( self.__auth_username): raise core_error.Config_Error( u'Invalid configuration value for \'auth_username\' parameter') self.__auth_password = core_file.cfg_get_auth_password(configParser) if self.__auth and not core_utilities.check_generic_string( self.__auth_password): raise core_error.Config_Error( u'Invalid configuration value for \'auth_password\' parameter') self.__default_header = core_file.cfg_get_default_header(configParser) if len(self.__default_header ) != 0 and not core_utilities.check_generic_string( self.__default_header): raise core_error.Config_Error( u'Invalid configuration value for \'default_header\' parameter' ) self.__default_header_value = core_file.cfg_get_default_header_value( configParser).strip() if not core_utilities.check_generic_string( self.__default_header_value): raise core_error.Config_Error( u'Invalid configuration value for \'default_header_value\' parameter' ) core_file.cfg_end_get(configParser) if self.__ssl and self.__ids and self.__ids_rp: raise core_error.Config_Error( u'Incompatible configuration settings: \'ssl\' and \'ids_rp\'.' ) if self.__proxy and self.__ids and self.__ids_rp: raise core_error.Config_Error( u'Incompatible configuration settings: \'proxy\' and \'ids_rp\'.' ) if self.__ids and self.__ids_rp: self.__ids_proxies = core_file.ids_load_proxy_list() if len(self.__ids_proxies) == 0: raise core_error.Config_Error(u'Proxy list is empty!') if self.__proxy: # proxy if self.__ssl and not override_disable_ssl: raise core_error.Config_Error(u'No support for HTTPS proxy.') # TODO: implement... else: self.__url_prefix = u'http://' + self.__host + u':' + unicode( self.__port) # proxy BASIC authentication # if self.__proxy_auth: self.__proxy_credentials = self.__basic_authentication( self.__proxy_username, self.__proxy_password) # connection to proxy self.__connection = httplib.HTTPConnection(self.__proxy_host, self.__proxy_port) else: # no proxy if self.__ids and self.__ids_rp: self.__connection = None else: if self.__ssl and not override_disable_ssl: # HTTPS connection self.__connection = httplib.HTTPSConnection( self.__host, self.__port) else: # HTTP connection self.__connection = httplib.HTTPConnection( self.__host, self.__port) self.__url_prefix = u'' socket.setdefaulttimeout(self.__socket_timeout)