def db_load_fuzzer_extensions():
entries = []
file = cfg_get_fuzzer_ext_list(cfg_start_get())
if file != u'':
if not core_utilities.check_file_path(file):
raise core_error.Config_Error(
u'Invalid configuration value for \'fuzzer_ext_list\' parameter'
)
else:
fileHandle = codecs.open(file, 'r', FILE_ENCODING)
for line in fileHandle.readlines():
if line[0] != FUZZ_DATABASE_COMMENT and len(line.split()) == 1:
# Note: len(u'\r\n') = len(u'\n') = 0, thus empty lines are implicitely not considered.
entries.append(
line.replace(u'\r', u'').replace(u'\n', u''))
else:
continue
fileHandle.close()
return entries
def ids_load_proxy_list():
field_separator = u'[::]'
entries = []
file = cfg_get_ids_rp_list(cfg_start_get())
if not core_utilities.check_file_path(file):
raise core_error.Config_Error(
u'Invalid configuration value for \'ids_rp_list\' parameter')
else:
fileHandle = codecs.open(file, 'r', FILE_ENCODING)
for line in fileHandle.readlines():
if line.strip()[0] == u'#':
continue
else:
params = [p.strip() for p in line.split(field_separator)]
if len(params) == 4:
if core_utilities.check_host(
params[0]) and core_utilities.check_port_string(
params[1]
) and core_utilities.check_generic_string(
params[2]
) and core_utilities.check_generic_string(
params[3]):
entries.append(params)
else:
continue
else:
continue
fileHandle.close()
return entries
def db_custom_update(tests, banners):
# load config parameters #
custom_db_dir = cfg_get_custom_db_dir(cfg_start_get())
if not core_utilities.check_dir_path(custom_db_dir):
raise core_error.Config_Error(
u'Invalid configuration value for \'custom_db_dir\' parameter')
else:
fileHandle = codecs.open(custom_db_dir + u'/' + CUSTOM_DB_BANNERS, 'w',
CUSTOM_DB_ENCODING)
fileHandle.write(banners)
fileHandle.close()
fileHandle = codecs.open(custom_db_dir + u'/' + CUSTOM_DB_TESTS, 'w',
CUSTOM_DB_ENCODING)
fileHandle.write(tests)
fileHandle.close()
def db_nikto_update(db_tests, db_variables):
# load config parameters #
nikto_db_dir = cfg_get_nikto_db_dir(cfg_start_get())
if not core_utilities.check_dir_path(nikto_db_dir):
raise core_error.Config_Error(
u'Invalid configuration value for \'nikto_db_dir\' parameter')
else:
fileHandle = codecs.open(nikto_db_dir + u'/' + NIKTO_DB_VARIABLES, 'w',
NIKTO_DB_ENCODING)
fileHandle.write(db_variables)
fileHandle.close()
fileHandle = codecs.open(nikto_db_dir + u'/' + NIKTO_DB_TESTS, 'w',
NIKTO_DB_ENCODING)
fileHandle.write(db_tests)
fileHandle.close()
def db_load_known_banners():
banners = {}
field_separator = u'[::]'
custom_db_dir = cfg_get_custom_db_dir(cfg_start_get())
if not core_utilities.check_dir_path(custom_db_dir) or not os.path.exists(
custom_db_dir + u'/' + CUSTOM_DB_BANNERS):
raise core_error.Config_Error(
u'Invalid configuration value for \'custom_db_dir\' parameter')
else:
fileHandle = codecs.open(custom_db_dir + u'/' + CUSTOM_DB_BANNERS, 'r',
CUSTOM_DB_ENCODING)
lines = fileHandle.readlines()
fileHandle.close()
for line in lines:
line = line.strip()
# skip comments or blank lines #
if len(line) == 0 or line[0] == u'#':
continue
else:
entry_fields = [e.strip() for e in line.split(field_separator)]
if len(entry_fields) != 2:
continue
else:
try:
banners[re.compile(entry_fields[0],
re.IGNORECASE)] = entry_fields[1]
except re.error:
print 'error'
continue
return banners
def db_load_nikto_tests():
nikto_variables = {}
nikto_tests = []
# load config parameters #
nikto_db_dir = cfg_get_nikto_db_dir(cfg_start_get())
if not core_utilities.check_dir_path(nikto_db_dir):
raise core_error.Config_Error(
u'Invalid configuration value for \'nikto_db_dir\' parameter')
elif not os.path.exists(nikto_db_dir + u'/' + NIKTO_DB_VARIABLES):
raise core_error.ConfigError(u'Inexistent file: ' + nikto_db_dir +
u'/' + NIKTO_DB_VARIABLES)
elif not os.path.exists(nikto_db_dir + u'/' + NIKTO_DB_TESTS):
raise core_error.Config_Error(u'Inexistent file: ' + nikto_db_dir +
u'/' + NIKTO_DB_TESTS)
else:
fileHandle = codecs.open(nikto_db_dir + u'/' + NIKTO_DB_VARIABLES, 'r',
NIKTO_DB_ENCODING)
lines = fileHandle.readlines()
fileHandle.close()
for line in lines:
line = line.strip()
if len(line) > 0 and line[0] == u'@':
variable, sep, values_str = line.partition(u'=')
values = values_str.split()
nikto_variables[variable] = values
else:
continue
fileHandle = codecs.open(nikto_db_dir + u'/' + NIKTO_DB_TESTS, 'r',
NIKTO_DB_ENCODING)
lines = fileHandle.readlines()
fileHandle.close()
for line in lines:
line = line.strip()
# skip comments or blank lines #
if len(line) == 0 or line[0] == u'#':
continue
else:
# replace the JUNK(X) entries in Nikto database by actual junk
junk_matches = NIKTO_JUNK_REGEXP.findall(line)
if len(junk_matches) > 0:
for match in junk_matches:
line = line.replace(
match[0],
core_utilities.random_string(int(match[1])))
else:
pass
# the code below 'extends' the test by replacing the nikto_variables by their possible values.
# It can support an arbitrary number of variables per line. A single test line containing
# X variables of Y possible values each will be 'extended' to produce Y^X test lines.
#
# @VAR1 = /value_11/ /value_21/
# @VAR2 = /value_12/ /value_22/
# LINE = "000000","0","x","@VAR1no_mather_what@VAR2","GET","200","","","","","example","",""
#
# ==> "000000","0","x","/value_11/no_mather_what/value_12/","GET","200","","","","","example","",""
# ==> "000000","0","x","/value_11/no_mather_what/value_22/","GET","200","","","","","example","",""
# ==> "000000","0","x","/value_21/no_mather_what/value_12/","GET","200","","","","","example","",""
# ==> "000000","0","x","/value_21/no_mather_what/value_22/","GET","200","","","","","example","",""
entries = [line]
extended_entries = []
for variable in nikto_variables.keys():
for entry in entries:
if variable in entry:
begin, var, end = entry.partition(variable)
extended_entries.extend([
begin + rpl + end
for rpl in nikto_variables[variable]
])
else:
extended_entries.append(entry)
entries = extended_entries
extended_entries = []
# once the line has been extended (if necessary) as multiple lines,
# all these lines are parsed to extract the various fields.
for entry in entries:
entry = entry[1:-1] # clean... remove 1st and last '"'
entry_fields = entry.split(u'","')
if len(entry_fields) != 13:
continue
else:
# ## NIKTO: db_tests ############################################################################################################################
# 0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 , 11 , 12
# "Test-ID","OSVDB-ID","Tuning Type","URI","HTTP Method","Match","Match And","Match Or","Fail","Fail Or","Summary","HTTP Data","Headers"
#
# ## WEBSHAG test ###############################################################################################################################
#
# OSVDB | SERVER | METHOD | PATH | HEADER | DATA | MATCH_CODE | MATCH | MATCH_AND | MATCH_OR | FAIL_CODE | FAIL | FAIL_OR | DESCRIPTION | TRIGGER
test = {}
test[u'OSVDB'] = entry_fields[1]
test[u'SERVER'] = u''
test[u'METHOD'] = entry_fields[4]
test[u'PATH'] = entry_fields[3]
test[u'HEADER'] = entry_fields[12]
test[u'DATA'] = entry_fields[11]
# only one of the fields is allowed to have a code...
if HTTP_CODE_REGEXP.match(entry_fields[5]):
test[u'MATCH_CODE'] = entry_fields[5]
test[u'MATCH'] = u''
test[u'MATCH_AND'] = entry_fields[6]
test[u'MATCH_OR'] = entry_fields[7]
elif HTTP_CODE_REGEXP.match(entry_fields[6]):
test[u'MATCH_CODE'] = entry_fields[6]
test[u'MATCH'] = entry_fields[5]
test[u'MATCH_AND'] = u''
test[u'MATCH_OR'] = entry_fields[7]
elif HTTP_CODE_REGEXP.match(entry_fields[7]):
test[u'MATCH_CODE'] = entry_fields[7]
test[u'MATCH'] = entry_fields[5]
test[u'MATCH_AND'] = entry_fields[6]
test[u'MATCH_OR'] = u''
else:
test[u'MATCH_CODE'] = u''
test[u'MATCH'] = entry_fields[5]
test[u'MATCH_AND'] = entry_fields[6]
test[u'MATCH_OR'] = entry_fields[7]
if HTTP_CODE_REGEXP.match(entry_fields[8]):
test[u'FAIL_CODE'] = entry_fields[8]
test[u'FAIL'] = u''
test[u'FAIL_OR'] = entry_fields[9]
elif HTTP_CODE_REGEXP.match(entry_fields[9]):
test[u'FAIL_CODE'] = entry_fields[9]
test[u'FAIL'] = entry_fields[8]
test[u'FAIL_OR'] = u''
else:
test[u'FAIL_CODE'] = u''
test[u'FAIL'] = entry_fields[8]
test[u'FAIL_OR'] = entry_fields[9]
test[u'DESCRIPTION'] = entry_fields[10]
test[u'TRIGGER'] = u''
nikto_tests.append(test)
return nikto_tests
def cfg_get_use_robots(configParser):
try:
return configParser.getboolean(CFG_SECTION_SPIDER, u'use_robots')
except ValueError:
raise core_error.Config_Error(
'Invalid configuration value for \'use_robots\' parameter')
def cfg_get_spider_threads(configParser):
try:
return configParser.getint(CFG_SECTION_SPIDER, u'spider_threads')
except ValueError:
raise core_error.Config_Error(
'Invalid configuration value for \'spider_threads\' parameter')
def cfg_get_use_db_custom(configParser):
try:
return configParser.getboolean(CFG_SECTION_URLSCAN, u'use_db_custom')
except ValueError:
raise core_error.Config_Error(
'Invalid configuration value for \'use_db_custom\' parameter')
def cfg_get_nmap(configParser):
try:
return configParser.getboolean(CFG_SECTION_PORTSCAN, u'nmap')
except ValueError:
raise core_error.Config_Error(
'Invalid configuration value for \'nmap\' parameter')
def cfg_get_ssl(configParser):
try:
return configParser.getboolean(CFG_SECTION_HTTP, u'ssl')
except ValueError:
raise core_error.Config_Error(
'Invalid configuration value for \'ssl\' parameter')
def cfg_get_ids_pause_time(configParser):
try:
return configParser.getint(CFG_SECTION_HTTP, u'ids_pause_time')
except ValueError:
raise core_error.Config_Error(
'Invalid configuration value for \'ids_pause_time\' parameter')
def cfg_get_socket_timeout(configParser):
try:
return configParser.getint(CFG_SECTION_HTTP, u'socket_timeout')
except ValueError:
raise core_error.Config_Error(
'Invalid configuration value for \'socket_timeout\' parameter')
def cfg_get_proxy_port(configParser):
try:
return configParser.getint(CFG_SECTION_HTTP, u'proxy_port')
except ValueError:
raise core_error.Config_Error(
'Invalid configuration value for \'proxy_port\' parameter')
def db_load_custom_tests(triggers=[], host=None, load_generic=True):
empty_field = u'__empty__'
host_field = u'__host__'
field_separator = u'[::]'
custom_tests = []
custom_db_dir = cfg_get_custom_db_dir(cfg_start_get())
if not core_utilities.check_dir_path(custom_db_dir) or not os.path.exists(
custom_db_dir + u'/' + CUSTOM_DB_TESTS):
raise core_error.Config_Error(
u'Invalid configuration value for \'custom_db_dir\' parameter')
else:
# ## custom_test.db ####################################################################################################################################################
# 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14
# id[::]server[::]method[::]path[::]header[::]data[::]match_code[::]indicator[::]indicator_and[::]indicator_or[::]fail_code[::]fail[::]fail_or[::]trigger[::]description
#
# ## WEBSHAG test ###############################################################################################################################
#
# OSVDB | SERVER | METHOD | PATH | HEADER | DATA | MATCH_CODE | MATCH | MATCH_AND | MATCH_OR | FAIL_CODE | FAIL | FAIL_OR | DESCRIPTION | TRIGGER
fileHandle = codecs.open(custom_db_dir + u'/' + CUSTOM_DB_TESTS, 'r',
CUSTOM_DB_ENCODING)
lines = fileHandle.readlines()
fileHandle.close()
for line in lines:
line = line.strip()
# skip comments or blank lines #
if len(line) == 0 or line[0] == u'#':
continue
else:
# maybe stripping is not a good idea...
entry_fields = [e.strip() for e in line.split(field_separator)]
if len(entry_fields) != 15:
continue
else:
if (entry_fields[1] == empty_field and not load_generic):
continue
if (entry_fields[1] != empty_field) and (
entry_fields[1].lower()
not in [t.lower() for t in triggers]):
continue
else:
for i in range(0, len(entry_fields)):
if entry_fields[i] == empty_field:
entry_fields[i] = u''
elif host_field in entry_fields[i]:
if host != None:
entry_fields[i] = entry_fields[i].replace(
host_field, host)
else:
entry_fields[i] = entry_fields[i].replace(
host_field, u'')
else:
pass
test = {}
test[u'OSVDB'] = u''
test[u'SERVER'] = entry_fields[1]
test[u'METHOD'] = entry_fields[2]
test[u'PATH'] = entry_fields[3]
test[u'HEADER'] = entry_fields[4]
test[u'DATA'] = entry_fields[5]
test[u'MATCH_CODE'] = entry_fields[6]
test[u'MATCH'] = entry_fields[7]
test[u'MATCH_AND'] = entry_fields[8]
test[u'MATCH_OR'] = entry_fields[9]
test[u'FAIL_CODE'] = entry_fields[10]
test[u'FAIL'] = entry_fields[11]
test[u'FAIL_OR'] = entry_fields[12]
test[u'DESCRIPTION'] = entry_fields[14]
test[u'TRIGGER'] = entry_fields[13]
custom_tests.append(test)
return custom_tests
def __init__(self, host, port, override_disable_ssl=False):
self.__host = host
self.__port = port
# Loading and checking the configuration settings
configParser = core_file.cfg_start_get()
self.__user_agent = core_file.cfg_get_user_agent(configParser)
if not core_utilities.check_ascii_string(self.__user_agent):
raise core_error.Config_Error(
u'Invalid configuration value for \'user_agent\' parameter')
self.__socket_timeout = core_file.cfg_get_socket_timeout(configParser)
self.__ssl = core_file.cfg_get_ssl(configParser)
self.__ids = core_file.cfg_get_ids(configParser)
self.__ids_rp = core_file.cfg_get_ids_rp(configParser)
if self.__ids and not core_utilities.check_boolean(self.__ids_rp):
raise core_error.Config_Error(
u'Invalid configuration value for \'ids_rp\' parameter')
self.__ids_rp_list = core_file.cfg_get_ids_rp_list(configParser)
if self.__ids and self.__ids_rp and not core_utilities.check_file_path(
self.__ids_rp_list):
raise core_error.Config_Error(
u'Invalid configuration value for \'ids_rp_list\' parameter')
self.__ids_pause = core_file.cfg_get_ids_pause(configParser)
self.__ids_pause_time = core_file.cfg_get_ids_pause_time(configParser)
self.__proxy = core_file.cfg_get_proxy(configParser)
self.__proxy_host = core_file.cfg_get_proxy_host(configParser)
if self.__proxy and not core_utilities.check_host(self.__proxy_host):
raise core_error.Config_Error(
u'Invalid configuration value for \'proxy_host\' parameter')
self.__proxy_port = core_file.cfg_get_proxy_port(configParser)
self.__proxy_auth = core_file.cfg_get_proxy_auth(configParser)
if self.__proxy and not core_utilities.check_boolean(
self.__proxy_auth):
raise core_error.Config_Error(
u'Invalid configuration value for \'proxy_auth\' parameter')
self.__proxy_username = core_file.cfg_get_proxy_username(configParser)
if self.__proxy and self.__proxy_auth and not core_utilities.check_generic_string(
self.__proxy_username):
raise core_error.Config_Error(
u'Invalid configuration value for \'proxy_username\' parameter'
)
self.__proxy_password = core_file.cfg_get_proxy_password(configParser)
if self.__proxy and self.__proxy_auth and not core_utilities.check_generic_string(
self.__proxy_password):
raise core_error.Config_Error(
u'Invalid configuration value for \'proxy_password\' parameter'
)
self.__auth = core_file.cfg_get_auth(configParser)
self.__auth_username = core_file.cfg_get_auth_username(configParser)
if self.__auth and not core_utilities.check_generic_string(
self.__auth_username):
raise core_error.Config_Error(
u'Invalid configuration value for \'auth_username\' parameter')
self.__auth_password = core_file.cfg_get_auth_password(configParser)
if self.__auth and not core_utilities.check_generic_string(
self.__auth_password):
raise core_error.Config_Error(
u'Invalid configuration value for \'auth_password\' parameter')
self.__default_header = core_file.cfg_get_default_header(configParser)
if len(self.__default_header
) != 0 and not core_utilities.check_generic_string(
self.__default_header):
raise core_error.Config_Error(
u'Invalid configuration value for \'default_header\' parameter'
)
self.__default_header_value = core_file.cfg_get_default_header_value(
configParser).strip()
if not core_utilities.check_generic_string(
self.__default_header_value):
raise core_error.Config_Error(
u'Invalid configuration value for \'default_header_value\' parameter'
)
core_file.cfg_end_get(configParser)
if self.__ssl and self.__ids and self.__ids_rp:
raise core_error.Config_Error(
u'Incompatible configuration settings: \'ssl\' and \'ids_rp\'.'
)
if self.__proxy and self.__ids and self.__ids_rp:
raise core_error.Config_Error(
u'Incompatible configuration settings: \'proxy\' and \'ids_rp\'.'
)
if self.__ids and self.__ids_rp:
self.__ids_proxies = core_file.ids_load_proxy_list()
if len(self.__ids_proxies) == 0:
raise core_error.Config_Error(u'Proxy list is empty!')
if self.__proxy: # proxy
if self.__ssl and not override_disable_ssl:
raise core_error.Config_Error(u'No support for HTTPS proxy.')
# TODO: implement...
else:
self.__url_prefix = u'http://' + self.__host + u':' + unicode(
self.__port)
# proxy BASIC authentication #
if self.__proxy_auth:
self.__proxy_credentials = self.__basic_authentication(
self.__proxy_username, self.__proxy_password)
# connection to proxy
self.__connection = httplib.HTTPConnection(self.__proxy_host,
self.__proxy_port)
else: # no proxy
if self.__ids and self.__ids_rp:
self.__connection = None
else:
if self.__ssl and not override_disable_ssl: # HTTPS connection
self.__connection = httplib.HTTPSConnection(
self.__host, self.__port)
else: # HTTP connection
self.__connection = httplib.HTTPConnection(
self.__host, self.__port)
self.__url_prefix = u''
socket.setdefaulttimeout(self.__socket_timeout)
