def _determine_ok_200(self, requests_verb, url):
if common.is_string(self.regular_file_url):
reg_url = url + self.regular_file_url
ok_resp = requests_verb(reg_url)
ok_200 = ok_resp.status_code == 200
else:
ok_200 = False
for path in self.regular_file_url:
reg_url = url + path
ok_resp = requests_verb(reg_url)
if ok_resp.status_code == 200:
ok_200 = True
break
len_content = len(ok_resp.content)
return ok_200, len_content
def _determine_ok_200(self, requests_verb, url):
if common.is_string(self.regular_file_url):
reg_url = url + self.regular_file_url
ok_resp = requests_verb(reg_url)
ok_200 = ok_resp.status_code == 200
else:
ok_200 = False
for path in self.regular_file_url:
reg_url = url + path
ok_resp = requests_verb(reg_url)
if ok_resp.status_code == 200:
ok_200 = True
break
len_content = len(ok_resp.content)
return ok_200, len_content
def enumerate(self, url, base_url_supplied, scanning_method,
iterator_returning_method, iterator_len, max_iterator=500, threads=10,
verb='head', timeout=15, hide_progressbar=False, imu=None, headers={}):
'''
@param url: base URL for the website.
@param base_url_supplied: Base url for themes, plugins. E.g. '%ssites/all/modules/%s/'
@param scanning_method: see ScanningMethod
@param iterator_returning_method: a function which returns an
element that, when iterated, will return a full list of plugins
@param iterator_len: the number of items the above iterator can
return, regardless of user preference.
@param max_iterator: integer that will be passed unto iterator_returning_method
@param threads: number of threads
@param verb: what HTTP verb. Valid options are 'get' and 'head'.
@param timeout: the time, in seconds, that requests should wait
before throwing an exception.
@param hide_progressbar: if true, the progressbar will not be
displayed.
@param imu: Interesting module urls. A list containing tuples in the
following format [('readme.txt', 'default readme')].
@param headers: List of custom headers as expected by requests.
'''
if common.is_string(base_url_supplied):
base_urls = [base_url_supplied]
else:
base_urls = base_url_supplied
requests_verb = getattr(self.session, verb)
futures = []
with ThreadPoolExecutor(max_workers=threads) as executor:
for base_url in base_urls:
plugins = iterator_returning_method(max_iterator)
if scanning_method == ScanningMethod.not_found:
url_template = base_url + self.module_common_file
else:
url_template = base_url
for plugin_name in plugins:
plugin_url = url_template % (url, plugin_name)
future = executor.submit(requests_verb, plugin_url,
timeout=timeout, headers=headers)
if plugin_url.endswith('/'):
final_url = plugin_url
else:
final_url = dirname(plugin_url) + "/"
futures.append({
'base_url': base_url,
'future': future,
'plugin_name': plugin_name,
'plugin_url': final_url,
})
if not hide_progressbar:
max_possible = max_iterator if int(max_iterator) < int(iterator_len) else iterator_len
items_total = int(max_possible) * len(base_urls)
p = ProgressBar(sys.stderr, items_total, "modules")
if scanning_method == ScanningMethod.forbidden:
expected_status = [403]
else:
expected_status = [200, 403]
no_results = True
found = []
for future_array in futures:
if common.shutdown:
future_array['future'].cancel()
continue
if not hide_progressbar:
p.increment_progress()
r = future_array['future'].result()
if r.status_code in expected_status:
plugin_url = future_array['plugin_url']
plugin_name = future_array['plugin_name']
no_results = False
found.append({
'name': plugin_name,
'url': plugin_url
})
elif r.status_code >= 500:
self.out.warn('\rGot a 500 error. Is the server overloaded?')
if not hide_progressbar:
p.hide()
if not common.shutdown and (imu != None and not no_results):
found = self._enumerate_plugin_if(found, verb, threads, imu,
hide_progressbar, timeout=timeout, headers=headers)
return found, no_results
def enumerate(self,
url,
base_url_supplied,
scanning_method,
iterator_returning_method,
iterator_len,
max_iterator=500,
threads=10,
verb='head',
timeout=15,
hide_progressbar=False,
imu=None,
headers={}):
"""
@param url: base URL for the website.
@param base_url_supplied: Base url for themes, plugins. E.g. '%ssites/all/modules/%s/'
@param scanning_method: see ScanningMethod
@param iterator_returning_method: a function which returns an
element that, when iterated, will return a full list of plugins
@param iterator_len: the number of items the above iterator can
return, regardless of user preference.
@param max_iterator: integer that will be passed unto iterator_returning_method
@param threads: number of threads
@param verb: what HTTP verb. Valid options are 'get' and 'head'.
@param timeout: the time, in seconds, that requests should wait
before throwing an exception.
@param hide_progressbar: if true, the progressbar will not be
displayed.
@param imu: Interesting module urls. A list containing tuples in the
following format [('readme.txt', 'default readme')].
@param headers: List of custom headers as expected by requests.
"""
if common.is_string(base_url_supplied):
base_urls = [base_url_supplied]
else:
base_urls = base_url_supplied
requests_verb = getattr(self.session, verb)
futures = []
fake_200 = False
with ThreadPoolExecutor(max_workers=threads) as executor:
for base_url in base_urls:
plugins = iterator_returning_method(max_iterator)
if scanning_method == ScanningMethod.not_found:
url_template = base_url + self.module_common_file
else:
url_template = base_url
fake_200_inst = self._determine_fake_200_module(
requests_verb, url_template, url)
if fake_200_inst:
fake_200 = fake_200_inst
for plugin_name in plugins:
plugin_url = url_template % (url, plugin_name)
future = executor.submit(requests_verb,
plugin_url,
timeout=timeout,
headers=headers)
if plugin_url.endswith('/'):
final_url = plugin_url
else:
final_url = dirname(plugin_url) + "/"
futures.append({
'base_url': base_url,
'future': future,
'plugin_name': plugin_name,
'plugin_url': final_url,
})
if not hide_progressbar:
max_possible = max_iterator if int(max_iterator) < int(
iterator_len) else iterator_len
items_total = int(max_possible) * len(base_urls)
p = ProgressBar(sys.stderr, items_total, "modules")
if not fake_200:
expected_status = [200, 403, 500]
else:
expected_status = [403, 500]
no_results = True
found = []
for future_array in futures:
if common.shutdown:
future_array['future'].cancel()
continue
if not hide_progressbar:
p.increment_progress()
try:
r = future_array['future'].result()
except requests.exceptions.ReadTimeout:
self.out.warn(
'\rGot a read timeout. Is the server overloaded? This may affect the results of your scan'
)
continue
if r.status_code in expected_status:
plugin_url = future_array['plugin_url']
plugin_name = future_array['plugin_name']
no_results = False
found.append({'name': plugin_name, 'url': plugin_url})
if r.status_code >= 500:
self.out.warn('\rGot an HTTP 500 response.')
if not hide_progressbar:
p.hide()
if not common.shutdown and (imu != None and not no_results):
found = self._enumerate_plugin_if(found,
verb,
threads,
imu,
hide_progressbar,
timeout=timeout,
headers=headers)
return found, no_results
