def test_progressbar(self):
u = MockBuffer()
p = ProgressBar(u)
p.set(10, 100)
a = u.get()[-4:]
assert a == '10%)'
assert " ===== " in u.get()
def test_progressbar(self):
u = MockBuffer()
p = ProgressBar(u)
p.set(10, 100)
a = u.get()[-4:]
assert a == '10%)'
assert " ===== " in u.get()
def enumerate(self, url, base_url_supplied, scanning_method,
iterator_returning_method, iterator_len, max_iterator=500, threads=10,
verb='head', timeout=15, hide_progressbar=False, imu=None):
'''
@param url base URL for the website.
@param base_url_supplied Base url for themes, plugins. E.g. '%ssites/all/modules/%s/'
@param scanning_method see ScanningMethod
@param iterator_returning_method a function which returns an
element that, when iterated, will return a full list of plugins
@param iterator_len the number of items the above iterator can
return, regardless of user preference.
@param max_iterator integer that will be passed unto iterator_returning_method
@param threads number of threads
@param verb what HTTP verb. Valid options are 'get' and 'head'.
@param timeout the time, in seconds, that requests should wait
before throwing an exception.
@param hide_progressbar if true, the progressbar will not be
displayed.
@param imu Interesting module urls. A list containing tuples in the
following format [('readme.txt', 'default readme')].
'''
if common.is_string(base_url_supplied):
base_urls = [base_url_supplied]
else:
base_urls = base_url_supplied
requests_verb = getattr(self.session, verb)
futures = []
with ThreadPoolExecutor(max_workers=threads) as executor:
for base_url in base_urls:
plugins = iterator_returning_method(max_iterator)
if scanning_method == ScanningMethod.not_found:
url_template = base_url + self.module_common_file
else:
url_template = base_url
for plugin_name in plugins:
plugin_url = url_template % (url, plugin_name)
future = executor.submit(requests_verb, plugin_url,
timeout=timeout)
if plugin_url.endswith('/'):
final_url = plugin_url
else:
final_url = dirname(plugin_url) + "/"
futures.append({
'base_url': base_url,
'future': future,
'plugin_name': plugin_name,
'plugin_url': final_url,
})
if not hide_progressbar:
p = ProgressBar(sys.stderr)
items_progressed = 0
max_possible = max_iterator if int(max_iterator) < int(iterator_len) else iterator_len
items_total = int(max_possible) * len(base_urls)
no_results = True
found = []
for future_array in futures:
if not hide_progressbar:
items_progressed += 1
p.set(items_progressed, items_total)
r = future_array['future'].result()
if r.status_code in [200, 403]:
plugin_url = future_array['plugin_url']
plugin_name = future_array['plugin_name']
no_results = False
found.append({
'name': plugin_name,
'url': plugin_url
})
elif r.status_code >= 500:
self.out.warn('\rGot a 500 error. Is the server overloaded?')
if not hide_progressbar:
p.hide()
if imu != None and not no_results:
found = self._enumerate_plugin_if(found, verb, threads, imu)
return found, no_results
def enumerate(self,
url,
base_url_supplied,
scanning_method,
iterator_returning_method,
max_iterator=500,
threads=10,
verb='head',
timeout=15):
'''
@param url base URL for the website.
@param base_url_supplied Base url for themes, plugins. E.g. '%ssites/all/modules/%s/'
@param scanning_method see ScanningMethod
@param iterator_returning_method a function which returns an
element that, when iterated, will return a full list of plugins
@param max_iterator integer that will be passed unto iterator_returning_method
@param threads number of threads
@param verb what HTTP verb. Valid options are 'get' and 'head'.
@param timeout the time, in seconds, that requests should wait
before throwing an exception.
'''
if common.is_string(base_url_supplied):
base_urls = [base_url_supplied]
else:
base_urls = base_url_supplied
requests_verb = getattr(self.session, verb)
futures = []
with ThreadPoolExecutor(max_workers=threads) as executor:
for base_url in base_urls:
plugins = iterator_returning_method(max_iterator)
if scanning_method == ScanningMethod.not_found:
url_template = base_url + self.module_readme_file
expected_status = 200
else:
url_template = base_url
expected_status = common.scan_http_status(scanning_method)
for plugin_name in plugins:
plugin_url = url_template % (url, plugin_name)
future = executor.submit(requests_verb,
plugin_url,
timeout=timeout)
futures.append({
'base_url': base_url,
'future': future,
'plugin_name': plugin_name,
'plugin_url': plugin_url,
})
p = ProgressBar(sys.stderr)
items_progressed = 0
items_total = len(base_urls) * int(max_iterator)
no_results = True
found = []
for future_array in futures:
items_progressed += 1
p.set(items_progressed, items_total)
r = future_array['future'].result()
if r.status_code == expected_status:
plugin_url = future_array['plugin_url']
plugin_name = future_array['plugin_name']
no_results = False
found.append({'name': plugin_name, 'url': plugin_url})
elif r.status_code >= 500:
self.out.warn('Got a 500 error. Is the server overloaded?')
p.hide()
return found, no_results
def enumerate(self, url, base_url_supplied, scanning_method, iterator_returning_method, max_iterator=500, threads=10, verb='head', timeout=15):
'''
@param url base URL for the website.
@param base_url_supplied Base url for themes, plugins. E.g. '%ssites/all/modules/%s/'
@param scanning_method see ScanningMethod
@param iterator_returning_method a function which returns an
element that, when iterated, will return a full list of plugins
@param max_iterator integer that will be passed unto iterator_returning_method
@param threads number of threads
@param verb what HTTP verb. Valid options are 'get' and 'head'.
@param timeout the time, in seconds, that requests should wait
before throwing an exception.
'''
if common.is_string(base_url_supplied):
base_urls = [base_url_supplied]
else:
base_urls = base_url_supplied
requests_verb = getattr(self.session, verb)
futures = []
with ThreadPoolExecutor(max_workers=threads) as executor:
for base_url in base_urls:
plugins = iterator_returning_method(max_iterator)
if scanning_method == ScanningMethod.not_found:
url_template = base_url + self.module_readme_file
expected_status = 200
else:
url_template = base_url
expected_status = common.scan_http_status(scanning_method)
for plugin_name in plugins:
plugin_url = url_template % (url, plugin_name)
future = executor.submit(requests_verb, plugin_url,
timeout=timeout)
futures.append({
'base_url': base_url,
'future': future,
'plugin_name': plugin_name,
'plugin_url': plugin_url,
})
p = ProgressBar(sys.stderr)
items_progressed = 0
items_total = len(base_urls) * int(max_iterator)
no_results = True
found = []
for future_array in futures:
items_progressed += 1
p.set(items_progressed, items_total)
r = future_array['future'].result()
if r.status_code == expected_status:
plugin_url = future_array['plugin_url']
plugin_name = future_array['plugin_name']
no_results = False
found.append({
'name': plugin_name,
'url': plugin_url
})
elif r.status_code >= 500:
self.out.warn('Got a 500 error. Is the server overloaded?')
p.hide()
return found, no_results