def enumerate_interesting(self, url, interesting_urls, threads=10,
verb='head', timeout=15, hide_progressbar=False):
requests_verb = getattr(self.session, verb)
if not hide_progressbar:
p = ProgressBar(sys.stderr, len(interesting_urls),
"interesting")
found = []
for path, description in interesting_urls:
if shutdown:
continue
interesting_url = url + path
resp = requests_verb(interesting_url, timeout=timeout)
if resp.status_code == 200 or resp.status_code == 301:
found.append({
'url': interesting_url,
'description': description
})
if not hide_progressbar:
p.increment_progress()
if not hide_progressbar:
p.hide()
return found, len(found) == 0
def _enumerate_plugin_if(self, found_list, verb, threads, imu_list, hide_progressbar):
"""
Finds interesting urls within a plugin folder which respond with 200 OK.
@param found_list: as returned in self.enumerate. E.g. [{'name':
'this_exists', 'url': 'http://adhwuiaihduhaknbacnckajcwnncwkakncw.com/sites/all/modules/this_exists/'}]
@param verb: the verb to use.
@param threads: the number of threads to use.
@param imu_list: Interesting module urls.
@param hide_progressbar: whether to display a progressbar.
"""
if not hide_progressbar:
p = ProgressBar(sys.stderr, len(found_list) *
len(imu_list), name="IMU")
requests_verb = getattr(self.session, verb)
with ThreadPoolExecutor(max_workers=threads) as executor:
futures = []
for i, found in enumerate(found_list):
found_list[i]['imu'] = []
for imu in imu_list:
interesting_url = found['url'] + imu[0]
future = executor.submit(requests_verb, interesting_url)
futures.append({
'url': interesting_url,
'future': future,
'description': imu[1],
'i': i
})
for f in futures:
if shutdown:
futures[file_url].cancel()
continue
r = f['future'].result()
if r.status_code == 200:
found_list[f['i']]['imu'].append({
'url': f['url'],
'description': f['description']
})
if not hide_progressbar:
p.increment_progress()
if not hide_progressbar:
p.hide()
return found_list
def enumerate_version(self, url, versions_file, threads=10, verb='head',
timeout=15, hide_progressbar=False):
vf = VersionsFile(versions_file)
files = vf.files_get()
changelogs = vf.changelogs_get()
if not hide_progressbar:
p = ProgressBar(sys.stderr, len(files) +
len(changelogs), "version")
hashes = {}
futures = {}
with ThreadPoolExecutor(max_workers=threads) as executor:
for file_url in files:
futures[file_url] = executor.submit(self.enumerate_file_hash,
url, file_url=file_url, timeout=timeout)
for file_url in futures:
if shutdown:
futures[file_url].cancel()
continue
try:
hsh = futures[file_url].result()
hashes[file_url] = hsh
except RuntimeError:
pass
if not hide_progressbar:
p.increment_progress()
version = vf.version_get(hashes)
# Narrow down using changelog, if accurate.
if vf.has_changelog():
version = self.enumerate_version_changelog(url, version, vf, timeout)
if not hide_progressbar:
p.increment_progress()
p.hide()
return version, len(version) == 0
def enumerate(self, url, base_url_supplied, scanning_method,
iterator_returning_method, iterator_len, max_iterator=500, threads=10,
verb='head', timeout=15, hide_progressbar=False, imu=None):
'''
@param url: base URL for the website.
@param base_url_supplied: Base url for themes, plugins. E.g. '%ssites/all/modules/%s/'
@param scanning_method: see ScanningMethod
@param iterator_returning_method: a function which returns an
element that, when iterated, will return a full list of plugins
@param iterator_len: the number of items the above iterator can
return, regardless of user preference.
@param max_iterator: integer that will be passed unto iterator_returning_method
@param threads: number of threads
@param verb: what HTTP verb. Valid options are 'get' and 'head'.
@param timeout: the time, in seconds, that requests should wait
before throwing an exception.
@param hide_progressbar: if true, the progressbar will not be
displayed.
@param imu: Interesting module urls. A list containing tuples in the
following format [('readme.txt', 'default readme')].
'''
if common.is_string(base_url_supplied):
base_urls = [base_url_supplied]
else:
base_urls = base_url_supplied
requests_verb = getattr(self.session, verb)
futures = []
with ThreadPoolExecutor(max_workers=threads) as executor:
for base_url in base_urls:
plugins = iterator_returning_method(max_iterator)
if scanning_method == ScanningMethod.not_found:
url_template = base_url + self.module_common_file
else:
url_template = base_url
for plugin_name in plugins:
plugin_url = url_template % (url, plugin_name)
future = executor.submit(requests_verb, plugin_url,
timeout=timeout)
if plugin_url.endswith('/'):
final_url = plugin_url
else:
final_url = dirname(plugin_url) + "/"
futures.append({
'base_url': base_url,
'future': future,
'plugin_name': plugin_name,
'plugin_url': final_url,
})
if not hide_progressbar:
max_possible = max_iterator if int(max_iterator) < int(iterator_len) else iterator_len
items_total = int(max_possible) * len(base_urls)
p = ProgressBar(sys.stderr, items_total, "modules")
no_results = True
found = []
for future_array in futures:
if shutdown:
future_array['future'].cancel()
continue
if not hide_progressbar:
p.increment_progress()
r = future_array['future'].result()
if r.status_code in [200, 403]:
plugin_url = future_array['plugin_url']
plugin_name = future_array['plugin_name']
no_results = False
found.append({
'name': plugin_name,
'url': plugin_url
})
elif r.status_code >= 500:
self.out.warn('\rGot a 500 error. Is the server overloaded?')
if not hide_progressbar:
p.hide()
if not shutdown and (imu != None and not no_results):
found = self._enumerate_plugin_if(found, verb, threads, imu,
hide_progressbar)
return found, no_results