def enumerate_interesting(self, url, interesting_urls, threads=10,
verb='head', timeout=15, hide_progressbar=False, headers={}):
requests_verb = getattr(self.session, verb)
if not hide_progressbar:
p = ProgressBar(sys.stderr, len(interesting_urls),
"interesting")
found = []
for path, description in interesting_urls:
if common.shutdown:
continue
interesting_url = url + path
resp = requests_verb(interesting_url, timeout=timeout,
headers=headers)
if resp.status_code == 200 or resp.status_code == 301:
found.append({
'url': interesting_url,
'description': description
})
if not hide_progressbar:
p.increment_progress()
if not hide_progressbar:
p.hide()
return found, len(found) == 0
def _enumerate_plugin_if(self, found_list, verb, threads, imu_list,
hide_progressbar, timeout=15, headers={}):
"""
Finds interesting urls within a plugin folder which respond with 200 OK.
@param found_list: as returned in self.enumerate. E.g. [{'name':
'this_exists', 'url': 'http://adhwuiaihduhaknbacnckajcwnncwkakncw.com/sites/all/modules/this_exists/'}]
@param verb: the verb to use.
@param threads: the number of threads to use.
@param imu_list: Interesting module urls.
@param hide_progressbar: whether to display a progressbar.
@param timeout: timeout in seconds for http requests.
@param headers: custom headers as expected by requests.
"""
if not hide_progressbar:
p = ProgressBar(sys.stderr, len(found_list) *
len(imu_list), name="IMU")
requests_verb = getattr(self.session, verb)
with ThreadPoolExecutor(max_workers=threads) as executor:
futures = []
for i, found in enumerate(found_list):
found_list[i]['imu'] = []
for imu in imu_list:
interesting_url = found['url'] + imu[0]
future = executor.submit(requests_verb, interesting_url,
timeout=timeout, headers=headers)
futures.append({
'url': interesting_url,
'future': future,
'description': imu[1],
'i': i
})
for f in futures:
if common.shutdown:
futures[file_url].cancel()
continue
r = f['future'].result()
if r.status_code == 200:
found_list[f['i']]['imu'].append({
'url': f['url'],
'description': f['description']
})
if not hide_progressbar:
p.increment_progress()
if not hide_progressbar:
p.hide()
return found_list
def enumerate_version(self, url, threads=10, verb='head',
timeout=15, hide_progressbar=False, headers={}):
"""
Determines which version of CMS is installed at url. This is done by
comparing file hashes against the database of hashes in
self.version_file, which is located at dscan/plugins/<plugin_name>/versions.xml
@param url: the url to check.
@param threads: the number of threads to use for this scan.
@param verb: HTTP verb to use.
@param timeout: time, in seconds, before timing out a request.
@param hide_progressbar: should the function hide the progressbar?
@param headers: a dict of headers to pass to requests.get.
@return (possible_versions, is_empty)
"""
files = self.vf.files_get()
changelogs = self.vf.changelogs_get()
if not hide_progressbar:
p = ProgressBar(sys.stderr, len(files) +
len(changelogs), "version")
hashes = {}
futures = {}
with ThreadPoolExecutor(max_workers=threads) as executor:
for file_url in files:
futures[file_url] = executor.submit(self.enumerate_file_hash,
url, file_url=file_url, timeout=timeout, headers=headers)
for file_url in futures:
if common.shutdown:
futures[file_url].cancel()
continue
try:
hsh = futures[file_url].result()
hashes[file_url] = hsh
except RuntimeError:
pass
if not hide_progressbar:
p.increment_progress()
version = self.vf.version_get(hashes)
# Narrow down using changelog, if accurate.
if self.vf.has_changelog():
version = self.enumerate_version_changelog(url, version, timeout, headers=headers)
if not hide_progressbar:
p.increment_progress()
p.hide()
return version, len(version) == 0
def enumerate(self, url, base_url_supplied, scanning_method,
iterator_returning_method, iterator_len, max_iterator=500, threads=10,
verb='head', timeout=15, hide_progressbar=False, imu=None, headers={}):
'''
@param url: base URL for the website.
@param base_url_supplied: Base url for themes, plugins. E.g. '%ssites/all/modules/%s/'
@param scanning_method: see ScanningMethod
@param iterator_returning_method: a function which returns an
element that, when iterated, will return a full list of plugins
@param iterator_len: the number of items the above iterator can
return, regardless of user preference.
@param max_iterator: integer that will be passed unto iterator_returning_method
@param threads: number of threads
@param verb: what HTTP verb. Valid options are 'get' and 'head'.
@param timeout: the time, in seconds, that requests should wait
before throwing an exception.
@param hide_progressbar: if true, the progressbar will not be
displayed.
@param imu: Interesting module urls. A list containing tuples in the
following format [('readme.txt', 'default readme')].
@param headers: List of custom headers as expected by requests.
'''
if common.is_string(base_url_supplied):
base_urls = [base_url_supplied]
else:
base_urls = base_url_supplied
requests_verb = getattr(self.session, verb)
futures = []
with ThreadPoolExecutor(max_workers=threads) as executor:
for base_url in base_urls:
plugins = iterator_returning_method(max_iterator)
if scanning_method == ScanningMethod.not_found:
url_template = base_url + self.module_common_file
else:
url_template = base_url
for plugin_name in plugins:
plugin_url = url_template % (url, plugin_name)
future = executor.submit(requests_verb, plugin_url,
timeout=timeout, headers=headers)
if plugin_url.endswith('/'):
final_url = plugin_url
else:
final_url = dirname(plugin_url) + "/"
futures.append({
'base_url': base_url,
'future': future,
'plugin_name': plugin_name,
'plugin_url': final_url,
})
if not hide_progressbar:
max_possible = max_iterator if int(max_iterator) < int(iterator_len) else iterator_len
items_total = int(max_possible) * len(base_urls)
p = ProgressBar(sys.stderr, items_total, "modules")
if scanning_method == ScanningMethod.forbidden:
expected_status = [403]
else:
expected_status = [200, 403]
no_results = True
found = []
for future_array in futures:
if common.shutdown:
future_array['future'].cancel()
continue
if not hide_progressbar:
p.increment_progress()
r = future_array['future'].result()
if r.status_code in expected_status:
plugin_url = future_array['plugin_url']
plugin_name = future_array['plugin_name']
no_results = False
found.append({
'name': plugin_name,
'url': plugin_url
})
elif r.status_code >= 500:
self.out.warn('\rGot a 500 error. Is the server overloaded?')
if not hide_progressbar:
p.hide()
if not common.shutdown and (imu != None and not no_results):
found = self._enumerate_plugin_if(found, verb, threads, imu,
hide_progressbar, timeout=timeout, headers=headers)
return found, no_results
def _enumerate_plugin_if(self,
found_list,
verb,
threads,
imu_list,
hide_progressbar,
timeout=15,
headers={}):
"""
Finds interesting urls within a plugin folder which respond with 200 OK.
@param found_list: as returned in self.enumerate. E.g. [{'name':
'this_exists', 'url': 'http://adhwuiaihduhaknbacnckajcwnncwkakncw.com/sites/all/modules/this_exists/'}]
@param verb: the verb to use.
@param threads: the number of threads to use.
@param imu_list: Interesting module urls.
@param hide_progressbar: whether to display a progressbar.
@param timeout: timeout in seconds for http requests.
@param headers: custom headers as expected by requests.
"""
if not hide_progressbar:
p = ProgressBar(sys.stderr,
len(found_list) * len(imu_list),
name="IMU")
requests_verb = getattr(self.session, verb)
with ThreadPoolExecutor(max_workers=threads) as executor:
futures = []
for i, found in enumerate(found_list):
found_list[i]['imu'] = []
for imu in imu_list:
interesting_url = found['url'] + imu[0]
future = executor.submit(requests_verb,
interesting_url,
timeout=timeout,
headers=headers)
futures.append({
'url': interesting_url,
'future': future,
'description': imu[1],
'i': i
})
for f in futures:
if common.shutdown:
futures[file_url].cancel()
continue
r = f['future'].result()
if r.status_code == 200:
found_list[f['i']]['imu'].append({
'url':
f['url'],
'description':
f['description']
})
if not hide_progressbar:
p.increment_progress()
if not hide_progressbar:
p.hide()
return found_list
def enumerate_version(self,
url,
threads=10,
verb='head',
timeout=15,
hide_progressbar=False,
headers={}):
"""
Determines which version of CMS is installed at url. This is done by
comparing file hashes against the database of hashes in
self.version_file, which is located at dscan/plugins/<plugin_name>/versions.xml
@param url: the url to check.
@param threads: the number of threads to use for this scan.
@param verb: HTTP verb to use.
@param timeout: time, in seconds, before timing out a request.
@param hide_progressbar: should the function hide the progressbar?
@param headers: a dict of headers to pass to requests.get.
@return (possible_versions, is_empty)
"""
files = self.vf.files_get()
changelogs = self.vf.changelogs_get()
if not hide_progressbar:
p = ProgressBar(sys.stderr,
len(files) + len(changelogs), "version")
hashes = {}
futures = {}
with ThreadPoolExecutor(max_workers=threads) as executor:
for file_url in files:
futures[file_url] = executor.submit(self.enumerate_file_hash,
url,
file_url=file_url,
timeout=timeout,
headers=headers)
for file_url in futures:
if common.shutdown:
futures[file_url].cancel()
continue
try:
hsh = futures[file_url].result()
hashes[file_url] = hsh
except RuntimeError:
pass
if not hide_progressbar:
p.increment_progress()
version = self.vf.version_get(hashes)
# Narrow down using changelog, if accurate.
if self.vf.has_changelog():
version = self.enumerate_version_changelog(url,
version,
timeout,
headers=headers)
if not hide_progressbar:
p.increment_progress()
p.hide()
return version, len(version) == 0
def enumerate(self,
url,
base_url_supplied,
scanning_method,
iterator_returning_method,
iterator_len,
max_iterator=500,
threads=10,
verb='head',
timeout=15,
hide_progressbar=False,
imu=None,
headers={}):
"""
@param url: base URL for the website.
@param base_url_supplied: Base url for themes, plugins. E.g. '%ssites/all/modules/%s/'
@param scanning_method: see ScanningMethod
@param iterator_returning_method: a function which returns an
element that, when iterated, will return a full list of plugins
@param iterator_len: the number of items the above iterator can
return, regardless of user preference.
@param max_iterator: integer that will be passed unto iterator_returning_method
@param threads: number of threads
@param verb: what HTTP verb. Valid options are 'get' and 'head'.
@param timeout: the time, in seconds, that requests should wait
before throwing an exception.
@param hide_progressbar: if true, the progressbar will not be
displayed.
@param imu: Interesting module urls. A list containing tuples in the
following format [('readme.txt', 'default readme')].
@param headers: List of custom headers as expected by requests.
"""
if common.is_string(base_url_supplied):
base_urls = [base_url_supplied]
else:
base_urls = base_url_supplied
requests_verb = getattr(self.session, verb)
futures = []
fake_200 = False
with ThreadPoolExecutor(max_workers=threads) as executor:
for base_url in base_urls:
plugins = iterator_returning_method(max_iterator)
if scanning_method == ScanningMethod.not_found:
url_template = base_url + self.module_common_file
else:
url_template = base_url
fake_200_inst = self._determine_fake_200_module(
requests_verb, url_template, url)
if fake_200_inst:
fake_200 = fake_200_inst
for plugin_name in plugins:
plugin_url = url_template % (url, plugin_name)
future = executor.submit(requests_verb,
plugin_url,
timeout=timeout,
headers=headers)
if plugin_url.endswith('/'):
final_url = plugin_url
else:
final_url = dirname(plugin_url) + "/"
futures.append({
'base_url': base_url,
'future': future,
'plugin_name': plugin_name,
'plugin_url': final_url,
})
if not hide_progressbar:
max_possible = max_iterator if int(max_iterator) < int(
iterator_len) else iterator_len
items_total = int(max_possible) * len(base_urls)
p = ProgressBar(sys.stderr, items_total, "modules")
if not fake_200:
expected_status = [200, 403, 500]
else:
expected_status = [403, 500]
no_results = True
found = []
for future_array in futures:
if common.shutdown:
future_array['future'].cancel()
continue
if not hide_progressbar:
p.increment_progress()
try:
r = future_array['future'].result()
except requests.exceptions.ReadTimeout:
self.out.warn(
'\rGot a read timeout. Is the server overloaded? This may affect the results of your scan'
)
continue
if r.status_code in expected_status:
plugin_url = future_array['plugin_url']
plugin_name = future_array['plugin_name']
no_results = False
found.append({'name': plugin_name, 'url': plugin_url})
if r.status_code >= 500:
self.out.warn('\rGot an HTTP 500 response.')
if not hide_progressbar:
p.hide()
if not common.shutdown and (imu != None and not no_results):
found = self._enumerate_plugin_if(found,
verb,
threads,
imu,
hide_progressbar,
timeout=timeout,
headers=headers)
return found, no_results
