def _determine_ok_200(self, requests_verb, url): if common.is_string(self.regular_file_url): reg_url = url + self.regular_file_url ok_resp = requests_verb(reg_url) ok_200 = ok_resp.status_code == 200 else: ok_200 = False for path in self.regular_file_url: reg_url = url + path ok_resp = requests_verb(reg_url) if ok_resp.status_code == 200: ok_200 = True break len_content = len(ok_resp.content) return ok_200, len_content
def enumerate(self, url, base_url_supplied, scanning_method, iterator_returning_method, iterator_len, max_iterator=500, threads=10, verb='head', timeout=15, hide_progressbar=False, imu=None, headers={}): ''' @param url: base URL for the website. @param base_url_supplied: Base url for themes, plugins. E.g. '%ssites/all/modules/%s/' @param scanning_method: see ScanningMethod @param iterator_returning_method: a function which returns an element that, when iterated, will return a full list of plugins @param iterator_len: the number of items the above iterator can return, regardless of user preference. @param max_iterator: integer that will be passed unto iterator_returning_method @param threads: number of threads @param verb: what HTTP verb. Valid options are 'get' and 'head'. @param timeout: the time, in seconds, that requests should wait before throwing an exception. @param hide_progressbar: if true, the progressbar will not be displayed. @param imu: Interesting module urls. A list containing tuples in the following format [('readme.txt', 'default readme')]. @param headers: List of custom headers as expected by requests. ''' if common.is_string(base_url_supplied): base_urls = [base_url_supplied] else: base_urls = base_url_supplied requests_verb = getattr(self.session, verb) futures = [] with ThreadPoolExecutor(max_workers=threads) as executor: for base_url in base_urls: plugins = iterator_returning_method(max_iterator) if scanning_method == ScanningMethod.not_found: url_template = base_url + self.module_common_file else: url_template = base_url for plugin_name in plugins: plugin_url = url_template % (url, plugin_name) future = executor.submit(requests_verb, plugin_url, timeout=timeout, headers=headers) if plugin_url.endswith('/'): final_url = plugin_url else: final_url = dirname(plugin_url) + "/" futures.append({ 'base_url': base_url, 'future': future, 'plugin_name': plugin_name, 'plugin_url': final_url, }) if not hide_progressbar: max_possible = max_iterator if int(max_iterator) < int(iterator_len) else iterator_len items_total = int(max_possible) * len(base_urls) p = ProgressBar(sys.stderr, items_total, "modules") if scanning_method == ScanningMethod.forbidden: expected_status = [403] else: expected_status = [200, 403] no_results = True found = [] for future_array in futures: if common.shutdown: future_array['future'].cancel() continue if not hide_progressbar: p.increment_progress() r = future_array['future'].result() if r.status_code in expected_status: plugin_url = future_array['plugin_url'] plugin_name = future_array['plugin_name'] no_results = False found.append({ 'name': plugin_name, 'url': plugin_url }) elif r.status_code >= 500: self.out.warn('\rGot a 500 error. Is the server overloaded?') if not hide_progressbar: p.hide() if not common.shutdown and (imu != None and not no_results): found = self._enumerate_plugin_if(found, verb, threads, imu, hide_progressbar, timeout=timeout, headers=headers) return found, no_results
def enumerate(self, url, base_url_supplied, scanning_method, iterator_returning_method, iterator_len, max_iterator=500, threads=10, verb='head', timeout=15, hide_progressbar=False, imu=None, headers={}): """ @param url: base URL for the website. @param base_url_supplied: Base url for themes, plugins. E.g. '%ssites/all/modules/%s/' @param scanning_method: see ScanningMethod @param iterator_returning_method: a function which returns an element that, when iterated, will return a full list of plugins @param iterator_len: the number of items the above iterator can return, regardless of user preference. @param max_iterator: integer that will be passed unto iterator_returning_method @param threads: number of threads @param verb: what HTTP verb. Valid options are 'get' and 'head'. @param timeout: the time, in seconds, that requests should wait before throwing an exception. @param hide_progressbar: if true, the progressbar will not be displayed. @param imu: Interesting module urls. A list containing tuples in the following format [('readme.txt', 'default readme')]. @param headers: List of custom headers as expected by requests. """ if common.is_string(base_url_supplied): base_urls = [base_url_supplied] else: base_urls = base_url_supplied requests_verb = getattr(self.session, verb) futures = [] fake_200 = False with ThreadPoolExecutor(max_workers=threads) as executor: for base_url in base_urls: plugins = iterator_returning_method(max_iterator) if scanning_method == ScanningMethod.not_found: url_template = base_url + self.module_common_file else: url_template = base_url fake_200_inst = self._determine_fake_200_module( requests_verb, url_template, url) if fake_200_inst: fake_200 = fake_200_inst for plugin_name in plugins: plugin_url = url_template % (url, plugin_name) future = executor.submit(requests_verb, plugin_url, timeout=timeout, headers=headers) if plugin_url.endswith('/'): final_url = plugin_url else: final_url = dirname(plugin_url) + "/" futures.append({ 'base_url': base_url, 'future': future, 'plugin_name': plugin_name, 'plugin_url': final_url, }) if not hide_progressbar: max_possible = max_iterator if int(max_iterator) < int( iterator_len) else iterator_len items_total = int(max_possible) * len(base_urls) p = ProgressBar(sys.stderr, items_total, "modules") if not fake_200: expected_status = [200, 403, 500] else: expected_status = [403, 500] no_results = True found = [] for future_array in futures: if common.shutdown: future_array['future'].cancel() continue if not hide_progressbar: p.increment_progress() try: r = future_array['future'].result() except requests.exceptions.ReadTimeout: self.out.warn( '\rGot a read timeout. Is the server overloaded? This may affect the results of your scan' ) continue if r.status_code in expected_status: plugin_url = future_array['plugin_url'] plugin_name = future_array['plugin_name'] no_results = False found.append({'name': plugin_name, 'url': plugin_url}) if r.status_code >= 500: self.out.warn('\rGot an HTTP 500 response.') if not hide_progressbar: p.hide() if not common.shutdown and (imu != None and not no_results): found = self._enumerate_plugin_if(found, verb, threads, imu, hide_progressbar, timeout=timeout, headers=headers) return found, no_results