def test_up_requests_counter(self):
Registry().set('proxies', ProxiesMock())
http = Http()
sess1 = http.session
http.up_requests_counter()
assert 1 == http.requests_counter
assert sess1 == http.session
http.requests_per_session = 2
http.up_requests_counter()
assert 0 == http.requests_counter
assert sess1 != http.session
def _insert_hosts(self, hosts):
""" Add found hosts in db """
pid = Registry().get('pData')['id']
Hosts = HostsModel()
Ips = IpsModel()
added = 0
for host in hosts:
ip_id = Ips.get_id_or_add(pid, host['ip'])
if Hosts.add(pid, ip_id, host['name'], founder='dnsbrute'):
added += 1
return added
def __init__(self, job, host, src, not_found_re, delay, ddos_phrase,
ddos_human, recreate_re, counter):
super(SSpiderThread, self).__init__()
self._db = Registry().get('mongo')
self.job = job
self.host = host
self.links_parser = SpiderLinksParser()
self.not_found_re = False if not len(not_found_re) else re.compile(
not_found_re)
self.http = Registry().get('http')
self.delay = int(delay)
self.counter = counter
self.src = src
self.running = True
self.ddos_phrase = ddos_phrase
self.ddos_human = ddos_human
self.recreate_re = False if not len(recreate_re) else re.compile(
recreate_re)
Registry().set('url_for_proxy_check', "{0}://{1}".format('http', host))
self.browser_create()
def prepare_first_pages(host):
""" Prepare link on first page in MongoDB. Add root url if urls for this host not exists. """
pid = Registry().get('pData')['id']
coll = Registry().get('mongo').spider_urls
coll.drop()
Urls = UrlsModel()
urls = Urls.list_by_host_name_for_spider(pid, host)
if not len(urls):
Registry().get('logger').log(
"Spider: Root URL was added automaticaly")
Urls.add(pid,
HostsModel().get_id_by_name(pid, host),
'/',
who_add='spider')
urls = Urls.list_by_host_name_for_spider(pid, host)
for url in urls:
url = urlparse(url['url'])
data = {
'hash': md5(str(url.path + url.query)),
'path': url.path,
'query': url.query,
'time': 0,
'code': 0,
'checked': 0,
'getted': 0,
'referer': '',
'size': 0,
'founder': 'spider'
}
coll.insert(data)
coll.create_index([('hash', 1)], unique=True, dropDups=True)
coll.create_index([('checked', 1)])
def scan_action(self):
""" Scan action of module """
self.enable_logger()
self.validate_main()
self.pre_start_inf()
if self.options['proxies'].value:
Registry().get('proxies').load(self.options['proxies'].value)
SpiderCommon.clear_old_data(self.options['host'].value)
self.result = SpiderResult()
self._options_to_registry()
if self.options['full-new'].value:
SpiderCommon.make_full_new_scan()
SpiderCommon.prepare_first_pages(self.options['host'].value)
if not os.path.exists(Registry().get('data_path') +
self.options['host'].value):
os.mkdir(Registry().get('data_path') + self.options['host'].value)
os.chmod(Registry().get('data_path') + self.options['host'].value,
0o777)
job = SpiderJob()
src = SpiderRequestsCounter()
counter = WSCounter(5, 300, 0)
workers = []
for _ in range(int(self.options['threads'].value)):
if self.options['selenium'].value:
worker = SSpiderThread(
job, self.options['host'].value, src,
self.options['not-found-re'].value,
self.options['delay'].value,
self.options['ddos-detect-phrase'].value,
self.options['ddos-human-action'].value,
self.options['browser-recreate-re'].value, counter)
else:
worker = SpiderThread(job, self.options['host'].value, src,
self.options['delay'].value, counter)
worker.setDaemon(True)
workers.append(worker)
time.sleep(1)
self.kernel.create_threads(workers)
while not self.kernel.finished():
time.sleep(2)
self.logger.log("\nPut results into DB...")
SpiderCommon.links_in_database(Registry().get('pData')['id'],
self.options['host'].value)
self.logger.log("\nTotal links count: " +
str(Registry().get('mongo').spider_urls.count()))
self.logger.log(str(self.result))
def insert_links(links, referer, site):
""" Put links data in MongoDB """
links = SpiderCommon.prepare_links_for_insert(links, urlparse(referer),
site)
if not len(links):
return
denied_schemas = SpiderCommon.denied_schemas
for link in links:
if 'scheme' in link and link['scheme'] in denied_schemas:
continue
insert = {
'hash': SpiderCommon.get_url_hash(link.path, link.query),
'path': link.path.strip(),
'query': link.query.strip(),
'referer': referer,
'founder': 'spider',
'checked': 0 if SpiderCommon._link_allowed(link) else 1,
'getted': 0,
'code': 0,
'time': 0,
'size': 0
}
try:
Registry().get('mongo').spider_urls.insert(insert)
except pymongo.errors.DuplicateKeyError:
pass
except BaseException as e:
Registry().get('logger').log(
"Can`t insert link " + insert['path'] + " " +
insert['query'] + ") in db. "
"May be it have non-utf8 symbols or somethink else. Exception message:"
)
Registry().get('logger').ex(e)
def __init__(self):
config = configparser.ConfigParser()
config.read(os.getcwd() + '/' + 'config.ini')
R = Registry()
R.set('config', config)
R.set('wr_path', os.getcwd())
R.set('data_path', os.getcwd() + '/data/')
R.set('http', Http())
R.set('ua', random_ua())
R.set('proxies', Proxies())
R.set('tmp_files', [])
R.set(
'fuzzer_evil_value',
file_get_contents(Registry().get('wr_path') +
"/bases/fuzzer/evil-value.txt").strip())
R.set('proxy_many_died', False)
R.set('positive_limit_stop', False)
if " ".join(sys.argv).count('selenium') and int(
config['selenium']['virtual_display']):
display = Display(visible=0, size=(800, 600))
display.start()
R.set('display', display)
def test_load_proxies(self, param_name):
file_put_contents("/tmp/test.txt", "")
proxies_mock = ProxiesMock()
Registry().set('proxies', proxies_mock)
module = ModuleMock(False)
module.options = {
param_name:
WSOption("proxies", "File with list of proxies", "", False,
['--proxies'])
}
module.options[param_name].value = "/tmp/test.txt"
module.load_proxies()
assert "/tmp/test.txt" in proxies_mock.loaded_path
def export_action(self):
""" Action list of module """
self.validate_main()
urls = self.model.list_by_host_name(Registry().get('pData')['id'],
self.options['host'].value,
self.options['like'].value)
for url in urls:
if not int(self.options['without-host'].value):
link = url['url']
else:
link = "{0}://{1}{2}".format(self.options['protocol'].value,
self.options['host'].value,
url['url'])
print link
def get_current_proxy(self):
""" Check current proxy, get next if need (max requests per proxy made) """
if Registry().isset('tor'):
proxy_str = Registry().get('config')['tor']['ip'] + ":" + Registry(
).get('config')['tor']['port']
return {
"http": "socks5h://" + proxy_str,
"https": "socks5h://" + proxy_str,
}
if self.current_proxy_counter >= int(
Registry().get('config')['main']['requests_per_proxy']):
self.current_proxy = None
self.current_proxy_counter = 0
if not self.current_proxy:
self.change_proxy()
self.current_proxy_counter += 1
return {
"http": "http://" + self.current_proxy,
"https": "http://" + self.current_proxy,
} if self.current_proxy else None
def __init__(self, queue, protocol, host, url, false_phrase, true_phrase,
retest_codes, delay, confstr, first_stop, login, pass_found,
counter, result):
threading.Thread.__init__(self)
self.retested_words = {}
self.queue = queue
self.protocol = protocol.lower()
self.host = host
self.url = url
self.false_phrase = false_phrase
self.true_phrase = true_phrase
self.delay = int(delay)
self.confstr = confstr
self.first_stop = first_stop
self.login = login
self.counter = counter
self.result = result
self.retest_codes = list(set(
retest_codes.split(','))) if len(retest_codes) else []
self.pass_found = pass_found
self.done = False
self.logger = Registry().get('logger')
self.http = copy.deepcopy(Registry().get('http'))
self.http.every_request_new_session = True
def validate_main(self):
""" Check users params """
super(UrlsModules, self).validate_main()
skip_listings = Registry().get('config')['main']['skip_listings']
standart_msymbol = Registry().get('config')['main']['standart_msymbol']
source_url = self.options['template'].value.replace(standart_msymbol, "")
try:
resp = Registry().get('http').get(source_url)
response_text = resp.text
resp.close()
if ("<title>Index of" in response_text or "<h1>Index of" in response_text) and skip_listings == "1":
raise WSException("Source URL is listing, check it. Or change 'skip_listings' param in config")
except requests.exceptions.ConnectionError:
raise WSException("Target web-site not available")
parsed_url = urlparse(self.options['template'].value)
if not len(parsed_url.scheme) or not len(parsed_url.netloc):
raise WSException("Target URL not valid")
if self.options['not-found-size'].value != "-1" and self.options['method'].value.lower() == 'head':
raise WSException(
"You can`t use HEAD method with --false-size param"
)
def __init__(self):
threading.Thread.__init__(self)
self._db = Factory().new_db_connect()
config = Registry().get('config')
self.tmp_dir = config['main']['tmp_dir']
self.dicts_path = config['main']['dicts_path']
self.outs_path = config['main']['outs_path']
self.rules_path = config['main']['rules_path']
self.path_to_hc = config['main']['path_to_hc']
self.hc_bin = config['main']['hc_bin']
self.finder_key = config['main']['finder_key']
self.config = config
def do_work(self):
""" Start working """
self.validate_main()
dict_path = "/tmp/web-scout-content-discovery-%d.txt" % random.randint(
1000000, 9000000)
self.build_attack_list(dict_path)
dict_path_uniq = dict_path + "-uniq"
subprocess.check_output("sort -u {0} > {1}".format(
dict_path, dict_path_uniq),
shell=True)
tmp_files = Registry().get('tmp_files')
tmp_files.append(dict_path)
tmp_files.append(dict_path_uniq)
Registry().set('tmp_files', tmp_files)
self.options['dict'] = WSOption("dict", "Dictionary for work", "",
True, ['--dict'])
self.options['dict'].value = dict_path_uniq
WSModule.do_work(self)
def validate_main(self):
""" Check users params """
super(HttpAuth, self).validate_main()
try:
resp = Registry().get('http').get(self.options['url'].value)
if resp.status_code != 401:
raise WSException(
"Target URL has response code {0}, not 401".format(
resp.status_code))
except requests.exceptions.ConnectionError:
raise WSException("Target web-site not available")
parsed_url = urlparse(self.options['url'].value)
if not len(parsed_url.scheme) or not len(parsed_url.netloc):
raise WSException("Target URL not valid")
def _put(self, item):
if self.unique:
if item not in self.all_items:
WSJob._put(self, item)
self.all_items.add(item)
else:
_str = "WARNING: try to add not unique item `{0}`".format(item)
if Registry().isset('logger'):
#Registry().get('logger').log(_str)
pass
else:
#print _str
pass
else:
WSJob._put(self, item)
def pre_start_inf(self):
""" Show options values before work start """
log_str = ""
log_str += "---------------------------------\n"
for option in self.options:
log_str += "Option '{0}': {1}\n".format(option,
self.options[option].value)
log_str += "Logs dir: {0}\n".format(self.logger.logs_dir)
log_str += "---------------------------------"
self.logger.log(log_str)
if int(Registry().get('config')['main']['confirm']):
tmp = raw_input("Do you have continue? [Y/n]")
if len(tmp.strip()) and tmp.lower() != 'y':
self.logger.log("Aborted...")
self.work_end_error()
def _get_codes_stat(self):
""" Build dict with http-codes and their counts """
coll = Registry().get('mongo').spider_urls
result = {}
codes = coll.group({'code': True}, '', {}, 'function () {}')
for code in codes:
links = []
code = code['code']
data = coll.find({'code': code}, {'path': 1, 'query': 1})
for link in mongo_result_to_list(data):
links.append(link['path'] + '?' +
link['query'] if link['query'] else link['path'])
result[int(code)] = links
return result
def test_factory(self):
Registry().set('config', {
'main': {
'counter_step': '1',
'counter_steps_for_new_string': '2'
}
})
counter = WSCounter.factory(3)
assert 1 == counter.point
assert 2 == counter.new_str
assert 3 == counter.all
assert int(time.time()) == counter.start_time
assert int(time.time()) == counter.last_point_time
def add_action(self):
""" Action add of module """
self.validate_main()
pData = Registry().get('pData')
ip = self.options['ip'].value
descr = self.options['descr'].value
if self.model.exists(pData['id'], ip):
raise WSException(
"IP '{0}' already exists in project '{1}'!".format(
ip, pData['name']))
self.model.add(pData['id'], ip, descr)
print " IP '{0}' successfully added to project '{1}' ! ".format(
ip, pData['name'])
def scan_links(self, links):
""" Scan links """
req_func = getattr(self.http, 'get')
for link in links:
self.last_action = int(time.time())
self.counter.up()
url = SpiderCommon.gen_url(link, self.host)
start_time = int(round(time.time() * 1000))
pre_url = link['path'] + '?' + link['query'] if len(
link['query']) else link['path']
if self.delay:
time.sleep(self.delay)
response = req_func(url)
self.src.up()
if response is not None:
result_time = int(round(time.time() * 1000)) - start_time
if 'content-type' in response.headers:
content_type = response.headers['content-type'].split(";")[0] \
if (response.headers['content-type'].find(";") != -1) \
else response.headers['content-type']
else:
content_type = 'unknown/unknown'
if 299 < response.status_code < 400:
SpiderCommon.insert_links([response.headers['Location']],
url, self.host)
else:
new_links = self.links_parser.parse_links(
content_type, str(response.content), link)
SpiderCommon.insert_links(new_links, url, self.host)
file_put_contents(
"{0}{1}/{2}".format(Registry().get('data_path'), self.host,
md5(pre_url)), str(response.content))
link['size'] = len(response.content) if response is not None else 0
link['code'] = response.status_code if response is not None else 0
link['time'] = result_time if response is not None else 0
SpiderCommon.links_checked(links)
def get_current_proxy(self):
""" Check current proxy, get next if need (max requests per proxy made) """
if self.current_proxy_count >= int(
Registry().get('config')['main']['requests_per_proxy']):
self.current_proxy = None
self.current_proxy_count = 0
if not self.current_proxy:
#self.current_proxy = Registry().get('proxies').get_proxy()
self.change_proxy()
self.current_proxy_count += 1
return {
"http": "http://" + self.current_proxy,
"https": "http://" + self.current_proxy,
} if self.current_proxy else None
def validate_main(self):
""" Check users params """
super(HostsModules, self).validate_main()
try:
resp = Registry().get(
'http').get(self.options['http-protocol'].value + "://" +
self.options['ip'].value)
except requests.exceptions.ConnectionError:
raise WSException("Target web-site not available")
if (not self.options['false-re'].value
or not len(self.options['false-re'].value)) and (
not self.options['false-size'].value
or not len(self.options['false-size'].value)):
raise WSException(
"You must specify --false-re param or --false-size param!")
def pre_start_inf(self):
""" Show options values before work start """
log_str = ""
log_str += "---------------------------------\n"
for option in self.options:
log_str += "Option '{0}': {1}\n".format(option,
self.options[option].value)
log_str += "---------------------------------"
print log_str
if int(Registry().get('config')['main']['confirm']):
tmp = raw_input("Do you have continue? [Y/n]")
if len(tmp.strip()) and tmp.lower() != 'y':
print "Aborted..."
exit(0)
self.logger.log(log_str + '\n', new_str=False, _print=False)
def write_tools_variants(self, base_fh):
"""
Write in file variants from tools dict.
Using for write one big dictionary.
:param base_fh:
:return:
"""
fh = open(Registry().get('wr_path') +
"/bases/content-discovery/tools-and-others.txt")
while True:
line = fh.readline()
if not line:
break
line = line.strip()
base_fh.write(line + "\n")
fh.close()
def _insert_urls(self, urls):
""" Insert found urls in DB """
pid = Registry().get('pData')['id']
host_id = HostsModel().get_id_by_name(pid, self.options['host'].value)
U = UrlsModel()
added = 0
for url in urls:
if isinstance(url, str) or isinstance(url, unicode):
if U.add(pid, host_id, url, '', 0, 0, 'pre'):
added += 1
else:
if U.add(pid, host_id, url['url'], '', url['code'],
url['time'], 'pre'):
added += 1
return added
def __init__(self, queue, counter, result, params):
"""
:type params: FuzzerThreadParams
"""
AbstractSeleniumThread.__init__(self)
self.queue = queue
self.method = params.method
self.result = result
self.counter = counter
self.bad_words = params.bad_words
self.delay = params.delay
self.browser_wait_re = params.browser_wait_re
self.browser_recreate_re = params.browser_recreate_re
Registry().set('url_for_proxy_check', "https://google.com")
self.browser_create()
def gen_backups_variants(basename, may_be_file):
"""
Generate backups variants by schemas
:param basename:
:param may_be_file:
:return:
"""
results = []
basenames = [basename]
if may_be_file and basename.count("."):
basenames.append(basename[:basename.rfind(".")])
schemas = file_to_list(Registry().get('wr_path') + "/bases/content-discovery/backup-schemas.txt")
for basename in basenames:
for schema in schemas:
results.append(schema.replace("|name|", basename))
return results
def additional_hostname_validation(self, name):
""" If we found hostname, be better if we re-check it some times """
dns_server = Registry().get(
'config')['dns']['additional_domains_check_dns']
myResolver = dns.resolver.Resolver()
myResolver.nameservers = [dns_server]
result = False
for i in range(0, 3):
time.sleep(3)
try:
myResolver.query(name, 'A')
result = True
except dns.resolver.NXDOMAIN:
result = False
break
return result
def test_maskdict_task(self):
""" Run hybride mask+dict task """
self._add_hashlist(alg_id=0, uncracked=4)
self._add_hash(hash=md5('123'))
self._add_hash(hash=md5('456'))
self._add_hash(hash=md5('1ccc'))
self._add_hash(hash=md5('789'))
self._add_work_task()
self._add_task(source=json.dumps({
'mask': '?d',
'dict': 1
}),
type='maskdict')
dicts_path = Registry().get('config')['main']['dicts_path']
self._add_dict_group()
self._add_dict()
self._add_dict(id=2, hash='2')
file_put_contents(dicts_path + "/1.dict", "aaa\nbbb\n")
file_put_contents(dicts_path + "/2.dict", "ccc\nddd\n")
self.thrd = WorkerThread(
self.db.fetch_row("SELECT * FROM task_works WHERE id = 1"))
self.thrd.catch_exceptions = False
self.thrd.start()
start_time = int(time.time())
while True:
if self.thrd.done:
break
if int(time.time()) - start_time > 5:
pytest.fail("Long time of WorkerThread")
time.sleep(1)
wtask = self.db.fetch_row("SELECT * FROM task_works WHERE id = 1")
assert wtask['status'] == 'waitoutparse'
assert wtask['uncracked_before'] == 4
assert os.path.exists(wtask['out_file'])
assert file_get_contents(
wtask['out_file']) == '49a14108270c0596ac1d70c3c4f82a10:31636363\n'
