def up():
ErrorsCounter.counter += 1
if not ErrorsCounter.reported and ErrorsCounter.counter / (int(
Registry().get('config')['main']['errors_limit']) / 100) > 70:
Registry().get("logger").log(
"\nAttention! Too many errors, 70% of limit!")
ErrorsCounter.reported = True
def _get_extensions(self):
""" Build files extensions list """
result = {}
coll = Registry().get('mongo').spider_urls
links = coll.group({'path': True}, '', {}, 'function () {}')
links = mongo_result_to_list(links)
exts = []
for link in links:
if link['path'].rfind('.') > -1 and len(
link['path']) - link['path'].rfind('.') <= 5:
exts.append(link['path'][link['path'].rfind('.'):])
for ext in list(set(exts)):
if ext not in result:
result[ext] = []
links = coll.find({'path': re.compile('\\' + ext + '$')})
links = mongo_result_to_list(links)
for link in links:
result[ext].append(
link['path'] + '?' +
link['query'] if link['query'] else link['path'])
return result
def add_action(self):
""" Action add of module """
pid = Registry().get('pData')['id']
self.validate_main()
if self.model.exists(pid, self.options['host'].value,
self.options['url'].value):
raise WSException(
"URL '{0}' already exists in this project in host '{1}'!".
format(self.options['url'].value, self.options['host'].value))
host_id = HostsModel().get_id_by_name(pid, self.options['host'].value)
if (self.options['url'].value[-1] == '/' and
self.model.exists(pid, self.options['host'].value, self.options['url'].value[:-1])) or\
(self.options['url'].value[-1] != '/' and
self.model.exists(pid, self.options['host'].value, self.options['url'].value + "/")):
if raw_input(
'Url {0} have analogue in database (with or without end slash). '
'Are you realy want to add it (y/n)?'.format(
self.options['url'].value)).lower()[0] != 'y':
print "Url {0} was not added!".format(
self.options['url'].value)
return
self.model.add(Registry().get('pData')['id'], host_id,
self.options['url'].value)
print " URL '{0}' successfully added to host '{1}'".\
format(self.options['url'].value, self.options['host'].value)
def q(self, sql, return_curs=False):
""" Usual query, return cursor """
for i in range(1, self._restart_by_deadlock_limit + 1):
try:
curs = self._db.cursor(buffered=True)
curs.execute(sql)
except mysql.connector.errors.OperationalError as ex:
if "MySQL Connection not available" in str(ex):
self.connect()
Registry().get('logger').log(
"database", "Reconnect on '{0}'".format(sql))
return self.q(sql, return_curs)
else:
raise ex
except mysql.connector.errors.DatabaseError as e:
if str(e).count("Lock wait timeout exceeded") or str(e).count(
"Deadlock found when trying to get lock"):
Registry().get('logger').log(
"database", "Deadlock '{0}', try {1} ".format(sql, i))
if i == self._restart_by_deadlock_limit:
curs = self._db.cursor()
curs.execute(sql)
else:
time.sleep(self._sleep_by_deadlock_restart)
continue
else:
raise e
break
if return_curs:
return curs
else:
curs.close()
def __init__(
self, queue, protocol, host, url, method, mask_symbol, not_found_re,
not_found_codes, retest_codes, delay, counter, result):
threading.Thread.__init__(self)
self.retested_words = {}
self.queue = queue
self.protocol = protocol.lower()
self.host = host
self.method = method if not (len(not_found_re) and method.lower() == 'head') else 'get'
self.url = url
self.mask_symbol = mask_symbol
self.counter = counter
self.result = result
self.done = False
self.not_found_re = False if not len(not_found_re) else re.compile(not_found_re)
not_found_codes = not_found_codes.split(',')
not_found_codes.append('404')
self.not_found_codes = list(set(not_found_codes))
self.retest_codes = list(set(retest_codes.split(','))) if len(retest_codes) else []
self.delay = int(delay)
self.http = copy.deepcopy(Registry().get('http'))
self.logger = Registry().get('logger')
def factory(_all=0):
""" Build counter object by WS config """
return WSCounter(
int(Registry().get('config')['main']['counter_step']),
int(Registry().get('config')['main']['counter_step']) *
int(Registry().get('config')['main']
['counter_steps_for_new_string']), _all)
def __init__(self, queue, domain, protocol, method, not_found_re, not_found_codes,
not_found_size, delay, counter, result):
super(BackupsFinderThread, self).__init__()
self.queue = queue
self.domain = domain
self.result = result
self.counter = counter
self.protocol = protocol
self.not_found_re = False if not len(not_found_re) else re.compile(not_found_re)
self.not_found_size = int(not_found_size)
self.method = method.lower()
self.method = method if \
not ((len(not_found_re) or self.not_found_size != -1) and method.lower() == 'head') else \
'get'
not_found_codes = not_found_codes.split(',')
not_found_codes.append('404')
self.not_found_codes = list(set(not_found_codes))
self.delay = int(delay)
self.done = False
self.http = Registry().get('http')
self.logger = Registry().get('logger')
def __init__(self):
self.tags = {}
tags = Registry().get('config')['spider']['tags']
tags = tags.split(",")
for tag in tags:
tag = tag.split("|")
self.tags[tag[0]] = tag[1]
def __init__(self):
self.tags = {}
tags = Registry().get('config')['spider']['tags']
tags = tags.split(",")
for tag in tags:
tag = tag.split("|")
self.tags[tag[0]] = tag[1]
def is_critical_stop(self):
"""
Is we has critical sign for immidiately stop?
:return:
"""
return Registry().get('proxy_many_died') or Registry().get(
'positive_limit_stop') or ErrorsCounter.is_limit()
def load_file(self, _file):
""" Fill queue from text file """
self.collection.drop()
fh = open(_file)
last = "START OF FILE"
while True:
line = fh.readline()
if not line:
break
if not line.strip() and self.skip_blank_rows:
continue
try:
line = line.strip()
unicode(line)
self.collection.insert(self.build_row(line))
except UnicodeDecodeError:
_str = " UNICODE ERROR: In file '{0}' skip word '{1}', after word '{2}' !".format(_file, line, last)
if Registry().isset('logger'):
Registry().get('logger').log(_str)
else:
print _str
continue
last = line
fh.close()
self.collection.create_index('name', drop_dups=True, unique=self.unique)
self.load_data()
return self.collection.count()
def head(self, url, verify=None, allow_redirects=None, headers=None):
""" HTTP HEAD request """
if self.every_request_new_session:
self.session = requests.Session()
verify = self.verify if verify is None else verify
allow_redirects = self.allow_redirects if allow_redirects is None else allow_redirects
headers = self.headers if headers is None else headers
resp = self.session.head(url,
verify=verify,
allow_redirects=allow_redirects,
headers=headers,
proxies=self.get_current_proxy())
if 'content-length' in resp.headers and \
int(resp.headers['content-length']) > int(Registry().get('config')['main']['max_size']):
self.errors['maxsize'].append(
"URL {0} has size {1} bytes, but limit in config - {2} bytes".\
format(
url,
resp.headers['content-length'],
Registry().get('config')['main']['max_size']
)
)
resp = None
return resp
def _options_to_registry(self):
if self.options['ignore'].value:
Registry().set('ignore_regexp',
re.compile(self.options['ignore'].value))
expr = ''
for ext in Registry().get('config')['spider']['allow_exts'].split(','):
expr += r'\.' + ext.strip() + '$|'
expr = expr.rstrip('|')
Registry().set('allow_regexp', re.compile(expr, re.I))
if self.options['only_one'].value:
tmp = self.options['only_one'].value.split("||")
if len(tmp):
only_one = []
for regex in tmp:
only_one.append({'regex': regex, 'block': False})
Registry().set('only_one', only_one)
denied_schemes = parse_split_conf(
Registry().get('config')['spider']['denied_schemes'])
Registry().set('denied_schemes', denied_schemes)
Registry().get('http').set_allowed_types(
parse_split_conf(
Registry().get('config')['spider']['scan_content_types']))
Registry().get('http').set_denied_types(
parse_split_conf(
Registry().get('config')['spider']['noscan_content_types']))
def load_dict(self, dict_for_load, drop=True):
""" Fill collection from dict """
if drop:
self.collection.drop()
counter = 0
last = "START OF FILE"
for line in dict_for_load:
try:
line = line.strip()
unicode(line)
self.collection.insert(self.build_row(line))
except UnicodeDecodeError:
_str = " UNICODE ERROR: In file '{0}' skip word '{1}', after word '{2}' !".format(file, line, last)
if Registry().isset('logger'):
Registry().get('logger').log(_str)
else:
print _str
continue
counter += 1
last = line
self.load_data()
return counter
def list_action(self):
""" Action list of module """
if self.options['ip'].value:
print "{0:=^51}".format("")
print "|{0: ^49}|".format("Hosts for IP '{0}'".format(
self.options['ip'].value))
print "{0:=^51}".format("")
print "| {0: ^23}| {1: ^23}|".format('Title', 'Description')
print "{0:=^51}".format("")
for host in self.model.list(Registry().get('pData')['id'],
self.options['ip'].value):
print "| {0: <23}| {1: <23}|".format(host['name'],
host['descr'])
print "{0:=^51}".format("")
else:
print "{0:=^76}".format("")
print "|{0: ^74}|".format("All host for project '{0}'".format(
Registry().get('pData')['name']))
print "{0:=^76}".format("")
print "| {0: ^23}| {1: ^23}| {2: ^23}|".format(
'Title', 'Description', 'IP')
print "{0:=^76}".format("")
for host in self.model.list_without_ip(
Registry().get('pData')['id']):
print "| {0: <23}| {1: <23}| {2: <23}|".format(
host['name'], host['descr'], host['ip'])
print "{0:=^76}".format("")
def load_file_in_db(self, file_path):
"""
Import in-db file in database with mysqlimport util
:param file_path:
:return:
"""
self.update_status('putindb')
self.log("Data go to DB")
if os.path.exists(self.tmp_dir + "/hashes"):
os.remove(self.tmp_dir + "/hashes")
hashes_file_path = self.tmp_dir + "/hashes"
shutil.move(file_path, hashes_file_path)
importcmd = "mysqlimport --lock-tables --user {0} -p{1} --local " \
"--columns hashlist_id,hash,salt,summ --fields-enclosed-by '\"'" \
" --fields-terminated-by ',' --fields-escaped-by \"\\\\\" {2} {3}"\
.format(
Registry().get('config')['main']['mysql_user'],
Registry().get('config')['main']['mysql_pass'],
Registry().get('config')['main']['mysql_dbname'],
self.tmp_dir + "/hashes"
)
subprocess.check_output(importcmd, shell=True)
os.remove(hashes_file_path)
def add(self,
pid,
host_id,
url,
referer='',
response_code=0,
response_time=0,
who_add='human',
spidered=0,
size=0,
descr=''):
""" Add url to table """
try:
return self._db.insert(
"urls", {
"project_id": pid,
"host_id": host_id,
"hash": md5(url),
"url": url,
"referer": referer,
"response_code": response_code,
"response_time": response_time,
"when_add": int(time.time()),
"who_add": who_add,
"spidered": spidered,
"size": size,
"descr": descr
}, 1)
except BaseException as e:
if Registry().isset('logger'):
Registry().get('logger').ex(e)
else:
print "Can`t add url: " + str(e)
def check_backups(self):
""" Simple check backups """
Http = Registry().get('http')
check = []
result = []
dirs = file_to_list(Registry().get('wr_path') + "/bases/pre-backups-dirs.txt")
dirs.append(self.options['host'].value)
files = file_to_list(Registry().get('wr_path') + "/bases/pre-backups-files.txt")
dirs_exts = file_to_list(Registry().get('wr_path') + "/bases/bf-dirs.txt")
files_exts = file_to_list(Registry().get('wr_path') + "/bases/bf-files.txt")
for _dir in dirs:
for dir_ext in dirs_exts:
check.append(dir_ext.replace('|name|', _dir))
for _file in files:
for file_ext in files_exts:
check.append(file_ext.replace('|name|', _file))
for url in check:
r = Http.get(self.root_url + url)
if not self._response_404(r):
result.append({'url': self.root_url + url, 'code': r.status_code})
return result
def __init__(self, queue, domain, protocol, method, not_found_re, delay,
ddos_phrase, ddos_human, recreate_re, counter, result):
super(SBackupsFinderThread, self).__init__()
self.queue = queue
self.method = method if not (len(not_found_re)
and method.lower() == 'head') else 'get'
self.domain = domain
self.result = result
self.counter = counter
self.protocol = protocol
self.not_found_re = False if not len(not_found_re) else re.compile(
not_found_re)
self.done = False
self.http = Registry().get('http')
self.delay = int(delay)
self.ddos_phrase = ddos_phrase
self.ddos_human = ddos_human
self.recreate_re = False if not len(recreate_re) else re.compile(
recreate_re)
self.logger = Registry().get('logger')
Registry().set('url_for_proxy_check',
"{0}://{1}".format(protocol, domain))
self.browser_create()
def links_in_spider_base(pid, host):
""" Put found links in MySQL """
links_per_time_limit = 50
c = WSCounter(
1, 60,
int(Registry().get('mongo').spider_urls.count() /
links_per_time_limit))
Urls = UrlsModel()
host_id = HostsModel().get_id_by_name(pid, host)
urls_add = []
skip = 0
while True:
links = mongo_result_to_list(
Registry().get('mongo').spider_urls.find().skip(skip).limit(
links_per_time_limit))
for link in links:
url = link['path'] + '?' + link['query'] if len(
link['query']) else link['path']
urls_add.append({
'url': url,
'referer': link['referer'],
'response_code': link['code'],
'response_time': link['time'],
'size': link['size'],
'who_add': 'spider',
'spidered': link['checked']
})
Urls.add_mass(pid, host_id, urls_add)
urls_add = []
to_update = {'spidered': [], 'code': [], 'time': [], 'size': []}
for link in links:
url = link['path'] + '?' + link['query'] if len(
link['query']) else link['path']
if link['checked']:
to_update['spidered'].append({'url': url, 'value': 1})
to_update['code'].append({'url': url, 'value': link['code']})
to_update['time'].append({'url': url, 'value': link['time']})
to_update['size'].append({'url': url, 'value': link['size']})
Urls.update_url_field_mass(pid, host, 'spidered',
to_update['spidered'])
Urls.update_url_field_mass(pid, host, 'response_code',
to_update['code'])
Urls.update_url_field_mass(pid, host, 'response_time',
to_update['time'])
Urls.update_url_field_mass(pid, host, 'size', to_update['size'])
skip += len(links)
c.up()
if len(links) < links_per_time_limit:
break
class DnsBruteThread(threading.Thread):
""" Thread class for DnsBrute* modules """
done = False
def __init__(self, queue, domain, proto, msymbol, dns_srv, delay, result,
counter):
threading.Thread.__init__(self)
self.queue = queue
self.domain = domain
self.proto = proto
self.dns_srv = dns_srv
self.counter = counter
self.msymbol = msymbol
self.result = result
self.delay = int(delay)
self.done = False
self.logger = Registry().get('logger')
def run(self):
""" Run thread """
ip_re = re.compile(r"(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})")
ns_resp_re = re.compile(r";ANSWER\s(?P<data>(.|\s)*)\s;AUTHORITY",
re.M)
req_func = getattr(dns.query, self.proto.lower())
while True:
host = None
if self.delay:
time.sleep(self.delay)
try:
host = self.queue.get()
self.counter.up()
check_name = self.domain.replace(self.msymbol, host)
query = dns.message.make_query(check_name, 'A')
result = req_func(query, self.dns_srv, timeout=5)
response = ns_resp_re.search(result.to_text())
if response is not None:
for ip in ip_re.findall(response.group('data')):
self.result.append({
'name': check_name,
'ip': ip,
'dns': self.dns_srv
})
break
if len(self.result) >= int(Registry().get('config')['main']
['positive_limit_stop']):
Registry().set('positive_limit_stop', True)
except Queue.Empty:
self.done = True
break
except BaseException as e:
self.logger.ex(e)
time.sleep(5)
def __init__(self):
threading.Thread.__init__(self)
self.logger = Registry().get('logger')
self.retest_limit = int(
Registry().get('config')['main']['retest_limit'])
self.retest_delay = int(
Registry().get('config')['main']['retest_delay'])
def test_is_critical_stop(self, proxy_many_died, positive_limit_stop,
errors_limit, expected):
Registry().set('config', {"main": {"errors_limit": "10"}})
Registry().set('proxy_many_died', proxy_many_died)
Registry().set('positive_limit_stop', positive_limit_stop)
ErrorsCounter.counter = 100 if errors_limit else 0
module = ModuleMock(False)
assert expected == module.is_critical_stop()
def get_denied_schemas():
""" Get list of denied schemas """
if SpiderCommon.denied_schemas is None:
denied_schemas = Registry().get('config')['spider']['denied_schemes'].split(',')
for dschema in denied_schemas:
index = denied_schemas.index(dschema)
denied_schemas[index] = dschema.encode('utf8')
SpiderCommon.denied_schemas = list(map(str.strip, denied_schemas))
return SpiderCommon.denied_schemas
def links_in_database(pid, host):
""" Method for insert all found links in MySQL in work end """
Registry().get(
'logger').log("\nInsert links in DB..." +
time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
SpiderCommon.links_in_spider_base(pid, host)
Registry().get(
'logger').log("\nInsert links in DB (base)..." +
time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
SpiderCommon.links_in_urls_base(pid, host)
def get_denied_schemas():
""" Get list of denied schemas """
if SpiderCommon.denied_schemas is None:
denied_schemas = Registry().get(
'config')['spider']['denied_schemes'].split(',')
for dschema in denied_schemas:
index = denied_schemas.index(dschema)
denied_schemas[index] = dschema.encode('utf8')
SpiderCommon.denied_schemas = list(map(str.strip, denied_schemas))
return SpiderCommon.denied_schemas
def is_response_length_less_than_limit(self, url, resp):
""" Checking response length by Content-Length header """
if 'content-length' in resp.headers and \
int(resp.headers['content-length']) > int(Registry().get('config')['main']['max_size']):
self.errors['maxsize'].append(
"URL {0} has size {1} bytes, but limit in config - {2} bytes".
format(url, resp.headers['content-length'],
Registry().get('config')['main']['max_size']))
return False
return True
def prepare_first_pages(host):
""" Prepare link on first page in MongoDB. Add root url if urls for this host not exists. """
pid = Registry().get('pData')['id']
coll = Registry().get('mongo').spider_urls
coll.drop()
Urls = UrlsModel()
urls = Urls.list_by_host_name_for_spider(pid, host)
if not len(urls):
Registry().get('logger').log("Spider: Root URL was added automaticaly")
Urls.add(
pid, HostsModel().get_id_by_name(pid, host), '/', who_add='spider'
)
urls = Urls.list_by_host_name_for_spider(pid, host)
for url in urls:
url = urlparse(url['url'])
data = {
'hash': md5(str(url.path + url.query)),
'path': url.path,
'query': url.query,
'time': 0,
'code':0,
'checked': 0,
'getted' : 0,
'referer': '',
'size': 0,
'founder': 'spider'
}
coll.insert(data)
coll.create_index([('hash', 1)], unique=True, dropDups=True)
coll.create_index([('checked', 1)])
def check_dafs(self, _type):
""" Simple dafs search """
Http = Registry().get('http')
result = []
_dict = open(Registry().get('wr_path') + '/bases/pre-dafs-{0}.txt'.format(_type), 'r').read().split("\n")
for obj in _dict:
r = Http.get(self.root_url + obj)
if r is not None and not self._response_404(r):
result.append({'url': "/" + obj, 'code': r.status_code, 'time': 0})
return result
def __init__(self, job, host, src, delay, counter):
WSThread.__init__(self, None, None)
self.job = job
self.host = host
self.links_parser = SpiderLinksParser()
#self.not_found_phrase = not_found_phrase
self.http = Registry().get('http')
self.src = src
self.delay = int(delay)
self.counter = counter
self._db = Registry().get('mongo')
def check_positive_limit_stop(self, result, rate=1):
"""
Does we have too many positive results now?
:param result:
:param rate:
:return:
"""
if len(result) >= (
int(Registry().get('config')['main']['positive_limit_stop']) *
rate):
Registry().set('positive_limit_stop', True)
def parse_links(self, content_type, content, link):
"""
Method parse a page content and extract links from it. Can parse xml, html and css.
Other content parse how plain text
:param content_type: type of content (text/xml, application/xml, text/html, text/css or other one)
:param content: content of page
:param link: url of current page
:return:
"""
try:
if not len(content.strip()):
return []
if content_type in ['text/xml', 'application/xml'
] or content[:6] == '<?xml ':
links = self.parse_links_xml(content)
text_links = self.parse_links_text_re(content)
links.extend(text_links)
elif content_type == 'text/css':
links = self.parse_links_css(content)
text_links = self.parse_links_text_re(content)
links.extend(text_links)
elif content_type == 'text/html':
links = self.parse_links_html_re(content)
text_links = self.parse_links_text_re(content)
links.extend(text_links)
else:
links = self.parse_links_html_re(content)
text_links = self.parse_links_text_re(content)
links.extend(text_links)
if Registry().isset('ignore_regexp'):
links = self._clear_ignore(links)
if Registry().isset('only_one'):
links = self._clear_only_one(links)
if Registry().get('config')['spider']['denied_schemes']:
links = self._clear_by_schema(links)
except etree.XMLSyntaxError:
links = self.parse_links_text_re(content)
Registry().get('logger').log(
" Document syntax error {0}, parsing as text".format(
link['path'] + '?' +
link['query'] if len(link['query']) else link['path']))
except etree.DocumentInvalid:
links = self.parse_links_text_re(content)
Registry().get('logger').log(
" Document invalid {0}, parsing as text".format(
link['path'] + '?' +
link['query'] if len(link['query']) else link['path']))
return list(set(links))
def _insert_infos(self, result):
""" Insert found infos in db """
h_id = HostsModel().get_id_by_name(Registry().get('pData')['id'],
self.options['host'].value)
HI = HostsInfoModel()
for k in result:
HI.set_info(
Registry().get('pData')['id'], h_id, k,
json.dumps(result[k])
if result[k] and len(str(result[k])) else '')
def run(self):
""" Run thread """
while not self.done:
self.last_action = int(time.time())
if self.delay:
time.sleep(self.delay)
try:
word = self.queue.get()
self.counter.up()
url = "{0}://{1}{2}".format(self.protocol, self.domain, word)
self.browser.get(url)
if self.recreate_re and self.recreate_re.findall(
self.browser.page_source):
#self.queue.task_done(word)
#self.queue.put(word)
self.browser_close()
self.browser_create()
continue
if not self.not_found_re.findall(self.browser.page_source):
self.result.append(word)
self.logger.item(word, self.browser.page_source, True)
if len(self.result) >= int(Registry().get('config')['main']
['positive_limit_stop']):
Registry().set('positive_limit_stop', True)
#self.queue.task_done(word)
except Queue.Empty:
self.done = True
break
except TimeoutException as e:
self.queue.put(word)
self.browser_close()
self.browser_create()
continue
except BaseException as e:
#self.queue.task_done(word)
if not str(e).count('Timed out waiting for page load'):
self.logger.ex(e)
if str(e).count("Connection refused"):
self.queue.put(word)
self.browser_close()
self.browser_create()
self.up_requests_count()
self.browser_close()
def __init__(self, queue, domain, protocol, method, delay, counter, result):
threading.Thread.__init__(self)
self.queue = queue
self.method = method.lower()
self.domain = domain
self.result = result
self.counter = counter
self.protocol = protocol
self.done = False
self.bad_words = file_to_list(Registry().get('wr_path') + "/bases/bad-words.txt")
self.http = Registry().get('http')
self.delay = int(delay)
def __init__(self, host, user, password, basename):
self._host = host
self._user = user
self._password = password
self._basename = basename
self.connect()
self._restart_by_deadlock_limit = int(
Registry().get('config')['main']['restarts_by_deadlock_limit'])
self._sleep_by_deadlock_restart = int(
Registry().get('config')['main']['sleep_by_deadlock_restart'])
def _get_codes_stat(self):
""" Build dict with http-codes and their counts """
coll = Registry().get('mongo').spider_urls
result = {}
codes = coll.group({'code': True}, '', {}, 'function () {}')
for code in codes:
links = []
code = code['code']
data = coll.find({'code': code}, {'path': 1, 'query': 1})
for link in mongo_result_to_list(data):
links.append(link['path'] + '?' + link['query'] if link['query'] else link['path'])
result[int(code)] = links
return result
def __init__(
self, queue, protocol, host, url, max_params_length, value, method, mask_symbol, not_found_re,
not_found_size, not_found_codes, retest_codes, delay, ignore_words_re,
counter, result):
super(ParamsBruterThread, self).__init__()
self.retested_words = {}
self.queue = queue
self.protocol = protocol.lower()
self.host = host
self.url = url
self.mask_symbol = mask_symbol
self.counter = counter
self.result = result
self.value = value
self.done = False
self.max_params_length = int(max_params_length)
self.ignore_words_re = False if not len(ignore_words_re) else re.compile(ignore_words_re)
self.not_found_re = False if not len(not_found_re) else re.compile(not_found_re)
self.not_found_size = int(not_found_size)
self.method = method.lower()
not_found_codes = not_found_codes.split(',')
not_found_codes.append('404')
self.not_found_codes = list(set(not_found_codes))
self.retest_codes = list(set(retest_codes.split(','))) if len(retest_codes) else []
self.delay = int(delay)
self.retest_delay = int(Registry().get('config')['params_bruter']['retest_delay'])
self.http = copy.deepcopy(Registry().get('http'))
self.logger = Registry().get('logger')
self.retest_limit = int(Registry().get('config')['dafs']['retest_limit'])
def __init__(
self, queue, protocol, host, template, method, mask_symbol, not_found_re,
not_found_size, not_found_codes, retest_codes, delay, ignore_words_re,
counter, result):
threading.Thread.__init__(self)
self.retested_words = {}
self.queue = queue
self.protocol = protocol.lower()
self.host = host
self.template = template
self.mask_symbol = mask_symbol
self.counter = counter
self.result = result
self.done = False
self.ignore_words_re = False if not len(ignore_words_re) else re.compile(ignore_words_re)
self.not_found_re = False if not len(not_found_re) else re.compile(not_found_re)
self.not_found_size = int(not_found_size)
self.method = method
if method == 'head' and (len(not_found_re) or self.not_found_size != -1):
self.method = 'get'
not_found_codes = not_found_codes.split(',')
not_found_codes.append('404')
self.not_found_codes = list(set(not_found_codes))
self.retest_codes = list(set(retest_codes.split(','))) if len(retest_codes) else []
self.delay = int(delay)
self.retest_delay = int(Registry().get('config')['dafs']['retest_delay'])
self.http = copy.deepcopy(Registry().get('http'))
self.logger = Registry().get('logger')
self.retest_limit = int(Registry().get('config')['dafs']['retest_limit'])
def __init__(
self, queue, protocol, host, url, false_phrase, true_phrase, retest_codes, delay,
confstr, first_stop, login, pass_min_len, pass_max_len, pass_found, counter, result
):
threading.Thread.__init__(self)
self.retested_words = {}
self.queue = queue
self.protocol = protocol.lower()
self.host = host
self.url = url
self.false_phrase = false_phrase
self.true_phrase = true_phrase
self.delay = int(delay)
self.confstr = confstr
self.first_stop = first_stop
self.login = login
self.counter = counter
self.result = result
self.retest_codes = list(set(retest_codes.split(','))) if len(retest_codes) else []
self.pass_found = pass_found
self.done = False
self.logger = Registry().get('logger')
self.http = copy.deepcopy(Registry().get('http'))
self.http.every_request_new_session = True
self.pass_min_len = int(pass_min_len)
self.pass_max_len = int(pass_max_len)
self.retest_delay = int(Registry().get('config')['form_bruter']['retest_delay'])
self.retest_limit = int(Registry().get('config')['form_bruter']['retest_limit'])
def __init__(
self, queue, protocol, host, template, method, mask_symbol, not_found_re,
delay, ddos_phrase, ddos_human, recreate_re, ignore_words_re, counter, result
):
super(SDafsThread, self).__init__()
self.queue = queue
self.protocol = protocol.lower()
self.host = host
self.method = method if not (len(not_found_re) and method.lower() == 'head') else 'get'
self.template = template
self.mask_symbol = mask_symbol
self.counter = counter
self.result = result
self.done = False
self.not_found_re = False if not len(not_found_re) else re.compile(not_found_re)
self.recreate_re = False if not len(recreate_re) else re.compile(recreate_re)
self.http = Registry().get('http')
self.delay = int(delay)
self.ddos_phrase = ddos_phrase
self.ddos_human = ddos_human
self.ignore_words_re = False if not len(ignore_words_re) else re.compile(ignore_words_re)
Registry().set('url_for_proxy_check', "{0}://{1}".format(protocol, host))
self.browser_create()
self.logger = Registry().get('logger')
def __init__(
self, queue, protocol, host, url, false_phrase, true_phrase, delay, ddos_phrase, ddos_human, recreate_phrase,
conffile, first_stop, login, #reload_form_page,
pass_found, counter, result
):
super(SFormBruterThread, self).__init__()
self.retested_words = {}
self.queue = queue
self.protocol = protocol.lower()
self.host = host
self.url = url
self.delay = int(delay)
self.ddos_phrase = ddos_phrase
self.ddos_human = ddos_human
self.recreate_phrase = recreate_phrase
self.conffile = conffile
self.false_phrase = false_phrase
self.true_phrase = true_phrase
self.first_stop = first_stop
self.login = login
self.pass_found = pass_found
self.logger = Registry().get('logger')
#self.reload_form_page = int(reload_form_page)
self.browser_create()
self.counter = counter
self.result = result
self.done = False
Registry().set('url_for_proxy_check', "{0}://{1}".format(protocol, host))
def __init__(
self, queue, protocol, host, template, mask_symbol,
false_phrase, retest_codes, delay, ignore_words_re, counter, result):
threading.Thread.__init__(self)
self.retested_words = {}
self.queue = queue
self.protocol = protocol.lower()
self.host = host
self.template = template
self.mask_symbol = mask_symbol
self.counter = counter
self.result = result
self.done = False
self.false_phrase = false_phrase
self.retest_codes = list(set(retest_codes.split(','))) if len(retest_codes) else []
self.delay = int(delay)
self.retest_delay = int(Registry().get('config')['hosts_brute']['retest_delay'])
self.http = copy.deepcopy(Registry().get('http'))
self.logger = Registry().get('logger')
self.method = 'get'
self.ignore_words_re = False if not len(ignore_words_re) else re.compile(ignore_words_re)
self.retest_limit = int(Registry().get('config')['hosts_brute']['retest_limit'])
def __init__(self, queue, domain, url, protocol, method, not_found_re, not_found_codes, delay, counter, result):
threading.Thread.__init__(self)
self.queue = queue
self.method = method if not (len(not_found_re) and method.lower() == 'head') else 'get'
self.domain = domain
self.url = url
self.result = result
self.counter = counter
self.protocol = protocol
self.not_found_re = False if not len(not_found_re) else re.compile(not_found_re)
not_found_codes = not_found_codes.split(',')
not_found_codes.append('404')
self.not_found_codes = list(set(not_found_codes))
self.delay = int(delay)
self.done = False
self.http = Registry().get('http')
self.logger = Registry().get('logger')
def __init__(self, queue, domain, protocol, method, delay, counter, result):
threading.Thread.__init__(self)
self.queue = queue
self.method = method.lower()
self.domain = domain
self.result = result
self.counter = counter
self.protocol = protocol
self.done = False
self.bad_words = file_to_list(Registry().get('wr_path') + "/bases/bad-words.txt")
self.http = Registry().get('http')
self.delay = int(delay)
def _get_extensions(self):
""" Build files extensions list """
result = {}
coll = Registry().get('mongo').spider_urls
links = coll.group({'path': True}, '', {}, 'function () {}')
links = mongo_result_to_list(links)
exts = []
for link in links:
if link['path'].rfind('.') > -1 and len(link['path']) - link['path'].rfind('.') <= 5:
exts.append(link['path'][link['path'].rfind('.'):])
for ext in list(set(exts)):
if ext not in result:
result[ext] = []
links = coll.find({'path': re.compile('\\' + ext + '$')})
links = mongo_result_to_list(links)
for link in links:
result[ext].append(link['path'] + '?' + link['query'] if link['query'] else link['path'])
return result
def __init__(self, queue, domains, template, proto, msymbol, ignore_ip, dns_srv, delay, http_nf_re, ignore_words_re, result, counter):
threading.Thread.__init__(self)
self.queue = queue
self.domains = domains
self.proto = proto
self.dns_srv = dns_srv
self.counter = counter
self.msymbol = msymbol
self.template = template
self.result = result
self.delay = int(delay)
self.done = False
self.logger = Registry().get('logger')
self.ignore_ip = ignore_ip
self.http_nf_re = re.compile(http_nf_re) if len(http_nf_re) else None
self.ignore_words_re = False if not len(ignore_words_re) else re.compile(ignore_words_re)
def setup_class(self):
""" Prepare class for run tests """
CURPATH = os.path.dirname(__file__) + "/"
config = configparser.ConfigParser()
config.read(CURPATH + 'config.ini')
Registry().set('config', config)
Registry().set('logger', LoggerMock())
db = Database(
config['main']['mysql_host'],
config['main']['mysql_user'],
config['main']['mysql_pass'],
config['main']['mysql_dbname'],
)
Registry().set('db', db)
self.db = Registry().get('db') # type: Database
def __init__(
self, queue, domain, protocol, method, not_found_re,
delay, ddos_phrase, ddos_human, recreate_re, counter, result
):
super(SBackupsFinderThread, self).__init__()
self.queue = queue
self.method = method if not (len(not_found_re) and method.lower() == 'head') else 'get'
self.domain = domain
self.result = result
self.counter = counter
self.protocol = protocol
self.not_found_re = False if not len(not_found_re) else re.compile(not_found_re)
self.done = False
self.http = Registry().get('http')
self.delay = int(delay)
self.ddos_phrase = ddos_phrase
self.ddos_human = ddos_human
self.recreate_re = False if not len(recreate_re) else re.compile(recreate_re)
self.logger = Registry().get('logger')
Registry().set('url_for_proxy_check', "{0}://{1}".format(protocol, domain))
self.browser_create()
class SBackupsFinderThread(SeleniumThread):
""" Thread class for BF module (selenium) """
queue = None
method = None
url = None
counter = None
last_action = 0
def __init__(
self, queue, domain, protocol, method, not_found_re,
delay, ddos_phrase, ddos_human, recreate_re, counter, result
):
super(SBackupsFinderThread, self).__init__()
self.queue = queue
self.method = method if not (len(not_found_re) and method.lower() == 'head') else 'get'
self.domain = domain
self.result = result
self.counter = counter
self.protocol = protocol
self.not_found_re = False if not len(not_found_re) else re.compile(not_found_re)
self.done = False
self.http = Registry().get('http')
self.delay = int(delay)
self.ddos_phrase = ddos_phrase
self.ddos_human = ddos_human
self.recreate_re = False if not len(recreate_re) else re.compile(recreate_re)
self.logger = Registry().get('logger')
Registry().set('url_for_proxy_check', "{0}://{1}".format(protocol, domain))
self.browser_create()
def run(self):
""" Run thread """
while not self.done:
self.last_action = int(time.time())
if self.delay:
time.sleep(self.delay)
try:
word = self.queue.get()
self.counter.up()
url = "{0}://{1}{2}".format(self.protocol, self.domain, word)
self.browser.get(url)
if self.recreate_re and self.recreate_re.findall(self.browser.page_source):
#self.queue.task_done(word)
#self.queue.put(word)
self.browser_close()
self.browser_create()
continue
positive_item = False
if not self.not_found_re.findall(self.browser.page_source):
self.result.append(word)
positive_item = True
self.logger.item(word, self.browser.page_source, True, positive=positive_item)
if len(self.result) >= int(Registry().get('config')['main']['positive_limit_stop']):
Registry().set('positive_limit_stop', True)
#self.queue.task_done(word)
except Queue.Empty:
self.done = True
break
except TimeoutException as e:
self.queue.put(word)
self.browser_close()
self.browser_create()
continue
except BaseException as e:
#self.queue.task_done(word)
if not str(e).count('Timed out waiting for page load'):
self.logger.ex(e)
if str(e).count("Connection refused"):
self.queue.put(word)
self.browser_close()
self.browser_create()
self.up_requests_count()
self.browser_close()
class BackupsFinderThread(HttpThread):
""" Thread class for BF module """
queue = None
method = None
url = None
counter = None
last_action = 0
def __init__(self, queue, domain, protocol, method, not_found_re, not_found_codes,
not_found_size, delay, counter, result):
super(BackupsFinderThread, self).__init__()
self.queue = queue
self.domain = domain
self.result = result
self.counter = counter
self.protocol = protocol
self.not_found_re = False if not len(not_found_re) else re.compile(not_found_re)
self.not_found_size = int(not_found_size)
self.method = method.lower()
self.method = method if \
not ((len(not_found_re) or self.not_found_size != -1) and method.lower() == 'head') else \
'get'
not_found_codes = not_found_codes.split(',')
not_found_codes.append('404')
self.not_found_codes = list(set(not_found_codes))
self.delay = int(delay)
self.done = False
self.http = Registry().get('http')
self.logger = Registry().get('logger')
def run(self):
""" Run thread """
req_func = getattr(self.http, self.method)
need_retest = False
while not self.done:
self.last_action = int(time.time())
if self.delay:
time.sleep(self.delay)
try:
if not need_retest:
word = self.queue.get()
self.counter.up()
url = "{0}://{1}{2}".format(self.protocol, self.domain, word)
try:
resp = req_func(url)
except ConnectionError:
need_retest = True
self.http.change_proxy()
continue
positive_item = False
if self.is_response_right(resp):
self.result.append(word)
positive_item = True
self.log_item(word, resp, positive_item)
self.check_positive_limit_stop(self.result)
need_retest = False
except Queue.Empty:
self.done = True
break
except ChunkedEncodingError as e:
self.logger.ex(e)
except BaseException as e:
self.logger.ex(e)
class HostsBruteThread(HttpThread):
""" Thread class for HostsBrute modules """
queue = None
method = None
url = None
mask_symbol = None
counter = None
retested_words = None
last_action = 0
def __init__(
self, queue, protocol, host, template, mask_symbol,
false_phrase, retest_codes, delay, ignore_words_re, counter, result):
threading.Thread.__init__(self)
self.retested_words = {}
self.queue = queue
self.protocol = protocol.lower()
self.host = host
self.template = template
self.mask_symbol = mask_symbol
self.counter = counter
self.result = result
self.done = False
self.false_phrase = false_phrase
self.retest_codes = list(set(retest_codes.split(','))) if len(retest_codes) else []
self.delay = int(delay)
self.retest_delay = int(Registry().get('config')['hosts_brute']['retest_delay'])
self.http = copy.deepcopy(Registry().get('http'))
self.logger = Registry().get('logger')
self.method = 'get'
self.ignore_words_re = False if not len(ignore_words_re) else re.compile(ignore_words_re)
self.retest_limit = int(Registry().get('config')['hosts_brute']['retest_limit'])
def run(self):
""" Run thread """
req_func = getattr(self.http, self.method)
need_retest = False
word = False
while not self.done:
self.last_action = int(time.time())
if self.delay:
time.sleep(self.delay)
try:
if not need_retest:
word = self.queue.get()
self.counter.up()
if not len(word.strip()) or (self.ignore_words_re and self.ignore_words_re.findall(word)):
continue
try:
hostname = self.template.replace(self.mask_symbol, word)
except UnicodeDecodeError:
self.logger.log(
"URL build error (UnicodeDecodeError) with word '{0}', skip it".format(pprint.pformat(word)),
_print=False
)
continue
try:
resp = req_func(self.protocol + "://" + self.host, headers={'host': hostname})
except ConnectionError:
need_retest = True
self.http.change_proxy()
continue
if self.is_retest_need(word, resp):
time.sleep(self.retest_delay)
need_retest = True
continue
search_scope = ""
for header in resp.headers:
search_scope += "{0}: {1}\r\n".format(header.title(), resp.headers[header])
search_scope += '\r\n\r\n' + resp.text
positive_item = False
if resp is not None and not search_scope.count(self.false_phrase):
self.result.append(hostname)
positive_item = True
self.log_item(word, resp, positive_item)
self.check_positive_limit_stop(self.result)
need_retest = False
except Queue.Empty:
self.done = True
break
except ChunkedEncodingError as e:
self.logger.ex(e)
except BaseException as e:
try:
if str(e).count('Cannot connect to proxy'):
need_retest = True
else:
self.logger.ex(e)
except UnicodeDecodeError:
pass
except UnboundLocalError:
self.logger.ex(e)
finally:
pass
def setup_class(self):
config = configparser.ConfigParser()
config.read('config.ini')
try:
db = mysql.connector.connect(
host=config['db']['host'],
user=config['db']['user'],
password=config['db']['pass'],
database=config['db']['database']
)
db.autocommit = True
except mysql.connector.errors.ProgrammingError as e:
print " ERROR: Can`t connect to MySQL server! ({0})".format(str(e))
exit(0)
try:
mc = MongoClient(host=config['mongo']['host'], port=int(config['mongo']['port']))
mongo_collection = getattr(mc, config['mongo']['collection'])
except pymongo.errors.ConnectionFailure as e:
print " ERROR: Can`t connect to MongoDB server! ({0})".format(str(e))
exit(0)
R = Registry()
R.set('config', config)
R.set('db', db)
R.set('mongo', mongo_collection)
R.set('wr_path', wrpath)
R.set('data_path', wrpath + '/data/')
R.set('ndb',
Database(config['db']['host'], config['db']['user'], config['db']['pass'], config['db']['database']))
R.set('proxies', Proxies())
self.db = R.get('ndb')
class CommonTest(object):
""" Common class tests """
db = None
def setup_class(self):
open(wrpath + '/proxies.list', 'w').close()
config = configparser.ConfigParser()
config.read(os.getcwd() + '/' + 'config.ini')
try:
db = mysql.connector.connect(
host=config['db']['host'],
user=config['db']['user'],
password=config['db']['pass'],
database=config['db']['database']
)
db.autocommit = True
except mysql.connector.errors.ProgrammingError as e:
print " ERROR: Can`t connect to MySQL server! ({0})".format(str(e))
exit(0)
try:
mc = MongoClient(host=config['mongo']['host'], port=int(config['mongo']['port']))
mongo_collection = getattr(mc, config['mongo']['collection'])
except pymongo.errors.ConnectionFailure as e:
print " ERROR: Can`t connect to MongoDB server! ({0})".format(str(e))
exit(0)
R = Registry()
R.set('config', config)
R.set('db', db)
R.set('mongo', mongo_collection)
R.set('wr_path', wrpath)
R.set('data_path', wrpath + '/data/')
R.set('ndb',
Database(config['db']['host'], config['db']['user'], config['db']['pass'], config['db']['database']))
self.db = Registry().get('ndb')
def setup(self):
self.clear_db()
def clear_db(self):
""" Clearing db """
self.db.q("TRUNCATE TABLE `hosts`")
self.db.q("TRUNCATE TABLE `ips`")
self.db.q("TRUNCATE TABLE `projects`")
self.db.q("TRUNCATE TABLE `urls`")
self.db.q("TRUNCATE TABLE `requests`")
self.db.q("TRUNCATE TABLE `hosts_info`")
self.db.q("TRUNCATE TABLE `urls_base`")
self.db.q("TRUNCATE TABLE `urls_base_params`")
self.db.q("TRUNCATE TABLE `cms`")
self.db.q("TRUNCATE TABLE `cms_paths`")
self.db.q("TRUNCATE TABLE `cms_paths_hashes`")
def output_errors(self, out):
"""
Check output for errors
:param out: output
:return:
"""
assert out.find("Traceback") == -1
assert "killed by time" not in out
def _replace_config(self, name):
"""
Change work config
:param name: config name (*.ini)
:return:
"""
shutil.copyfile("{0}/config.ini".format(wrpath), "{0}/config.ini.bak".format(wrpath))
shutil.copyfile("{0}/configs/{1}.ini".format(testpath, name), "{0}/config.ini".format(wrpath))
def _restore_config(self):
""" Restore old original config """
shutil.move("{0}/config.ini.bak".format(wrpath), "{0}/config.ini".format(wrpath))
def _run(self, config_name, run_params):
"""
Run WS
:param config_name: name of config for test
:param run_params: params for run WS-process
:return: output of WS
"""
self._replace_config(config_name)
os.chdir(wrpath)
out = subprocess.check_output(run_params)
self._restore_config()
self.output_errors(out)
return out
class FuzzerUrlsThread(threading.Thread):
""" Thread class for FuzzerUrls module """
queue = None
method = None
url = None
counter = None
last_action = 0
def __init__(self, queue, domain, protocol, method, delay, counter, result):
threading.Thread.__init__(self)
self.queue = queue
self.method = method.lower()
self.domain = domain
self.result = result
self.counter = counter
self.protocol = protocol
self.done = False
self.bad_words = file_to_list(Registry().get('wr_path') + "/bases/bad-words.txt")
self.http = Registry().get('http')
self.delay = int(delay)
def run(self):
""" Run thread """
req_func = getattr(self.http, self.method)
need_retest = False
while True:
self.last_action = int(time.time())
if self.delay:
time.sleep(self.delay)
try:
if not need_retest:
url = self.queue.get()
try:
resp = req_func(
"{0}://{1}{2}".format(self.protocol, self.domain, url)
)
except ConnectionError:
need_retest = True
self.http.change_proxy()
continue
if resp is None:
continue
if resp.status_code > 499 and resp.status_code < 600:
self.result.append({"url": url, "words": ["{0} Status code".format(resp.status_code)]})
continue
found_words = []
for bad_word in self.bad_words:
if resp.content.count(bad_word):
found_words.append(bad_word)
if len(found_words):
self.result.append({"url": url, "words": found_words})
self.counter.up()
need_retest = False
except Queue.Empty:
self.done = True
break
except BaseException as e:
print url + " " + str(e)
def __init__(self):
config = configparser.ConfigParser()
config.read(os.getcwd() + '/' + 'config.ini')
try:
db = mysql.connector.connect(
host=config['db']['host'],
user=config['db']['user'],
password=config['db']['pass'],
database=config['db']['database']
)
db.autocommit = True
except mysql.connector.errors.ProgrammingError as e:
print " ERROR: Can`t connect to MySQL server! ({0})".format(str(e))
exit(0)
try:
mc = MongoClient(host=config['mongo']['host'], port=int(config['mongo']['port']))
mongo_collection = getattr(mc, config['mongo']['collection'])
except pymongo.errors.ConnectionFailure as e:
print " ERROR: Can`t connect to MongoDB server! ({0})".format(str(e))
exit(0)
R = Registry()
R.set('config', config)
R.set('db', db)
R.set('mongo', mongo_collection)
R.set('wr_path', os.getcwd())
R.set('data_path', os.getcwd() + '/data/')
R.set('http', Http())
R.set('proxies', Proxies())
R.set(
'ndb',
Database(config['db']['host'], config['db']['user'], config['db']['pass'], config['db']['database'])
)
R.set(
'fuzzer_evil_value',
file_get_contents(Registry().get('wr_path') + "/bases/fuzzer-evil-value.txt").strip()
)
R.set('proxy_many_died', False)
R.set('positive_limit_stop', False)
if " ".join(sys.argv).count('selenium') and int(config['selenium']['virtual_display']):
display = Display(visible=0, size=(800, 600))
display.start()
R.set('display', display)
class MongoJob(WSJob):
""" Common class for jobs works with MongoDB """
unique = True
collection = None
select_limit = 50
skip_blank_rows = True
counter = 0
collection_name = None
def __init__(self, maxsize=0):
WSJob.__init__(self, maxsize)
self.collection = Registry().get('mongo')[self.collection_name]
def build_row(self, _str):
""" Common build row method for MongoDB """
return {
"name": _str.strip(),
"checked": 0,
"getted": 0
}
def qsize(self):
""" Size of queue """
return self.collection.find({"checked": 0}).count()
def set_unique(self, unique=True):
""" Enable remove dups in queue """
self.unique = unique
def set_skip_blank_rows(self, value=True):
""" If True - we will skip blank rows then fill queue from dict or file """
self.skip_blank_rows = value
def task_done(self, name):
""" Mark current row as done """
self.counter += 1
self.collection.update({'name': str(unicode(name)), "getted": 1}, {"$set": {"checked": 1}})
WSJob.task_done(self)
def get(self, block=False, timeout=None):
""" Get next item from queue """
if self.empty() or self.qsize() < 50:
self.load_data()
if self.empty():
raise Queue.Empty
return WSJob.get(self, block, timeout)
def load_data(self):
""" Load data into queue from MongoDB """
data = self.collection.find(
{"checked": 0, "getted": 0},
limit=int(Registry().get('config')['main']['mongo_data_load_per_once'])
)
for row in data:
self.put(row['name'])
self.collection.update({"name": row['name']}, {"$set": {"getted": 1}})
return True
def load_dict(self, dict_for_load, drop=True):
""" Fill collection from dict """
if drop:
self.collection.drop()
counter = 0
last = "START OF FILE"
for line in dict_for_load:
try:
line = line.strip()
unicode(line)
self.collection.insert(self.build_row(line))
except UnicodeDecodeError:
_str = " UNICODE ERROR: In file '{0}' skip word '{1}', after word '{2}' !".format(file, line, last)
if Registry().isset('logger'):
Registry().get('logger').log(_str)
else:
print _str
continue
counter += 1
last = line
self.load_data()
return counter
def load_dom(self, dom):
""" Fill queue from DictOfMask """
self.collection.drop()
while True:
word = dom.get()
if word is None:
break
self.collection.insert(self.build_row(word))
self.collection.create_index('name', drop_dups=True, unique=self.unique)
self.load_data()
return self.collection.count()
def load_file(self, _file):
""" Fill queue from text file """
self.collection.drop()
fh = open(_file)
last = "START OF FILE"
while True:
line = fh.readline()
if not line:
break
if not line.strip() and self.skip_blank_rows:
continue
try:
line = line.strip()
unicode(line)
self.collection.insert(self.build_row(line))
except UnicodeDecodeError:
_str = " UNICODE ERROR: In file '{0}' skip word '{1}', after word '{2}' !".format(_file, line, last)
if Registry().isset('logger'):
Registry().get('logger').log(_str)
else:
print _str
continue
last = line
fh.close()
self.collection.create_index('name', drop_dups=True, unique=self.unique)
self.load_data()
return self.collection.count()
# 2 метода ниже взяты с
# http://stackoverflow.com/questions/1581895/how-check-if-a-task-is-already-in-python-queue
# Рецепт для уникальных задачь в очереди
def _init(self, maxsize):
WSJob._init(self, maxsize)
if self.unique:
self.all_items = set()
def _put(self, item):
if self.unique:
if item not in self.all_items:
WSJob._put(self, item)
self.all_items.add(item)
else:
_str = "WARNING: try to add not unique item `{0}`".format(item)
if Registry().isset('logger'):
#Registry().get('logger').log(_str)
pass
else:
#print _str
pass
else:
WSJob._put(self, item)
class SFormBruterThread(SeleniumThread):
""" Thread class for FormBruter module (selenium) """
queue = None
method = None
url = None
mask_symbol = None
counter = None
retested_words = None
logger = None
last_action = 0
first_page_load = False
def __init__(
self, queue, protocol, host, url, false_phrase, true_phrase, delay, ddos_phrase, ddos_human, recreate_phrase,
conffile, first_stop, login, #reload_form_page,
pass_found, counter, result
):
super(SFormBruterThread, self).__init__()
self.retested_words = {}
self.queue = queue
self.protocol = protocol.lower()
self.host = host
self.url = url
self.delay = int(delay)
self.ddos_phrase = ddos_phrase
self.ddos_human = ddos_human
self.recreate_phrase = recreate_phrase
self.conffile = conffile
self.false_phrase = false_phrase
self.true_phrase = true_phrase
self.first_stop = first_stop
self.login = login
self.pass_found = pass_found
self.logger = Registry().get('logger')
#self.reload_form_page = int(reload_form_page)
self.browser_create()
self.counter = counter
self.result = result
self.done = False
Registry().set('url_for_proxy_check', "{0}://{1}".format(protocol, host))
def parse_brute_config(self, path):
""" Parse conf file to dict """
to_return = {}
have_user = False
have_pass = False
have_submit = False
fh = open(path)
for line in fh.readlines():
if not len(line.strip()):
continue
point, selector = line.strip().split(" ")
if point == "^USER^":
have_user = True
if point == "^PASS^":
have_pass = True
if point == "^SUBMIT^":
have_submit = True
to_return[point] = selector
return to_return
def run(self):
""" Run thread """
need_retest = False
word = False
brute_conf = self.parse_brute_config(self.conffile)
while not self.pass_found and not self.done:
try:
self.last_action = int(time.time())
if self.pass_found:
self.done = True
break
if self.delay:
time.sleep(self.delay)
if not need_retest:
word = self.queue.get()
self.counter.up()
#if self.reload_form_page or \
# (not self.browser.element_exists(By.CSS_SELECTOR, brute_conf['^USER^']) or
# not self.browser.element_exists(By.CSS_SELECTOR, brute_conf['^PASS^'])) :
#self.browser.get(self.protocol + "://" + self.host + self.url)
self.browser.get(self.protocol + "://" + self.host + self.url)
if len(self.recreate_phrase) and self.browser.page_source.lower().count(self.recreate_phrase.lower()):
need_retest = True
self.browser_close()
self.browser_create()
continue
self.browser.find_element(By.CSS_SELECTOR, brute_conf['^USER^']).clear()
self.browser.find_element(By.CSS_SELECTOR, brute_conf['^USER^']).send_keys(self.login)
self.browser.find_element(By.CSS_SELECTOR, brute_conf['^PASS^']).clear()
self.browser.find_element(By.CSS_SELECTOR, brute_conf['^PASS^']).send_keys(word)
self.browser.find_element(By.CSS_SELECTOR, brute_conf['^SUBMIT^']).click()
time.sleep(1)
self.logger.item(word, self.browser.page_source, True)
if ( (len(self.false_phrase) and not self.browser.page_source.count(self.false_phrase)) or
(len(self.true_phrase) and self.browser.page_source.count(self.true_phrase)) ):
self.result.append({'word': word, 'content': self.browser.page_source})
#self.logger.log("Result: {0}".format(word))
if len(self.result) >= int(Registry().get('config')['main']['positive_limit_stop']):
Registry().set('positive_limit_stop', True)
if int(self.first_stop):
self.done = True
self.pass_found = True
break
else:
# Иначе старая сессия останется и будет куча false-positive
self.browser_close()
self.browser_create()
need_retest = False
except Queue.Empty:
self.done = True
break
except TimeoutException as e:
need_retest = True
self.browser_close()
self.browser_create()
continue
except UnicodeDecodeError as e:
self.logger.ex(e)
need_retest = False
except BaseException as e:
try:
need_retest = True
if len(e.args) and e.args[0] == 111:
self.browser_close()
self.browser_create()
elif not str(e).count('Timed out waiting for page load'):
self.logger.ex(e)
except UnicodeDecodeError:
need_retest = False
self.up_requests_count()
self.browser_close()
def __init__(self, maxsize=0):
WSJob.__init__(self, maxsize)
self.collection = Registry().get('mongo')[self.collection_name]
