def _insert_hosts(self, hosts):
""" Insert found hosts in db """
pid = Registry().get('pData')['id']
H = HostsModel()
I = IpsModel()
added = 0
for host in hosts:
ip_id = I.get_id_or_add(pid, host['ip'])
if H.add(pid, ip_id, host['name'], founder='pre'):
added += 1
return added
def _insert_hosts(self, hosts):
""" Add found hosts in db """
pid = Registry().get('pData')['id']
Hosts = HostsModel()
Ips = IpsModel()
added = 0
for host in hosts:
ip_id = Ips.get_id_or_add(pid, host['ip'])
if Hosts.add(pid, ip_id, host['name'], founder='dnsbrute'):
added += 1
return added
def _insert_hosts(self, hosts):
""" Insert found hosts in db """
pid = Registry().get('pData')['id']
H = HostsModel()
I = IpsModel()
added = 0
for host in hosts:
ip_id = I.get_id_or_add(pid, host['ip'])
if H.add(pid, ip_id, host['name'], founder='pre'):
added += 1
return added
def _insert_hosts(self, hosts):
""" Add found hosts in db """
pid = Registry().get('pData')['id']
Hosts = HostsModel()
Ips = IpsModel()
added = 0
for host in hosts:
ip_id = Ips.get_id_or_add(pid, host['ip'])
if Hosts.add(pid, ip_id, host['name'], founder='dnsbrute'):
added += 1
return added
def exists(self, project_id, host, url):
""" Is url exists? """
host_id = HostsModel().get_id_by_name(project_id, host)
return self._db.fetch_one(
"SELECT 1 FROM urls WHERE project_id = {0} AND host_id={1} AND hash = '{2}'"
.format(project_id, host_id, md5(url)))
def update_url_field(self, project_id, host, url, field, value):
""" Update custom field of current url """
host_id = HostsModel().get_id_by_name(project_id, host)
return self._db.update(
"urls", {field: value},
"hash = '{0}' AND project_id = {1} AND host_id = {2}".format(
md5(url), project_id, host_id))
def add_action(self):
""" Action add of module """
pid = Registry().get('pData')['id']
self.validate_main()
if self.model.exists(pid, self.options['host'].value,
self.options['url'].value):
raise WSException(
"URL '{0}' already exists in this project in host '{1}'!".
format(self.options['url'].value, self.options['host'].value))
host_id = HostsModel().get_id_by_name(pid, self.options['host'].value)
if (self.options['url'].value[-1] == '/' and
self.model.exists(pid, self.options['host'].value, self.options['url'].value[:-1])) or\
(self.options['url'].value[-1] != '/' and
self.model.exists(pid, self.options['host'].value, self.options['url'].value + "/")):
if raw_input(
'Url {0} have analogue in database (with or without end slash). '
'Are you realy want to add it (y/n)?'.format(
self.options['url'].value)).lower()[0] != 'y':
print "Url {0} was not added!".format(
self.options['url'].value)
return
self.model.add(Registry().get('pData')['id'], host_id,
self.options['url'].value)
print " URL '{0}' successfully added to host '{1}'".\
format(self.options['url'].value, self.options['host'].value)
def list_by_host_name_for_spider(self, project_id, host):
""" Get urls list by host name and project_id, but in special format for spider """
host_id = HostsModel().get_id_by_name(project_id, host)
return self._db.fetch_all(
"SELECT url, response_code as code, response_time as time, when_add, who_add, descr FROM urls "
"WHERE project_id = {0} AND host_id = {1} AND !spidered "
"ORDER BY url".format(project_id, host_id))
def links_in_spider_base(pid, host):
""" Put found links in MySQL """
links_per_time_limit = 50
c = WSCounter(
1, 60,
int(Registry().get('mongo').spider_urls.count() /
links_per_time_limit))
Urls = UrlsModel()
host_id = HostsModel().get_id_by_name(pid, host)
urls_add = []
skip = 0
while True:
links = mongo_result_to_list(
Registry().get('mongo').spider_urls.find().skip(skip).limit(
links_per_time_limit))
for link in links:
url = link['path'] + '?' + link['query'] if len(
link['query']) else link['path']
urls_add.append({
'url': url,
'referer': link['referer'],
'response_code': link['code'],
'response_time': link['time'],
'size': link['size'],
'who_add': 'spider',
'spidered': link['checked']
})
Urls.add_mass(pid, host_id, urls_add)
urls_add = []
to_update = {'spidered': [], 'code': [], 'time': [], 'size': []}
for link in links:
url = link['path'] + '?' + link['query'] if len(
link['query']) else link['path']
if link['checked']:
to_update['spidered'].append({'url': url, 'value': 1})
to_update['code'].append({'url': url, 'value': link['code']})
to_update['time'].append({'url': url, 'value': link['time']})
to_update['size'].append({'url': url, 'value': link['size']})
Urls.update_url_field_mass(pid, host, 'spidered',
to_update['spidered'])
Urls.update_url_field_mass(pid, host, 'response_code',
to_update['code'])
Urls.update_url_field_mass(pid, host, 'response_time',
to_update['time'])
Urls.update_url_field_mass(pid, host, 'size', to_update['size'])
skip += len(links)
c.up()
if len(links) < links_per_time_limit:
break
def update_url_field_mass(self, project_id, host, field, data):
""" Mass update custom field of many urls """
host_id = HostsModel().get_id_by_name(project_id, host)
update = {}
for row in data:
case = "host_id = '{0}' AND `hash` = '{1}' ".format(
host_id, md5(row['url']))
update[case] = row['value']
def validate_main(self):
""" Check users params """
if not HostsModel().exists(Registry().get('pData')['id'],
self.options['host'].value):
raise WSException("Host '{0}' not found in this project!".format(
self.options['host'].value))
if 'url' in self.options and self.options['url'].value[0] != '/':
raise WSException("URL must start from the root ('/') !")
def _insert_infos(self, result):
""" Insert found infos in db """
h_id = HostsModel().get_id_by_name(Registry().get('pData')['id'],
self.options['host'].value)
HI = HostsInfoModel()
for k in result:
HI.set_info(
Registry().get('pData')['id'], h_id, k,
json.dumps(result[k])
if result[k] and len(str(result[k])) else '')
def list_by_host_name(self, project_id, host, like=""):
""" Get urls list by host name and project_id """
host_id = HostsModel().get_id_by_name(project_id, host)
like_expr = "" \
if not len(like.strip()) \
else " AND url LIKE '%{0}%' ".format(self._db.escape(like.strip()))
return self._db.fetch_all(
"SELECT url, response_code as code, response_time as time, when_add, who_add, descr, size FROM urls "
"WHERE project_id = {0} AND host_id = {1} ".format(
project_id, host_id) + like_expr + "ORDER BY url")
def _insert_urls(self, urls):
""" Insert found urls in DB """
pid = Registry().get('pData')['id']
host_id = HostsModel().get_id_by_name(pid, self.options['host'].value)
U = UrlsModel()
added = 0
for url in urls:
if isinstance(url, str) or isinstance(url, unicode):
if U.add(pid, host_id, url, '', 0, 0, 'pre'):
added += 1
else:
if U.add(pid, host_id, url['url'], '', url['code'],
url['time'], 'pre'):
added += 1
return added
def links_in_urls_base(pid, host):
""" Put links in url_base table (MySQL) for site tree build """
links_per_time_limit = 50
c = WSCounter(
1, 60,
Registry().get('mongo').spider_urls.count() / links_per_time_limit)
UrlsBase = UrlsBaseModel()
host_id = HostsModel().get_id_by_name(pid, host)
skip = 0
while True:
links = mongo_result_to_list(
Registry().get('mongo').spider_urls.find().skip(skip).limit(
links_per_time_limit))
for link in links:
url = link['path'] + '?' + link['query'] if len(
link['query']) else link['path']
UrlsBase.add_url(host_id, url)
skip += len(links)
c.up()
if len(links) < links_per_time_limit:
break
def _insert_urls(self, urls):
""" Add found urls in db """
UrlsBase = UrlsBaseModel()
pid = Registry().get('pData')['id']
host_id = HostsModel().get_id_by_name(pid, self.options['host'].value)
Urls = UrlsModel()
added = 0
for url in urls:
if Urls.add(pid, host_id, url['url'], '', url['code'], url['time'],
'dafs'):
added += 1
paths = urlparse(url['url']).path.split("/")
while len(paths) != 1:
del paths[-1]
if Urls.add(pid, host_id, "/".join(paths) + "/", '', 0, 0,
'dafs'):
added += 1
UrlsBase.add_url(host_id, url['url'])
return added
def prepare_first_pages(host):
""" Prepare link on first page in MongoDB. Add root url if urls for this host not exists. """
pid = Registry().get('pData')['id']
coll = Registry().get('mongo').spider_urls
coll.drop()
Urls = UrlsModel()
urls = Urls.list_by_host_name_for_spider(pid, host)
if not len(urls):
Registry().get('logger').log(
"Spider: Root URL was added automaticaly")
Urls.add(pid,
HostsModel().get_id_by_name(pid, host),
'/',
who_add='spider')
urls = Urls.list_by_host_name_for_spider(pid, host)
for url in urls:
url = urlparse(url['url'])
data = {
'hash': md5(str(url.path + url.query)),
'path': url.path,
'query': url.query,
'time': 0,
'code': 0,
'checked': 0,
'getted': 0,
'referer': '',
'size': 0,
'founder': 'spider'
}
coll.insert(data)
coll.create_index([('hash', 1)], unique=True, dropDups=True)
coll.create_index([('checked', 1)])
def __init__(self, kernel):
WSModule.__init__(self, kernel)
self.model = HostsModel()
class Hosts(WSModule):
""" Class of Hosts module """
model = None
log_path = '/dev/null'
options = {}
options_sets = {
"list": {
"ip": WSOption("ip", "IP for host list", "", False, ['--ip']),
},
"delete": {
"host": WSOption("host", "Host for delete", "", True, ['--host']),
},
"add": {
"host": WSOption("host", "Host for add", "", True, ['--host']),
"ip": WSOption("ip", "Custom IP for this host", "", False, ['--ip']),
"descr": WSOption("descr", "Description of host", "", False, ['--descr'])
}
}
def __init__(self, kernel):
WSModule.__init__(self, kernel)
self.model = HostsModel()
def validate_main(self):
""" Check users params """
if not validate_host(self.options['host'].value):
raise WSException("'{0}' is not valid host name!".format(self.options['host'].value))
if 'ip' in self.options and self.options['ip'].value:
if not validate_ip(self.options['ip'].value):
raise WSException("IP '{0}' is not valid ip-address!".format(self.options['ip'].value))
def run(self, action):
""" Method of run the module """
WSModule.run(self, action)
self.done = True
def add_action(self):
""" Action add of module """
self.validate_main()
pData = Registry().get('pData')
name = self.options['host'].value
descr = self.options['descr'].value
if self.options['ip'].value:
ip = self.options['ip'].value
else:
try:
ip = socket.gethostbyname(name)
print " IP for host '{0}' is '{1}'".format(name, ip)
except socket.gaierror:
raise WSException("Can`t lookup hostname '{0}'. Check it or set ip-address in --ip param!".format(name))
IPs = IpsModel()
ip_exists = IPs.exists(pData['id'], ip)
ip_id = IPs.get_id_or_add(pData['id'], ip)
if not ip_exists:
print " IP '{0}' was automatically added to project '{1}'".format(ip, pData['name'])
if self.model.exists(pData['id'], name):
raise WSException("Host '{0}' already exists in project '{1}'!".format(name, pData['name']))
self.model.add(pData['id'], ip_id, name, descr)
print " Host '{0}' successfully added to project '{1}' with IP '{2}' ! ".format(
name, pData['name'], ip
)
def list_action(self):
""" Action list of module """
if self.options['ip'].value:
print "{0:=^51}".format("")
print "|{0: ^49}|".format("Hosts for IP '{0}'".format(self.options['ip'].value))
print "{0:=^51}".format("")
print "| {0: ^23}| {1: ^23}|".format('Title', 'Description')
print "{0:=^51}".format("")
for host in self.model.list(Registry().get('pData')['id'], self.options['ip'].value):
print "| {0: <23}| {1: <23}|".format(host['name'], host['descr'])
print "{0:=^51}".format("")
else:
print "{0:=^76}".format("")
print "|{0: ^74}|".format("All host for project '{0}'".format(Registry().get('pData')['name']))
print "{0:=^76}".format("")
print "| {0: ^23}| {1: ^23}| {2: ^23}|".format('Title', 'Description', 'IP')
print "{0:=^76}".format("")
for host in self.model.list_without_ip(Registry().get('pData')['id']):
print "| {0: <23}| {1: <23}| {2: <23}|".format(host['name'], host['descr'], host['ip'])
print "{0:=^76}".format("")
def delete_action(self):
""" Delete action of module """
self.validate_main()
name = self.options['host'].value
if not self.model.exists(Registry().get('pData')['id'], name):
raise WSException("Host '{0}' not exists in this project!".format(name))
answer = raw_input("You really want to delete host '{0}' [y/n]? ".format(name))
if answer.lower() == 'y':
self.model.delete(Registry().get('pData')['id'], name)
print "Host '{0}' successfully deleted.".format(name)
else:
print "Host '{0}' not deleted.".format(name)
def brute_action(self):
""" Action brute of module """
self.enable_logger()
self.validate_main()
self.pre_start_inf()
q = DnsBruteJob()
loaded = self.load_objects(q)
self.logger.log(
"Loaded {0} words ({1}-{2}) from all {3}.".format(
(loaded['end'] - loaded['start']), loaded['start'], loaded['end'], loaded['all'])
if (int(self.options['parts'].value) and int(self.options['part'].value)) else
"Loaded {0} words from source.".format(loaded['all'])
)
counter = WSCounter(5, 300, loaded['all'] if not loaded['end'] else loaded['end']-loaded['start'])
result = []
w_thrds = []
DnsRoller = Roller()
DnsRoller.load_file(Registry().get('wr_path') + '/bases/dns-servers.txt')
for _ in range(int(self.options['threads'].value)):
we_need_server = True
while we_need_server:
we_need_server = False
try:
next_server = DnsRoller.get()
#print "Next DNS " + next_server
if self.options['protocol'].value == 'auto':
try:
dns.query.tcp(dns.message.make_query('test.com', 'A'), next_server, timeout=5)
protocol = 'tcp'
except socket.error:
try:
dns.query.udp(dns.message.make_query('test.com', 'A'), next_server, timeout=5)
protocol = 'udp'
except socket.error:
#raise Exception('Can`t detect DNS-server protocol. Check addr.')
we_need_server = True
#print 'DNS protolol detected automaticaly: ' + protocol
else:
protocol = self.options['protocol'].value
except dns.exception.Timeout:
self.logger.log("Check server {0}. Don`t work.".format(next_server))
we_need_server = True
hosts = []
if self.options['host'].value == 'all':
hosts_model = HostsModel()
hosts.extend(hosts_model.list_of_names(Registry().get('pData')['id']))
else:
hosts.append(self.options['host'].value)
worker = DnsBruteThread(
q,
hosts,
self.options['template'].value,
protocol,
self.options['msymbol'].value,
self.options['ignore-ip'].value,
next_server,
self.options['delay'].value,
self.options['http-not-found-re'].value,
self.options['ignore-words-re'].value,
result,
counter
)
worker.setDaemon(True)
worker.start()
w_thrds.append(worker)
time.sleep(1)
while len(w_thrds):
for worker in w_thrds:
if worker.done or Registry().get('positive_limit_stop'):
del w_thrds[w_thrds.index(worker)]
time.sleep(2)
if self.options['host'].value == 'all':
self._output_zones(result)
else:
self._output(result)
self.done = True
class Hosts(WSModule):
""" Class of Hosts module """
model = None
log_path = '/dev/null'
options = {}
options_sets = {
"list": {
"ip": WSOption("ip", "IP for host list", "", False, ['--ip']),
},
"delete": {
"host": WSOption("host", "Host for delete", "", True, ['--host']),
},
"add": {
"host":
WSOption("host", "Host for add", "", True, ['--host']),
"ip":
WSOption("ip", "Custom IP for this host", "", False, ['--ip']),
"descr":
WSOption("descr", "Description of host", "", False, ['--descr'])
}
}
def __init__(self, kernel):
WSModule.__init__(self, kernel)
self.model = HostsModel()
def validate_main(self):
""" Check users params """
if not validate_host(self.options['host'].value):
raise WSException("'{0}' is not valid host name!".format(
self.options['host'].value))
if 'ip' in self.options and self.options['ip'].value:
if not validate_ip(self.options['ip'].value):
raise WSException("IP '{0}' is not valid ip-address!".format(
self.options['ip'].value))
def run(self, action):
""" Method of run the module """
WSModule.run(self, action)
self.done = True
def add_action(self):
""" Action add of module """
self.validate_main()
pData = Registry().get('pData')
name = self.options['host'].value
descr = self.options['descr'].value
if self.options['ip'].value:
ip = self.options['ip'].value
else:
try:
ip = socket.gethostbyname(name)
print " IP for host '{0}' is '{1}'".format(name, ip)
except socket.gaierror:
raise WSException(
"Can`t lookup hostname '{0}'. Check it or set ip-address in --ip param!"
.format(name))
IPs = IpsModel()
ip_exists = IPs.exists(pData['id'], ip)
ip_id = IPs.get_id_or_add(pData['id'], ip)
if not ip_exists:
print " IP '{0}' was automatically added to project '{1}'".format(
ip, pData['name'])
if self.model.exists(pData['id'], name):
raise WSException(
"Host '{0}' already exists in project '{1}'!".format(
name, pData['name']))
self.model.add(pData['id'], ip_id, name, descr)
print " Host '{0}' successfully added to project '{1}' with IP '{2}' ! ".format(
name, pData['name'], ip)
def list_action(self):
""" Action list of module """
if self.options['ip'].value:
print "{0:=^51}".format("")
print "|{0: ^49}|".format("Hosts for IP '{0}'".format(
self.options['ip'].value))
print "{0:=^51}".format("")
print "| {0: ^23}| {1: ^23}|".format('Title', 'Description')
print "{0:=^51}".format("")
for host in self.model.list(Registry().get('pData')['id'],
self.options['ip'].value):
print "| {0: <23}| {1: <23}|".format(host['name'],
host['descr'])
print "{0:=^51}".format("")
else:
print "{0:=^76}".format("")
print "|{0: ^74}|".format("All host for project '{0}'".format(
Registry().get('pData')['name']))
print "{0:=^76}".format("")
print "| {0: ^23}| {1: ^23}| {2: ^23}|".format(
'Title', 'Description', 'IP')
print "{0:=^76}".format("")
for host in self.model.list_without_ip(
Registry().get('pData')['id']):
print "| {0: <23}| {1: <23}| {2: <23}|".format(
host['name'], host['descr'], host['ip'])
print "{0:=^76}".format("")
def delete_action(self):
""" Delete action of module """
self.validate_main()
name = self.options['host'].value
if not self.model.exists(Registry().get('pData')['id'], name):
raise WSException(
"Host '{0}' not exists in this project!".format(name))
answer = raw_input(
"You really want to delete host '{0}' [y/n]? ".format(name))
if answer.lower() == 'y':
self.model.delete(Registry().get('pData')['id'], name)
print "Host '{0}' successfully deleted.".format(name)
else:
print "Host '{0}' not deleted.".format(name)
def delete(self, project_id, host, url):
""" Delete url from table """
host_id = HostsModel().get_id_by_name(project_id, host)
self._db.q(
"DELETE FROM urls WHERE project_id = {0} AND host_id = {1} AND hash = {2} "
.format(project_id, host_id, self._db.quote(md5(url))))
def setup(self):
self.model = HostsModel()
self.db.q("TRUNCATE TABLE `hosts`")
self.db.q("TRUNCATE TABLE `ips`")
def scan_action(self):
""" Scan action of module """
self.enable_logger()
self.validate_main()
self.pre_start_inf()
if self.options['proxies'].value:
Registry().get('proxies').load(self.options['proxies'].value)
result = []
q = FuzzerHeadersJob()
U = UrlsModel()
urls = U.list_by_host_name(Registry().get('pData')['id'],
self.options['host'].value)
to_scan = []
for url in urls:
to_scan.append(url['url'])
q.load_dict(to_scan)
self.logger.log("Loaded {0} variants.".format(len(to_scan)))
counter = WSCounter(1, 60, len(to_scan))
w_thrds = []
for _ in range(int(self.options['threads'].value)):
worker = FuzzerHeadersThread(
q, self.options['host'].value,
self.options['protocol'].value.lower(),
self.options['method'].value.lower(),
self.options['delay'].value, counter, result)
worker.setDaemon(True)
worker.start()
w_thrds.append(worker)
time.sleep(1)
while len(w_thrds):
for worker in w_thrds:
if worker.done:
del w_thrds[w_thrds.index(worker)]
time.sleep(2)
Requests = RequestsModel()
Hosts = HostsModel()
project_id = Registry().get('pData')['id']
host_id = Hosts.get_id_by_name(project_id, self.options['host'].value)
added = 0
for fuzz in result:
self.logger.log("{0} {1}://{2}{3} (Word: {4}, Header: {5})".format(
self.options['method'].value.upper(),
self.options['protocol'].value.lower(),
self.options['host'].value, fuzz['url'],
", ".join(fuzz['words']), fuzz['header']))
_id = Requests.add(
project_id, host_id,
urlparse(fuzz['url']).path,
urlparse(fuzz['url']).query,
{fuzz['header']: Registry().get('fuzzer_evil_value')},
self.options['method'].value,
self.options['protocol'].value.lower(), 'fuzzer',
'Found word(s): {0}'.format(", ".join(fuzz['words'])))
added += 1 if _id else 0
self.logger.log("\nAdded {0} new requests in database".format(added))
self.done = True
def scan_action(self):
""" Scan action of module """
self.enable_logger()
self.validate_main()
self.pre_start_inf()
if self.options['proxies'].value:
Registry().get('proxies').load(self.options['proxies'].value)
result = []
q = FuzzerUrlsJob()
U = UrlsModel()
if os.path.exists('/tmp/fuzzer-urls.txt'):
os.remove('/tmp/fuzzer-urls.txt')
urls = U.list_by_host_name(Registry().get('pData')['id'], self.options['host'].value)
for url in urls:
if url['url'].count('?'):
to_add = self._generate_fuzz_urls(url['url'])
for item in to_add:
file_put_contents('/tmp/fuzzer-urls.txt', item + "\n", True)
generator = FileGenerator('/tmp/fuzzer-urls.txt')
q.set_generator(generator)
self.logger.log("Loaded {0} variants.".format(generator.lines_count))
counter = WSCounter(1, 60, generator.lines_count)
w_thrds = []
for _ in range(int(self.options['threads'].value)):
if self.options['selenium'].value:
worker = SFuzzerUrlsThread(
q,
self.options['host'].value,
self.options['protocol'].value.lower(),
self.options['method'].value.lower(),
self.options['delay'].value,
self.options['ddos-detect-phrase'].value,
self.options['ddos-human-action'].value,
self.options['browser-recreate-phrase'].value,
counter,
result
)
else:
worker = FuzzerUrlsThread(
q,
self.options['host'].value,
self.options['protocol'].value.lower(),
self.options['method'].value.lower(),
self.options['delay'].value,
counter,
result
)
worker.setDaemon(True)
worker.start()
w_thrds.append(worker)
time.sleep(1)
timeout_threads_count = 0
while len(w_thrds):
for worker in w_thrds:
if worker.done or Registry().get('proxy_many_died'):
del w_thrds[w_thrds.index(worker)]
if int(time.time()) - worker.last_action > int(Registry().get('config')['main']['kill_thread_after_secs']):
self.logger.log(
"Thread killed by time, resurected {0} times from {1}".format(
timeout_threads_count,
Registry().get('config')['main']['timeout_threads_resurect_max_count']
)
)
del w_thrds[w_thrds.index(worker)]
if timeout_threads_count <= int(Registry().get('config')['main']['timeout_threads_resurect_max_count']):
if self.options['selenium'].value:
worker = SFuzzerUrlsThread(
q,
self.options['host'].value,
self.options['protocol'].value.lower(),
self.options['method'].value.lower(),
self.options['delay'].value,
self.options['ddos-detect-phrase'].value,
self.options['ddos-human-action'].value,
self.options['browser-recreate-phrase'].value,
counter,
result
)
else:
worker = FuzzerUrlsThread(
q,
self.options['host'].value,
self.options['protocol'].value.lower(),
self.options['method'].value.lower(),
self.options['delay'].value,
counter,
result
)
worker.setDaemon(True)
worker.start()
w_thrds.append(worker)
timeout_threads_count += 1
time.sleep(2)
if result:
self.logger.log("\nPut results into DB...")
Requests = RequestsModel()
Hosts = HostsModel()
project_id = Registry().get('pData')['id']
host_id = Hosts.get_id_by_name(project_id, self.options['host'].value)
added = 0
for fuzz in result:
self.logger.log("{0} {1}://{2}{3} (Word: {4})".format(
self.options['method'].value.upper(),
self.options['protocol'].value.lower(),
self.options['host'].value,
fuzz['url'],
", ".join(fuzz['words'])
))
if int(Registry().get('config')['main']['put_data_into_db']):
_id = Requests.add(
project_id,
host_id,
urlparse(fuzz['url']).path,
urlparse(fuzz['url']).query,
{},
self.options['method'].value,
self.options['protocol'].value.lower(),
'fuzzer',
'Found word: {0}'.format(", ".join(fuzz['words']))
)
added += 1 if _id else 0
self.logger.log("Added {0} new requests in database".format(added))
self.done = True
def scan_action(self):
""" Scan action """
self.enable_logger()
self.validate_main()
self.pre_start_inf()
if self.options['proxies'].value:
Registry().get('proxies').load(self.options['proxies'].value)
result = []
if os.path.exists('/tmp/bf-urls.txt'):
os.remove('/tmp/bf-urls.txt')
urls = self.build_objects(self.options['host'].value)
for url in urls:
file_put_contents('/tmp/bf-urls.txt', url + "\n", True)
q = BackupsFinderJob()
generator = FileGenerator('/tmp/bf-urls.txt')
q.set_generator(generator)
self.logger.log("Loaded {0} variants.".format(generator.lines_count))
counter = WSCounter(5, 300, generator.lines_count)
w_thrds = []
for _ in range(int(self.options['threads'].value)):
if self.options['selenium'].value:
worker = SBackupsFinderThread(
q,
self.options['host'].value,
self.options['protocol'].value.lower(),
self.options['method'].value.lower(),
self.options['not-found-re'].value,
self.options['delay'].value,
self.options['ddos-detect-phrase'].value,
self.options['ddos-human-action'].value,
self.options['browser-recreate-re'].value,
counter,
result
)
else:
worker = BackupsFinderThread(
q,
self.options['host'].value,
self.options['protocol'].value.lower(),
self.options['method'].value.lower(),
self.options['not-found-re'].value,
self.options['not-found-codes'].value.lower(),
self.options['not-found-size'].value,
self.options['delay'].value,
counter,
result
)
worker.setDaemon(True)
worker.start()
w_thrds.append(worker)
time.sleep(1)
timeout_threads_count = 0
while len(w_thrds):
for worker in w_thrds:
if Registry().get('proxy_many_died') or Registry().get('positive_limit_stop'):
worker.done = True
time.sleep(3)
if worker.done:
del w_thrds[w_thrds.index(worker)]
if int(time.time()) - worker.last_action > int(Registry().get('config')['main']['kill_thread_after_secs']):
self.logger.log(
"Thread killed by time, resurected {0} times from {1}".format(
timeout_threads_count,
Registry().get('config')['main']['timeout_threads_resurect_max_count']
)
)
del w_thrds[w_thrds.index(worker)]
if timeout_threads_count <= int(Registry().get('config')['main']['timeout_threads_resurect_max_count']):
if self.options['selenium'].value:
worker = SBackupsFinderThread(
q,
self.options['host'].value,
self.options['protocol'].value.lower(),
self.options['method'].value.lower(),
self.options['not-found-re'].value,
self.options['delay'].value,
self.options['ddos-detect-phrase'].value,
self.options['ddos-human-action'].value,
self.options['browser-recreate-re'].value,
counter,
result
)
else:
worker = BackupsFinderThread(
q,
self.options['host'].value,
self.options['protocol'].value.lower(),
self.options['method'].value.lower(),
self.options['not-found-re'].value,
self.options['not-found-codes'].value.lower(),
self.options['delay'].value,
counter,
result
)
worker.setDaemon(True)
worker.start()
w_thrds.append(worker)
timeout_threads_count += 1
time.sleep(2)
if Registry().get('positive_limit_stop'):
self.logger.log("\nMany positive detections. Please, look items logs")
self.logger.log("Last items:")
for i in range(1, 5):
print result[-i]
exit(0)
if result:
print "\n",
for item in result:
print item
if int(Registry().get('config')['main']['put_data_into_db']):
if result:
self.logger.log("\nPut found into DB...")
Requests = RequestsModel()
Hosts = HostsModel()
project_id = Registry().get('pData')['id']
host_id = Hosts.get_id_by_name(project_id, self.options['host'].value)
added = 0
for backup in result:
_id = Requests.add(
project_id, host_id, backup, "", {}, self.options['method'].value,
self.options['protocol'].value.lower(), 'backups', 'May be important backup'
)
added += 1 if _id else 0
self.logger.log("Found backups: {0}, new: {1}".format(len(result), added))
self.logger.log(str(result), _print=False)
self.done = True
def scan_action(self):
""" Scan action of module """
self.enable_logger()
self.validate_main()
self.pre_start_inf()
if self.options['proxies'].value:
Registry().get('proxies').load(self.options['proxies'].value)
result = []
q = FuzzerHeadersJob()
U = UrlsModel()
urls = U.list_by_host_name(Registry().get('pData')['id'], self.options['host'].value)
to_scan = []
for url in urls:
to_scan.append(url['url'])
q.load_dict(to_scan)
self.logger.log("Loaded {0} variants.".format(len(to_scan)))
counter = WSCounter(1, 60, len(to_scan))
w_thrds = []
for _ in range(int(self.options['threads'].value)):
worker = FuzzerHeadersThread(
q,
self.options['host'].value,
self.options['protocol'].value.lower(),
self.options['method'].value.lower(),
self.options['delay'].value,
counter,
result
)
worker.setDaemon(True)
worker.start()
w_thrds.append(worker)
time.sleep(1)
while len(w_thrds):
for worker in w_thrds:
if worker.done:
del w_thrds[w_thrds.index(worker)]
time.sleep(2)
Requests = RequestsModel()
Hosts = HostsModel()
project_id = Registry().get('pData')['id']
host_id = Hosts.get_id_by_name(project_id, self.options['host'].value)
added = 0
for fuzz in result:
self.logger.log("{0} {1}://{2}{3} (Word: {4}, Header: {5})".format(
self.options['method'].value.upper(),
self.options['protocol'].value.lower(),
self.options['host'].value,
fuzz['url'],
", ".join(fuzz['words']),
fuzz['header']
))
_id = Requests.add(
project_id,
host_id,
urlparse(fuzz['url']).path,
urlparse(fuzz['url']).query,
{fuzz['header']: Registry().get('fuzzer_evil_value')},
self.options['method'].value,
self.options['protocol'].value.lower(),
'fuzzer',
'Found word(s): {0}'.format(", ".join(fuzz['words']))
)
added += 1 if _id else 0
self.logger.log("\nAdded {0} new requests in database".format(added))
self.done = True
class Test_HostsModel(Common):
"""Unit tests for HostsModel"""
model = None
def setup(self):
self.model = HostsModel()
self.db.q("TRUNCATE TABLE `hosts`")
self.db.q("TRUNCATE TABLE `ips`")
def test_get_id_by_name(self):
self.db.q("INSERT INTO `hosts` (id, project_id, ip_id, name, descr) VALUES (2,1,1,'test.com', '')")
assert self.model.get_id_by_name(1, 'test.com') == 2
def test_exists(self):
self.db.q("INSERT INTO `hosts` (id, project_id, ip_id, name, descr) VALUES (2,1,1,'test.com', '')")
assert self.model.exists(1, 'test.com')
assert not self.model.exists(1, 'test1.com')
def test_add(self):
assert not self.model.exists(1, 'test.com')
assert self.model.add(1, 1, 'test.com', '') == 1
assert self.model.exists(1, 'test.com')
def test_list(self):
self.db.q("INSERT INTO `ips` (id, project_id, ip, descr) VALUES (1, 1, '111.111.111.111', '')")
self.db.q("INSERT INTO `hosts` (id, project_id, ip_id, name, descr) VALUES (1, 1, 1, 'test1.com', 'desc1')")
self.db.q("INSERT INTO `hosts` (id, project_id, ip_id, name, descr) VALUES (2, 1, 1, 'test2.com', 'desc2')")
self.db.q("INSERT INTO `hosts` (id, project_id, ip_id, name, descr) VALUES (3, 1, 1, 'test3.com', 'desc3')")
test_data = [
{'id': 1, 'name': 'test1.com', 'ip': '111.111.111.111', 'descr': 'desc1'},
{'id': 2, 'name': 'test2.com', 'ip': '111.111.111.111', 'descr': 'desc2'},
{'id': 3, 'name': 'test3.com', 'ip': '111.111.111.111', 'descr': 'desc3'},
]
assert test_data == self.model.list(1, '111.111.111.111')
def test_list_of_names(self):
self.db.q("INSERT INTO `ips` (id, project_id, ip, descr) VALUES (1, 1, '111.111.111.111', '')")
self.db.q("INSERT INTO `hosts` (id, project_id, ip_id, name, descr) VALUES (1, 1, 1, 'test1.com', 'desc1')")
self.db.q("INSERT INTO `hosts` (id, project_id, ip_id, name, descr) VALUES (2, 1, 1, 'test2.com', 'desc2')")
self.db.q("INSERT INTO `hosts` (id, project_id, ip_id, name, descr) VALUES (3, 1, 1, 'test3.com', 'desc3')")
test_data = ['test1.com', 'test2.com', 'test3.com']
assert test_data == self.model.list_of_names(1)
def test_delete(self):
self.db.q("INSERT INTO `hosts` (id, project_id, ip_id, name, descr) VALUES (2,1,1,'test.com', '')")
assert self.model.exists(1, 'test.com')
self.model.delete(1, 'test.com')
assert not self.model.exists(1, 'test.com')
def __init__(self, kernel):
WSModule.__init__(self, kernel)
self.model = HostsModel()
def validate_main(self):
""" Common user params validate functions """
options = self.options_sets[self.action].keys()
if 'selenium' in self.options.keys(
) and self.options['selenium'].value:
if 'not-found-re' in options and not self.options[
'not-found-re'].value:
raise WSException(
"Selenium enabled, module need a not found phrase (--not-found-re) for work!"
)
if int(self.options['threads'].value) > int(
Registry().get('config')['selenium']['max_threads']):
raise WSException(
"Selenium enabled, very many threads value ({0}), see docs."
.format(self.options['threads'].value))
if 'protocol' in self.options.keys(
) and self.options['protocol'].value.lower() not in ['http', 'https']:
raise WSException(
"Protocol param must be 'http' or 'https', but have value '{0}' !"
.format(self.options['protocol'].value))
if 'method' in self.options.keys(
) and self.options['method'].value.lower() not in [
'head', 'get', 'post'
]:
raise WSException(
"Method param must be only 'head', 'get' or 'post', but have value '{0}' !"
.format(self.options['method'].value))
if 'not-found-codes' in self.options.keys() and len(
self.options['not-found-codes'].value):
for code in self.options['not-found-codes'].value.strip().split(
","):
if len(code.strip()) and not re.match(r'^(\d+)$',
code.strip()):
raise WSException(
"Not-found code must be digital, but it is '{0}'".
format(code.strip()))
if 'retest-codes' in self.options.keys() and len(
self.options['retest-codes'].value):
for code in self.options['retest-codes'].value.strip().split(","):
if len(code.strip()) and not re.match(r'^(\d+)$',
code.strip()):
raise WSException(
"Retest code must be digital, but it is '{0}'".format(
code.strip()))
if 'proxies' in self.options.keys() and len(self.options['proxies'].value) and \
not os.path.exists(self.options['proxies'].value):
raise WSException("Proxy list not found: '{0}'".format(
self.options['proxies'].value))
if 'not-found-re' in self.options.keys() and len(
self.options['not-found-re'].value):
try:
re.compile(self.options['not-found-re'].value)
except re.error:
raise WSException("Invalid regex: '{0}'".format(
self.options['not-found-re'].value))
if 'browser-recreate-re' in self.options.keys() and len(
self.options['browser-recreate-re'].value):
try:
re.compile(self.options['browser-recreate-re'].value)
except re.error:
raise WSException("Invalid regex: '{0}'".format(
self.options['browser-recreate-re'].value))
if 'host' in self.options.keys() and \
not HostsModel().exists(Registry().get('pData')['id'], self.options['host'].value):
raise WSException("Host '{0}' not found in this project!".format(
self.options['host'].value))
if 'dict' in self.options.keys() and not os.path.exists(
self.options['dict'].value):
raise WSException(
"Dictionary '{0}' not exists or not readable!".format(
self.options['dict'].value))
if 'delay' in self.options.keys(
) and self.options['delay'].value != '0':
if not re.match(r'^(\d+)$', self.options['delay'].value):
raise WSException(
"Delay param must be digital, but it is '{0}'".format(
self.options['delay'].value.strip()))
if 'parts' in self.options.keys(
) and self.options['parts'].value != '0':
if not re.match(r'^(\d+)$', self.options['parts'].value):
raise WSException(
"Parts param must be digital, but it is '{0}'".format(
self.options['parts'].value.strip()))
if 'part' in self.options.keys() and self.options['part'].value != '0':
if not re.match(r'^(\d+)$', self.options['part'].value):
raise WSException(
"Part param must be digital, but it is '{0}'".format(
self.options['part'].value.strip()))
if 'parts' in self.options.keys(
) and self.options['parts'].value != '0':
if self.options['part'].value == '0':
raise WSException(
"If you use '--parts' param, you must specify '--part'")
if int(self.options['part'].value) > int(
self.options['parts'].value):
raise WSException(
"Number of part ({0}) more than parts count ({1})".format(
self.options['part'].value.strip(),
self.options['parts'].value.strip()))
if 'part' in self.options.keys() and self.options['part'].value != '0':
if self.options['parts'].value == '0':
raise WSException(
"If you use '--part' param, you must specify '--parts'")
def scan_action(self):
""" Scan action of module """
self.enable_logger()
self.validate_main()
self.pre_start_inf()
if self.options['proxies'].value:
Registry().get('proxies').load(self.options['proxies'].value)
result = []
q = FuzzerUrlsJob()
U = UrlsModel()
if os.path.exists('/tmp/fuzzer-urls.txt'):
os.remove('/tmp/fuzzer-urls.txt')
urls = U.list_by_host_name(Registry().get('pData')['id'],
self.options['host'].value)
for url in urls:
if url['url'].count('?'):
to_add = self._generate_fuzz_urls(url['url'])
for item in to_add:
file_put_contents('/tmp/fuzzer-urls.txt', item + "\n",
True)
generator = FileGenerator('/tmp/fuzzer-urls.txt')
q.set_generator(generator)
self.logger.log("Loaded {0} variants.".format(generator.lines_count))
counter = WSCounter(1, 60, generator.lines_count)
w_thrds = []
for _ in range(int(self.options['threads'].value)):
if self.options['selenium'].value:
worker = SFuzzerUrlsThread(
q, self.options['host'].value,
self.options['protocol'].value.lower(),
self.options['method'].value.lower(),
self.options['delay'].value,
self.options['ddos-detect-phrase'].value,
self.options['ddos-human-action'].value,
self.options['browser-recreate-phrase'].value, counter,
result)
else:
worker = FuzzerUrlsThread(
q, self.options['host'].value,
self.options['protocol'].value.lower(),
self.options['method'].value.lower(),
self.options['delay'].value, counter, result)
worker.setDaemon(True)
worker.start()
w_thrds.append(worker)
time.sleep(1)
while len(w_thrds):
for worker in w_thrds:
if worker.done or Registry().get('proxy_many_died'):
del w_thrds[w_thrds.index(worker)]
if int(time.time()) - worker.last_action > int(Registry().get(
'config')['main']['kill_thread_after_secs']):
self.logger.log("Thread killed by time")
del w_thrds[w_thrds.index(worker)]
time.sleep(2)
if result:
self.logger.log("\nPut results into DB...")
Requests = RequestsModel()
Hosts = HostsModel()
project_id = Registry().get('pData')['id']
host_id = Hosts.get_id_by_name(project_id, self.options['host'].value)
added = 0
for fuzz in result:
self.logger.log("{0} {1}://{2}{3} (Word: {4})".format(
self.options['method'].value.upper(),
self.options['protocol'].value.lower(),
self.options['host'].value, fuzz['url'],
", ".join(fuzz['words'])))
_id = Requests.add(
project_id, host_id,
urlparse(fuzz['url']).path,
urlparse(fuzz['url']).query, {}, self.options['method'].value,
self.options['protocol'].value.lower(), 'fuzzer',
'Found word: {0}'.format(", ".join(fuzz['words'])))
added += 1 if _id else 0
self.logger.log("Added {0} new requests in database".format(added))
self.done = True
def scan_action(self):
""" Scan action of module """
self.enable_logger()
self.validate_main()
self.pre_start_inf()
self.model = CmsModel()
if self.options['proxies'].value:
Registry().get('proxies').load(self.options['proxies'].value)
result = []
q = CmsJob()
for item in self.model.all_paths_list():
q.put(item.strip())
self.logger.log("Loaded {0} variants.".format(q.qsize()))
counter = WSCounter(1, 60, q.qsize())
w_thrds = []
for _ in range(int(self.options['threads'].value)):
if self.options['selenium'].value:
worker = SCmsThread(
q,
self.options['host'].value,
self.options['url'].value,
self.options['protocol'].value.lower(),
self.options['method'].value.lower(),
self.options['not-found-re'].value,
self.options['delay'].value,
self.options['ddos-detect-phrase'].value,
self.options['ddos-human-action'].value,
self.options['browser-recreate-re'].value,
counter,
result
)
else:
worker = CmsThread(
q,
self.options['host'].value,
self.options['url'].value,
self.options['protocol'].value.lower(),
self.options['method'].value.lower(),
self.options['not-found-re'].value,
self.options['not-found-codes'].value.lower(),
self.options['delay'].value,
counter,
result
)
worker.setDaemon(True)
worker.start()
w_thrds.append(worker)
time.sleep(1)
while len(w_thrds):
for worker in w_thrds:
if Registry().get('proxy_many_died'):
worker.done = True
time.sleep(3)
if worker.done:
del w_thrds[w_thrds.index(worker)]
if int(time.time()) - worker.last_action > int(Registry().get('config')['main']['kill_thread_after_secs']):
self.logger.log("Thread killed by time")
del w_thrds[w_thrds.index(worker)]
time.sleep(2)
pid = Registry().get('pData')['id']
host_id = HostsModel().get_id_by_name(pid, self.options['host'].value)
Urls = UrlsModel()
UrlsBase = UrlsBaseModel()
self.logger.log("\nInsert result info in DB...")
_all = 0
added = 0
HostsInfo = HostsInfoModel()
to_hosts_info = []
hash_ids = []
for link in result:
hash_ids.append(self.model.get_hash_id_by_path(link['path']))
_all += 1
if Urls.add(pid, host_id, link['path'], '', link['code'], 0, 'cms'):
added += 1
UrlsBase.add_url(host_id, link['path'])
self.logger.log("\nFound {0} URLs, inserted in database (new) - {1}.".format(_all, added))
cms_list = self.model.cms_list()
for cms_id in self.model.get_cms_by_hash_ids(hash_ids):
cms_paths = self.model.get_cms_paths(cms_id)
current_count = 0
for link in result:
if link['path'] in cms_paths:
current_count += 1
percent = int(current_count / len(cms_paths) * 100)
if int(Registry().get('config')['cms']['percent']) <= percent:
to_hosts_info.append({'name': cms_list[cms_id], 'percent': percent})
self.logger.log("{0}\t{1}%".format(cms_list[cms_id], percent))
if len(to_hosts_info):
HostsInfo.set_info(pid, host_id, 'cms', json.dumps(to_hosts_info))
self.done = True
