def test__check_bulk(self):
input_queue = PriorityQueue()
queue = PriorityQueue()
old_date_modified = datetime.now().isoformat()
id_1 = uuid4().hex
date_modified_1 = datetime.now().isoformat()
id_2 = uuid4().hex
date_modified_2 = datetime.now().isoformat()
id_3 = uuid4().hex
date_modified_3 = datetime.now().isoformat()
db = MagicMock()
bulk = {
id_1: date_modified_1,
id_2: date_modified_2,
id_3: date_modified_3
}
priority_cache = {id_1: 1, id_2: 1, id_3: 1}
return_value = {
u'docs': [
{
u'_type': u'Tender',
u'_source': {
u'dateModified': date_modified_1
},
u'_index': u'bridge_tenders',
u'_version': 1,
u'found': True,
u'_id': id_1
},
{
u'_type': u'Tender',
u'_source': {
u'dateModified': old_date_modified
},
u'_index': u'bridge_tenders',
u'_version': 1,
u'found': True,
u'_id': id_2
},
{
u'found': False,
u'_type': u'Tender',
u'_id': id_3,
u'_index': u'bridge_tenders'
}
]
}
db.mget.return_value = return_value
elastic_filter = BasicElasticSearchFilter(self.config, input_queue, queue, db)
self.assertEqual(queue.qsize(), 0)
elastic_filter._check_bulk(bulk, priority_cache)
self.assertEqual(queue.qsize(), 2)
def test_add_to_retry_queue(self, mocked_logger):
retry_items_queue = PriorityQueue()
worker = AgreementWorker(config_dict=self.worker_config,
retry_resource_items_queue=retry_items_queue)
resource_item = {'id': uuid.uuid4().hex}
priority = 1000
self.assertEqual(retry_items_queue.qsize(), 0)
# Add to retry_resource_items_queue
worker.add_to_retry_queue(resource_item, priority=priority)
self.assertEqual(retry_items_queue.qsize(), 1)
priority, retry_resource_item = retry_items_queue.get()
self.assertEqual((priority, retry_resource_item),
(1001, resource_item))
resource_item = {'id': 0}
# Add to retry_resource_items_queue with status_code '429'
worker.add_to_retry_queue(resource_item, priority, status_code=429)
self.assertEqual(retry_items_queue.qsize(), 1)
priority, retry_resource_item = retry_items_queue.get()
self.assertEqual((priority, retry_resource_item),
(1001, resource_item))
priority = 1002
worker.add_to_retry_queue(resource_item, priority=priority)
sleep(worker.config['retry_default_timeout'] * 2)
self.assertEqual(retry_items_queue.qsize(), 1)
priority, retry_resource_item = retry_items_queue.get()
self.assertEqual((priority, retry_resource_item),
(1003, resource_item))
worker.add_to_retry_queue(resource_item, priority=priority)
self.assertEqual(retry_items_queue.qsize(), 0)
mocked_logger.critical.assert_called_once_with(
'Tender {} reached limit retries count {} and droped from '
'retry_queue.'.format(resource_item['id'],
worker.config['retries_count']),
extra={
'MESSAGE_ID': 'dropped_documents',
'JOURNAL_TENDER_ID': resource_item['id']
})
del worker
class AutoHack():
def __init__(self, zoomeye_results, threads_num):
self.threads_num = threads_num
self.targets = PriorityQueue()
self.zoomeye_results = zoomeye_results
self.result = []
for zoomeye_result in zoomeye_results:
self.targets.put(zoomeye_result)
self.total = self.targets.qsize()
self.pbar = tqdm(total=self.total,ascii=True)
def check(self):
while self.targets.qsize() > 0:
target = self.targets.get().strip()
try:
self.pbar.update(1)
result = exp.exp(target)
if result:
self.result.append(result)
except Exception as e:
#print(e)
pass
def run(self):
threads = [gevent.spawn(self.check) for i in range(self.threads_num)]
try:
gevent.joinall(threads)
except KeyboardInterrupt as e:
print ('[WARNING] User aborted')
for res in self.result:
print (res)
self.pbar.close()
print ("Hack it!")
for res in self.result:
print (res)
print("Found ",len(self.result))
print ("End!")
def test_add_to_retry_queue(self, mocked_logger):
retry_items_queue = PriorityQueue()
worker = ResourceItemWorker(
config_dict=self.worker_config,
retry_resource_items_queue=retry_items_queue)
resource_item_id = uuid.uuid4().hex
priority = 1000
self.assertEqual(retry_items_queue.qsize(), 0)
# Add to retry_resource_items_queue
worker.add_to_retry_queue(resource_item_id, priority=priority)
# sleep(worker.config['retry_default_timeout'] * 0)
self.assertEqual(retry_items_queue.qsize(), 1)
priority, retry_resource_item_id = retry_items_queue.get()
self.assertEqual(priority, 1001)
self.assertEqual(retry_resource_item_id, resource_item_id)
# Add to retry_resource_items_queue with status_code '429'
worker.add_to_retry_queue(resource_item_id, priority, status_code=429)
self.assertEqual(retry_items_queue.qsize(), 1)
priority, retry_resource_item_id = retry_items_queue.get()
self.assertEqual(priority, 1001)
self.assertEqual(retry_resource_item_id, resource_item_id)
priority = 1002
worker.add_to_retry_queue(resource_item_id, priority=priority)
sleep(worker.config['retry_default_timeout'] * 2)
self.assertEqual(retry_items_queue.qsize(), 1)
priority, retry_resource_item_id = retry_items_queue.get()
self.assertEqual(priority, 1003)
self.assertEqual(retry_resource_item_id, resource_item_id)
worker.add_to_retry_queue(resource_item_id, priority=priority)
self.assertEqual(retry_items_queue.qsize(), 0)
mocked_logger.critical.assert_called_once_with(
'Tender {} reached limit retries count {} and droped from '
'retry_queue.'.format(resource_item_id,
worker.config['retries_count']),
extra={'MESSAGE_ID': 'dropped_documents'})
del worker
class TestBasicCouchDBFilter(unittest.TestCase):
config = deepcopy(TEST_CONFIG['main'])
config['storage_config']['bulk_query_limit'] = 1
def setUp(self):
self.old_date_modified = datetime.now().isoformat()
self.id_1 = uuid4().hex
self.date_modified_1 = datetime.now().isoformat()
self.id_2 = uuid4().hex
self.date_modified_2 = datetime.now().isoformat()
self.id_3 = uuid4().hex
self.date_modified_3 = datetime.now().isoformat()
self.queue = PriorityQueue()
self.input_queue = PriorityQueue()
self.db = MagicMock()
self.bulk = {
self.id_1: self.date_modified_1,
self.id_2: self.date_modified_2,
self.id_3: self.date_modified_3
}
self.priority_cache = {self.id_1: 1, self.id_2: 1, self.id_3: 1}
self.return_value = [
munchify({'id': self.id_1, 'key': self.date_modified_1}),
munchify({'id': self.id_2, 'key': self.old_date_modified}),
munchify({'id': self.id_3, 'key': self.old_date_modified})
]
self.db.db.view.return_value = self.return_value
def test__check_bulk(self):
self.queue.put((1000, self.id_3))
couchdb_filter = BasicCouchDBFilter(self.config, self.input_queue, self.queue, self.db)
self.assertEqual(self.queue.qsize(), 1)
couchdb_filter._check_bulk(self.bulk, self.priority_cache)
self.assertEqual(self.queue.qsize(), 2)
self.db.db.view.side_effect = [Exception(), Exception(), Exception('test')]
self.bulk = {}
with self.assertRaises(Exception) as e:
couchdb_filter._check_bulk(self.bulk, self.priority_cache)
self.assertEqual(e.exception.message, 'test')
@patch('openprocurement.bridge.basic.filters.INFINITY')
def test__run(self, mocked_infinity):
couchdb_filter = BasicCouchDBFilter(self.config, self.input_queue, self.queue, self.db)
self.input_queue.put((1, {'id': self.id_1, 'dateModified': self.date_modified_1}))
self.input_queue.put((1, {'id': self.id_2, 'dateModified': self.date_modified_2}))
self.input_queue.put((1, {'id': self.id_3, 'dateModified': self.date_modified_3}))
mocked_infinity.__nonzero__.side_effect = [True] * 5 + [False, False]
self.assertEqual(self.queue.qsize(), 0)
self.assertEqual(self.input_queue.qsize(), 3)
couchdb_filter._run()
self.assertEqual(self.queue.qsize(), 2)
self.assertEqual(self.input_queue.qsize(), 0)
class HttpTest(object):
def __init__(self,host,keyword,ips,timeout):
self.threads = 100
self.queue = PriorityQueue()
self.host = host
self.keyword = keyword
self.result = []
for ip in ips:
self.queue.put(ip)
self.num = self.queue.qsize()
self.i = 0
self.success = 0
self.timeout = timeout
self.filename = os.path.join(rootPath,"result",host + ".log")
self.outfile = open(self.filename, 'w')
def _scan(self,j):
while not self.queue.empty():
try:
item = self.queue.get(timeout=3.0)
if config.HTTPS_Support:
host, domain, port = item, self.host , 443
else:
host, domain, port = item, self.host , 80
html = httpServer((host, domain, port),self.timeout)
if html is not None and self.keyword in html:
self.outfile.write(item + '\n')
self.outfile.flush()
self.success += 1
except:
pass
finally:
self.i += 1
msg = '[*] %s found, %s scanned , %s groups left'%(self.success,self.i,self.num - self.i)
print_msg(msg)
time.sleep(1.0)
def run(self):
threads = [gevent.spawn(self._scan, i) for i in range(self.threads)]
gevent.joinall(threads)
msg = '[+] All Done. Success:%d Saved in:%s'%(self.success,self.filename)
print_msg(msg, line_feed=True)
class Scanner:
def __init__(self):
self.start_time = time.time()
self.queue = PriorityQueue()
self.history = []
self.total_count = 0
self.scan_count = 0
self._load_target()
self.outfile = open("log.log", 'w')
self.console_width = getTerminalSize()[0] - 2
def _print_msg(self, _msg=None, _found_msg=False):
if _msg is None:
msg = '%s TotalCount| %s Scanned in %.2f seconds' % (
self.total_count,self.total_count - self.queue.qsize(), time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) + msg)
else:
sys.stdout.write('\r' + _msg + ' ' * (self.console_width - len(_msg)) + '\n')
self.outfile.write(_msg + '\n')
self.outfile.flush()
if _found_msg:
msg = '%s TotalCount| %s Scanned in %.2f seconds' % (
self.total_count,self.total_count - self.queue.qsize(), time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) + msg)
sys.stdout.flush()
def _load_target(self):
print '[+] Read targets ...'
target_file = raw_input("Target File :")
with open(target_file) as f:
for line in f.xreadlines():
target = line.strip()
self.queue.put(target)
print "TotalCount is %d" % self.queue.qsize()
self.total_count = self.queue.qsize()
print "Now scanning ..."
def _scan(self,case):
while not self.queue.empty():
target = self.queue.get()
if case == "1":
self.vulnCheck(target)
if case == "2":
self.s2_045(target)
if case == "3":
self.headers(target)
if case == "4":
self.weakfile(target)
if case == "5":
self.portscan_c(target)
#####################################################################
# #
# Vuln poc by:xi4okv QQ:48011203 #
# #
#####################################################################
def vulnCheck(self,target):
if ":2375" in target:
try:
res = requests.head("http://" + str(target) + "/containers/json",timeout=2)
if res.headers['Content-Type'] == 'application/json':
self._print_msg(target + "==> docker api Vuln",True)
else:
self._print_msg()
except:
self._print_msg()
self._print_msg()
if ":27017" in target:
ip,port = target.split(":")
try:
socket.setdefaulttimeout(3)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((ip, int(port)))
data = binascii.a2b_hex("3a000000a741000000000000d40700000000000061646d696e2e24636d640000000000ffffffff130000001069736d6173746572000100000000")
s.send(data)
result = s.recv(1024)
if "ismaster" in result:
getlog_data = binascii.a2b_hex("480000000200000000000000d40700000000000061646d696e2e24636d6400000000000100000021000000026765744c6f670010000000737461727475705761726e696e67730000")
s.send(getlog_data)
result = s.recv(1024)
if "totalLinesWritten" in result:
self._print_msg(target + "==> mongodb Vuln",True)
except Exception, e:
pass
if ":6379" in target:
ip,port = target.split(":")
try:
socket.setdefaulttimeout(3)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((ip, int(port)))
s.send("INFO\r\n")
result = s.recv(1024)
if "redis_version" in result:
self._print_msg(target + "==> redis Vuln",True)
elif "Authentication" in result:
for pass_ in ['123456','redis','pass','password']:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((ip, int(port)))
s.send("AUTH %s\r\n" % (pass_))
result = s.recv(1024)
if '+OK' in result:
self._print_msg(target + "==> redis pass Vuln :" + pass_,True)
except Exception, e:
pass
class SubNameBrute:
def __init__(self, target, options):
self.start_time = time.time()
self.target = target.strip()
self.options = options
self.ignore_intranet = options.i
self.scan_count = self.found_count = 0
self.console_width = getTerminalSize()[0] - 2
self.resolvers = [dns.resolver.Resolver(configure=False) for _ in range(options.threads)]
for _ in self.resolvers:
_.lifetime = _.timeout = 10.0
self.print_count = 0
self.STOP_ME = False
self._load_dns_servers()
self._load_next_sub()
self.queue = PriorityQueue()
self.priority = 0
self._load_sub_names()
if options.output:
outfile = options.output
else:
_name = os.path.basename(self.options.file).replace('subnames', '')
if _name != '.txt':
_name = '_' + _name
outfile = target + _name if not options.full_scan else target + '_full' + _name
self.outfile = open(outfile, 'w')
self.ip_dict = {}
self.found_subs = set()
self.ex_resolver = dns.resolver.Resolver(configure=False)
self.ex_resolver.nameservers = self.dns_servers
def _load_dns_servers(self):
print '[+] Validate DNS servers ...'
self.dns_servers = []
pool = Pool(30)
for server in open('dict/dns_servers.txt').xreadlines():
server = server.strip()
if server:
pool.apply_async(self._test_server, (server,))
pool.join()
self.dns_count = len(self.dns_servers)
sys.stdout.write('\n')
print '[+] Found %s available DNS Servers in total' % self.dns_count
if self.dns_count == 0:
print '[ERROR] No DNS Servers available.'
sys.exit(-1)
def _test_server(self, server):
resolver = dns.resolver.Resolver()
resolver.lifetime = resolver.timeout = 10.0
try:
resolver.nameservers = [server]
answers = resolver.query('public-dns-a.baidu.com') # test lookup a existed domain
if answers[0].address != '180.76.76.76':
raise Exception('incorrect DNS response')
try:
resolver.query('test.bad.dns.lijiejie.com') # Non-existed domain test
with open('bad_dns_servers.txt', 'a') as f:
f.write(server + '\n')
self._print_msg('[+] Bad DNS Server found %s' % server)
except:
self.dns_servers.append(server)
self._print_msg('[+] Check DNS Server %s < OK > Found %s' % (server.ljust(16), len(self.dns_servers)))
except:
self._print_msg('[+] Check DNS Server %s <Fail> Found %s' % (server.ljust(16), len(self.dns_servers)))
def _load_sub_names(self):
self._print_msg('[+] Load sub names ...')
if self.options.full_scan and self.options.file == 'subnames.txt':
_file = 'dict/subnames_full.txt'
else:
if os.path.exists(self.options.file):
_file = self.options.file
elif os.path.exists('dict/%s' % self.options.file):
_file = 'dict/%s' % self.options.file
else:
self._print_msg('[ERROR] Names file not exists: %s' % self.options.file)
exit(-1)
normal_lines = []
wildcard_lines = []
wildcard_list = []
regex_list = []
lines = set()
with open(_file) as f:
for line in f.xreadlines():
sub = line.strip()
if not sub or sub in lines:
continue
lines.add(sub)
if sub.find('{alphnum}') >= 0 or sub.find('{alpha}') >= 0 or sub.find('{num}') >= 0:
wildcard_lines.append(sub)
sub = sub.replace('{alphnum}', '[a-z0-9]')
sub = sub.replace('{alpha}', '[a-z]')
sub = sub.replace('{num}', '[0-9]')
if sub not in wildcard_list:
wildcard_list.append(sub)
regex_list.append('^' + sub + '$')
else:
normal_lines.append(sub)
pattern = '|'.join(regex_list)
if pattern:
_regex = re.compile(pattern)
if _regex:
for line in normal_lines[:]:
if _regex.search(line):
normal_lines.remove(line)
for item in normal_lines:
self.priority += 1
self.queue.put((self.priority, item))
for item in wildcard_lines:
self.queue.put((88888888, item))
def _load_next_sub(self):
self._print_msg('[+] Load next level subs ...')
self.next_subs = []
_set = set()
_file = 'dict/next_sub.txt' if not self.options.full_scan else 'dict/next_sub_full.txt'
with open(_file) as f:
for line in f:
sub = line.strip()
if sub and sub not in self.next_subs:
tmp_set = {sub}
while len(tmp_set) > 0:
item = tmp_set.pop()
if item.find('{alphnum}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789':
tmp_set.add(item.replace('{alphnum}', _letter, 1))
elif item.find('{alpha}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz':
tmp_set.add(item.replace('{alpha}', _letter, 1))
elif item.find('{num}') >= 0:
for _letter in '0123456789':
tmp_set.add(item.replace('{num}', _letter, 1))
elif item not in _set:
_set.add(item)
self.next_subs.append(item)
def _print_msg(self, _msg=None, _found_msg=False):
if _msg is None:
self.print_count += 1
if self.print_count < 100:
return
self.print_count = 0
msg = '%s Found| %s Groups| %s scanned in %.1f seconds' % (
self.found_count, self.queue.qsize(), self.scan_count, time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) + msg)
elif _msg.startswith('[+] Check DNS Server'):
sys.stdout.write('\r' + _msg + ' ' * (self.console_width - len(_msg)))
else:
sys.stdout.write('\r' + _msg + ' ' * (self.console_width - len(_msg)) + '\n')
if _found_msg:
msg = '%s Found| %s Groups| %s scanned in %.1f seconds' % (
self.found_count, self.queue.qsize(), self.scan_count, time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) + msg)
sys.stdout.flush()
@staticmethod
def is_intranet(ip):
ret = ip.split('.')
if len(ret) != 4:
return True
if ret[0] == '10':
return True
if ret[0] == '172' and 16 <= int(ret[1]) <= 32:
return True
if ret[0] == '192' and ret[1] == '168':
return True
return False
def put_item(self, item):
num = item.count('{alphnum}') + item.count('{alpha}') + item.count('{num}')
if num == 0:
self.priority += 1
self.queue.put((self.priority, item))
else:
self.queue.put((self.priority + num * 10000000, item))
def _scan(self, j):
self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]]
while not self.queue.empty():
try:
item = self.queue.get(timeout=1.0)[1]
self.scan_count += 1
except:
break
self._print_msg()
try:
if item.find('{alphnum}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789':
self.put_item(item.replace('{alphnum}', _letter, 1))
continue
elif item.find('{alpha}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz':
self.put_item(item.replace('{alpha}', _letter, 1))
continue
elif item.find('{num}') >= 0:
for _letter in '0123456789':
self.put_item(item.replace('{num}', _letter, 1))
continue
elif item.find('{next_sub}') >= 0:
for _ in self.next_subs:
self.queue.put((0, item.replace('{next_sub}', _, 1)))
continue
else:
sub = item
if sub in self.found_subs:
continue
cur_sub_domain = sub + '.' + self.target
_sub = sub.split('.')[-1]
try:
answers = self.resolvers[j].query(cur_sub_domain)
except dns.resolver.NoAnswer, e:
answers = self.ex_resolver.query(cur_sub_domain)
if answers:
self.found_subs.add(sub)
ips = ', '.join(sorted([answer.address for answer in answers]))
if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0']:
continue
if self.ignore_intranet and SubNameBrute.is_intranet(answers[0].address):
continue
try:
self.scan_count += 1
answers = self.resolvers[j].query(cur_sub_domain, 'cname')
cname = answers[0].target.to_unicode().rstrip('.')
if cname.endswith(self.target) and cname not in self.found_subs:
self.found_subs.add(cname)
cname_sub = cname[:len(cname) - len(self.target) - 1] # new sub
self.queue.put((0, cname_sub))
except:
pass
if (_sub, ips) not in self.ip_dict:
self.ip_dict[(_sub, ips)] = 1
else:
self.ip_dict[(_sub, ips)] += 1
if ips not in self.ip_dict:
self.ip_dict[ips] = 1
else:
self.ip_dict[ips] += 1
if self.ip_dict[(_sub, ips)] > 3 or self.ip_dict[ips] > 6:
continue
self.found_count += 1
msg = cur_sub_domain.ljust(30) + ips
self._print_msg(msg, _found_msg=True)
self._print_msg()
self.outfile.write(cur_sub_domain.ljust(30) + '\t' + ips + '\n')
self.outfile.flush()
try:
self.resolvers[j].query('lijiejietest.' + cur_sub_domain)
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e:
self.queue.put((999999999, '{next_sub}.' + sub))
except:
pass
except (dns.resolver.NXDOMAIN, dns.name.EmptyLabel) as e:
pass
except (dns.resolver.NoNameservers, dns.resolver.NoAnswer, dns.exception.Timeout) as e:
pass
class SubNameBrute(object):
def __init__(self, *params):
(
self.domain,
self.options,
self.process_num,
self.dns_servers,
self.next_subs,
self.scan_count,
self.found_count,
self.queue_size_array,
tmp_dir,
) = params
self.dns_count = len(self.dns_servers)
self.scan_count_local = 0
self.found_count_local = 0
self.resolvers = [
dns.resolver.Resolver(configure=False)
for _ in range(self.options.threads)
]
for r in self.resolvers:
r.lifetime = r.timeout = 10.0
self.queue = PriorityQueue()
self.priority = 0
self.ip_dict = {}
self.found_subs = set()
self.timeout_subs = {}
self.count_time = time.time()
self.outfile = open(
"%s/%s_part_%s.txt" % (tmp_dir, self.domain, self.process_num),
"w")
self.normal_names_set = set()
self.load_sub_names()
self.lock = RLock()
def load_sub_names(self):
normal_lines = []
wildcard_lines = []
wildcard_set = set()
regex_list = []
lines = set()
with open(self.options.file) as inFile:
for line in inFile.readlines():
sub = line.strip()
if not sub or sub in lines:
continue
lines.add(sub)
brace_count = sub.count("{")
if brace_count > 0:
wildcard_lines.append((brace_count, sub))
sub = sub.replace("{alphnum}", "[a-z0-9]")
sub = sub.replace("{alpha}", "[a-z]")
sub = sub.replace("{num}", "[0-9]")
if sub not in wildcard_set:
wildcard_set.add(sub)
regex_list.append("^" + sub + "$")
else:
normal_lines.append(sub)
self.normal_names_set.add(sub)
if regex_list:
pattern = "|".join(regex_list)
_regex = re.compile(pattern)
for line in normal_lines:
if _regex.search(line):
normal_lines.remove(line)
for _ in normal_lines[self.process_num::self.options.process]:
self.queue.put((0, _)) # priority set to 0
for _ in wildcard_lines[self.process_num::self.options.process]:
self.queue.put(_)
def scan(self, j):
self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]
] + self.dns_servers
while True:
try:
self.lock.acquire()
if time.time() - self.count_time > 1.0:
self.scan_count.value += self.scan_count_local
self.scan_count_local = 0
self.queue_size_array[
self.process_num] = self.queue.qsize()
if self.found_count_local:
self.found_count.value += self.found_count_local
self.found_count_local = 0
self.count_time = time.time()
self.lock.release()
brace_count, sub = self.queue.get(timeout=3.0)
if brace_count > 0:
brace_count -= 1
if sub.find("{next_sub}") >= 0:
for _ in self.next_subs:
self.queue.put((0, sub.replace("{next_sub}", _)))
if sub.find("{alphnum}") >= 0:
for _ in "abcdefghijklmnopqrstuvwxyz0123456789":
self.queue.put(
(brace_count, sub.replace("{alphnum}", _, 1)))
elif sub.find("{alpha}") >= 0:
for _ in "abcdefghijklmnopqrstuvwxyz":
self.queue.put(
(brace_count, sub.replace("{alpha}", _, 1)))
elif sub.find("{num}") >= 0:
for _ in "0123456789":
self.queue.put(
(brace_count, sub.replace("{num}", _, 1)))
continue
except gevent.queue.Empty as e:
break
try:
if sub in self.found_subs:
continue
self.scan_count_local += 1
cur_domain = sub + "." + self.domain
answers = self.resolvers[j].query(cur_domain)
if answers:
self.found_subs.add(sub)
ips = ", ".join(
sorted([answer.address for answer in answers]))
if ips in ["1.1.1.1", "127.0.0.1", "0.0.0.0", "0.0.0.1"]:
continue
if self.options.i and is_intranet(answers[0].address):
continue
try:
self.scan_count_local += 1
answers = self.resolvers[j].query(cur_domain, "cname")
cname = answers[0].target.to_unicode().rstrip(".")
if cname.endswith(
self.domain) and cname not in self.found_subs:
cname_sub = cname[:len(cname) - len(self.domain) -
1] # new sub
if cname_sub not in self.normal_names_set:
self.found_subs.add(cname)
self.queue.put((0, cname_sub))
except Exception as e:
pass
first_level_sub = sub.split(".")[-1]
if (first_level_sub, ips) not in self.ip_dict:
self.ip_dict[(first_level_sub, ips)] = 1
else:
self.ip_dict[(first_level_sub, ips)] += 1
if self.ip_dict[(first_level_sub, ips)] > 30:
continue
self.found_count_local += 1
self.outfile.write(
cur_domain.ljust(30) + "\t" + ips + "\n")
self.outfile.flush()
try:
self.scan_count_local += 1
self.resolvers[j].query("test-not-existed." +
cur_domain)
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e:
if self.queue.qsize() < 10000:
for _ in self.next_subs:
self.queue.put((0, _ + "." + sub))
else:
self.queue.put((1, "{next_sub}." + sub))
except Exception as e:
pass
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e:
pass
except dns.resolver.NoNameservers as e:
self.queue.put((0, sub)) # Retry
except dns.exception.Timeout as e:
self.timeout_subs[sub] = self.timeout_subs.get(sub, 0) + 1
if self.timeout_subs[sub] <= 2:
self.queue.put((0, sub)) # Retry
except Exception as e:
import traceback
traceback.print_exc()
with open("errors.log", "a") as errFile:
errFile.write("[%s] %s\n" % (type(e), str(e)))
def run(self):
threads = [
gevent.spawn(self.scan, i) for i in range(self.options.threads)
]
gevent.joinall(threads)
class BasicDataBridge(object):
"""Basic Bridge"""
def __init__(self, config):
super(BasicDataBridge, self).__init__()
defaults = deepcopy(DEFAULTS)
defaults.update(config['main'])
self.config = defaults
# Init config
for key, value in defaults.items():
setattr(self, key, value)
self.bridge_id = uuid.uuid4().hex
self.api_host = self.config.get('resources_api_server')
self.api_version = self.config.get('resources_api_version')
self.retrievers_params = self.config.get('retrievers_params')
self.storage_type = self.config['storage_config'].get(
'storage_type', 'couchdb')
self.worker_type = self.config['worker_config'].get(
'worker_type', 'basic_couchdb')
self.filter_type = self.config['filter_config'].get(
'filter_type', 'basic_couchdb')
# Check up_wait_sleep
up_wait_sleep = self.retrievers_params.get('up_wait_sleep')
if up_wait_sleep is not None and up_wait_sleep < 30:
raise DataBridgeConfigError(
'Invalid \'up_wait_sleep\' in \'retrievers_params\'. Value must be grater than 30.'
)
# Pools
self.workers_pool = gevent.pool.Pool(self.workers_max)
self.retry_workers_pool = gevent.pool.Pool(self.retry_workers_max)
self.filter_workers_pool = gevent.pool.Pool(self.filter_workers_count)
# Queues
if self.input_queue_size == -1:
self.input_queue = PriorityQueue()
else:
self.input_queue = PriorityQueue(self.input_queue_size)
if self.resource_items_queue_size == -1:
self.resource_items_queue = PriorityQueue()
else:
self.resource_items_queue = PriorityQueue(
self.resource_items_queue_size)
self.api_clients_queue = Queue()
# self.retry_api_clients_queue = Queue()
if self.retry_resource_items_queue_size == -1:
self.retry_resource_items_queue = PriorityQueue()
else:
self.retry_resource_items_queue = PriorityQueue(
self.retry_resource_items_queue_size)
if self.api_host != '' and self.api_host is not None:
api_host = urlparse(self.api_host)
if api_host.scheme == '' and api_host.netloc == '':
raise DataBridgeConfigError(
'Invalid \'resources_api_server\' url.')
else:
raise DataBridgeConfigError(
'In config dictionary empty or missing \'resources_api_server\''
)
# Connecting storage plugin
self.db = None
for entry_point in iter_entry_points(
'openprocurement.bridge.basic.storage_plugins',
self.storage_type):
plugin = entry_point.load()
self.db = plugin(self.config)
# Register handlers
handlers = self.config.get('handlers', [])
for entry_point in iter_entry_points(
'openprocurement.bridge.basic.handlers'):
if not handlers or entry_point.name in handlers:
plugin = entry_point.load()
PROCUREMENT_METHOD_TYPE_HANDLERS[entry_point.name] = plugin(
self.config, self.db)
if hasattr(self, 'filter_type'):
for entry_point in iter_entry_points(
'openprocurement.bridge.basic.filter_plugins',
self.filter_type):
self.filter_greenlet = entry_point.load()
for entry_point in iter_entry_points(
'openprocurement.bridge.basic.worker_plugins',
self.worker_type):
self.worker_greenlet = entry_point.load()
self.feeder = ResourceFeeder(host=self.config.get(
'public_resources_api_server', self.api_host),
version=self.api_version,
key='',
resource=self.config['resource'],
extra_params=self.extra_params,
retrievers_params=self.retrievers_params,
adaptive=True,
with_priority=True)
self.api_clients_info = {}
def create_api_client(self):
client_user_agent = self.user_agent + '/' + self.bridge_id
timeout = 0.1
while 1:
try:
api_client = APIClient(host_url=self.api_host,
user_agent=client_user_agent,
api_version=self.api_version,
key='',
resource=self.resource)
client_id = uuid.uuid4().hex
logger.info('Started api_client {}'.format(
api_client.session.headers['User-Agent']),
extra={'MESSAGE_ID': 'create_api_clients'})
api_client_dict = {
'id': client_id,
'client': api_client,
'request_interval': 0,
'not_actual_count': 0
}
self.api_clients_info[api_client_dict['id']] = {
'drop_cookies': False,
'request_durations': {},
'request_interval': 0,
'avg_duration': 0
}
self.api_clients_queue.put(api_client_dict)
break
except RequestFailed as e:
logger.error(
'Failed start api_client with status code {}'.format(
e.status_code),
extra={'MESSAGE_ID': 'exceptions'})
timeout = timeout * 2
logger.info(
'create_api_client will be sleep {} sec.'.format(timeout))
sleep(timeout)
except Exception as e:
logger.error('Failed start api client with error: {}'.format(
e.message),
extra={'MESSAGE_ID': 'exceptions'})
timeout = timeout * 2
logger.info(
'create_api_client will be sleep {} sec.'.format(timeout))
sleep(timeout)
def fill_api_clients_queue(self):
while self.api_clients_queue.qsize() < self.workers_min:
self.create_api_client()
def fill_input_queue(self):
# if not hasattr(self.db, 'filter'):
# self.input_queue = self.resource_items_queue
for resource_item in self.feeder.get_resource_items():
self.input_queue.put(resource_item)
logger.debug('Add to temp queue from sync: {} {} {}'.format(
self.resource[:-1], resource_item[1]['id'],
resource_item[1]['dateModified']),
extra={
'MESSAGE_ID': 'received_from_sync',
'TEMP_QUEUE_SIZE': self.input_queue.qsize()
})
def _get_average_requests_duration(self):
req_durations = []
delta = timedelta(seconds=self.perfomance_window)
current_date = datetime.now() - delta
for cid, info in self.api_clients_info.items():
if len(info['request_durations']) > 0:
if min(info['request_durations'].keys()) <= current_date:
info['grown'] = True
avg = round(
sum(info['request_durations'].values()) * 1.0 /
len(info['request_durations']), 3)
req_durations.append(avg)
info['avg_duration'] = avg
if len(req_durations) > 0:
return round(sum(req_durations) / len(req_durations),
3), req_durations
else:
return 0, req_durations
# TODO: Add logic for restart sync if last response grater than some values
# and no active tasks specific for resource
def queues_controller(self):
while True:
if (self.workers_pool.free_count() > 0
and (self.resource_items_queue.qsize() >
((float(self.resource_items_queue_size) / 100) *
self.workers_inc_threshold))):
self.create_api_client()
w = self.worker_greenlet.spawn(self.api_clients_queue,
self.resource_items_queue,
self.db, self.config,
self.retry_resource_items_queue,
self.api_clients_info)
self.workers_pool.add(w)
logger.info('Queue controller: Create main queue worker.')
elif (self.resource_items_queue.qsize() <
((float(self.resource_items_queue_size) / 100) *
self.workers_dec_threshold)):
if len(self.workers_pool) > self.workers_min:
wi = self.workers_pool.greenlets.pop()
wi.shutdown()
api_client_dict = self.api_clients_queue.get()
del self.api_clients_info[api_client_dict['id']]
logger.info('Queue controller: Kill main queue worker.')
filled_resource_items_queue = round(
self.resource_items_queue.qsize() /
(float(self.resource_items_queue_size) / 100), 2)
logger.info('Resource items queue filled on {} %'.format(
filled_resource_items_queue))
filled_retry_resource_items_queue \
= round(self.retry_resource_items_queue.qsize() / float(self.retry_resource_items_queue_size) / 100, 2)
logger.info('Retry resource items queue filled on {} %'.format(
filled_retry_resource_items_queue))
sleep(self.queues_controller_timeout)
def gevent_watcher(self):
self.perfomance_watcher()
# Check fill threads
input_threads = 1
if self.input_queue_filler.ready():
input_threads = 0
logger.error('Temp queue filler error: {}'.format(
self.input_queue_filler.exception.message),
extra={'MESSAGE_ID': 'exception'})
self.input_queue_filler = spawn(self.fill_input_queue)
logger.info('Input threads {}'.format(input_threads),
extra={'INPUT_THREADS': input_threads})
fill_threads = 1
if hasattr(self, 'queue_filter') and self.queue_filter.ready():
fill_threads = 0
logger.error('Fill thread error: {}'.format(
self.queue_filter.exception.message),
extra={'MESSAGE_ID': 'exception'})
self.queue_filter = self.filter_greenlet.spawn(
self.config, self.input_queue, self.resource_items_queue,
self.db)
logger.info('Filter threads {}'.format(fill_threads),
extra={'FILTER_THREADS': fill_threads})
main_threads = self.workers_max - self.workers_pool.free_count()
logger.info('Main threads {}'.format(main_threads),
extra={'MAIN_THREADS': main_threads})
if len(self.workers_pool) < self.workers_min:
for i in xrange(0, (self.workers_min - len(self.workers_pool))):
self.create_api_client()
w = self.worker_greenlet.spawn(self.api_clients_queue,
self.resource_items_queue,
self.db, self.config,
self.retry_resource_items_queue,
self.api_clients_info)
self.workers_pool.add(w)
logger.info('Watcher: Create main queue worker.')
retry_threads = self.retry_workers_max - self.retry_workers_pool.free_count(
)
logger.info('Retry threads {}'.format(retry_threads),
extra={'RETRY_THREADS': retry_threads})
if len(self.retry_workers_pool) < self.retry_workers_min:
for i in xrange(
0, self.retry_workers_min - len(self.retry_workers_pool)):
self.create_api_client()
w = self.worker_greenlet.spawn(self.api_clients_queue,
self.retry_resource_items_queue,
self.db, self.config,
self.retry_resource_items_queue,
self.api_clients_info)
self.retry_workers_pool.add(w)
logger.info('Watcher: Create retry queue worker.')
# Log queues size and API clients count
main_queue_size = self.resource_items_queue.qsize()
logger.info('Resource items queue size {}'.format(main_queue_size),
extra={'MAIN_QUEUE_SIZE': main_queue_size})
retry_queue_size = self.retry_resource_items_queue.qsize()
logger.info(
'Resource items retry queue size {}'.format(retry_queue_size),
extra={'RETRY_QUEUE_SIZE': retry_queue_size})
api_clients_count = len(self.api_clients_info)
logger.info('API Clients count: {}'.format(api_clients_count),
extra={'API_CLIENTS': api_clients_count})
def _calculate_st_dev(self, values):
if len(values) > 0:
avg = sum(values) * 1.0 / len(values)
variance = map(lambda x: (x - avg)**2, values)
avg_variance = sum(variance) * 1.0 / len(variance)
st_dev = math.sqrt(avg_variance)
return round(st_dev, 3)
else:
return 0
def _mark_bad_clients(self, dev):
# Mark bad api clients
for cid, info in self.api_clients_info.items():
if info.get('grown', False) and info['avg_duration'] > dev:
info['drop_cookies'] = True
logger.debug(
'Perfomance watcher: Mark client {} as bad, avg. request_duration is {} sec.'
.format(cid, info['avg_duration']),
extra={'MESSAGE_ID': 'marked_as_bad'})
elif info['avg_duration'] < dev and info['request_interval'] > 0:
info['drop_cookies'] = True
logger.debug(
'Perfomance watcher: Mark client {} as bad, request_interval is {} sec.'
.format(cid, info['request_interval']),
extra={'MESSAGE_ID': 'marked_as_bad'})
def perfomance_watcher(self):
avg_duration, values = self._get_average_requests_duration()
for _, info in self.api_clients_info.items():
delta = timedelta(seconds=self.perfomance_window +
self.watch_interval)
current_date = datetime.now() - delta
delete_list = []
for key in info['request_durations']:
if key < current_date:
delete_list.append(key)
for k in delete_list:
del info['request_durations'][k]
delete_list = []
st_dev = self._calculate_st_dev(values)
if len(values) > 0:
min_avg = min(values) * 1000
max_avg = max(values) * 1000
else:
max_avg = 0
min_avg = 0
dev = round(st_dev + avg_duration, 3)
logger.info('Perfomance watcher:\nREQUESTS_STDEV - {} sec.\n'
'REQUESTS_DEV - {} ms.\nREQUESTS_MIN_AVG - {} ms.\n'
'REQUESTS_MAX_AVG - {} ms.\nREQUESTS_AVG - {} sec.'.format(
round(st_dev, 3), dev, min_avg, max_avg, avg_duration),
extra={
'REQUESTS_DEV': dev * 1000,
'REQUESTS_MIN_AVG': min_avg,
'REQUESTS_MAX_AVG': max_avg,
'REQUESTS_AVG': avg_duration * 1000
})
self._mark_bad_clients(dev)
def run(self):
logger.info('Start Basic Bridge',
extra={'MESSAGE_ID': 'start_basic_bridge'})
logger.info('Start data sync...',
extra={'MESSAGE_ID': 'basic_bridge__data_sync'})
self.input_queue_filler = spawn(self.fill_input_queue)
if hasattr(self, 'filter_greenlet'):
self.queue_filter = self.filter_greenlet.spawn(
self.config, self.input_queue, self.resource_items_queue,
self.db)
else:
self.resource_items_queue = self.input_queue
spawn(self.queues_controller)
while True:
self.gevent_watcher()
sleep(self.watch_interval)
class PriorityBlockingQueue(object):
"""
带优先级的阻塞队列。
优先级数字越小，优先级越高。
插入元素：
* put: 向队列尾部插入一个元素，如果该队列已满，则一直阻塞。
* offer: 向队列尾部插入一个元素，插入成功返回True。插入失败返回False。
获取元素：
* poll: 获取并移除队列的头元素，若队列为空，则返回null。
* take: 获取并移除队列的头元素，若队列为空，则一直阻塞。
* peek：获取但不移除队列的头元素，若队列为空，则返回null
队列状态状态：
* qsize：获取队列中当前元素数量
* maxsize：获取队列的最大容量
"""
def __init__(self, maxsize: int = None):
"""
init
:param maxsize: 队列的最大容量
"""
self.__queue = PriorityQueue(maxsize=maxsize)
def put(self, item, priority: int = 200) -> None:
"""
向队列尾部插入一个元素，如果该队列已满，则一直阻塞。
:param item:
:param priority: 优先级
:return:
"""
while True:
try:
self.__queue.put(PriorityEntry(priority, item))
break
except Exception as e:
logger.debug("put data failed error -> {0}".format(e))
time.sleep(0.5)
def offer(self, item, priority: int = 200) -> bool:
"""
向队列尾部插入一个元素，插入成功返回True。插入失败返回False。
:param item: 元素
:param priority: 优先级
:return:
"""
try:
self.__queue.put(PriorityEntry(priority, item), block=False)
return True
except Exception as e:
logger.debug("offer data failed error -> {0}".format(e))
return False
def poll(self):
"""
获取并移除队列的头元素，若队列为空，则返回null。
:return:
"""
try:
return self.__queue.get(block=False).data
except Exception as e:
logger.debug("poll data failed error -> {0}".format(e))
return None
def take(self):
"""
获取并移除队列的头元素，若队列为空，则一直阻塞。
:return:
"""
while True:
try:
return self.__queue.get().data
except Exception as e:
logger.debug("take data failed error -> {0}".format(e))
time.sleep(0.5)
def peek(self):
"""
获取但不移除队列的头元素，若队列为空，则返回null
:return:
"""
try:
return self.__queue.peek(block=False).data
except Exception as e:
logger.debug("peek data failed error -> {0}".format(e))
return None
def qsize(self) -> int:
"""
获取队列中当前元素数量
:return:
"""
return self.__queue.qsize()
def maxsize(self) -> int:
"""
获取队列的最大容量
:return:
"""
return self.__queue.maxsize
class SubNameBrute:
"""
receive commandline args and do some initialization work
"""
def __init__(self, target, options):
self.start_time = time.time()
self.target = target.strip()
self.options = options
self.scan_count = self.found_count = 0
self.console_width = os.get_terminal_size()[0] - 2
# create dns resolver pool ~ workers
self.resolvers = [
dns.resolver.Resolver(configure=False)
for _ in range(options.threads)
]
for resolver in self.resolvers:
resolver.lifetime = resolver.timeout = 10.0
self.print_count = 0
self.STOP_ME = False
# load dns servers and check whether these dns servers works fine ?
self._load_dns_servers()
# load sub names
self.subs = [] # subs in file
self.goodsubs = [] # checks ok for further exploitation
self._load_subname('dict/subnames.txt', self.subs)
# load sub.sub names
self.subsubs = []
self._load_subname('dict/next_sub.txt', self.subsubs)
# results will save to target.txt
global path
path = os.path.join("results", target)
if not os.path.exists(path):
os.makedirs(path)
self.outfile = open('%s/%s.txt' % (path, target), 'w')
self.ip_dict = set() #
self.found_sub = set()
# task queue
self.queue = PriorityQueue()
for sub in self.subs:
self.queue.put(sub)
"""
Load DNS Servers(ip saved in file), and check whether the DNS servers works fine
"""
def _load_dns_servers(self):
print('[*] Validate DNS servers ...')
self.dns_servers = []
# create a process pool for checking DNS servers, the number is your processors(cores) * 2, just change it!
processors = cpu_count() * 2
pool = Pool(processors)
# read dns ips and check one by one
for server in open('dict/dns_servers.txt').readlines():
server = server.strip()
if server:
pool.apply_async(self._test_server, (server, ))
pool.join() # waiting for process finish
self.dns_count = len(self.dns_servers)
sys.stdout.write('\n')
dns_info = '[+] Found {} available DNS Servers in total'.format(
self.dns_count)
print(dns_info)
if self.dns_count == 0:
print('[ERROR] No DNS Servers available.')
sys.exit(-1)
"""
test these dns servers whether works fine
"""
def _test_server(self, server):
# create a dns resolver and set timeout
resolver = dns.resolver.Resolver()
resolver.lifetime = resolver.timeout = 10.0
try:
resolver.nameservers = [server]
answers = resolver.query('public-dns-a.baidu.com')
if answers[0].address != '180.76.76.76':
raise Exception('incorrect DNS response')
self.dns_servers.append(server)
except:
self._print_msg('[-] Check DNS Server %s <Fail> Found %s' %
(server.ljust(16), len(self.dns_servers)))
self._print_msg('[+] Check DNS Server %s < OK > Found %s' %
(server.ljust(16), len(self.dns_servers)))
"""
load sub names in dict/*.txt, one function would be enough
file for read, subname_list for saving sub names
"""
def _load_subname(self, file, subname_list):
self._print_msg('[*] Load sub names ...')
with open(file) as f:
for line in f:
sub = line.strip()
if sub and sub not in subname_list:
tmp_set = {sub}
"""
in case of the sub names which contains the following expression
and replace them {alphnum}, {alpha}, {num} with character and num
"""
while len(tmp_set) > 0:
item = tmp_set.pop()
if item.find('{alphnum}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789':
tmp_set.add(
item.replace('{alphnum}', _letter, 1))
elif item.find('{alpha}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz':
tmp_set.add(item.replace(
'{alpha}', _letter, 1))
elif item.find('{num}') >= 0:
for _letter in '0123456789':
tmp_set.add(item.replace('{num}', _letter, 1))
elif item not in subname_list:
subname_list.append(item)
"""
for better presentation of brute force results, not really matters ...
"""
def _print_msg(self, _msg=None, _found_msg=False):
if _msg is None:
self.print_count += 1
if self.print_count < 100:
return
self.print_count = 0
msg = '%s Found| %s Groups| %s scanned in %.1f seconds' % (
self.found_count, self.queue.qsize(), self.scan_count,
time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) +
msg)
elif _msg.startswith('[+] Check DNS Server'):
sys.stdout.write('\r' + _msg + ' ' *
(self.console_width - len(_msg)))
else:
sys.stdout.write('\r' + _msg + ' ' *
(self.console_width - len(_msg)) + '\n')
if _found_msg:
msg = '%s Found| %s Groups| %s scanned in %.1f seconds' % (
self.found_count, self.queue.qsize(), self.scan_count,
time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) +
msg)
sys.stdout.flush()
def _print_domain(self, msg):
console_width = os.get_terminal_size()[0]
msg = '\r' + msg + ' ' * (console_width - len(msg))
# msg = '\033[0;31;47m%s{}\033[0m'.format(msg)
sys.stdout.write(msg)
def _print_progress(self):
"""
显示扫描进度,显示更美观
"""
msg = '\033[0;31;47m%s\033[0m found | %s remaining | %s scanned in %.2f seconds' % \
(self.found_count, self.queue.qsize(), self.scan_count, time.time()- self.start_time)
console_width = os.get_terminal_size()[0]
out = '\r' + ' ' * int((console_width - len(msg)) / 2) + msg
sys.stdout.write(out)
"""
important : assign task to resolvers
"""
def _scan(self, j):
self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]]
while not self.queue.empty():
sub = self.queue.get(timeout=1.0)
self.scan_count += 1
try:
cur_sub_domain = sub + '.' + self.target
answers = self.resolvers[j].query(cur_sub_domain)
except:
continue
if answers:
ips = ', '.join(sorted([answer.address for answer in answers]))
# exclude : intranet or kept addresses
if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0', '0.0.0.1']:
continue
if SubNameBrute.is_intranet(answers[0].address):
continue
self.found_sub.add(cur_sub_domain)
for answer in answers:
self.ip_dict.add(answer.address)
if sub not in self.goodsubs:
self.goodsubs.append(sub)
self.found_count += 1
ip_info = '{} \t {}'.format(cur_sub_domain, ips)
# print(ip_info)
self.outfile.write(cur_sub_domain + '\t' + ips + '\n')
self._print_domain(ip_info)
sys.stdout.flush()
self._print_progress()
sys.stdout.flush()
@staticmethod
def is_intranet(ip):
ret = ip.split('.')
if len(ret) != 4:
return True
if ret[0] == '10':
return True
if ret[0] == '172' and 16 <= int(ret[1]) <= 32:
return True
if ret[0] == '192' and ret[1] == '168':
return True
return False
"""
assign task to threads ...
"""
def run(self):
threads = [
gevent.spawn(self._scan, i) for i in range(self.options.threads)
]
print('[*] Initializing %d threads' % self.options.threads)
try:
gevent.joinall(threads)
except KeyboardInterrupt as e:
msg = '[WARNING] User aborted.'
sys.stdout.write('\r' + msg + ' ' *
(self.console_width - len(msg)) + '\n\r')
sys.stdout.flush()
class SubNameBrute:
def __init__(self, target, options):
self.start_time = time.time()
self.target = target.strip()
self.options = options
self.ignore_intranet = options.i
self.scan_count = self.found_count = 0
self.console_width = getTerminalSize()[0] - 2
self.resolvers = [
dns.resolver.Resolver(configure=False)
for _ in range(options.threads)
]
for _ in self.resolvers:
_.lifetime = _.timeout = 10.0
self.print_count = 0
self.STOP_ME = False
self._load_dns_servers()
self._load_next_sub()
self.queue = PriorityQueue()
self.priority = 0
self._load_sub_names()
if options.output:
outfile = options.output
else:
_name = os.path.basename(self.options.file).replace('subnames', '')
if _name != '.txt':
_name = '_' + _name
outfile = target + _name if not options.full_scan else target + '_full' + _name
self.outfile = open(outfile, 'w')
self.ip_dict = {}
self.found_subs = set()
self.ex_resolver = dns.resolver.Resolver(configure=False)
self.ex_resolver.nameservers = self.dns_servers
self.result_lines = []
self.result_domains = []
self.result_ips = []
def _load_dns_servers(self):
print '[+] Validate DNS servers ...'
self.dns_servers = []
pool = Pool(30)
# change file location
for server in open('lijiejie/' + 'dict/dns_servers.txt').xreadlines():
server = server.strip()
if server:
pool.apply_async(self._test_server, (server, ))
pool.join()
self.dns_count = len(self.dns_servers)
sys.stdout.write('\n')
print '[+] Found %s available DNS Servers in total' % self.dns_count
if self.dns_count == 0:
print '[ERROR] No DNS Servers available.'
sys.exit(-1)
def _test_server(self, server):
resolver = dns.resolver.Resolver()
resolver.lifetime = resolver.timeout = 10.0
try:
resolver.nameservers = [server]
answers = resolver.query(
'public-dns-a.baidu.com') # test lookup a existed domain
if answers[0].address != '180.76.76.76':
raise Exception('incorrect DNS response')
try:
resolver.query(
'test.bad.dns.lijiejie.com') # Non-existed domain test
with open('bad_dns_servers.txt', 'a') as f:
f.write(server + '\n')
self._print_msg('[+] Bad DNS Server found %s' % server)
except:
self.dns_servers.append(server)
self._print_msg('[+] Check DNS Server %s < OK > Found %s' %
(server.ljust(16), len(self.dns_servers)))
except:
self._print_msg('[+] Check DNS Server %s <Fail> Found %s' %
(server.ljust(16), len(self.dns_servers)))
def _load_sub_names(self):
self._print_msg('[+] Load sub names ...')
if self.options.full_scan and self.options.file == 'subnames.txt':
_file = 'lijiejie/' + 'dict/subnames_full.txt'
else:
if os.path.exists(self.options.file):
_file = self.options.file
elif os.path.exists('lijiejie/' + 'dict/%s' % self.options.file):
_file = 'lijiejie/' + 'dict/%s' % self.options.file
else:
self._print_msg('[ERROR] Names file not exists: %s' %
self.options.file)
exit(-1)
if self.options.debug:
_file = 'lijiejie/' + 'dict/debug.txt'
if not os.path.exists(_file):
self._print_msg('[ERROR] Names file not exists: %s' %
self.options.file)
exit(-1)
normal_lines = []
wildcard_lines = []
wildcard_list = []
regex_list = []
lines = set()
with open(_file) as f:
for line in f.xreadlines():
sub = line.strip()
if not sub or sub in lines:
continue
lines.add(sub)
if sub.find('{alphnum}') >= 0 or sub.find(
'{alpha}') >= 0 or sub.find('{num}') >= 0:
wildcard_lines.append(sub)
sub = sub.replace('{alphnum}', '[a-z0-9]')
sub = sub.replace('{alpha}', '[a-z]')
sub = sub.replace('{num}', '[0-9]')
if sub not in wildcard_list:
wildcard_list.append(sub)
regex_list.append('^' + sub + '$')
else:
normal_lines.append(sub)
pattern = '|'.join(regex_list)
if pattern:
_regex = re.compile(pattern)
if _regex:
for line in normal_lines[:]:
if _regex.search(line):
normal_lines.remove(line)
for item in normal_lines:
self.priority += 1
self.queue.put((self.priority, item))
for item in wildcard_lines:
self.queue.put((88888888, item))
def _load_next_sub(self):
self._print_msg('[+] Load next level subs ...')
self.next_subs = []
_set = set()
_file = 'lijiejie/' + 'dict/next_sub.txt' if not self.options.full_scan else 'dict/next_sub_full.txt'
with open(_file) as f:
for line in f:
sub = line.strip()
if sub and sub not in self.next_subs:
tmp_set = {sub}
while len(tmp_set) > 0:
item = tmp_set.pop()
if item.find('{alphnum}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789':
tmp_set.add(
item.replace('{alphnum}', _letter, 1))
elif item.find('{alpha}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz':
tmp_set.add(item.replace(
'{alpha}', _letter, 1))
elif item.find('{num}') >= 0:
for _letter in '0123456789':
tmp_set.add(item.replace('{num}', _letter, 1))
elif item not in _set:
_set.add(item)
self.next_subs.append(item)
def _print_msg(self, _msg=None, _found_msg=False):
if _msg is None:
self.print_count += 1
if self.print_count < 100:
return
self.print_count = 0
msg = '%s Found| %s Groups| %s scanned in %.1f seconds' % (
self.found_count, self.queue.qsize(), self.scan_count,
time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) +
msg)
elif _msg.startswith('[+] Check DNS Server'):
sys.stdout.write('\r' + _msg + ' ' *
(self.console_width - len(_msg)))
else:
sys.stdout.write('\r' + _msg + ' ' *
(self.console_width - len(_msg)) + '\n')
if _found_msg:
msg = '%s Found| %s Groups| %s scanned in %.1f seconds' % (
self.found_count, self.queue.qsize(), self.scan_count,
time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) +
msg)
sys.stdout.flush()
@staticmethod
def is_intranet(ip):
ret = ip.split('.')
if len(ret) != 4:
return True
if ret[0] == '10':
return True
if ret[0] == '172' and 16 <= int(ret[1]) <= 32:
return True
if ret[0] == '192' and ret[1] == '168':
return True
return False
def put_item(self, item):
num = item.count('{alphnum}') + item.count('{alpha}') + item.count(
'{num}')
if num == 0:
self.priority += 1
self.queue.put((self.priority, item))
else:
self.queue.put((self.priority + num * 10000000, item))
def _scan(self, j):
self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]]
while not self.queue.empty():
try:
item = self.queue.get(timeout=1.0)[1]
self.scan_count += 1
except:
break
self._print_msg()
try:
if item.find('{alphnum}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789':
self.put_item(item.replace('{alphnum}', _letter, 1))
continue
elif item.find('{alpha}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz':
self.put_item(item.replace('{alpha}', _letter, 1))
continue
elif item.find('{num}') >= 0:
for _letter in '0123456789':
self.put_item(item.replace('{num}', _letter, 1))
continue
elif item.find('{next_sub}') >= 0:
for _ in self.next_subs:
self.queue.put((0, item.replace('{next_sub}', _, 1)))
continue
else:
sub = item
if sub in self.found_subs:
continue
cur_sub_domain = sub + '.' + self.target
_sub = sub.split('.')[-1]
try:
answers = self.resolvers[j].query(cur_sub_domain)
except dns.resolver.NoAnswer, e:
answers = self.ex_resolver.query(cur_sub_domain)
if answers:
self.found_subs.add(sub)
ips = ', '.join(
sorted([answer.address for answer in answers]))
if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0']:
continue
if self.ignore_intranet and SubNameBrute.is_intranet(
answers[0].address):
continue
try:
self.scan_count += 1
answers = self.resolvers[j].query(
cur_sub_domain, 'cname')
cname = answers[0].target.to_unicode().rstrip('.')
if cname.endswith(
self.target) and cname not in self.found_subs:
self.found_subs.add(cname)
cname_sub = cname[:len(cname) - len(self.target) -
1] # new sub
self.queue.put((0, cname_sub))
except:
pass
if (_sub, ips) not in self.ip_dict:
self.ip_dict[(_sub, ips)] = 1
else:
self.ip_dict[(_sub, ips)] += 1
if ips not in self.ip_dict:
self.ip_dict[ips] = 1
else:
self.ip_dict[ips] += 1
if self.ip_dict[(_sub, ips)] > 3 or self.ip_dict[ips] > 6:
continue
self.found_count += 1
msg = cur_sub_domain.ljust(30) + ips
self._print_msg(msg, _found_msg=True)
self._print_msg()
# TODO: close write file
self.outfile.write(
cur_sub_domain.ljust(30) + '\t' + ips + '\n')
self.outfile.flush()
self.result_lines.append(
cur_sub_domain.ljust(30) + '\t' + ips)
self.result_domains.append(cur_sub_domain)
self.result_ips.extend(ips.split(", "))
try:
self.resolvers[j].query('lijiejietest.' +
cur_sub_domain)
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e:
self.queue.put((999999999, '{next_sub}.' + sub))
except:
pass
except (dns.resolver.NXDOMAIN, dns.name.EmptyLabel) as e:
pass
except (dns.resolver.NoNameservers, dns.resolver.NoAnswer,
dns.exception.Timeout) as e:
pass
class SubNameBrute(object):
def __init__(self, *params):
self.domain, self.options, self.process_num, self.dns_servers, self.next_subs, \
self.scan_count, self.found_count, self.queue_size_array, tmp_dir = params
self.dns_count = len(self.dns_servers)
self.scan_count_local = 0
self.found_count_local = 0
self.resolvers = [dns.resolver.Resolver(configure=False) for _ in range(self.options.threads)]
for r in self.resolvers:
r.lifetime = 6.0
r.timeout = 10.0
self.queue = PriorityQueue()
self.priority = 0
self.ip_dict = {}
self.found_subs = set()
self.cert_subs = set()
self.timeout_subs = {}
self.no_server_subs = {}
self.count_time = time.time()
self.outfile = open('%s/%s_part_%s.txt' % (tmp_dir, self.domain, self.process_num), 'w')
self.normal_names_set = set()
self.load_sub_names()
self.lock = RLock()
self.threads_status = ['1'] * self.options.threads
def load_sub_names(self):
normal_lines = []
wildcard_lines = []
wildcard_set = set()
regex_list = []
lines = set()
with open(self.options.file) as inFile:
for line in inFile.readlines():
sub = line.strip()
if not sub or sub in lines:
continue
lines.add(sub)
brace_count = sub.count('{')
if brace_count > 0:
wildcard_lines.append((brace_count, sub))
sub = sub.replace('{alphnum}', '[a-z0-9]')
sub = sub.replace('{alpha}', '[a-z]')
sub = sub.replace('{num}', '[0-9]')
if sub not in wildcard_set:
wildcard_set.add(sub)
regex_list.append('^' + sub + '$')
else:
normal_lines.append(sub)
self.normal_names_set.add(sub)
if regex_list:
pattern = '|'.join(regex_list)
_regex = re.compile(pattern)
for line in normal_lines:
if _regex.search(line):
normal_lines.remove(line)
for _ in normal_lines[self.process_num::self.options.process]:
self.queue.put((0, _)) # priority set to 0
for _ in wildcard_lines[self.process_num::self.options.process]:
self.queue.put(_)
def update_counter(self):
while True:
if '1' not in self.threads_status:
return
self.scan_count.value += self.scan_count_local
self.scan_count_local = 0
self.queue_size_array[self.process_num] = self.queue.qsize()
if self.found_count_local:
self.found_count.value += self.found_count_local
self.found_count_local = 0
self.count_time = time.time()
gevent.sleep(0.5)
def check_https_alt_names(self, domain):
try:
x509 = reqs.OpenSSL.crypto.load_certificate(
reqs.OpenSSL.crypto.FILETYPE_PEM,
reqs.ssl.get_server_certificate((domain, 443))
)
for item in reqs.get_subj_alt_name(x509):
if item[0].upper() == 'DNS':
name = item[1].lower()
if name.endswith(self.domain):
sub = name[:len(name) - len(self.domain) - 1] # new sub
sub = sub.replace('*', '')
sub = sub.strip('.')
if sub and sub not in self.found_subs and \
sub not in self.normal_names_set and sub not in self.cert_subs:
self.cert_subs.add(sub)
self.queue.put((0, sub))
except Exception as e:
pass
def scan(self, j):
self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]]
if self.dns_count > 1:
while True:
s = random.choice(self.dns_servers)
if s != self.dns_servers[j % self.dns_count]:
self.resolvers[j].nameservers.append(s)
break
empty_counter = 0
while True:
try:
brace_count, sub = self.queue.get_nowait()
self.threads_status[j] = '1'
empty_counter = 0
except gevent.queue.Empty as e:
empty_counter += 1
if empty_counter > 10:
self.threads_status[j] = '0'
if '1' not in self.threads_status:
break
else:
gevent.sleep(0.1)
continue
if brace_count > 0:
brace_count -= 1
if sub.find('{next_sub}') >= 0:
for _ in self.next_subs:
self.queue.put((0, sub.replace('{next_sub}', _)))
if sub.find('{alphnum}') >= 0:
for _ in 'abcdefghijklmnopqrstuvwxyz0123456789':
self.queue.put((brace_count, sub.replace('{alphnum}', _, 1)))
elif sub.find('{alpha}') >= 0:
for _ in 'abcdefghijklmnopqrstuvwxyz':
self.queue.put((brace_count, sub.replace('{alpha}', _, 1)))
elif sub.find('{num}') >= 0:
for _ in '0123456789':
self.queue.put((brace_count, sub.replace('{num}', _, 1)))
continue
try:
if sub in self.found_subs:
continue
self.scan_count_local += 1
cur_domain = sub + '.' + self.domain
answers = self.resolvers[j].query(cur_domain)
if answers:
self.found_subs.add(sub)
ips = ', '.join(sorted([answer.address for answer in answers]))
invalid_ip_found = False
for answer in answers:
if answer.address in ['1.1.1.1', '127.0.0.1', '0.0.0.0', '0.0.0.1']:
invalid_ip_found = True
if invalid_ip_found:
continue
if self.options.i and is_intranet(answers[0].address):
continue
try:
cname = str(answers.canonical_name)[:-1]
if cname != cur_domain and cname.endswith(self.domain):
cname_sub = cname[:len(cname) - len(self.domain) - 1] # new sub
if cname_sub not in self.found_subs and cname_sub not in self.normal_names_set:
self.queue.put((0, cname_sub))
except Exception as e:
print(e)
pass
try:
self.scan_count_local += 1
answers = self.resolvers[j].query(cur_domain, 'cname')
cname = answers[0].target.to_unicode().rstrip('.')
if cname.endswith(self.domain) and cname not in self.found_subs:
cname_sub = cname[:len(cname) - len(self.domain) - 1] # new sub
if cname_sub not in self.normal_names_set:
# self.found_subs.add(cname)
self.queue.put((0, cname_sub))
except Exception as e:
pass
first_level_sub = sub.split('.')[-1]
max_found = 20
if self.options.w:
first_level_sub = ''
max_found = 3
if (first_level_sub, ips) not in self.ip_dict:
self.ip_dict[(first_level_sub, ips)] = 1
else:
self.ip_dict[(first_level_sub, ips)] += 1
if self.ip_dict[(first_level_sub, ips)] > max_found:
continue
self.found_count_local += 1
self.outfile.write(cur_domain.ljust(30) + '\t' + ips + '\n')
self.outfile.flush()
if not self.options.no_cert_check:
with gevent.Timeout(10, False):
self.check_https_alt_names(cur_domain)
try:
self.scan_count_local += 1
self.resolvers[j].query('lijiejie-test-not-existed.' + cur_domain)
except (dns.resolver.NXDOMAIN, ) as e: # dns.resolver.NoAnswer
if self.queue.qsize() < 20000:
for _ in self.next_subs:
self.queue.put((0, _ + '.' + sub))
else:
self.queue.put((1, '{next_sub}.' + sub))
except Exception as e:
pass
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e:
pass
except dns.resolver.NoNameservers as e:
self.no_server_subs[sub] = self.no_server_subs.get(sub, 0) + 1
if self.no_server_subs[sub] <= 3:
self.queue.put((0, sub)) # Retry
except dns.exception.Timeout as e:
self.timeout_subs[sub] = self.timeout_subs.get(sub, 0) + 1
if self.timeout_subs[sub] <= 1:
self.queue.put((0, sub)) # Retry
except Exception as e:
import traceback
traceback.print_exc()
with open('errors.log', 'a') as errFile:
errFile.write('[%s] %s\n' % (type(e), str(e)))
def run(self):
threads = [gevent.spawn(self.scan, i) for i in range(self.options.threads)]
threads.insert(0, gevent.spawn(self.update_counter))
gevent.joinall(threads)
class EdgeDataBridge(object):
"""Edge Bridge"""
def __init__(self, config):
super(EdgeDataBridge, self).__init__()
self.config = config
self.workers_config = {}
self.bridge_id = uuid.uuid4().hex
self.api_host = self.config_get('resources_api_server')
self.api_version = self.config_get('resources_api_version')
self.retrievers_params = self.config_get('retrievers_params')
# Check up_wait_sleep
up_wait_sleep = self.retrievers_params.get('up_wait_sleep')
if up_wait_sleep is not None and up_wait_sleep < 30:
raise DataBridgeConfigError('Invalid \'up_wait_sleep\' in '
'\'retrievers_params\'. Value must be '
'grater than 30.')
# Workers settings
for key in WORKER_CONFIG:
self.workers_config[key] = (self.config_get(key)
or WORKER_CONFIG[key])
# Init config
for key in DEFAULTS:
setattr(self, key, self.config_get(key) or DEFAULTS[key])
# Pools
self.workers_pool = gevent.pool.Pool(self.workers_max)
self.retry_workers_pool = gevent.pool.Pool(self.retry_workers_max)
self.filter_workers_pool = gevent.pool.Pool(self.filter_workers_count)
# Queues
if self.input_queue_size == -1:
self.input_queue = PriorityQueue()
else:
self.input_queue = PriorityQueue(self.input_queue_size)
if self.resource_items_queue_size == -1:
self.resource_items_queue = PriorityQueue()
else:
self.resource_items_queue = PriorityQueue(
self.resource_items_queue_size)
self.api_clients_queue = Queue()
# self.retry_api_clients_queue = Queue()
if self.retry_resource_items_queue_size == -1:
self.retry_resource_items_queue = PriorityQueue()
else:
self.retry_resource_items_queue = PriorityQueue(
self.retry_resource_items_queue_size)
if self.api_host != '' and self.api_host is not None:
api_host = urlparse(self.api_host)
if api_host.scheme == '' and api_host.netloc == '':
raise DataBridgeConfigError(
'Invalid \'tenders_api_server\' url.')
else:
raise DataBridgeConfigError('In config dictionary empty or missing'
' \'tenders_api_server\'')
self.db = prepare_couchdb(self.couch_url, self.db_name, logger)
db_url = self.couch_url + '/' + self.db_name
prepare_couchdb_views(db_url, self.workers_config['resource'], logger)
self.server = Server(self.couch_url,
session=Session(retry_delays=range(10)))
self.view_path = '_design/{}/_view/by_dateModified'.format(
self.workers_config['resource'])
extra_params = {
'mode': self.retrieve_mode,
'limit': self.resource_items_limit
}
self.feeder = ResourceFeeder(host=self.api_host,
version=self.api_version,
key='',
resource=self.workers_config['resource'],
extra_params=extra_params,
retrievers_params=self.retrievers_params,
adaptive=True,
with_priority=True)
self.api_clients_info = {}
def config_get(self, name):
try:
return self.config.get('main').get(name)
except AttributeError:
raise DataBridgeConfigError('In config dictionary missed section'
' \'main\'')
def create_api_client(self):
client_user_agent = self.user_agent + '/' + self.bridge_id
timeout = 0.1
while 1:
try:
api_client = APIClient(
host_url=self.api_host,
user_agent=client_user_agent,
api_version=self.api_version,
key='',
resource=self.workers_config['resource'])
client_id = uuid.uuid4().hex
logger.info('Started api_client {}'.format(
api_client.session.headers['User-Agent']),
extra={'MESSAGE_ID': 'create_api_clients'})
api_client_dict = {
'id': client_id,
'client': api_client,
'request_interval': 0,
'not_actual_count': 0
}
self.api_clients_info[api_client_dict['id']] = {
'drop_cookies': False,
'request_durations': {},
'request_interval': 0,
'avg_duration': 0
}
self.api_clients_queue.put(api_client_dict)
break
except RequestFailed as e:
logger.error(
'Failed start api_client with status code {}'.format(
e.status_code),
extra={'MESSAGE_ID': 'exceptions'})
timeout = timeout * 2
logger.info(
'create_api_client will be sleep {} sec.'.format(timeout))
sleep(timeout)
except Exception as e:
logger.error('Failed start api client with error: {}'.format(
e.message),
extra={'MESSAGE_ID': 'exceptions'})
timeout = timeout * 2
logger.info(
'create_api_client will be sleep {} sec.'.format(timeout))
sleep(timeout)
def fill_api_clients_queue(self):
while self.api_clients_queue.qsize() < self.workers_min:
self.create_api_client()
def fill_input_queue(self):
for resource_item in self.feeder.get_resource_items():
self.input_queue.put(resource_item)
logger.debug('Add to temp queue from sync: {} {} {}'.format(
self.workers_config['resource'][:-1], resource_item[1]['id'],
resource_item[1]['dateModified']),
extra={
'MESSAGE_ID': 'received_from_sync',
'TEMP_QUEUE_SIZE': self.input_queue.qsize()
})
def send_bulk(self, input_dict, priority_cache):
sleep_before_retry = 2
for i in xrange(0, 3):
try:
logger.debug('Send check bulk: {}'.format(len(input_dict)),
extra={'CHECK_BULK_LEN': len(input_dict)})
start = time()
rows = self.db.view(self.view_path, keys=input_dict.values())
end = time() - start
logger.debug('Duration bulk check: {} sec.'.format(end),
extra={'CHECK_BULK_DURATION': end * 1000})
resp_dict = {k.id: k.key for k in rows}
break
except (IncompleteRead, Exception) as e:
logger.error('Error while send bulk {}'.format(e.message),
extra={'MESSAGE_ID': 'exceptions'})
if i == 2:
raise e
sleep(sleep_before_retry)
sleep_before_retry *= 2
for item_id, date_modified in input_dict.items():
if item_id in resp_dict and date_modified == resp_dict[item_id]:
logger.debug('Skipped {} {}: In db exist newest.'.format(
self.workers_config['resource'][:-1], item_id),
extra={'MESSAGE_ID': 'skipped'})
elif ((1, item_id) not in self.resource_items_queue.queue
and (1000, item_id) not in self.resource_items_queue.queue):
self.resource_items_queue.put(
(priority_cache[item_id], item_id))
logger.debug(
'Put to main queue {}: {}'.format(
self.workers_config['resource'][:-1], item_id),
extra={'MESSAGE_ID': 'add_to_resource_items_queue'})
else:
logger.debug(
'Skipped {} {}: In queue exist with same id'.format(
self.workers_config['resource'][:-1], item_id),
extra={'MESSAGE_ID': 'skipped'})
def fill_resource_items_queue(self):
start_time = datetime.now()
input_dict = {}
priority_cache = {}
while True:
# Get resource_item from temp queue
if not self.input_queue.empty():
priority, resource_item = self.input_queue.get()
else:
timeout = self.bulk_query_interval -\
(datetime.now() - start_time).total_seconds()
if timeout > self.bulk_query_interval:
timeout = self.bulk_query_interval
try:
priority, resource_item = self.input_queue.get(
timeout=timeout)
except Empty:
resource_item = None
# Add resource_item to bulk
if resource_item is not None:
logger.debug('Add to input_dict {}'.format(
resource_item['id']))
input_dict[resource_item['id']] = resource_item['dateModified']
priority_cache[resource_item['id']] = priority
if (len(input_dict) >= self.bulk_query_limit
or (datetime.now() - start_time).total_seconds() >=
self.bulk_query_interval):
if len(input_dict) > 0:
self.send_bulk(input_dict, priority_cache)
input_dict = {}
priority_cache = {}
start_time = datetime.now()
def resource_items_filter(self, r_id, r_date_modified):
try:
local_document = self.db.get(r_id)
if local_document:
if local_document['dateModified'] < r_date_modified:
return True
else:
return False
else:
return True
except Exception as e:
logger.error(
'Filter error: Error while getting {} {} from couchdb: '
'{}'.format(self.workers_config['resource'][:-1], r_id,
e.message),
extra={'MESSAGE_ID': 'exceptions'})
return True
def _get_average_requests_duration(self):
req_durations = []
delta = timedelta(seconds=self.perfomance_window)
current_date = datetime.now() - delta
for cid, info in self.api_clients_info.items():
if len(info['request_durations']) > 0:
if min(info['request_durations'].keys()) <= current_date:
info['grown'] = True
avg = round(
sum(info['request_durations'].values()) * 1.0 /
len(info['request_durations']), 3)
req_durations.append(avg)
info['avg_duration'] = avg
if len(req_durations) > 0:
return round(sum(req_durations) / len(req_durations),
3), req_durations
else:
return 0, req_durations
# TODO: Add logic for restart sync if last response grater than some values
# and no active tasks specific for resource
def queues_controller(self):
while True:
if (self.workers_pool.free_count() > 0
and (self.resource_items_queue.qsize() >
((float(self.resource_items_queue_size) / 100) *
self.workers_inc_threshold))):
self.create_api_client()
w = ResourceItemWorker.spawn(self.api_clients_queue,
self.resource_items_queue,
self.db, self.workers_config,
self.retry_resource_items_queue,
self.api_clients_info)
self.workers_pool.add(w)
logger.info('Queue controller: Create main queue worker.')
elif (self.resource_items_queue.qsize() <
((float(self.resource_items_queue_size) / 100) *
self.workers_dec_threshold)):
if len(self.workers_pool) > self.workers_min:
wi = self.workers_pool.greenlets.pop()
wi.shutdown()
api_client_dict = self.api_clients_queue.get()
del self.api_clients_info[api_client_dict['id']]
logger.info('Queue controller: Kill main queue worker.')
filled_resource_items_queue = round(
self.resource_items_queue.qsize() /
(float(self.resource_items_queue_size) / 100), 2)
logger.info('Resource items queue filled on {} %'.format(
filled_resource_items_queue))
filled_retry_resource_items_queue \
= round(self.retry_resource_items_queue.qsize() / float(
self.retry_resource_items_queue_size) / 100, 2)
logger.info('Retry resource items queue filled on {} %'.format(
filled_retry_resource_items_queue))
sleep(self.queues_controller_timeout)
def gevent_watcher(self):
self.perfomance_watcher()
for t in self.server.tasks():
if (t['type'] == 'indexer' and t['database'] == self.db_name
and t.get('design_document', None) == '_design/{}'.format(
self.workers_config['resource'])):
logger.info(
'Watcher: Waiting for end of view indexing. Current'
' progress: {} %'.format(t['progress']))
# Check fill threads
input_threads = 1
if self.input_queue_filler.ready():
input_threads = 0
logger.error('Temp queue filler error: {}'.format(
self.input_queue_filler.exception.message),
extra={'MESSAGE_ID': 'exception'})
self.input_queue_filler = spawn(self.fill_input_queue)
logger.info('Input threads {}'.format(input_threads),
extra={'INPUT_THREADS': input_threads})
fill_threads = 1
if self.filler.ready():
fill_threads = 0
logger.error('Fill thread error: {}'.format(
self.filler.exception.message),
extra={'MESSAGE_ID': 'exception'})
self.filler = spawn(self.fill_resource_items_queue)
logger.info('Filter threads {}'.format(fill_threads),
extra={'FILTER_THREADS': fill_threads})
main_threads = self.workers_max - self.workers_pool.free_count()
logger.info('Main threads {}'.format(main_threads),
extra={'MAIN_THREADS': main_threads})
if len(self.workers_pool) < self.workers_min:
for i in xrange(0, (self.workers_min - len(self.workers_pool))):
w = ResourceItemWorker.spawn(self.api_clients_queue,
self.resource_items_queue,
self.db, self.workers_config,
self.retry_resource_items_queue,
self.api_clients_info)
self.workers_pool.add(w)
logger.info('Watcher: Create main queue worker.')
self.create_api_client()
retry_threads = self.retry_workers_max -\
self.retry_workers_pool.free_count()
logger.info('Retry threads {}'.format(retry_threads),
extra={'RETRY_THREADS': retry_threads})
if len(self.retry_workers_pool) < self.retry_workers_min:
for i in xrange(
0, self.retry_workers_min - len(self.retry_workers_pool)):
self.create_api_client()
w = ResourceItemWorker.spawn(self.api_clients_queue,
self.retry_resource_items_queue,
self.db, self.workers_config,
self.retry_resource_items_queue,
self.api_clients_info)
self.retry_workers_pool.add(w)
logger.info('Watcher: Create retry queue worker.')
# Log queues size and API clients count
main_queue_size = self.resource_items_queue.qsize()
logger.info('Resource items queue size {}'.format(main_queue_size),
extra={'MAIN_QUEUE_SIZE': main_queue_size})
retry_queue_size = self.retry_resource_items_queue.qsize()
logger.info(
'Resource items retry queue size {}'.format(retry_queue_size),
extra={'RETRY_QUEUE_SIZE': retry_queue_size})
api_clients_count = len(self.api_clients_info)
logger.info('API Clients count: {}'.format(api_clients_count),
extra={'API_CLIENTS': api_clients_count})
def _calculate_st_dev(self, values):
if len(values) > 0:
avg = sum(values) * 1.0 / len(values)
variance = map(lambda x: (x - avg)**2, values)
avg_variance = sum(variance) * 1.0 / len(variance)
st_dev = math.sqrt(avg_variance)
return round(st_dev, 3)
else:
return 0
def _mark_bad_clients(self, dev):
# Mark bad api clients
for cid, info in self.api_clients_info.items():
if info.get('grown', False) and info['avg_duration'] > dev:
info['drop_cookies'] = True
logger.debug('Perfomance watcher: Mark client {} as bad, avg.'
' request_duration is {} sec.'.format(
cid, info['avg_duration']),
extra={'MESSAGE_ID': 'marked_as_bad'})
elif info['avg_duration'] < dev and info['request_interval'] > 0:
info['drop_cookies'] = True
logger.debug('Perfomance watcher: Mark client {} as bad,'
' request_interval is {} sec.'.format(
cid, info['request_interval']),
extra={'MESSAGE_ID': 'marked_as_bad'})
def perfomance_watcher(self):
avg_duration, values = self._get_average_requests_duration()
for _, info in self.api_clients_info.items():
delta = timedelta(seconds=self.perfomance_window +
self.watch_interval)
current_date = datetime.now() - delta
delete_list = []
for key in info['request_durations']:
if key < current_date:
delete_list.append(key)
for k in delete_list:
del info['request_durations'][k]
delete_list = []
st_dev = self._calculate_st_dev(values)
if len(values) > 0:
min_avg = min(values) * 1000
max_avg = max(values) * 1000
else:
max_avg = 0
min_avg = 0
dev = round(st_dev + avg_duration, 3)
logger.info('Perfomance watcher:\nREQUESTS_STDEV - {} sec.\n'
'REQUESTS_DEV - {} ms.\nREQUESTS_MIN_AVG - {} ms.\n'
'REQUESTS_MAX_AVG - {} ms.\nREQUESTS_AVG - {} sec.'.format(
round(st_dev, 3), dev, min_avg, max_avg, avg_duration),
extra={
'REQUESTS_DEV': dev * 1000,
'REQUESTS_MIN_AVG': min_avg,
'REQUESTS_MAX_AVG': max_avg,
'REQUESTS_AVG': avg_duration * 1000
})
self._mark_bad_clients(dev)
def run(self):
logger.info('Start Edge Bridge',
extra={'MESSAGE_ID': 'edge_bridge_start_bridge'})
logger.info('Start data sync...',
extra={'MESSAGE_ID': 'edge_bridge__data_sync'})
self.input_queue_filler = spawn(self.fill_input_queue)
self.filler = spawn(self.fill_resource_items_queue)
spawn(self.queues_controller)
while True:
self.gevent_watcher()
sleep(self.watch_interval)
class Scanner:
def __init__(self):
self.start_time = time.time()
self.queue = PriorityQueue()
self.history = []
self.total_count = 0
self.scan_count = 0
self._load_target()
self.outfile = open("log.log", 'w')
self.console_width = getTerminalSize()[0] - 2
def _print_msg(self, _msg=None, _found_msg=False):
if _msg is None:
msg = '%s TotalCount| %s Scanned in %.2f seconds' % (
self.total_count, self.total_count - self.queue.qsize(),
time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) +
msg)
else:
sys.stdout.write('\r' + _msg + ' ' *
(self.console_width - len(_msg)) + '\n')
self.outfile.write(_msg + '\n')
self.outfile.flush()
if _found_msg:
msg = '%s TotalCount| %s Scanned in %.2f seconds' % (
self.total_count, self.total_count - self.queue.qsize(),
time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) +
msg)
sys.stdout.flush()
def _load_target(self):
print('[+] Read targets ...')
target_file = raw_input("Target File :")
with open(target_file) as f:
for line in f.xreadlines():
target = line.strip()
self.queue.put(target)
print("TotalCount is %d" % self.queue.qsize())
self.total_count = self.queue.qsize()
print("Now scanning ...")
def _scan(self, case):
while not self.queue.empty():
target = self.queue.get()
if case == "1":
self.vulnCheck(target)
if case == "2":
self.s2_045(target)
if case == "3":
self.headers(target)
if case == "4":
self.weakfile(target)
if case == "5":
self.portscan_c(target)
def vulnCheck(self, target):
if ":2375" in target:
try:
res = requests.head("http://" + str(target) +
"/containers/json",
timeout=2)
if res.headers['Content-Type'] == 'application/json':
self._print_msg(target + "==> docker api Vuln", True)
else:
self._print_msg()
except:
self._print_msg()
self._print_msg()
if ":27017" in target:
ip, port = target.split(":")
try:
socket.setdefaulttimeout(3)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((ip, int(port)))
data = binascii.a2b_hex(
"3a000000a741000000000000d40700000000000061646d696e2e24636d640000000000ffffffff130000001069736d6173746572000100000000"
)
s.send(data)
result = s.recv(1024)
if "ismaster" in result:
getlog_data = binascii.a2b_hex(
"480000000200000000000000d40700000000000061646d696e2e24636d6400000000000100000021000000026765744c6f670010000000737461727475705761726e696e67730000"
)
s.send(getlog_data)
result = s.recv(1024)
if "totalLinesWritten" in result:
self._print_msg(target + "==> mongodb Vuln", True)
except Exception as e:
pass
if ":6379" in target:
ip, port = target.split(":")
try:
socket.setdefaulttimeout(3)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((ip, int(port)))
s.send("INFO\r\n")
result = s.recv(1024)
if "redis_version" in result:
self._print_msg(target + "==> redis Vuln", True)
elif "Authentication" in result:
for pass_ in ['123456', 'redis', 'pass', 'password']:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((ip, int(port)))
s.send("AUTH %s\r\n" % (pass_))
result = s.recv(1024)
if '+OK' in result:
self._print_msg(
target + "==> redis pass Vuln :" + pass_,
True)
except Exception as e:
pass
if ":11211" in target:
ip, port = target.split(":")
try:
socket.setdefaulttimeout(3)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((ip, int(port)))
s.send("stats\r\n")
result = s.recv(1024)
if "STAT version" in result:
self._print_msg(target + "==> memcache Vuln", True)
except Exception as e:
pass
if ":9200" in target:
try:
res = requests.head("http://" + str(target) +
"/_rvier/_search",
timeout=2)
if res.status_code == 200:
self._print_msg(target + "==> Elasticsearch Vuln", True)
else:
self._print_msg()
except:
self._print_msg()
self._print_msg()
def headers(self, target):
try:
res = requests.head("http://" + str(target), timeout=1)
self._print_msg(target + "==>" + str(res.headers), True)
except:
self._print_msg()
self._print_msg()
def s2_045(self, target):
try:
data = {"image": " "}
headers = {
"User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36",
"Content-Type":
"%{#context['com.opensymphony.xwork2.dispatcher.HttpServletResponse'].addHeader('vul','s2-045')}.multtargetart/form-data"
}
req = requests.post("http://" + str(target),
data=data,
headers=headers)
if req.headers["vul"] == "s2-045":
self._print_msg(target + "==>" + "S2-045 Vuln", True)
except:
self._print_msg()
self._print_msg()
def weakfile(self, target):
weaklist = ["robots.txt", "/i.php", "/phpinfo.php"]
for weakfile in weaklist:
try:
res = requests.head("http://" + str(target) + weakfile,
timeout=1)
if res.status_code == 200:
if ("User-agent" in res.content) or ("phpinfo"
in res.content):
self._print_msg("http://" + target + weakfile, True)
except:
self._print_msg()
self._print_msg()
def portscan_c(self, target):
import socket
ip = socket.gethostbyname(target)
ports = [1433, 2375, 3306, 6379, 9200, 11211, 27017]
ip = ip.split(".")
ipc = ip[0] + "." + ip[1] + "." + ip[2] + "."
if ipc in self.history:
return
else:
self.history.append(ipc)
for port in ports:
for i in range(255):
try:
cs = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
cs.settimeout(float(2.5))
address = (ipc + str(i), int(port))
status = cs.connect_ex((address))
if status == 0:
self._print_msg(ipc + str(i) + ":" + str(port), True)
except Exception as e:
pass
finally:
cs.close()
self._print_msg()
def run(self, case):
threads = [gevent.spawn(self._scan, case) for i in xrange(1000)]
class SubNameBrute:
def __init__(self, target, args, process_num, dns_servers, next_subs,
scan_count, found_count, queue_size_list, tmp_dir):
self.target = target.strip()
self.args = args
self.process_num = process_num
self.dns_servers = dns_servers
self.dns_count = len(dns_servers)
self.next_subs = next_subs
self.scan_count = scan_count
self.scan_count_local = 0
self.found_count = found_count
self.found_count_local = 0
self.queue_size_list = queue_size_list
self.resolvers = [
dns.resolver.Resolver(configure=False) for _ in range(args.threads)
]
for _r in self.resolvers:
_r.lifetime = _r.timeout = 6.0
self.queue = PriorityQueue()
self.item_index = 0
self.priority = 0
self._load_sub_names()
self.ip_dict = {}
self.found_subs = set()
self.ex_resolver = dns.resolver.Resolver(configure=False)
self.ex_resolver.nameservers = dns_servers
self.local_time = time.time()
self.outfile = open(
'%s/%s_part_%s.txt' % (tmp_dir, target, process_num), 'w')
self.outfile_html = open('tmp/%s_html_%s.txt' % (target, process_num),
'w')
def _load_sub_names(self):
if self.args.full_scan and self.args.file == 'subnames.txt':
_file = 'Dict/subnames_full.txt'
else:
if os.path.exists(self.args.file):
_file = self.args.file
elif os.path.exists('Dict/%s' % self.args.file):
_file = 'Dict/%s' % self.args.file
else:
print_msg('[ERROR] Names file not found: %s' % self.args.file)
exit(-1)
normal_lines = []
lines = set()
with open(_file) as f:
for line in f.xreadlines():
sub = line.strip()
if not sub or sub in lines:
continue
lines.add(sub)
normal_lines.append(sub)
for item in normal_lines[self.process_num::self.args.process]:
self.priority += 1
self.queue.put((self.priority, item))
def _scan(self, j):
self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]]
while not self.queue.empty():
try:
item = self.queue.get()[1]
self.scan_count_local += 1
if time.time() - self.local_time > 3.0:
self.scan_count.value += self.scan_count_local
self.scan_count_local = 0
self.queue_size_list[self.process_num] = self.queue.qsize()
except Exception as e:
break
try:
if item.find('{next_sub}') >= 0:
for _ in self.next_subs:
self.queue.put((0, item.replace('{next_sub}', _, 1)))
continue
else:
sub = item
if sub in self.found_subs:
continue
cur_sub_domain = sub + '.' + self.target
_sub = sub.split('.')[-1]
try:
answers = self.resolvers[j].query(cur_sub_domain)
except dns.resolver.NoAnswer, e:
answers = self.ex_resolver.query(cur_sub_domain)
if answers:
self.found_subs.add(sub)
ips = ', '.join(
sorted([answer.address for answer in answers]))
if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0']:
continue
if self.args.intranet and is_intranet(answers[0].address):
continue
try:
self.scan_count_local += 1
answers = self.resolvers[j].query(
cur_sub_domain, 'cname')
cname = answers[0].target.to_unicode().rstrip('.')
if cname.endswith(
self.target) and cname not in self.found_subs:
self.found_subs.add(cname)
cname_sub = cname[:len(cname) - len(self.target) -
1]
self.queue.put((0, cname_sub))
except:
pass
if (_sub, ips) not in self.ip_dict:
self.ip_dict[(_sub, ips)] = 1
else:
self.ip_dict[(_sub, ips)] += 1
if self.ip_dict[(_sub, ips)] > 30:
continue
self.found_count_local += 1
if time.time() - self.local_time > 3.0:
self.found_count.value += self.found_count_local
self.found_count_local = 0
self.queue_size_list[
self.process_num] = self.queue.qsize()
self.local_time = time.time()
self.outfile_html.write(
PrintHtml.Sub_html_print(cur_sub_domain, ips))
self.outfile_html.flush()
self.outfile.write(
cur_sub_domain.ljust(30) + '\t' + ips + '\n')
self.outfile.flush()
try:
self.resolvers[j].query('myzxcghelloha.' +
cur_sub_domain)
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e:
self.queue.put((999999999, '{next_sub}.' + sub))
except:
pass
except (dns.resolver.NXDOMAIN, dns.name.EmptyLabel) as e:
pass
except (dns.resolver.NoNameservers, dns.resolver.NoAnswer,
dns.exception.Timeout) as e:
pass
class SyncClient:
idle = idle
backward_class = BackwardWorker
forward_class = ForwardWorker
def __init__(self,
host_url,
resource,
auth=None,
params={},
headers=None,
retrievers_params=DEFAULT_RETRIEVERS_PARAMS,
adaptive=False,
with_priority=False):
LOGGER.info(f'Init SyncClient for resource {resource}')
self.host = host_url
self.auth = auth
self.resource = resource
self.adaptive = adaptive
self.headers = headers
self.params = params
self.retrievers_params = retrievers_params
self.queue = PriorityQueue(maxsize=retrievers_params['queue_size'])
def init_clients(self):
self.backward_client = ResourceClient(self.host, self.resource,
self.params, self.auth,
self.headers)
self.forward_client = ResourceClient(self.host, self.resource,
self.params, self.auth,
self.headers)
def handle_response_data(self, data):
for resource_item in data:
self.queue.put(PrioritizedItem(1, resource_item))
def worker_watcher(self):
while True:
if time() - self.heartbeat > DEFAULT_FORWARD_HEARTBEAT:
self.restart_sync()
LOGGER.warning(
'Restart sync, reason: Last response from workers greater than 15 min ago.'
)
sleep(300)
def start_sync(self):
LOGGER.info('Start sync...')
data = self.backward_client.get_resource_items(self.params)
self.handle_response_data(data[f'{self.resource}s'])
forward_params = deepcopy(self.params)
forward_params.update({
k: v[0]
for k, v in parse.parse_qs(parse.urlparse(
data.links.prev).query).items()
})
backward_params = deepcopy(self.params)
backward_params.update({
k: v[0]
for k, v in parse.parse_qs(parse.urlparse(
data.links.next).query).items()
})
self.forward_worker = self.forward_class(sync_client=self,
client=self.forward_client,
params=forward_params)
self.backward_worker = self.backward_class(sync_client=self,
client=self.backward_client,
params=backward_params)
self.workers = [self.forward_worker, self.backward_worker]
for worker in self.workers:
worker.start()
self.heartbeat = time()
self.watcher = spawn(self.worker_watcher)
def restart_sync(self):
"""
Restart retrieving from OCDS API.
"""
LOGGER.info('Restart workers')
for worker in self.workers:
worker.kill()
self.watcher.kill()
self.init_clients()
self.start_sync()
def get_resource_items(self):
self.init_clients()
self.start_sync()
while True:
if self.forward_worker.check() or self.backward_worker.check():
self.restart_sync()
while not self.queue.empty():
LOGGER.debug(f'Sync queue size: {self.queue.qsize()}',
extra={'SYNC_QUEUE_SIZE': self.queue.qsize()})
LOGGER.debug('Yield resource item',
extra={'MESSAGE_ID': 'sync_yield'})
item = self.queue.get()
yield item.data
LOGGER.debug(f'Sync queue size: {self.queue.qsize()}',
extra={'SYNC_QUEUE_SIZE': self.queue.qsize()})
try:
self.queue.peek(block=True, timeout=0.1)
except Empty:
pass
class SubNameBrute(object):
def __init__(self, *params):
self.domain, self.options, self.process_num, self.dns_servers, self.next_subs, \
self.scan_count, self.found_count, self.queue_size_array, tmp_dir = params
self.dns_count = len(self.dns_servers)
self.scan_count_local = 0
self.found_count_local = 0
self.resolvers = [dns.resolver.Resolver(configure=False) for _ in range(self.options.threads)]
for r in self.resolvers:
r.lifetime = 4
r.timeout = 10.0
self.queue = PriorityQueue()
self.priority = 0
self.ip_dict = {}
self.found_subs = set()
self.timeout_subs = {}
self.count_time = time.time()
self.outfile = open('%s/%s_part_%s.txt' % (tmp_dir, self.domain, self.process_num), 'w')
self.normal_names_set = set()
self.load_sub_names()
self.lock = RLock()
self.threads_status = ['1'] * self.options.threads
def load_sub_names(self):
normal_lines = []
wildcard_lines = []
wildcard_set = set()
regex_list = []
lines = set()
with open(self.options.file) as inFile:
for line in inFile.readlines():
sub = line.strip()
if not sub or sub in lines:
continue
lines.add(sub)
brace_count = sub.count('{')
if brace_count > 0:
wildcard_lines.append((brace_count, sub))
sub = sub.replace('{alphnum}', '[a-z0-9]')
sub = sub.replace('{alpha}', '[a-z]')
sub = sub.replace('{num}', '[0-9]')
if sub not in wildcard_set:
wildcard_set.add(sub)
regex_list.append('^' + sub + '$')
else:
normal_lines.append(sub)
self.normal_names_set.add(sub)
if regex_list:
pattern = '|'.join(regex_list)
_regex = re.compile(pattern)
for line in normal_lines:
if _regex.search(line):
normal_lines.remove(line)
for _ in normal_lines[self.process_num::self.options.process]:
self.queue.put((0, _)) # priority set to 0
for _ in wildcard_lines[self.process_num::self.options.process]:
self.queue.put(_)
def scan(self, j):
self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]] + self.dns_servers
while True:
try:
if time.time() - self.count_time > 1.0:
self.lock.acquire()
self.scan_count.value += self.scan_count_local
self.scan_count_local = 0
self.queue_size_array[self.process_num] = self.queue.qsize()
if self.found_count_local:
self.found_count.value += self.found_count_local
self.found_count_local = 0
self.count_time = time.time()
self.lock.release()
brace_count, sub = self.queue.get_nowait()
self.threads_status[j] = '1'
if brace_count > 0:
brace_count -= 1
if sub.find('{next_sub}') >= 0:
for _ in self.next_subs:
self.queue.put((0, sub.replace('{next_sub}', _)))
if sub.find('{alphnum}') >= 0:
for _ in 'abcdefghijklmnopqrstuvwxyz0123456789':
self.queue.put((brace_count, sub.replace('{alphnum}', _, 1)))
elif sub.find('{alpha}') >= 0:
for _ in 'abcdefghijklmnopqrstuvwxyz':
self.queue.put((brace_count, sub.replace('{alpha}', _, 1)))
elif sub.find('{num}') >= 0:
for _ in '0123456789':
self.queue.put((brace_count, sub.replace('{num}', _, 1)))
continue
except gevent.queue.Empty as e:
self.threads_status[j] = '0'
gevent.sleep(0.5)
if '1' not in self.threads_status:
break
else:
continue
try:
if sub in self.found_subs:
continue
self.scan_count_local += 1
cur_domain = sub + '.' + self.domain
answers = self.resolvers[j].query(cur_domain)
if answers:
self.found_subs.add(sub)
ips = ', '.join(sorted([answer.address for answer in answers]))
if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0', '0.0.0.1']:
continue
if self.options.i and is_intranet(answers[0].address):
continue
try:
self.scan_count_local += 1
answers = self.resolvers[j].query(cur_domain, 'cname')
cname = answers[0].target.to_unicode().rstrip('.')
if cname.endswith(self.domain) and cname not in self.found_subs:
cname_sub = cname[:len(cname) - len(self.domain) - 1] # new sub
if cname_sub not in self.normal_names_set:
self.found_subs.add(cname)
self.queue.put((0, cname_sub))
except Exception as e:
pass
first_level_sub = sub.split('.')[-1]
max_found = 20
if self.options.w:
first_level_sub = ''
max_found = 3
if (first_level_sub, ips) not in self.ip_dict:
self.ip_dict[(first_level_sub, ips)] = 1
else:
self.ip_dict[(first_level_sub, ips)] += 1
if self.ip_dict[(first_level_sub, ips)] > max_found:
continue
self.found_count_local += 1
self.outfile.write(cur_domain.ljust(30) + '\t' + ips + '\n')
self.outfile.flush()
try:
self.scan_count_local += 1
self.resolvers[j].query('lijiejie-test-not-existed.' + cur_domain)
except (dns.resolver.NXDOMAIN, ) as e: # dns.resolver.NoAnswer
if self.queue.qsize() < 50000:
for _ in self.next_subs:
self.queue.put((0, _ + '.' + sub))
else:
self.queue.put((1, '{next_sub}.' + sub))
except Exception as e:
pass
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e:
pass
except dns.resolver.NoNameservers as e:
self.queue.put((0, sub)) # Retry
except dns.exception.Timeout as e:
self.timeout_subs[sub] = self.timeout_subs.get(sub, 0) + 1
if self.timeout_subs[sub] <= 1:
self.queue.put((0, sub)) # Retry
except Exception as e:
import traceback
traceback.print_exc()
with open('errors.log', 'a') as errFile:
errFile.write('[%s] %s\n' % (type(e), str(e)))
def run(self):
threads = [gevent.spawn(self.scan, i) for i in range(self.options.threads)]
gevent.joinall(threads)
class SubNameBrute:
def __init__(self, target, options, process_num, dns_servers, cdns,next_subs,
scan_count, found_count, queue_size_list, tmp_dir):
self.target = target.strip()
self.options = options
self.process_num = process_num
self.dns_servers = dns_servers
self.cdns = cdns
self.dns_count = len(dns_servers)
self.next_subs = next_subs
self.scan_count = scan_count
self.scan_count_local = 0
self.found_count = found_count
self.found_count_local = 0
self.queue_size_list = queue_size_list
self.resolvers = [dns.resolver.Resolver(configure=False) for _ in range(options.threads)]
for _r in self.resolvers:
_r.lifetime = _r.timeout = 6.0
self.queue = PriorityQueue()
self.item_index = 0
self.priority = 0
self._load_sub_names()
self.ip_dict = {}
self.found_subs = set()
self.ex_resolver = dns.resolver.Resolver(configure=False)
self.ex_resolver.nameservers = dns_servers
self.local_time = time.time()
self.outfile = open('%s/%s_part_%s.txt' % (tmp_dir, target, process_num), 'w')
def _load_sub_names(self):
if self.options.full_scan and self.options.file == 'subnames.txt':
_file = 'dict/subnames_full.txt'
else:
if os.path.exists(self.options.file):
_file = self.options.file
elif os.path.exists('dict/%s' % self.options.file):
_file = 'dict/%s' % self.options.file
else:
print_msg('[ERROR] Names file not found: %s' % self.options.file)
exit(-1)
normal_lines = []
wildcard_lines = []
wildcard_list = []
regex_list = []
lines = set()
with open(_file) as f:
for line in f.xreadlines():
sub = line.strip()
if not sub or sub in lines:
continue
lines.add(sub)
if sub.find('{alphnum}') >= 0 or sub.find('{alpha}') >= 0 or sub.find('{num}') >= 0:
wildcard_lines.append(sub)
sub = sub.replace('{alphnum}', '[a-z0-9]')
sub = sub.replace('{alpha}', '[a-z]')
sub = sub.replace('{num}', '[0-9]')
if sub not in wildcard_list:
wildcard_list.append(sub)
regex_list.append('^' + sub + '$')
else:
normal_lines.append(sub)
if regex_list:
pattern = '|'.join(regex_list)
_regex = re.compile(pattern)
for line in normal_lines[:]:
if _regex.search(line):
normal_lines.remove(line)
for item in normal_lines[self.process_num::self.options.process]:
self.priority += 1
self.queue.put((self.priority, item))
for item in wildcard_lines[self.process_num::self.options.process]:
self.queue.put((88888888, item))
def put_item(self, item):
num = item.count('{alphnum}') + item.count('{alpha}') + item.count('{num}')
if num == 0:
self.priority += 1
self.queue.put((self.priority, item))
else:
self.queue.put((self.priority + num * 10000000, item))
def check_cdn(self, cname):
'''
bTrue = True
bFound = False
i = 0
while bTrue:
cdn = self.cdns[i]
i += 1
if (cdn in cname) or (i == len(self.cdns)):
if (cdn in cname): bFound = True
bTrue = False
return bFound
'''
for cdn in self.cdns:
if cdn in cname:
return True
return False
def _scan(self, j):
self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]]
while not self.queue.empty():
try:
item = self.queue.get(timeout=3.0)[1]
self.scan_count_local += 1
if time.time() - self.local_time > 3.0:
self.scan_count.value += self.scan_count_local
self.scan_count_local = 0
self.queue_size_list[self.process_num] = self.queue.qsize()
except Exception as e:
break
try:
if item.find('{alphnum}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789':
self.put_item(item.replace('{alphnum}', _letter, 1))
continue
elif item.find('{alpha}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz':
self.put_item(item.replace('{alpha}', _letter, 1))
continue
elif item.find('{num}') >= 0:
for _letter in '0123456789':
self.put_item(item.replace('{num}', _letter, 1))
continue
elif item.find('{next_sub}') >= 0:
for _ in self.next_subs:
self.queue.put((0, item.replace('{next_sub}', _, 1)))
continue
else:
sub = item
if sub in self.found_subs:
continue
cur_sub_domain = sub + '.' + self.target
_sub = sub.split('.')[-1]
try:
answers = self.resolvers[j].query(cur_sub_domain)
except dns.resolver.NoAnswer, e:
answers = self.ex_resolver.query(cur_sub_domain)
if answers:
ans = self.resolvers[j].query(cur_sub_domain,'cname')
cname = ans[0].target.to_unicode().rstrip('.')
if self.check_cdn(cname):
continue
self.found_subs.add(sub)
ips = ', '.join(sorted([answer.address for answer in answers]))
if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0']:
continue
if self.options.i and is_intranet(answers[0].address):
continue
try:
self.scan_count_local += 1
answers = self.resolvers[j].query(cur_sub_domain, 'cname')
cname = answers[0].target.to_unicode().rstrip('.')
if cname.endswith(self.target) and cname not in self.found_subs:
self.found_subs.add(cname)
cname_sub = cname[:len(cname) - len(self.target) - 1] # new sub
self.queue.put((0, cname_sub))
except:
pass
if (_sub, ips) not in self.ip_dict:
self.ip_dict[(_sub, ips)] = 1
else:
self.ip_dict[(_sub, ips)] += 1
if self.ip_dict[(_sub, ips)] > 30:
continue
self.found_count_local += 1
if time.time() - self.local_time > 3.0:
self.found_count.value += self.found_count_local
self.found_count_local = 0
self.queue_size_list[self.process_num] = self.queue.qsize()
self.local_time = time.time()
msg = cur_sub_domain.ljust(30) + ips
# print_msg(msg, line_feed=True)
self.outfile.write(cur_sub_domain.ljust(30) + '\t' + ips + '\n')
self.outfile.flush()
try:
self.resolvers[j].query('lijiejietest.' + cur_sub_domain)
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e:
self.queue.put((999999999, '{next_sub}.' + sub))
except:
pass
except (dns.resolver.NXDOMAIN, dns.name.EmptyLabel) as e:
pass
except (dns.resolver.NoNameservers, dns.resolver.NoAnswer, dns.exception.Timeout) as e:
pass
class SubNameBrute:
def __init__(self, target, options, process_num, dns_servers, next_subs,
scan_count, found_count, queue_size_list, tmp_dir):
self.target = target.strip()
self.options = options
self.process_num = process_num
self.dns_servers = dns_servers
self.dns_count = len(dns_servers)
self.next_subs = next_subs
self.scan_count = scan_count
self.scan_count_local = 0
self.found_count = found_count
self.found_count_local = 0
self.queue_size_list = queue_size_list
self.resolvers = [dns.resolver.Resolver(configure=False) for _ in range(options.threads)]
for _r in self.resolvers:
_r.lifetime = _r.timeout = 6.0
self.queue = PriorityQueue()
self.item_index = 0
self.priority = 0
self._load_sub_names()
self.ip_dict = {}
self.found_subs = set()
self.ex_resolver = dns.resolver.Resolver(configure=False)
self.ex_resolver.nameservers = dns_servers
self.local_time = time.time()
self.outfile = open('%s/%s_part_%s.txt' % (tmp_dir, target, process_num), 'w')
def _load_sub_names(self):
if self.options.full_scan and self.options.file == 'subnames.txt':
_file = 'dict/subnames_full.txt'
else:
if os.path.exists(self.options.file):
_file = self.options.file
elif os.path.exists('dict/%s' % self.options.file):
_file = 'dict/%s' % self.options.file
else:
print_msg('[ERROR] Names file not found: %s' % self.options.file)
exit(-1)
normal_lines = []
wildcard_lines = []
wildcard_list = []
regex_list = []
lines = set()
with open(_file) as f:
for line in f.xreadlines():
sub = line.strip()
if not sub or sub in lines:
continue
lines.add(sub)
if sub.find('{alphnum}') >= 0 or sub.find('{alpha}') >= 0 or sub.find('{num}') >= 0:
wildcard_lines.append(sub)
sub = sub.replace('{alphnum}', '[a-z0-9]')
sub = sub.replace('{alpha}', '[a-z]')
sub = sub.replace('{num}', '[0-9]')
if sub not in wildcard_list:
wildcard_list.append(sub)
regex_list.append('^' + sub + '$')
else:
normal_lines.append(sub)
if regex_list:
pattern = '|'.join(regex_list)
_regex = re.compile(pattern)
for line in normal_lines[:]:
if _regex.search(line):
normal_lines.remove(line)
for item in normal_lines[self.process_num::self.options.process]:
self.priority += 1
self.queue.put((self.priority, item))
for item in wildcard_lines[self.process_num::self.options.process]:
self.queue.put((88888888, item))
def put_item(self, item):
num = item.count('{alphnum}') + item.count('{alpha}') + item.count('{num}')
if num == 0:
self.priority += 1
self.queue.put((self.priority, item))
else:
self.queue.put((self.priority + num * 10000000, item))
def _scan(self, j):
self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]]
while not self.queue.empty():
try:
item = self.queue.get(timeout=3.0)[1]
self.scan_count_local += 1
if time.time() - self.local_time > 3.0:
self.scan_count.value += self.scan_count_local
self.scan_count_local = 0
self.queue_size_list[self.process_num] = self.queue.qsize()
except Exception as e:
break
try:
if item.find('{alphnum}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789':
self.put_item(item.replace('{alphnum}', _letter, 1))
continue
elif item.find('{alpha}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz':
self.put_item(item.replace('{alpha}', _letter, 1))
continue
elif item.find('{num}') >= 0:
for _letter in '0123456789':
self.put_item(item.replace('{num}', _letter, 1))
continue
elif item.find('{next_sub}') >= 0:
for _ in self.next_subs:
self.queue.put((0, item.replace('{next_sub}', _, 1)))
continue
else:
sub = item
if sub in self.found_subs:
continue
cur_sub_domain = sub + '.' + self.target
_sub = sub.split('.')[-1]
try:
answers = self.resolvers[j].query(cur_sub_domain)
except dns.resolver.NoAnswer, e:
answers = self.ex_resolver.query(cur_sub_domain)
if answers:
self.found_subs.add(sub)
ips = ', '.join(sorted([answer.address for answer in answers]))
if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0']:
continue
if self.options.i and is_intranet(answers[0].address):
continue
try:
self.scan_count_local += 1
answers = self.resolvers[j].query(cur_sub_domain, 'cname')
cname = answers[0].target.to_unicode().rstrip('.')
if cname.endswith(self.target) and cname not in self.found_subs:
self.found_subs.add(cname)
cname_sub = cname[:len(cname) - len(self.target) - 1] # new sub
self.queue.put((0, cname_sub))
except:
pass
if (_sub, ips) not in self.ip_dict:
self.ip_dict[(_sub, ips)] = 1
else:
self.ip_dict[(_sub, ips)] += 1
if self.ip_dict[(_sub, ips)] > 30:
continue
self.found_count_local += 1
if time.time() - self.local_time > 3.0:
self.found_count.value += self.found_count_local
self.found_count_local = 0
self.queue_size_list[self.process_num] = self.queue.qsize()
self.local_time = time.time()
msg = cur_sub_domain.ljust(30) + ips
# print_msg(msg, line_feed=True)
self.outfile.write(cur_sub_domain.ljust(30) + '\t' + ips + '\n')
self.outfile.flush()
try:
self.resolvers[j].query('lijiejietest.' + cur_sub_domain)
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e:
self.queue.put((999999999, '{next_sub}.' + sub))
except:
pass
except (dns.resolver.NXDOMAIN, dns.name.EmptyLabel) as e:
pass
except (dns.resolver.NoNameservers, dns.resolver.NoAnswer, dns.exception.Timeout) as e:
pass
