def test__check_bulk(self): input_queue = PriorityQueue() queue = PriorityQueue() old_date_modified = datetime.now().isoformat() id_1 = uuid4().hex date_modified_1 = datetime.now().isoformat() id_2 = uuid4().hex date_modified_2 = datetime.now().isoformat() id_3 = uuid4().hex date_modified_3 = datetime.now().isoformat() db = MagicMock() bulk = { id_1: date_modified_1, id_2: date_modified_2, id_3: date_modified_3 } priority_cache = {id_1: 1, id_2: 1, id_3: 1} return_value = { u'docs': [ { u'_type': u'Tender', u'_source': { u'dateModified': date_modified_1 }, u'_index': u'bridge_tenders', u'_version': 1, u'found': True, u'_id': id_1 }, { u'_type': u'Tender', u'_source': { u'dateModified': old_date_modified }, u'_index': u'bridge_tenders', u'_version': 1, u'found': True, u'_id': id_2 }, { u'found': False, u'_type': u'Tender', u'_id': id_3, u'_index': u'bridge_tenders' } ] } db.mget.return_value = return_value elastic_filter = BasicElasticSearchFilter(self.config, input_queue, queue, db) self.assertEqual(queue.qsize(), 0) elastic_filter._check_bulk(bulk, priority_cache) self.assertEqual(queue.qsize(), 2)
def test_add_to_retry_queue(self, mocked_logger): retry_items_queue = PriorityQueue() worker = AgreementWorker(config_dict=self.worker_config, retry_resource_items_queue=retry_items_queue) resource_item = {'id': uuid.uuid4().hex} priority = 1000 self.assertEqual(retry_items_queue.qsize(), 0) # Add to retry_resource_items_queue worker.add_to_retry_queue(resource_item, priority=priority) self.assertEqual(retry_items_queue.qsize(), 1) priority, retry_resource_item = retry_items_queue.get() self.assertEqual((priority, retry_resource_item), (1001, resource_item)) resource_item = {'id': 0} # Add to retry_resource_items_queue with status_code '429' worker.add_to_retry_queue(resource_item, priority, status_code=429) self.assertEqual(retry_items_queue.qsize(), 1) priority, retry_resource_item = retry_items_queue.get() self.assertEqual((priority, retry_resource_item), (1001, resource_item)) priority = 1002 worker.add_to_retry_queue(resource_item, priority=priority) sleep(worker.config['retry_default_timeout'] * 2) self.assertEqual(retry_items_queue.qsize(), 1) priority, retry_resource_item = retry_items_queue.get() self.assertEqual((priority, retry_resource_item), (1003, resource_item)) worker.add_to_retry_queue(resource_item, priority=priority) self.assertEqual(retry_items_queue.qsize(), 0) mocked_logger.critical.assert_called_once_with( 'Tender {} reached limit retries count {} and droped from ' 'retry_queue.'.format(resource_item['id'], worker.config['retries_count']), extra={ 'MESSAGE_ID': 'dropped_documents', 'JOURNAL_TENDER_ID': resource_item['id'] }) del worker
class AutoHack(): def __init__(self, zoomeye_results, threads_num): self.threads_num = threads_num self.targets = PriorityQueue() self.zoomeye_results = zoomeye_results self.result = [] for zoomeye_result in zoomeye_results: self.targets.put(zoomeye_result) self.total = self.targets.qsize() self.pbar = tqdm(total=self.total,ascii=True) def check(self): while self.targets.qsize() > 0: target = self.targets.get().strip() try: self.pbar.update(1) result = exp.exp(target) if result: self.result.append(result) except Exception as e: #print(e) pass def run(self): threads = [gevent.spawn(self.check) for i in range(self.threads_num)] try: gevent.joinall(threads) except KeyboardInterrupt as e: print ('[WARNING] User aborted') for res in self.result: print (res) self.pbar.close() print ("Hack it!") for res in self.result: print (res) print("Found ",len(self.result)) print ("End!")
def test_add_to_retry_queue(self, mocked_logger): retry_items_queue = PriorityQueue() worker = ResourceItemWorker( config_dict=self.worker_config, retry_resource_items_queue=retry_items_queue) resource_item_id = uuid.uuid4().hex priority = 1000 self.assertEqual(retry_items_queue.qsize(), 0) # Add to retry_resource_items_queue worker.add_to_retry_queue(resource_item_id, priority=priority) # sleep(worker.config['retry_default_timeout'] * 0) self.assertEqual(retry_items_queue.qsize(), 1) priority, retry_resource_item_id = retry_items_queue.get() self.assertEqual(priority, 1001) self.assertEqual(retry_resource_item_id, resource_item_id) # Add to retry_resource_items_queue with status_code '429' worker.add_to_retry_queue(resource_item_id, priority, status_code=429) self.assertEqual(retry_items_queue.qsize(), 1) priority, retry_resource_item_id = retry_items_queue.get() self.assertEqual(priority, 1001) self.assertEqual(retry_resource_item_id, resource_item_id) priority = 1002 worker.add_to_retry_queue(resource_item_id, priority=priority) sleep(worker.config['retry_default_timeout'] * 2) self.assertEqual(retry_items_queue.qsize(), 1) priority, retry_resource_item_id = retry_items_queue.get() self.assertEqual(priority, 1003) self.assertEqual(retry_resource_item_id, resource_item_id) worker.add_to_retry_queue(resource_item_id, priority=priority) self.assertEqual(retry_items_queue.qsize(), 0) mocked_logger.critical.assert_called_once_with( 'Tender {} reached limit retries count {} and droped from ' 'retry_queue.'.format(resource_item_id, worker.config['retries_count']), extra={'MESSAGE_ID': 'dropped_documents'}) del worker
class TestBasicCouchDBFilter(unittest.TestCase): config = deepcopy(TEST_CONFIG['main']) config['storage_config']['bulk_query_limit'] = 1 def setUp(self): self.old_date_modified = datetime.now().isoformat() self.id_1 = uuid4().hex self.date_modified_1 = datetime.now().isoformat() self.id_2 = uuid4().hex self.date_modified_2 = datetime.now().isoformat() self.id_3 = uuid4().hex self.date_modified_3 = datetime.now().isoformat() self.queue = PriorityQueue() self.input_queue = PriorityQueue() self.db = MagicMock() self.bulk = { self.id_1: self.date_modified_1, self.id_2: self.date_modified_2, self.id_3: self.date_modified_3 } self.priority_cache = {self.id_1: 1, self.id_2: 1, self.id_3: 1} self.return_value = [ munchify({'id': self.id_1, 'key': self.date_modified_1}), munchify({'id': self.id_2, 'key': self.old_date_modified}), munchify({'id': self.id_3, 'key': self.old_date_modified}) ] self.db.db.view.return_value = self.return_value def test__check_bulk(self): self.queue.put((1000, self.id_3)) couchdb_filter = BasicCouchDBFilter(self.config, self.input_queue, self.queue, self.db) self.assertEqual(self.queue.qsize(), 1) couchdb_filter._check_bulk(self.bulk, self.priority_cache) self.assertEqual(self.queue.qsize(), 2) self.db.db.view.side_effect = [Exception(), Exception(), Exception('test')] self.bulk = {} with self.assertRaises(Exception) as e: couchdb_filter._check_bulk(self.bulk, self.priority_cache) self.assertEqual(e.exception.message, 'test') @patch('openprocurement.bridge.basic.filters.INFINITY') def test__run(self, mocked_infinity): couchdb_filter = BasicCouchDBFilter(self.config, self.input_queue, self.queue, self.db) self.input_queue.put((1, {'id': self.id_1, 'dateModified': self.date_modified_1})) self.input_queue.put((1, {'id': self.id_2, 'dateModified': self.date_modified_2})) self.input_queue.put((1, {'id': self.id_3, 'dateModified': self.date_modified_3})) mocked_infinity.__nonzero__.side_effect = [True] * 5 + [False, False] self.assertEqual(self.queue.qsize(), 0) self.assertEqual(self.input_queue.qsize(), 3) couchdb_filter._run() self.assertEqual(self.queue.qsize(), 2) self.assertEqual(self.input_queue.qsize(), 0)
class HttpTest(object): def __init__(self,host,keyword,ips,timeout): self.threads = 100 self.queue = PriorityQueue() self.host = host self.keyword = keyword self.result = [] for ip in ips: self.queue.put(ip) self.num = self.queue.qsize() self.i = 0 self.success = 0 self.timeout = timeout self.filename = os.path.join(rootPath,"result",host + ".log") self.outfile = open(self.filename, 'w') def _scan(self,j): while not self.queue.empty(): try: item = self.queue.get(timeout=3.0) if config.HTTPS_Support: host, domain, port = item, self.host , 443 else: host, domain, port = item, self.host , 80 html = httpServer((host, domain, port),self.timeout) if html is not None and self.keyword in html: self.outfile.write(item + '\n') self.outfile.flush() self.success += 1 except: pass finally: self.i += 1 msg = '[*] %s found, %s scanned , %s groups left'%(self.success,self.i,self.num - self.i) print_msg(msg) time.sleep(1.0) def run(self): threads = [gevent.spawn(self._scan, i) for i in range(self.threads)] gevent.joinall(threads) msg = '[+] All Done. Success:%d Saved in:%s'%(self.success,self.filename) print_msg(msg, line_feed=True)
class Scanner: def __init__(self): self.start_time = time.time() self.queue = PriorityQueue() self.history = [] self.total_count = 0 self.scan_count = 0 self._load_target() self.outfile = open("log.log", 'w') self.console_width = getTerminalSize()[0] - 2 def _print_msg(self, _msg=None, _found_msg=False): if _msg is None: msg = '%s TotalCount| %s Scanned in %.2f seconds' % ( self.total_count,self.total_count - self.queue.qsize(), time.time() - self.start_time) sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) + msg) else: sys.stdout.write('\r' + _msg + ' ' * (self.console_width - len(_msg)) + '\n') self.outfile.write(_msg + '\n') self.outfile.flush() if _found_msg: msg = '%s TotalCount| %s Scanned in %.2f seconds' % ( self.total_count,self.total_count - self.queue.qsize(), time.time() - self.start_time) sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) + msg) sys.stdout.flush() def _load_target(self): print '[+] Read targets ...' target_file = raw_input("Target File :") with open(target_file) as f: for line in f.xreadlines(): target = line.strip() self.queue.put(target) print "TotalCount is %d" % self.queue.qsize() self.total_count = self.queue.qsize() print "Now scanning ..." def _scan(self,case): while not self.queue.empty(): target = self.queue.get() if case == "1": self.vulnCheck(target) if case == "2": self.s2_045(target) if case == "3": self.headers(target) if case == "4": self.weakfile(target) if case == "5": self.portscan_c(target) ##################################################################### # # # Vuln poc by:xi4okv QQ:48011203 # # # ##################################################################### def vulnCheck(self,target): if ":2375" in target: try: res = requests.head("http://" + str(target) + "/containers/json",timeout=2) if res.headers['Content-Type'] == 'application/json': self._print_msg(target + "==> docker api Vuln",True) else: self._print_msg() except: self._print_msg() self._print_msg() if ":27017" in target: ip,port = target.split(":") try: socket.setdefaulttimeout(3) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((ip, int(port))) data = binascii.a2b_hex("3a000000a741000000000000d40700000000000061646d696e2e24636d640000000000ffffffff130000001069736d6173746572000100000000") s.send(data) result = s.recv(1024) if "ismaster" in result: getlog_data = binascii.a2b_hex("480000000200000000000000d40700000000000061646d696e2e24636d6400000000000100000021000000026765744c6f670010000000737461727475705761726e696e67730000") s.send(getlog_data) result = s.recv(1024) if "totalLinesWritten" in result: self._print_msg(target + "==> mongodb Vuln",True) except Exception, e: pass if ":6379" in target: ip,port = target.split(":") try: socket.setdefaulttimeout(3) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((ip, int(port))) s.send("INFO\r\n") result = s.recv(1024) if "redis_version" in result: self._print_msg(target + "==> redis Vuln",True) elif "Authentication" in result: for pass_ in ['123456','redis','pass','password']: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((ip, int(port))) s.send("AUTH %s\r\n" % (pass_)) result = s.recv(1024) if '+OK' in result: self._print_msg(target + "==> redis pass Vuln :" + pass_,True) except Exception, e: pass
class SubNameBrute: def __init__(self, target, options): self.start_time = time.time() self.target = target.strip() self.options = options self.ignore_intranet = options.i self.scan_count = self.found_count = 0 self.console_width = getTerminalSize()[0] - 2 self.resolvers = [dns.resolver.Resolver(configure=False) for _ in range(options.threads)] for _ in self.resolvers: _.lifetime = _.timeout = 10.0 self.print_count = 0 self.STOP_ME = False self._load_dns_servers() self._load_next_sub() self.queue = PriorityQueue() self.priority = 0 self._load_sub_names() if options.output: outfile = options.output else: _name = os.path.basename(self.options.file).replace('subnames', '') if _name != '.txt': _name = '_' + _name outfile = target + _name if not options.full_scan else target + '_full' + _name self.outfile = open(outfile, 'w') self.ip_dict = {} self.found_subs = set() self.ex_resolver = dns.resolver.Resolver(configure=False) self.ex_resolver.nameservers = self.dns_servers def _load_dns_servers(self): print '[+] Validate DNS servers ...' self.dns_servers = [] pool = Pool(30) for server in open('dict/dns_servers.txt').xreadlines(): server = server.strip() if server: pool.apply_async(self._test_server, (server,)) pool.join() self.dns_count = len(self.dns_servers) sys.stdout.write('\n') print '[+] Found %s available DNS Servers in total' % self.dns_count if self.dns_count == 0: print '[ERROR] No DNS Servers available.' sys.exit(-1) def _test_server(self, server): resolver = dns.resolver.Resolver() resolver.lifetime = resolver.timeout = 10.0 try: resolver.nameservers = [server] answers = resolver.query('public-dns-a.baidu.com') # test lookup a existed domain if answers[0].address != '180.76.76.76': raise Exception('incorrect DNS response') try: resolver.query('test.bad.dns.lijiejie.com') # Non-existed domain test with open('bad_dns_servers.txt', 'a') as f: f.write(server + '\n') self._print_msg('[+] Bad DNS Server found %s' % server) except: self.dns_servers.append(server) self._print_msg('[+] Check DNS Server %s < OK > Found %s' % (server.ljust(16), len(self.dns_servers))) except: self._print_msg('[+] Check DNS Server %s <Fail> Found %s' % (server.ljust(16), len(self.dns_servers))) def _load_sub_names(self): self._print_msg('[+] Load sub names ...') if self.options.full_scan and self.options.file == 'subnames.txt': _file = 'dict/subnames_full.txt' else: if os.path.exists(self.options.file): _file = self.options.file elif os.path.exists('dict/%s' % self.options.file): _file = 'dict/%s' % self.options.file else: self._print_msg('[ERROR] Names file not exists: %s' % self.options.file) exit(-1) normal_lines = [] wildcard_lines = [] wildcard_list = [] regex_list = [] lines = set() with open(_file) as f: for line in f.xreadlines(): sub = line.strip() if not sub or sub in lines: continue lines.add(sub) if sub.find('{alphnum}') >= 0 or sub.find('{alpha}') >= 0 or sub.find('{num}') >= 0: wildcard_lines.append(sub) sub = sub.replace('{alphnum}', '[a-z0-9]') sub = sub.replace('{alpha}', '[a-z]') sub = sub.replace('{num}', '[0-9]') if sub not in wildcard_list: wildcard_list.append(sub) regex_list.append('^' + sub + '$') else: normal_lines.append(sub) pattern = '|'.join(regex_list) if pattern: _regex = re.compile(pattern) if _regex: for line in normal_lines[:]: if _regex.search(line): normal_lines.remove(line) for item in normal_lines: self.priority += 1 self.queue.put((self.priority, item)) for item in wildcard_lines: self.queue.put((88888888, item)) def _load_next_sub(self): self._print_msg('[+] Load next level subs ...') self.next_subs = [] _set = set() _file = 'dict/next_sub.txt' if not self.options.full_scan else 'dict/next_sub_full.txt' with open(_file) as f: for line in f: sub = line.strip() if sub and sub not in self.next_subs: tmp_set = {sub} while len(tmp_set) > 0: item = tmp_set.pop() if item.find('{alphnum}') >= 0: for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789': tmp_set.add(item.replace('{alphnum}', _letter, 1)) elif item.find('{alpha}') >= 0: for _letter in 'abcdefghijklmnopqrstuvwxyz': tmp_set.add(item.replace('{alpha}', _letter, 1)) elif item.find('{num}') >= 0: for _letter in '0123456789': tmp_set.add(item.replace('{num}', _letter, 1)) elif item not in _set: _set.add(item) self.next_subs.append(item) def _print_msg(self, _msg=None, _found_msg=False): if _msg is None: self.print_count += 1 if self.print_count < 100: return self.print_count = 0 msg = '%s Found| %s Groups| %s scanned in %.1f seconds' % ( self.found_count, self.queue.qsize(), self.scan_count, time.time() - self.start_time) sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) + msg) elif _msg.startswith('[+] Check DNS Server'): sys.stdout.write('\r' + _msg + ' ' * (self.console_width - len(_msg))) else: sys.stdout.write('\r' + _msg + ' ' * (self.console_width - len(_msg)) + '\n') if _found_msg: msg = '%s Found| %s Groups| %s scanned in %.1f seconds' % ( self.found_count, self.queue.qsize(), self.scan_count, time.time() - self.start_time) sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) + msg) sys.stdout.flush() @staticmethod def is_intranet(ip): ret = ip.split('.') if len(ret) != 4: return True if ret[0] == '10': return True if ret[0] == '172' and 16 <= int(ret[1]) <= 32: return True if ret[0] == '192' and ret[1] == '168': return True return False def put_item(self, item): num = item.count('{alphnum}') + item.count('{alpha}') + item.count('{num}') if num == 0: self.priority += 1 self.queue.put((self.priority, item)) else: self.queue.put((self.priority + num * 10000000, item)) def _scan(self, j): self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]] while not self.queue.empty(): try: item = self.queue.get(timeout=1.0)[1] self.scan_count += 1 except: break self._print_msg() try: if item.find('{alphnum}') >= 0: for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789': self.put_item(item.replace('{alphnum}', _letter, 1)) continue elif item.find('{alpha}') >= 0: for _letter in 'abcdefghijklmnopqrstuvwxyz': self.put_item(item.replace('{alpha}', _letter, 1)) continue elif item.find('{num}') >= 0: for _letter in '0123456789': self.put_item(item.replace('{num}', _letter, 1)) continue elif item.find('{next_sub}') >= 0: for _ in self.next_subs: self.queue.put((0, item.replace('{next_sub}', _, 1))) continue else: sub = item if sub in self.found_subs: continue cur_sub_domain = sub + '.' + self.target _sub = sub.split('.')[-1] try: answers = self.resolvers[j].query(cur_sub_domain) except dns.resolver.NoAnswer, e: answers = self.ex_resolver.query(cur_sub_domain) if answers: self.found_subs.add(sub) ips = ', '.join(sorted([answer.address for answer in answers])) if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0']: continue if self.ignore_intranet and SubNameBrute.is_intranet(answers[0].address): continue try: self.scan_count += 1 answers = self.resolvers[j].query(cur_sub_domain, 'cname') cname = answers[0].target.to_unicode().rstrip('.') if cname.endswith(self.target) and cname not in self.found_subs: self.found_subs.add(cname) cname_sub = cname[:len(cname) - len(self.target) - 1] # new sub self.queue.put((0, cname_sub)) except: pass if (_sub, ips) not in self.ip_dict: self.ip_dict[(_sub, ips)] = 1 else: self.ip_dict[(_sub, ips)] += 1 if ips not in self.ip_dict: self.ip_dict[ips] = 1 else: self.ip_dict[ips] += 1 if self.ip_dict[(_sub, ips)] > 3 or self.ip_dict[ips] > 6: continue self.found_count += 1 msg = cur_sub_domain.ljust(30) + ips self._print_msg(msg, _found_msg=True) self._print_msg() self.outfile.write(cur_sub_domain.ljust(30) + '\t' + ips + '\n') self.outfile.flush() try: self.resolvers[j].query('lijiejietest.' + cur_sub_domain) except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e: self.queue.put((999999999, '{next_sub}.' + sub)) except: pass except (dns.resolver.NXDOMAIN, dns.name.EmptyLabel) as e: pass except (dns.resolver.NoNameservers, dns.resolver.NoAnswer, dns.exception.Timeout) as e: pass
class SubNameBrute(object): def __init__(self, *params): ( self.domain, self.options, self.process_num, self.dns_servers, self.next_subs, self.scan_count, self.found_count, self.queue_size_array, tmp_dir, ) = params self.dns_count = len(self.dns_servers) self.scan_count_local = 0 self.found_count_local = 0 self.resolvers = [ dns.resolver.Resolver(configure=False) for _ in range(self.options.threads) ] for r in self.resolvers: r.lifetime = r.timeout = 10.0 self.queue = PriorityQueue() self.priority = 0 self.ip_dict = {} self.found_subs = set() self.timeout_subs = {} self.count_time = time.time() self.outfile = open( "%s/%s_part_%s.txt" % (tmp_dir, self.domain, self.process_num), "w") self.normal_names_set = set() self.load_sub_names() self.lock = RLock() def load_sub_names(self): normal_lines = [] wildcard_lines = [] wildcard_set = set() regex_list = [] lines = set() with open(self.options.file) as inFile: for line in inFile.readlines(): sub = line.strip() if not sub or sub in lines: continue lines.add(sub) brace_count = sub.count("{") if brace_count > 0: wildcard_lines.append((brace_count, sub)) sub = sub.replace("{alphnum}", "[a-z0-9]") sub = sub.replace("{alpha}", "[a-z]") sub = sub.replace("{num}", "[0-9]") if sub not in wildcard_set: wildcard_set.add(sub) regex_list.append("^" + sub + "$") else: normal_lines.append(sub) self.normal_names_set.add(sub) if regex_list: pattern = "|".join(regex_list) _regex = re.compile(pattern) for line in normal_lines: if _regex.search(line): normal_lines.remove(line) for _ in normal_lines[self.process_num::self.options.process]: self.queue.put((0, _)) # priority set to 0 for _ in wildcard_lines[self.process_num::self.options.process]: self.queue.put(_) def scan(self, j): self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count] ] + self.dns_servers while True: try: self.lock.acquire() if time.time() - self.count_time > 1.0: self.scan_count.value += self.scan_count_local self.scan_count_local = 0 self.queue_size_array[ self.process_num] = self.queue.qsize() if self.found_count_local: self.found_count.value += self.found_count_local self.found_count_local = 0 self.count_time = time.time() self.lock.release() brace_count, sub = self.queue.get(timeout=3.0) if brace_count > 0: brace_count -= 1 if sub.find("{next_sub}") >= 0: for _ in self.next_subs: self.queue.put((0, sub.replace("{next_sub}", _))) if sub.find("{alphnum}") >= 0: for _ in "abcdefghijklmnopqrstuvwxyz0123456789": self.queue.put( (brace_count, sub.replace("{alphnum}", _, 1))) elif sub.find("{alpha}") >= 0: for _ in "abcdefghijklmnopqrstuvwxyz": self.queue.put( (brace_count, sub.replace("{alpha}", _, 1))) elif sub.find("{num}") >= 0: for _ in "0123456789": self.queue.put( (brace_count, sub.replace("{num}", _, 1))) continue except gevent.queue.Empty as e: break try: if sub in self.found_subs: continue self.scan_count_local += 1 cur_domain = sub + "." + self.domain answers = self.resolvers[j].query(cur_domain) if answers: self.found_subs.add(sub) ips = ", ".join( sorted([answer.address for answer in answers])) if ips in ["1.1.1.1", "127.0.0.1", "0.0.0.0", "0.0.0.1"]: continue if self.options.i and is_intranet(answers[0].address): continue try: self.scan_count_local += 1 answers = self.resolvers[j].query(cur_domain, "cname") cname = answers[0].target.to_unicode().rstrip(".") if cname.endswith( self.domain) and cname not in self.found_subs: cname_sub = cname[:len(cname) - len(self.domain) - 1] # new sub if cname_sub not in self.normal_names_set: self.found_subs.add(cname) self.queue.put((0, cname_sub)) except Exception as e: pass first_level_sub = sub.split(".")[-1] if (first_level_sub, ips) not in self.ip_dict: self.ip_dict[(first_level_sub, ips)] = 1 else: self.ip_dict[(first_level_sub, ips)] += 1 if self.ip_dict[(first_level_sub, ips)] > 30: continue self.found_count_local += 1 self.outfile.write( cur_domain.ljust(30) + "\t" + ips + "\n") self.outfile.flush() try: self.scan_count_local += 1 self.resolvers[j].query("test-not-existed." + cur_domain) except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e: if self.queue.qsize() < 10000: for _ in self.next_subs: self.queue.put((0, _ + "." + sub)) else: self.queue.put((1, "{next_sub}." + sub)) except Exception as e: pass except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e: pass except dns.resolver.NoNameservers as e: self.queue.put((0, sub)) # Retry except dns.exception.Timeout as e: self.timeout_subs[sub] = self.timeout_subs.get(sub, 0) + 1 if self.timeout_subs[sub] <= 2: self.queue.put((0, sub)) # Retry except Exception as e: import traceback traceback.print_exc() with open("errors.log", "a") as errFile: errFile.write("[%s] %s\n" % (type(e), str(e))) def run(self): threads = [ gevent.spawn(self.scan, i) for i in range(self.options.threads) ] gevent.joinall(threads)
class BasicDataBridge(object): """Basic Bridge""" def __init__(self, config): super(BasicDataBridge, self).__init__() defaults = deepcopy(DEFAULTS) defaults.update(config['main']) self.config = defaults # Init config for key, value in defaults.items(): setattr(self, key, value) self.bridge_id = uuid.uuid4().hex self.api_host = self.config.get('resources_api_server') self.api_version = self.config.get('resources_api_version') self.retrievers_params = self.config.get('retrievers_params') self.storage_type = self.config['storage_config'].get( 'storage_type', 'couchdb') self.worker_type = self.config['worker_config'].get( 'worker_type', 'basic_couchdb') self.filter_type = self.config['filter_config'].get( 'filter_type', 'basic_couchdb') # Check up_wait_sleep up_wait_sleep = self.retrievers_params.get('up_wait_sleep') if up_wait_sleep is not None and up_wait_sleep < 30: raise DataBridgeConfigError( 'Invalid \'up_wait_sleep\' in \'retrievers_params\'. Value must be grater than 30.' ) # Pools self.workers_pool = gevent.pool.Pool(self.workers_max) self.retry_workers_pool = gevent.pool.Pool(self.retry_workers_max) self.filter_workers_pool = gevent.pool.Pool(self.filter_workers_count) # Queues if self.input_queue_size == -1: self.input_queue = PriorityQueue() else: self.input_queue = PriorityQueue(self.input_queue_size) if self.resource_items_queue_size == -1: self.resource_items_queue = PriorityQueue() else: self.resource_items_queue = PriorityQueue( self.resource_items_queue_size) self.api_clients_queue = Queue() # self.retry_api_clients_queue = Queue() if self.retry_resource_items_queue_size == -1: self.retry_resource_items_queue = PriorityQueue() else: self.retry_resource_items_queue = PriorityQueue( self.retry_resource_items_queue_size) if self.api_host != '' and self.api_host is not None: api_host = urlparse(self.api_host) if api_host.scheme == '' and api_host.netloc == '': raise DataBridgeConfigError( 'Invalid \'resources_api_server\' url.') else: raise DataBridgeConfigError( 'In config dictionary empty or missing \'resources_api_server\'' ) # Connecting storage plugin self.db = None for entry_point in iter_entry_points( 'openprocurement.bridge.basic.storage_plugins', self.storage_type): plugin = entry_point.load() self.db = plugin(self.config) # Register handlers handlers = self.config.get('handlers', []) for entry_point in iter_entry_points( 'openprocurement.bridge.basic.handlers'): if not handlers or entry_point.name in handlers: plugin = entry_point.load() PROCUREMENT_METHOD_TYPE_HANDLERS[entry_point.name] = plugin( self.config, self.db) if hasattr(self, 'filter_type'): for entry_point in iter_entry_points( 'openprocurement.bridge.basic.filter_plugins', self.filter_type): self.filter_greenlet = entry_point.load() for entry_point in iter_entry_points( 'openprocurement.bridge.basic.worker_plugins', self.worker_type): self.worker_greenlet = entry_point.load() self.feeder = ResourceFeeder(host=self.config.get( 'public_resources_api_server', self.api_host), version=self.api_version, key='', resource=self.config['resource'], extra_params=self.extra_params, retrievers_params=self.retrievers_params, adaptive=True, with_priority=True) self.api_clients_info = {} def create_api_client(self): client_user_agent = self.user_agent + '/' + self.bridge_id timeout = 0.1 while 1: try: api_client = APIClient(host_url=self.api_host, user_agent=client_user_agent, api_version=self.api_version, key='', resource=self.resource) client_id = uuid.uuid4().hex logger.info('Started api_client {}'.format( api_client.session.headers['User-Agent']), extra={'MESSAGE_ID': 'create_api_clients'}) api_client_dict = { 'id': client_id, 'client': api_client, 'request_interval': 0, 'not_actual_count': 0 } self.api_clients_info[api_client_dict['id']] = { 'drop_cookies': False, 'request_durations': {}, 'request_interval': 0, 'avg_duration': 0 } self.api_clients_queue.put(api_client_dict) break except RequestFailed as e: logger.error( 'Failed start api_client with status code {}'.format( e.status_code), extra={'MESSAGE_ID': 'exceptions'}) timeout = timeout * 2 logger.info( 'create_api_client will be sleep {} sec.'.format(timeout)) sleep(timeout) except Exception as e: logger.error('Failed start api client with error: {}'.format( e.message), extra={'MESSAGE_ID': 'exceptions'}) timeout = timeout * 2 logger.info( 'create_api_client will be sleep {} sec.'.format(timeout)) sleep(timeout) def fill_api_clients_queue(self): while self.api_clients_queue.qsize() < self.workers_min: self.create_api_client() def fill_input_queue(self): # if not hasattr(self.db, 'filter'): # self.input_queue = self.resource_items_queue for resource_item in self.feeder.get_resource_items(): self.input_queue.put(resource_item) logger.debug('Add to temp queue from sync: {} {} {}'.format( self.resource[:-1], resource_item[1]['id'], resource_item[1]['dateModified']), extra={ 'MESSAGE_ID': 'received_from_sync', 'TEMP_QUEUE_SIZE': self.input_queue.qsize() }) def _get_average_requests_duration(self): req_durations = [] delta = timedelta(seconds=self.perfomance_window) current_date = datetime.now() - delta for cid, info in self.api_clients_info.items(): if len(info['request_durations']) > 0: if min(info['request_durations'].keys()) <= current_date: info['grown'] = True avg = round( sum(info['request_durations'].values()) * 1.0 / len(info['request_durations']), 3) req_durations.append(avg) info['avg_duration'] = avg if len(req_durations) > 0: return round(sum(req_durations) / len(req_durations), 3), req_durations else: return 0, req_durations # TODO: Add logic for restart sync if last response grater than some values # and no active tasks specific for resource def queues_controller(self): while True: if (self.workers_pool.free_count() > 0 and (self.resource_items_queue.qsize() > ((float(self.resource_items_queue_size) / 100) * self.workers_inc_threshold))): self.create_api_client() w = self.worker_greenlet.spawn(self.api_clients_queue, self.resource_items_queue, self.db, self.config, self.retry_resource_items_queue, self.api_clients_info) self.workers_pool.add(w) logger.info('Queue controller: Create main queue worker.') elif (self.resource_items_queue.qsize() < ((float(self.resource_items_queue_size) / 100) * self.workers_dec_threshold)): if len(self.workers_pool) > self.workers_min: wi = self.workers_pool.greenlets.pop() wi.shutdown() api_client_dict = self.api_clients_queue.get() del self.api_clients_info[api_client_dict['id']] logger.info('Queue controller: Kill main queue worker.') filled_resource_items_queue = round( self.resource_items_queue.qsize() / (float(self.resource_items_queue_size) / 100), 2) logger.info('Resource items queue filled on {} %'.format( filled_resource_items_queue)) filled_retry_resource_items_queue \ = round(self.retry_resource_items_queue.qsize() / float(self.retry_resource_items_queue_size) / 100, 2) logger.info('Retry resource items queue filled on {} %'.format( filled_retry_resource_items_queue)) sleep(self.queues_controller_timeout) def gevent_watcher(self): self.perfomance_watcher() # Check fill threads input_threads = 1 if self.input_queue_filler.ready(): input_threads = 0 logger.error('Temp queue filler error: {}'.format( self.input_queue_filler.exception.message), extra={'MESSAGE_ID': 'exception'}) self.input_queue_filler = spawn(self.fill_input_queue) logger.info('Input threads {}'.format(input_threads), extra={'INPUT_THREADS': input_threads}) fill_threads = 1 if hasattr(self, 'queue_filter') and self.queue_filter.ready(): fill_threads = 0 logger.error('Fill thread error: {}'.format( self.queue_filter.exception.message), extra={'MESSAGE_ID': 'exception'}) self.queue_filter = self.filter_greenlet.spawn( self.config, self.input_queue, self.resource_items_queue, self.db) logger.info('Filter threads {}'.format(fill_threads), extra={'FILTER_THREADS': fill_threads}) main_threads = self.workers_max - self.workers_pool.free_count() logger.info('Main threads {}'.format(main_threads), extra={'MAIN_THREADS': main_threads}) if len(self.workers_pool) < self.workers_min: for i in xrange(0, (self.workers_min - len(self.workers_pool))): self.create_api_client() w = self.worker_greenlet.spawn(self.api_clients_queue, self.resource_items_queue, self.db, self.config, self.retry_resource_items_queue, self.api_clients_info) self.workers_pool.add(w) logger.info('Watcher: Create main queue worker.') retry_threads = self.retry_workers_max - self.retry_workers_pool.free_count( ) logger.info('Retry threads {}'.format(retry_threads), extra={'RETRY_THREADS': retry_threads}) if len(self.retry_workers_pool) < self.retry_workers_min: for i in xrange( 0, self.retry_workers_min - len(self.retry_workers_pool)): self.create_api_client() w = self.worker_greenlet.spawn(self.api_clients_queue, self.retry_resource_items_queue, self.db, self.config, self.retry_resource_items_queue, self.api_clients_info) self.retry_workers_pool.add(w) logger.info('Watcher: Create retry queue worker.') # Log queues size and API clients count main_queue_size = self.resource_items_queue.qsize() logger.info('Resource items queue size {}'.format(main_queue_size), extra={'MAIN_QUEUE_SIZE': main_queue_size}) retry_queue_size = self.retry_resource_items_queue.qsize() logger.info( 'Resource items retry queue size {}'.format(retry_queue_size), extra={'RETRY_QUEUE_SIZE': retry_queue_size}) api_clients_count = len(self.api_clients_info) logger.info('API Clients count: {}'.format(api_clients_count), extra={'API_CLIENTS': api_clients_count}) def _calculate_st_dev(self, values): if len(values) > 0: avg = sum(values) * 1.0 / len(values) variance = map(lambda x: (x - avg)**2, values) avg_variance = sum(variance) * 1.0 / len(variance) st_dev = math.sqrt(avg_variance) return round(st_dev, 3) else: return 0 def _mark_bad_clients(self, dev): # Mark bad api clients for cid, info in self.api_clients_info.items(): if info.get('grown', False) and info['avg_duration'] > dev: info['drop_cookies'] = True logger.debug( 'Perfomance watcher: Mark client {} as bad, avg. request_duration is {} sec.' .format(cid, info['avg_duration']), extra={'MESSAGE_ID': 'marked_as_bad'}) elif info['avg_duration'] < dev and info['request_interval'] > 0: info['drop_cookies'] = True logger.debug( 'Perfomance watcher: Mark client {} as bad, request_interval is {} sec.' .format(cid, info['request_interval']), extra={'MESSAGE_ID': 'marked_as_bad'}) def perfomance_watcher(self): avg_duration, values = self._get_average_requests_duration() for _, info in self.api_clients_info.items(): delta = timedelta(seconds=self.perfomance_window + self.watch_interval) current_date = datetime.now() - delta delete_list = [] for key in info['request_durations']: if key < current_date: delete_list.append(key) for k in delete_list: del info['request_durations'][k] delete_list = [] st_dev = self._calculate_st_dev(values) if len(values) > 0: min_avg = min(values) * 1000 max_avg = max(values) * 1000 else: max_avg = 0 min_avg = 0 dev = round(st_dev + avg_duration, 3) logger.info('Perfomance watcher:\nREQUESTS_STDEV - {} sec.\n' 'REQUESTS_DEV - {} ms.\nREQUESTS_MIN_AVG - {} ms.\n' 'REQUESTS_MAX_AVG - {} ms.\nREQUESTS_AVG - {} sec.'.format( round(st_dev, 3), dev, min_avg, max_avg, avg_duration), extra={ 'REQUESTS_DEV': dev * 1000, 'REQUESTS_MIN_AVG': min_avg, 'REQUESTS_MAX_AVG': max_avg, 'REQUESTS_AVG': avg_duration * 1000 }) self._mark_bad_clients(dev) def run(self): logger.info('Start Basic Bridge', extra={'MESSAGE_ID': 'start_basic_bridge'}) logger.info('Start data sync...', extra={'MESSAGE_ID': 'basic_bridge__data_sync'}) self.input_queue_filler = spawn(self.fill_input_queue) if hasattr(self, 'filter_greenlet'): self.queue_filter = self.filter_greenlet.spawn( self.config, self.input_queue, self.resource_items_queue, self.db) else: self.resource_items_queue = self.input_queue spawn(self.queues_controller) while True: self.gevent_watcher() sleep(self.watch_interval)
class PriorityBlockingQueue(object): """ 带优先级的阻塞队列。 优先级数字越小,优先级越高。 插入元素: * put: 向队列尾部插入一个元素,如果该队列已满,则一直阻塞。 * offer: 向队列尾部插入一个元素,插入成功返回True。插入失败返回False。 获取元素: * poll: 获取并移除队列的头元素,若队列为空,则返回null。 * take: 获取并移除队列的头元素,若队列为空,则一直阻塞。 * peek:获取但不移除队列的头元素,若队列为空,则返回null 队列状态状态: * qsize:获取队列中当前元素数量 * maxsize:获取队列的最大容量 """ def __init__(self, maxsize: int = None): """ init :param maxsize: 队列的最大容量 """ self.__queue = PriorityQueue(maxsize=maxsize) def put(self, item, priority: int = 200) -> None: """ 向队列尾部插入一个元素,如果该队列已满,则一直阻塞。 :param item: :param priority: 优先级 :return: """ while True: try: self.__queue.put(PriorityEntry(priority, item)) break except Exception as e: logger.debug("put data failed error -> {0}".format(e)) time.sleep(0.5) def offer(self, item, priority: int = 200) -> bool: """ 向队列尾部插入一个元素,插入成功返回True。插入失败返回False。 :param item: 元素 :param priority: 优先级 :return: """ try: self.__queue.put(PriorityEntry(priority, item), block=False) return True except Exception as e: logger.debug("offer data failed error -> {0}".format(e)) return False def poll(self): """ 获取并移除队列的头元素,若队列为空,则返回null。 :return: """ try: return self.__queue.get(block=False).data except Exception as e: logger.debug("poll data failed error -> {0}".format(e)) return None def take(self): """ 获取并移除队列的头元素,若队列为空,则一直阻塞。 :return: """ while True: try: return self.__queue.get().data except Exception as e: logger.debug("take data failed error -> {0}".format(e)) time.sleep(0.5) def peek(self): """ 获取但不移除队列的头元素,若队列为空,则返回null :return: """ try: return self.__queue.peek(block=False).data except Exception as e: logger.debug("peek data failed error -> {0}".format(e)) return None def qsize(self) -> int: """ 获取队列中当前元素数量 :return: """ return self.__queue.qsize() def maxsize(self) -> int: """ 获取队列的最大容量 :return: """ return self.__queue.maxsize
class SubNameBrute: """ receive commandline args and do some initialization work """ def __init__(self, target, options): self.start_time = time.time() self.target = target.strip() self.options = options self.scan_count = self.found_count = 0 self.console_width = os.get_terminal_size()[0] - 2 # create dns resolver pool ~ workers self.resolvers = [ dns.resolver.Resolver(configure=False) for _ in range(options.threads) ] for resolver in self.resolvers: resolver.lifetime = resolver.timeout = 10.0 self.print_count = 0 self.STOP_ME = False # load dns servers and check whether these dns servers works fine ? self._load_dns_servers() # load sub names self.subs = [] # subs in file self.goodsubs = [] # checks ok for further exploitation self._load_subname('dict/subnames.txt', self.subs) # load sub.sub names self.subsubs = [] self._load_subname('dict/next_sub.txt', self.subsubs) # results will save to target.txt global path path = os.path.join("results", target) if not os.path.exists(path): os.makedirs(path) self.outfile = open('%s/%s.txt' % (path, target), 'w') self.ip_dict = set() # self.found_sub = set() # task queue self.queue = PriorityQueue() for sub in self.subs: self.queue.put(sub) """ Load DNS Servers(ip saved in file), and check whether the DNS servers works fine """ def _load_dns_servers(self): print('[*] Validate DNS servers ...') self.dns_servers = [] # create a process pool for checking DNS servers, the number is your processors(cores) * 2, just change it! processors = cpu_count() * 2 pool = Pool(processors) # read dns ips and check one by one for server in open('dict/dns_servers.txt').readlines(): server = server.strip() if server: pool.apply_async(self._test_server, (server, )) pool.join() # waiting for process finish self.dns_count = len(self.dns_servers) sys.stdout.write('\n') dns_info = '[+] Found {} available DNS Servers in total'.format( self.dns_count) print(dns_info) if self.dns_count == 0: print('[ERROR] No DNS Servers available.') sys.exit(-1) """ test these dns servers whether works fine """ def _test_server(self, server): # create a dns resolver and set timeout resolver = dns.resolver.Resolver() resolver.lifetime = resolver.timeout = 10.0 try: resolver.nameservers = [server] answers = resolver.query('public-dns-a.baidu.com') if answers[0].address != '180.76.76.76': raise Exception('incorrect DNS response') self.dns_servers.append(server) except: self._print_msg('[-] Check DNS Server %s <Fail> Found %s' % (server.ljust(16), len(self.dns_servers))) self._print_msg('[+] Check DNS Server %s < OK > Found %s' % (server.ljust(16), len(self.dns_servers))) """ load sub names in dict/*.txt, one function would be enough file for read, subname_list for saving sub names """ def _load_subname(self, file, subname_list): self._print_msg('[*] Load sub names ...') with open(file) as f: for line in f: sub = line.strip() if sub and sub not in subname_list: tmp_set = {sub} """ in case of the sub names which contains the following expression and replace them {alphnum}, {alpha}, {num} with character and num """ while len(tmp_set) > 0: item = tmp_set.pop() if item.find('{alphnum}') >= 0: for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789': tmp_set.add( item.replace('{alphnum}', _letter, 1)) elif item.find('{alpha}') >= 0: for _letter in 'abcdefghijklmnopqrstuvwxyz': tmp_set.add(item.replace( '{alpha}', _letter, 1)) elif item.find('{num}') >= 0: for _letter in '0123456789': tmp_set.add(item.replace('{num}', _letter, 1)) elif item not in subname_list: subname_list.append(item) """ for better presentation of brute force results, not really matters ... """ def _print_msg(self, _msg=None, _found_msg=False): if _msg is None: self.print_count += 1 if self.print_count < 100: return self.print_count = 0 msg = '%s Found| %s Groups| %s scanned in %.1f seconds' % ( self.found_count, self.queue.qsize(), self.scan_count, time.time() - self.start_time) sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) + msg) elif _msg.startswith('[+] Check DNS Server'): sys.stdout.write('\r' + _msg + ' ' * (self.console_width - len(_msg))) else: sys.stdout.write('\r' + _msg + ' ' * (self.console_width - len(_msg)) + '\n') if _found_msg: msg = '%s Found| %s Groups| %s scanned in %.1f seconds' % ( self.found_count, self.queue.qsize(), self.scan_count, time.time() - self.start_time) sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) + msg) sys.stdout.flush() def _print_domain(self, msg): console_width = os.get_terminal_size()[0] msg = '\r' + msg + ' ' * (console_width - len(msg)) # msg = '\033[0;31;47m%s{}\033[0m'.format(msg) sys.stdout.write(msg) def _print_progress(self): """ 显示扫描进度,显示更美观 """ msg = '\033[0;31;47m%s\033[0m found | %s remaining | %s scanned in %.2f seconds' % \ (self.found_count, self.queue.qsize(), self.scan_count, time.time()- self.start_time) console_width = os.get_terminal_size()[0] out = '\r' + ' ' * int((console_width - len(msg)) / 2) + msg sys.stdout.write(out) """ important : assign task to resolvers """ def _scan(self, j): self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]] while not self.queue.empty(): sub = self.queue.get(timeout=1.0) self.scan_count += 1 try: cur_sub_domain = sub + '.' + self.target answers = self.resolvers[j].query(cur_sub_domain) except: continue if answers: ips = ', '.join(sorted([answer.address for answer in answers])) # exclude : intranet or kept addresses if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0', '0.0.0.1']: continue if SubNameBrute.is_intranet(answers[0].address): continue self.found_sub.add(cur_sub_domain) for answer in answers: self.ip_dict.add(answer.address) if sub not in self.goodsubs: self.goodsubs.append(sub) self.found_count += 1 ip_info = '{} \t {}'.format(cur_sub_domain, ips) # print(ip_info) self.outfile.write(cur_sub_domain + '\t' + ips + '\n') self._print_domain(ip_info) sys.stdout.flush() self._print_progress() sys.stdout.flush() @staticmethod def is_intranet(ip): ret = ip.split('.') if len(ret) != 4: return True if ret[0] == '10': return True if ret[0] == '172' and 16 <= int(ret[1]) <= 32: return True if ret[0] == '192' and ret[1] == '168': return True return False """ assign task to threads ... """ def run(self): threads = [ gevent.spawn(self._scan, i) for i in range(self.options.threads) ] print('[*] Initializing %d threads' % self.options.threads) try: gevent.joinall(threads) except KeyboardInterrupt as e: msg = '[WARNING] User aborted.' sys.stdout.write('\r' + msg + ' ' * (self.console_width - len(msg)) + '\n\r') sys.stdout.flush()
class SubNameBrute: def __init__(self, target, options): self.start_time = time.time() self.target = target.strip() self.options = options self.ignore_intranet = options.i self.scan_count = self.found_count = 0 self.console_width = getTerminalSize()[0] - 2 self.resolvers = [ dns.resolver.Resolver(configure=False) for _ in range(options.threads) ] for _ in self.resolvers: _.lifetime = _.timeout = 10.0 self.print_count = 0 self.STOP_ME = False self._load_dns_servers() self._load_next_sub() self.queue = PriorityQueue() self.priority = 0 self._load_sub_names() if options.output: outfile = options.output else: _name = os.path.basename(self.options.file).replace('subnames', '') if _name != '.txt': _name = '_' + _name outfile = target + _name if not options.full_scan else target + '_full' + _name self.outfile = open(outfile, 'w') self.ip_dict = {} self.found_subs = set() self.ex_resolver = dns.resolver.Resolver(configure=False) self.ex_resolver.nameservers = self.dns_servers self.result_lines = [] self.result_domains = [] self.result_ips = [] def _load_dns_servers(self): print '[+] Validate DNS servers ...' self.dns_servers = [] pool = Pool(30) # change file location for server in open('lijiejie/' + 'dict/dns_servers.txt').xreadlines(): server = server.strip() if server: pool.apply_async(self._test_server, (server, )) pool.join() self.dns_count = len(self.dns_servers) sys.stdout.write('\n') print '[+] Found %s available DNS Servers in total' % self.dns_count if self.dns_count == 0: print '[ERROR] No DNS Servers available.' sys.exit(-1) def _test_server(self, server): resolver = dns.resolver.Resolver() resolver.lifetime = resolver.timeout = 10.0 try: resolver.nameservers = [server] answers = resolver.query( 'public-dns-a.baidu.com') # test lookup a existed domain if answers[0].address != '180.76.76.76': raise Exception('incorrect DNS response') try: resolver.query( 'test.bad.dns.lijiejie.com') # Non-existed domain test with open('bad_dns_servers.txt', 'a') as f: f.write(server + '\n') self._print_msg('[+] Bad DNS Server found %s' % server) except: self.dns_servers.append(server) self._print_msg('[+] Check DNS Server %s < OK > Found %s' % (server.ljust(16), len(self.dns_servers))) except: self._print_msg('[+] Check DNS Server %s <Fail> Found %s' % (server.ljust(16), len(self.dns_servers))) def _load_sub_names(self): self._print_msg('[+] Load sub names ...') if self.options.full_scan and self.options.file == 'subnames.txt': _file = 'lijiejie/' + 'dict/subnames_full.txt' else: if os.path.exists(self.options.file): _file = self.options.file elif os.path.exists('lijiejie/' + 'dict/%s' % self.options.file): _file = 'lijiejie/' + 'dict/%s' % self.options.file else: self._print_msg('[ERROR] Names file not exists: %s' % self.options.file) exit(-1) if self.options.debug: _file = 'lijiejie/' + 'dict/debug.txt' if not os.path.exists(_file): self._print_msg('[ERROR] Names file not exists: %s' % self.options.file) exit(-1) normal_lines = [] wildcard_lines = [] wildcard_list = [] regex_list = [] lines = set() with open(_file) as f: for line in f.xreadlines(): sub = line.strip() if not sub or sub in lines: continue lines.add(sub) if sub.find('{alphnum}') >= 0 or sub.find( '{alpha}') >= 0 or sub.find('{num}') >= 0: wildcard_lines.append(sub) sub = sub.replace('{alphnum}', '[a-z0-9]') sub = sub.replace('{alpha}', '[a-z]') sub = sub.replace('{num}', '[0-9]') if sub not in wildcard_list: wildcard_list.append(sub) regex_list.append('^' + sub + '$') else: normal_lines.append(sub) pattern = '|'.join(regex_list) if pattern: _regex = re.compile(pattern) if _regex: for line in normal_lines[:]: if _regex.search(line): normal_lines.remove(line) for item in normal_lines: self.priority += 1 self.queue.put((self.priority, item)) for item in wildcard_lines: self.queue.put((88888888, item)) def _load_next_sub(self): self._print_msg('[+] Load next level subs ...') self.next_subs = [] _set = set() _file = 'lijiejie/' + 'dict/next_sub.txt' if not self.options.full_scan else 'dict/next_sub_full.txt' with open(_file) as f: for line in f: sub = line.strip() if sub and sub not in self.next_subs: tmp_set = {sub} while len(tmp_set) > 0: item = tmp_set.pop() if item.find('{alphnum}') >= 0: for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789': tmp_set.add( item.replace('{alphnum}', _letter, 1)) elif item.find('{alpha}') >= 0: for _letter in 'abcdefghijklmnopqrstuvwxyz': tmp_set.add(item.replace( '{alpha}', _letter, 1)) elif item.find('{num}') >= 0: for _letter in '0123456789': tmp_set.add(item.replace('{num}', _letter, 1)) elif item not in _set: _set.add(item) self.next_subs.append(item) def _print_msg(self, _msg=None, _found_msg=False): if _msg is None: self.print_count += 1 if self.print_count < 100: return self.print_count = 0 msg = '%s Found| %s Groups| %s scanned in %.1f seconds' % ( self.found_count, self.queue.qsize(), self.scan_count, time.time() - self.start_time) sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) + msg) elif _msg.startswith('[+] Check DNS Server'): sys.stdout.write('\r' + _msg + ' ' * (self.console_width - len(_msg))) else: sys.stdout.write('\r' + _msg + ' ' * (self.console_width - len(_msg)) + '\n') if _found_msg: msg = '%s Found| %s Groups| %s scanned in %.1f seconds' % ( self.found_count, self.queue.qsize(), self.scan_count, time.time() - self.start_time) sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) + msg) sys.stdout.flush() @staticmethod def is_intranet(ip): ret = ip.split('.') if len(ret) != 4: return True if ret[0] == '10': return True if ret[0] == '172' and 16 <= int(ret[1]) <= 32: return True if ret[0] == '192' and ret[1] == '168': return True return False def put_item(self, item): num = item.count('{alphnum}') + item.count('{alpha}') + item.count( '{num}') if num == 0: self.priority += 1 self.queue.put((self.priority, item)) else: self.queue.put((self.priority + num * 10000000, item)) def _scan(self, j): self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]] while not self.queue.empty(): try: item = self.queue.get(timeout=1.0)[1] self.scan_count += 1 except: break self._print_msg() try: if item.find('{alphnum}') >= 0: for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789': self.put_item(item.replace('{alphnum}', _letter, 1)) continue elif item.find('{alpha}') >= 0: for _letter in 'abcdefghijklmnopqrstuvwxyz': self.put_item(item.replace('{alpha}', _letter, 1)) continue elif item.find('{num}') >= 0: for _letter in '0123456789': self.put_item(item.replace('{num}', _letter, 1)) continue elif item.find('{next_sub}') >= 0: for _ in self.next_subs: self.queue.put((0, item.replace('{next_sub}', _, 1))) continue else: sub = item if sub in self.found_subs: continue cur_sub_domain = sub + '.' + self.target _sub = sub.split('.')[-1] try: answers = self.resolvers[j].query(cur_sub_domain) except dns.resolver.NoAnswer, e: answers = self.ex_resolver.query(cur_sub_domain) if answers: self.found_subs.add(sub) ips = ', '.join( sorted([answer.address for answer in answers])) if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0']: continue if self.ignore_intranet and SubNameBrute.is_intranet( answers[0].address): continue try: self.scan_count += 1 answers = self.resolvers[j].query( cur_sub_domain, 'cname') cname = answers[0].target.to_unicode().rstrip('.') if cname.endswith( self.target) and cname not in self.found_subs: self.found_subs.add(cname) cname_sub = cname[:len(cname) - len(self.target) - 1] # new sub self.queue.put((0, cname_sub)) except: pass if (_sub, ips) not in self.ip_dict: self.ip_dict[(_sub, ips)] = 1 else: self.ip_dict[(_sub, ips)] += 1 if ips not in self.ip_dict: self.ip_dict[ips] = 1 else: self.ip_dict[ips] += 1 if self.ip_dict[(_sub, ips)] > 3 or self.ip_dict[ips] > 6: continue self.found_count += 1 msg = cur_sub_domain.ljust(30) + ips self._print_msg(msg, _found_msg=True) self._print_msg() # TODO: close write file self.outfile.write( cur_sub_domain.ljust(30) + '\t' + ips + '\n') self.outfile.flush() self.result_lines.append( cur_sub_domain.ljust(30) + '\t' + ips) self.result_domains.append(cur_sub_domain) self.result_ips.extend(ips.split(", ")) try: self.resolvers[j].query('lijiejietest.' + cur_sub_domain) except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e: self.queue.put((999999999, '{next_sub}.' + sub)) except: pass except (dns.resolver.NXDOMAIN, dns.name.EmptyLabel) as e: pass except (dns.resolver.NoNameservers, dns.resolver.NoAnswer, dns.exception.Timeout) as e: pass
class SubNameBrute(object): def __init__(self, *params): self.domain, self.options, self.process_num, self.dns_servers, self.next_subs, \ self.scan_count, self.found_count, self.queue_size_array, tmp_dir = params self.dns_count = len(self.dns_servers) self.scan_count_local = 0 self.found_count_local = 0 self.resolvers = [dns.resolver.Resolver(configure=False) for _ in range(self.options.threads)] for r in self.resolvers: r.lifetime = 6.0 r.timeout = 10.0 self.queue = PriorityQueue() self.priority = 0 self.ip_dict = {} self.found_subs = set() self.cert_subs = set() self.timeout_subs = {} self.no_server_subs = {} self.count_time = time.time() self.outfile = open('%s/%s_part_%s.txt' % (tmp_dir, self.domain, self.process_num), 'w') self.normal_names_set = set() self.load_sub_names() self.lock = RLock() self.threads_status = ['1'] * self.options.threads def load_sub_names(self): normal_lines = [] wildcard_lines = [] wildcard_set = set() regex_list = [] lines = set() with open(self.options.file) as inFile: for line in inFile.readlines(): sub = line.strip() if not sub or sub in lines: continue lines.add(sub) brace_count = sub.count('{') if brace_count > 0: wildcard_lines.append((brace_count, sub)) sub = sub.replace('{alphnum}', '[a-z0-9]') sub = sub.replace('{alpha}', '[a-z]') sub = sub.replace('{num}', '[0-9]') if sub not in wildcard_set: wildcard_set.add(sub) regex_list.append('^' + sub + '$') else: normal_lines.append(sub) self.normal_names_set.add(sub) if regex_list: pattern = '|'.join(regex_list) _regex = re.compile(pattern) for line in normal_lines: if _regex.search(line): normal_lines.remove(line) for _ in normal_lines[self.process_num::self.options.process]: self.queue.put((0, _)) # priority set to 0 for _ in wildcard_lines[self.process_num::self.options.process]: self.queue.put(_) def update_counter(self): while True: if '1' not in self.threads_status: return self.scan_count.value += self.scan_count_local self.scan_count_local = 0 self.queue_size_array[self.process_num] = self.queue.qsize() if self.found_count_local: self.found_count.value += self.found_count_local self.found_count_local = 0 self.count_time = time.time() gevent.sleep(0.5) def check_https_alt_names(self, domain): try: x509 = reqs.OpenSSL.crypto.load_certificate( reqs.OpenSSL.crypto.FILETYPE_PEM, reqs.ssl.get_server_certificate((domain, 443)) ) for item in reqs.get_subj_alt_name(x509): if item[0].upper() == 'DNS': name = item[1].lower() if name.endswith(self.domain): sub = name[:len(name) - len(self.domain) - 1] # new sub sub = sub.replace('*', '') sub = sub.strip('.') if sub and sub not in self.found_subs and \ sub not in self.normal_names_set and sub not in self.cert_subs: self.cert_subs.add(sub) self.queue.put((0, sub)) except Exception as e: pass def scan(self, j): self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]] if self.dns_count > 1: while True: s = random.choice(self.dns_servers) if s != self.dns_servers[j % self.dns_count]: self.resolvers[j].nameservers.append(s) break empty_counter = 0 while True: try: brace_count, sub = self.queue.get_nowait() self.threads_status[j] = '1' empty_counter = 0 except gevent.queue.Empty as e: empty_counter += 1 if empty_counter > 10: self.threads_status[j] = '0' if '1' not in self.threads_status: break else: gevent.sleep(0.1) continue if brace_count > 0: brace_count -= 1 if sub.find('{next_sub}') >= 0: for _ in self.next_subs: self.queue.put((0, sub.replace('{next_sub}', _))) if sub.find('{alphnum}') >= 0: for _ in 'abcdefghijklmnopqrstuvwxyz0123456789': self.queue.put((brace_count, sub.replace('{alphnum}', _, 1))) elif sub.find('{alpha}') >= 0: for _ in 'abcdefghijklmnopqrstuvwxyz': self.queue.put((brace_count, sub.replace('{alpha}', _, 1))) elif sub.find('{num}') >= 0: for _ in '0123456789': self.queue.put((brace_count, sub.replace('{num}', _, 1))) continue try: if sub in self.found_subs: continue self.scan_count_local += 1 cur_domain = sub + '.' + self.domain answers = self.resolvers[j].query(cur_domain) if answers: self.found_subs.add(sub) ips = ', '.join(sorted([answer.address for answer in answers])) invalid_ip_found = False for answer in answers: if answer.address in ['1.1.1.1', '127.0.0.1', '0.0.0.0', '0.0.0.1']: invalid_ip_found = True if invalid_ip_found: continue if self.options.i and is_intranet(answers[0].address): continue try: cname = str(answers.canonical_name)[:-1] if cname != cur_domain and cname.endswith(self.domain): cname_sub = cname[:len(cname) - len(self.domain) - 1] # new sub if cname_sub not in self.found_subs and cname_sub not in self.normal_names_set: self.queue.put((0, cname_sub)) except Exception as e: print(e) pass try: self.scan_count_local += 1 answers = self.resolvers[j].query(cur_domain, 'cname') cname = answers[0].target.to_unicode().rstrip('.') if cname.endswith(self.domain) and cname not in self.found_subs: cname_sub = cname[:len(cname) - len(self.domain) - 1] # new sub if cname_sub not in self.normal_names_set: # self.found_subs.add(cname) self.queue.put((0, cname_sub)) except Exception as e: pass first_level_sub = sub.split('.')[-1] max_found = 20 if self.options.w: first_level_sub = '' max_found = 3 if (first_level_sub, ips) not in self.ip_dict: self.ip_dict[(first_level_sub, ips)] = 1 else: self.ip_dict[(first_level_sub, ips)] += 1 if self.ip_dict[(first_level_sub, ips)] > max_found: continue self.found_count_local += 1 self.outfile.write(cur_domain.ljust(30) + '\t' + ips + '\n') self.outfile.flush() if not self.options.no_cert_check: with gevent.Timeout(10, False): self.check_https_alt_names(cur_domain) try: self.scan_count_local += 1 self.resolvers[j].query('lijiejie-test-not-existed.' + cur_domain) except (dns.resolver.NXDOMAIN, ) as e: # dns.resolver.NoAnswer if self.queue.qsize() < 20000: for _ in self.next_subs: self.queue.put((0, _ + '.' + sub)) else: self.queue.put((1, '{next_sub}.' + sub)) except Exception as e: pass except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e: pass except dns.resolver.NoNameservers as e: self.no_server_subs[sub] = self.no_server_subs.get(sub, 0) + 1 if self.no_server_subs[sub] <= 3: self.queue.put((0, sub)) # Retry except dns.exception.Timeout as e: self.timeout_subs[sub] = self.timeout_subs.get(sub, 0) + 1 if self.timeout_subs[sub] <= 1: self.queue.put((0, sub)) # Retry except Exception as e: import traceback traceback.print_exc() with open('errors.log', 'a') as errFile: errFile.write('[%s] %s\n' % (type(e), str(e))) def run(self): threads = [gevent.spawn(self.scan, i) for i in range(self.options.threads)] threads.insert(0, gevent.spawn(self.update_counter)) gevent.joinall(threads)
class EdgeDataBridge(object): """Edge Bridge""" def __init__(self, config): super(EdgeDataBridge, self).__init__() self.config = config self.workers_config = {} self.bridge_id = uuid.uuid4().hex self.api_host = self.config_get('resources_api_server') self.api_version = self.config_get('resources_api_version') self.retrievers_params = self.config_get('retrievers_params') # Check up_wait_sleep up_wait_sleep = self.retrievers_params.get('up_wait_sleep') if up_wait_sleep is not None and up_wait_sleep < 30: raise DataBridgeConfigError('Invalid \'up_wait_sleep\' in ' '\'retrievers_params\'. Value must be ' 'grater than 30.') # Workers settings for key in WORKER_CONFIG: self.workers_config[key] = (self.config_get(key) or WORKER_CONFIG[key]) # Init config for key in DEFAULTS: setattr(self, key, self.config_get(key) or DEFAULTS[key]) # Pools self.workers_pool = gevent.pool.Pool(self.workers_max) self.retry_workers_pool = gevent.pool.Pool(self.retry_workers_max) self.filter_workers_pool = gevent.pool.Pool(self.filter_workers_count) # Queues if self.input_queue_size == -1: self.input_queue = PriorityQueue() else: self.input_queue = PriorityQueue(self.input_queue_size) if self.resource_items_queue_size == -1: self.resource_items_queue = PriorityQueue() else: self.resource_items_queue = PriorityQueue( self.resource_items_queue_size) self.api_clients_queue = Queue() # self.retry_api_clients_queue = Queue() if self.retry_resource_items_queue_size == -1: self.retry_resource_items_queue = PriorityQueue() else: self.retry_resource_items_queue = PriorityQueue( self.retry_resource_items_queue_size) if self.api_host != '' and self.api_host is not None: api_host = urlparse(self.api_host) if api_host.scheme == '' and api_host.netloc == '': raise DataBridgeConfigError( 'Invalid \'tenders_api_server\' url.') else: raise DataBridgeConfigError('In config dictionary empty or missing' ' \'tenders_api_server\'') self.db = prepare_couchdb(self.couch_url, self.db_name, logger) db_url = self.couch_url + '/' + self.db_name prepare_couchdb_views(db_url, self.workers_config['resource'], logger) self.server = Server(self.couch_url, session=Session(retry_delays=range(10))) self.view_path = '_design/{}/_view/by_dateModified'.format( self.workers_config['resource']) extra_params = { 'mode': self.retrieve_mode, 'limit': self.resource_items_limit } self.feeder = ResourceFeeder(host=self.api_host, version=self.api_version, key='', resource=self.workers_config['resource'], extra_params=extra_params, retrievers_params=self.retrievers_params, adaptive=True, with_priority=True) self.api_clients_info = {} def config_get(self, name): try: return self.config.get('main').get(name) except AttributeError: raise DataBridgeConfigError('In config dictionary missed section' ' \'main\'') def create_api_client(self): client_user_agent = self.user_agent + '/' + self.bridge_id timeout = 0.1 while 1: try: api_client = APIClient( host_url=self.api_host, user_agent=client_user_agent, api_version=self.api_version, key='', resource=self.workers_config['resource']) client_id = uuid.uuid4().hex logger.info('Started api_client {}'.format( api_client.session.headers['User-Agent']), extra={'MESSAGE_ID': 'create_api_clients'}) api_client_dict = { 'id': client_id, 'client': api_client, 'request_interval': 0, 'not_actual_count': 0 } self.api_clients_info[api_client_dict['id']] = { 'drop_cookies': False, 'request_durations': {}, 'request_interval': 0, 'avg_duration': 0 } self.api_clients_queue.put(api_client_dict) break except RequestFailed as e: logger.error( 'Failed start api_client with status code {}'.format( e.status_code), extra={'MESSAGE_ID': 'exceptions'}) timeout = timeout * 2 logger.info( 'create_api_client will be sleep {} sec.'.format(timeout)) sleep(timeout) except Exception as e: logger.error('Failed start api client with error: {}'.format( e.message), extra={'MESSAGE_ID': 'exceptions'}) timeout = timeout * 2 logger.info( 'create_api_client will be sleep {} sec.'.format(timeout)) sleep(timeout) def fill_api_clients_queue(self): while self.api_clients_queue.qsize() < self.workers_min: self.create_api_client() def fill_input_queue(self): for resource_item in self.feeder.get_resource_items(): self.input_queue.put(resource_item) logger.debug('Add to temp queue from sync: {} {} {}'.format( self.workers_config['resource'][:-1], resource_item[1]['id'], resource_item[1]['dateModified']), extra={ 'MESSAGE_ID': 'received_from_sync', 'TEMP_QUEUE_SIZE': self.input_queue.qsize() }) def send_bulk(self, input_dict, priority_cache): sleep_before_retry = 2 for i in xrange(0, 3): try: logger.debug('Send check bulk: {}'.format(len(input_dict)), extra={'CHECK_BULK_LEN': len(input_dict)}) start = time() rows = self.db.view(self.view_path, keys=input_dict.values()) end = time() - start logger.debug('Duration bulk check: {} sec.'.format(end), extra={'CHECK_BULK_DURATION': end * 1000}) resp_dict = {k.id: k.key for k in rows} break except (IncompleteRead, Exception) as e: logger.error('Error while send bulk {}'.format(e.message), extra={'MESSAGE_ID': 'exceptions'}) if i == 2: raise e sleep(sleep_before_retry) sleep_before_retry *= 2 for item_id, date_modified in input_dict.items(): if item_id in resp_dict and date_modified == resp_dict[item_id]: logger.debug('Skipped {} {}: In db exist newest.'.format( self.workers_config['resource'][:-1], item_id), extra={'MESSAGE_ID': 'skipped'}) elif ((1, item_id) not in self.resource_items_queue.queue and (1000, item_id) not in self.resource_items_queue.queue): self.resource_items_queue.put( (priority_cache[item_id], item_id)) logger.debug( 'Put to main queue {}: {}'.format( self.workers_config['resource'][:-1], item_id), extra={'MESSAGE_ID': 'add_to_resource_items_queue'}) else: logger.debug( 'Skipped {} {}: In queue exist with same id'.format( self.workers_config['resource'][:-1], item_id), extra={'MESSAGE_ID': 'skipped'}) def fill_resource_items_queue(self): start_time = datetime.now() input_dict = {} priority_cache = {} while True: # Get resource_item from temp queue if not self.input_queue.empty(): priority, resource_item = self.input_queue.get() else: timeout = self.bulk_query_interval -\ (datetime.now() - start_time).total_seconds() if timeout > self.bulk_query_interval: timeout = self.bulk_query_interval try: priority, resource_item = self.input_queue.get( timeout=timeout) except Empty: resource_item = None # Add resource_item to bulk if resource_item is not None: logger.debug('Add to input_dict {}'.format( resource_item['id'])) input_dict[resource_item['id']] = resource_item['dateModified'] priority_cache[resource_item['id']] = priority if (len(input_dict) >= self.bulk_query_limit or (datetime.now() - start_time).total_seconds() >= self.bulk_query_interval): if len(input_dict) > 0: self.send_bulk(input_dict, priority_cache) input_dict = {} priority_cache = {} start_time = datetime.now() def resource_items_filter(self, r_id, r_date_modified): try: local_document = self.db.get(r_id) if local_document: if local_document['dateModified'] < r_date_modified: return True else: return False else: return True except Exception as e: logger.error( 'Filter error: Error while getting {} {} from couchdb: ' '{}'.format(self.workers_config['resource'][:-1], r_id, e.message), extra={'MESSAGE_ID': 'exceptions'}) return True def _get_average_requests_duration(self): req_durations = [] delta = timedelta(seconds=self.perfomance_window) current_date = datetime.now() - delta for cid, info in self.api_clients_info.items(): if len(info['request_durations']) > 0: if min(info['request_durations'].keys()) <= current_date: info['grown'] = True avg = round( sum(info['request_durations'].values()) * 1.0 / len(info['request_durations']), 3) req_durations.append(avg) info['avg_duration'] = avg if len(req_durations) > 0: return round(sum(req_durations) / len(req_durations), 3), req_durations else: return 0, req_durations # TODO: Add logic for restart sync if last response grater than some values # and no active tasks specific for resource def queues_controller(self): while True: if (self.workers_pool.free_count() > 0 and (self.resource_items_queue.qsize() > ((float(self.resource_items_queue_size) / 100) * self.workers_inc_threshold))): self.create_api_client() w = ResourceItemWorker.spawn(self.api_clients_queue, self.resource_items_queue, self.db, self.workers_config, self.retry_resource_items_queue, self.api_clients_info) self.workers_pool.add(w) logger.info('Queue controller: Create main queue worker.') elif (self.resource_items_queue.qsize() < ((float(self.resource_items_queue_size) / 100) * self.workers_dec_threshold)): if len(self.workers_pool) > self.workers_min: wi = self.workers_pool.greenlets.pop() wi.shutdown() api_client_dict = self.api_clients_queue.get() del self.api_clients_info[api_client_dict['id']] logger.info('Queue controller: Kill main queue worker.') filled_resource_items_queue = round( self.resource_items_queue.qsize() / (float(self.resource_items_queue_size) / 100), 2) logger.info('Resource items queue filled on {} %'.format( filled_resource_items_queue)) filled_retry_resource_items_queue \ = round(self.retry_resource_items_queue.qsize() / float( self.retry_resource_items_queue_size) / 100, 2) logger.info('Retry resource items queue filled on {} %'.format( filled_retry_resource_items_queue)) sleep(self.queues_controller_timeout) def gevent_watcher(self): self.perfomance_watcher() for t in self.server.tasks(): if (t['type'] == 'indexer' and t['database'] == self.db_name and t.get('design_document', None) == '_design/{}'.format( self.workers_config['resource'])): logger.info( 'Watcher: Waiting for end of view indexing. Current' ' progress: {} %'.format(t['progress'])) # Check fill threads input_threads = 1 if self.input_queue_filler.ready(): input_threads = 0 logger.error('Temp queue filler error: {}'.format( self.input_queue_filler.exception.message), extra={'MESSAGE_ID': 'exception'}) self.input_queue_filler = spawn(self.fill_input_queue) logger.info('Input threads {}'.format(input_threads), extra={'INPUT_THREADS': input_threads}) fill_threads = 1 if self.filler.ready(): fill_threads = 0 logger.error('Fill thread error: {}'.format( self.filler.exception.message), extra={'MESSAGE_ID': 'exception'}) self.filler = spawn(self.fill_resource_items_queue) logger.info('Filter threads {}'.format(fill_threads), extra={'FILTER_THREADS': fill_threads}) main_threads = self.workers_max - self.workers_pool.free_count() logger.info('Main threads {}'.format(main_threads), extra={'MAIN_THREADS': main_threads}) if len(self.workers_pool) < self.workers_min: for i in xrange(0, (self.workers_min - len(self.workers_pool))): w = ResourceItemWorker.spawn(self.api_clients_queue, self.resource_items_queue, self.db, self.workers_config, self.retry_resource_items_queue, self.api_clients_info) self.workers_pool.add(w) logger.info('Watcher: Create main queue worker.') self.create_api_client() retry_threads = self.retry_workers_max -\ self.retry_workers_pool.free_count() logger.info('Retry threads {}'.format(retry_threads), extra={'RETRY_THREADS': retry_threads}) if len(self.retry_workers_pool) < self.retry_workers_min: for i in xrange( 0, self.retry_workers_min - len(self.retry_workers_pool)): self.create_api_client() w = ResourceItemWorker.spawn(self.api_clients_queue, self.retry_resource_items_queue, self.db, self.workers_config, self.retry_resource_items_queue, self.api_clients_info) self.retry_workers_pool.add(w) logger.info('Watcher: Create retry queue worker.') # Log queues size and API clients count main_queue_size = self.resource_items_queue.qsize() logger.info('Resource items queue size {}'.format(main_queue_size), extra={'MAIN_QUEUE_SIZE': main_queue_size}) retry_queue_size = self.retry_resource_items_queue.qsize() logger.info( 'Resource items retry queue size {}'.format(retry_queue_size), extra={'RETRY_QUEUE_SIZE': retry_queue_size}) api_clients_count = len(self.api_clients_info) logger.info('API Clients count: {}'.format(api_clients_count), extra={'API_CLIENTS': api_clients_count}) def _calculate_st_dev(self, values): if len(values) > 0: avg = sum(values) * 1.0 / len(values) variance = map(lambda x: (x - avg)**2, values) avg_variance = sum(variance) * 1.0 / len(variance) st_dev = math.sqrt(avg_variance) return round(st_dev, 3) else: return 0 def _mark_bad_clients(self, dev): # Mark bad api clients for cid, info in self.api_clients_info.items(): if info.get('grown', False) and info['avg_duration'] > dev: info['drop_cookies'] = True logger.debug('Perfomance watcher: Mark client {} as bad, avg.' ' request_duration is {} sec.'.format( cid, info['avg_duration']), extra={'MESSAGE_ID': 'marked_as_bad'}) elif info['avg_duration'] < dev and info['request_interval'] > 0: info['drop_cookies'] = True logger.debug('Perfomance watcher: Mark client {} as bad,' ' request_interval is {} sec.'.format( cid, info['request_interval']), extra={'MESSAGE_ID': 'marked_as_bad'}) def perfomance_watcher(self): avg_duration, values = self._get_average_requests_duration() for _, info in self.api_clients_info.items(): delta = timedelta(seconds=self.perfomance_window + self.watch_interval) current_date = datetime.now() - delta delete_list = [] for key in info['request_durations']: if key < current_date: delete_list.append(key) for k in delete_list: del info['request_durations'][k] delete_list = [] st_dev = self._calculate_st_dev(values) if len(values) > 0: min_avg = min(values) * 1000 max_avg = max(values) * 1000 else: max_avg = 0 min_avg = 0 dev = round(st_dev + avg_duration, 3) logger.info('Perfomance watcher:\nREQUESTS_STDEV - {} sec.\n' 'REQUESTS_DEV - {} ms.\nREQUESTS_MIN_AVG - {} ms.\n' 'REQUESTS_MAX_AVG - {} ms.\nREQUESTS_AVG - {} sec.'.format( round(st_dev, 3), dev, min_avg, max_avg, avg_duration), extra={ 'REQUESTS_DEV': dev * 1000, 'REQUESTS_MIN_AVG': min_avg, 'REQUESTS_MAX_AVG': max_avg, 'REQUESTS_AVG': avg_duration * 1000 }) self._mark_bad_clients(dev) def run(self): logger.info('Start Edge Bridge', extra={'MESSAGE_ID': 'edge_bridge_start_bridge'}) logger.info('Start data sync...', extra={'MESSAGE_ID': 'edge_bridge__data_sync'}) self.input_queue_filler = spawn(self.fill_input_queue) self.filler = spawn(self.fill_resource_items_queue) spawn(self.queues_controller) while True: self.gevent_watcher() sleep(self.watch_interval)
class Scanner: def __init__(self): self.start_time = time.time() self.queue = PriorityQueue() self.history = [] self.total_count = 0 self.scan_count = 0 self._load_target() self.outfile = open("log.log", 'w') self.console_width = getTerminalSize()[0] - 2 def _print_msg(self, _msg=None, _found_msg=False): if _msg is None: msg = '%s TotalCount| %s Scanned in %.2f seconds' % ( self.total_count, self.total_count - self.queue.qsize(), time.time() - self.start_time) sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) + msg) else: sys.stdout.write('\r' + _msg + ' ' * (self.console_width - len(_msg)) + '\n') self.outfile.write(_msg + '\n') self.outfile.flush() if _found_msg: msg = '%s TotalCount| %s Scanned in %.2f seconds' % ( self.total_count, self.total_count - self.queue.qsize(), time.time() - self.start_time) sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) + msg) sys.stdout.flush() def _load_target(self): print('[+] Read targets ...') target_file = raw_input("Target File :") with open(target_file) as f: for line in f.xreadlines(): target = line.strip() self.queue.put(target) print("TotalCount is %d" % self.queue.qsize()) self.total_count = self.queue.qsize() print("Now scanning ...") def _scan(self, case): while not self.queue.empty(): target = self.queue.get() if case == "1": self.vulnCheck(target) if case == "2": self.s2_045(target) if case == "3": self.headers(target) if case == "4": self.weakfile(target) if case == "5": self.portscan_c(target) def vulnCheck(self, target): if ":2375" in target: try: res = requests.head("http://" + str(target) + "/containers/json", timeout=2) if res.headers['Content-Type'] == 'application/json': self._print_msg(target + "==> docker api Vuln", True) else: self._print_msg() except: self._print_msg() self._print_msg() if ":27017" in target: ip, port = target.split(":") try: socket.setdefaulttimeout(3) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((ip, int(port))) data = binascii.a2b_hex( "3a000000a741000000000000d40700000000000061646d696e2e24636d640000000000ffffffff130000001069736d6173746572000100000000" ) s.send(data) result = s.recv(1024) if "ismaster" in result: getlog_data = binascii.a2b_hex( "480000000200000000000000d40700000000000061646d696e2e24636d6400000000000100000021000000026765744c6f670010000000737461727475705761726e696e67730000" ) s.send(getlog_data) result = s.recv(1024) if "totalLinesWritten" in result: self._print_msg(target + "==> mongodb Vuln", True) except Exception as e: pass if ":6379" in target: ip, port = target.split(":") try: socket.setdefaulttimeout(3) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((ip, int(port))) s.send("INFO\r\n") result = s.recv(1024) if "redis_version" in result: self._print_msg(target + "==> redis Vuln", True) elif "Authentication" in result: for pass_ in ['123456', 'redis', 'pass', 'password']: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((ip, int(port))) s.send("AUTH %s\r\n" % (pass_)) result = s.recv(1024) if '+OK' in result: self._print_msg( target + "==> redis pass Vuln :" + pass_, True) except Exception as e: pass if ":11211" in target: ip, port = target.split(":") try: socket.setdefaulttimeout(3) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((ip, int(port))) s.send("stats\r\n") result = s.recv(1024) if "STAT version" in result: self._print_msg(target + "==> memcache Vuln", True) except Exception as e: pass if ":9200" in target: try: res = requests.head("http://" + str(target) + "/_rvier/_search", timeout=2) if res.status_code == 200: self._print_msg(target + "==> Elasticsearch Vuln", True) else: self._print_msg() except: self._print_msg() self._print_msg() def headers(self, target): try: res = requests.head("http://" + str(target), timeout=1) self._print_msg(target + "==>" + str(res.headers), True) except: self._print_msg() self._print_msg() def s2_045(self, target): try: data = {"image": " "} headers = { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36", "Content-Type": "%{#context['com.opensymphony.xwork2.dispatcher.HttpServletResponse'].addHeader('vul','s2-045')}.multtargetart/form-data" } req = requests.post("http://" + str(target), data=data, headers=headers) if req.headers["vul"] == "s2-045": self._print_msg(target + "==>" + "S2-045 Vuln", True) except: self._print_msg() self._print_msg() def weakfile(self, target): weaklist = ["robots.txt", "/i.php", "/phpinfo.php"] for weakfile in weaklist: try: res = requests.head("http://" + str(target) + weakfile, timeout=1) if res.status_code == 200: if ("User-agent" in res.content) or ("phpinfo" in res.content): self._print_msg("http://" + target + weakfile, True) except: self._print_msg() self._print_msg() def portscan_c(self, target): import socket ip = socket.gethostbyname(target) ports = [1433, 2375, 3306, 6379, 9200, 11211, 27017] ip = ip.split(".") ipc = ip[0] + "." + ip[1] + "." + ip[2] + "." if ipc in self.history: return else: self.history.append(ipc) for port in ports: for i in range(255): try: cs = socket.socket(socket.AF_INET, socket.SOCK_STREAM) cs.settimeout(float(2.5)) address = (ipc + str(i), int(port)) status = cs.connect_ex((address)) if status == 0: self._print_msg(ipc + str(i) + ":" + str(port), True) except Exception as e: pass finally: cs.close() self._print_msg() def run(self, case): threads = [gevent.spawn(self._scan, case) for i in xrange(1000)]
class SubNameBrute: def __init__(self, target, args, process_num, dns_servers, next_subs, scan_count, found_count, queue_size_list, tmp_dir): self.target = target.strip() self.args = args self.process_num = process_num self.dns_servers = dns_servers self.dns_count = len(dns_servers) self.next_subs = next_subs self.scan_count = scan_count self.scan_count_local = 0 self.found_count = found_count self.found_count_local = 0 self.queue_size_list = queue_size_list self.resolvers = [ dns.resolver.Resolver(configure=False) for _ in range(args.threads) ] for _r in self.resolvers: _r.lifetime = _r.timeout = 6.0 self.queue = PriorityQueue() self.item_index = 0 self.priority = 0 self._load_sub_names() self.ip_dict = {} self.found_subs = set() self.ex_resolver = dns.resolver.Resolver(configure=False) self.ex_resolver.nameservers = dns_servers self.local_time = time.time() self.outfile = open( '%s/%s_part_%s.txt' % (tmp_dir, target, process_num), 'w') self.outfile_html = open('tmp/%s_html_%s.txt' % (target, process_num), 'w') def _load_sub_names(self): if self.args.full_scan and self.args.file == 'subnames.txt': _file = 'Dict/subnames_full.txt' else: if os.path.exists(self.args.file): _file = self.args.file elif os.path.exists('Dict/%s' % self.args.file): _file = 'Dict/%s' % self.args.file else: print_msg('[ERROR] Names file not found: %s' % self.args.file) exit(-1) normal_lines = [] lines = set() with open(_file) as f: for line in f.xreadlines(): sub = line.strip() if not sub or sub in lines: continue lines.add(sub) normal_lines.append(sub) for item in normal_lines[self.process_num::self.args.process]: self.priority += 1 self.queue.put((self.priority, item)) def _scan(self, j): self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]] while not self.queue.empty(): try: item = self.queue.get()[1] self.scan_count_local += 1 if time.time() - self.local_time > 3.0: self.scan_count.value += self.scan_count_local self.scan_count_local = 0 self.queue_size_list[self.process_num] = self.queue.qsize() except Exception as e: break try: if item.find('{next_sub}') >= 0: for _ in self.next_subs: self.queue.put((0, item.replace('{next_sub}', _, 1))) continue else: sub = item if sub in self.found_subs: continue cur_sub_domain = sub + '.' + self.target _sub = sub.split('.')[-1] try: answers = self.resolvers[j].query(cur_sub_domain) except dns.resolver.NoAnswer, e: answers = self.ex_resolver.query(cur_sub_domain) if answers: self.found_subs.add(sub) ips = ', '.join( sorted([answer.address for answer in answers])) if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0']: continue if self.args.intranet and is_intranet(answers[0].address): continue try: self.scan_count_local += 1 answers = self.resolvers[j].query( cur_sub_domain, 'cname') cname = answers[0].target.to_unicode().rstrip('.') if cname.endswith( self.target) and cname not in self.found_subs: self.found_subs.add(cname) cname_sub = cname[:len(cname) - len(self.target) - 1] self.queue.put((0, cname_sub)) except: pass if (_sub, ips) not in self.ip_dict: self.ip_dict[(_sub, ips)] = 1 else: self.ip_dict[(_sub, ips)] += 1 if self.ip_dict[(_sub, ips)] > 30: continue self.found_count_local += 1 if time.time() - self.local_time > 3.0: self.found_count.value += self.found_count_local self.found_count_local = 0 self.queue_size_list[ self.process_num] = self.queue.qsize() self.local_time = time.time() self.outfile_html.write( PrintHtml.Sub_html_print(cur_sub_domain, ips)) self.outfile_html.flush() self.outfile.write( cur_sub_domain.ljust(30) + '\t' + ips + '\n') self.outfile.flush() try: self.resolvers[j].query('myzxcghelloha.' + cur_sub_domain) except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e: self.queue.put((999999999, '{next_sub}.' + sub)) except: pass except (dns.resolver.NXDOMAIN, dns.name.EmptyLabel) as e: pass except (dns.resolver.NoNameservers, dns.resolver.NoAnswer, dns.exception.Timeout) as e: pass
class SyncClient: idle = idle backward_class = BackwardWorker forward_class = ForwardWorker def __init__(self, host_url, resource, auth=None, params={}, headers=None, retrievers_params=DEFAULT_RETRIEVERS_PARAMS, adaptive=False, with_priority=False): LOGGER.info(f'Init SyncClient for resource {resource}') self.host = host_url self.auth = auth self.resource = resource self.adaptive = adaptive self.headers = headers self.params = params self.retrievers_params = retrievers_params self.queue = PriorityQueue(maxsize=retrievers_params['queue_size']) def init_clients(self): self.backward_client = ResourceClient(self.host, self.resource, self.params, self.auth, self.headers) self.forward_client = ResourceClient(self.host, self.resource, self.params, self.auth, self.headers) def handle_response_data(self, data): for resource_item in data: self.queue.put(PrioritizedItem(1, resource_item)) def worker_watcher(self): while True: if time() - self.heartbeat > DEFAULT_FORWARD_HEARTBEAT: self.restart_sync() LOGGER.warning( 'Restart sync, reason: Last response from workers greater than 15 min ago.' ) sleep(300) def start_sync(self): LOGGER.info('Start sync...') data = self.backward_client.get_resource_items(self.params) self.handle_response_data(data[f'{self.resource}s']) forward_params = deepcopy(self.params) forward_params.update({ k: v[0] for k, v in parse.parse_qs(parse.urlparse( data.links.prev).query).items() }) backward_params = deepcopy(self.params) backward_params.update({ k: v[0] for k, v in parse.parse_qs(parse.urlparse( data.links.next).query).items() }) self.forward_worker = self.forward_class(sync_client=self, client=self.forward_client, params=forward_params) self.backward_worker = self.backward_class(sync_client=self, client=self.backward_client, params=backward_params) self.workers = [self.forward_worker, self.backward_worker] for worker in self.workers: worker.start() self.heartbeat = time() self.watcher = spawn(self.worker_watcher) def restart_sync(self): """ Restart retrieving from OCDS API. """ LOGGER.info('Restart workers') for worker in self.workers: worker.kill() self.watcher.kill() self.init_clients() self.start_sync() def get_resource_items(self): self.init_clients() self.start_sync() while True: if self.forward_worker.check() or self.backward_worker.check(): self.restart_sync() while not self.queue.empty(): LOGGER.debug(f'Sync queue size: {self.queue.qsize()}', extra={'SYNC_QUEUE_SIZE': self.queue.qsize()}) LOGGER.debug('Yield resource item', extra={'MESSAGE_ID': 'sync_yield'}) item = self.queue.get() yield item.data LOGGER.debug(f'Sync queue size: {self.queue.qsize()}', extra={'SYNC_QUEUE_SIZE': self.queue.qsize()}) try: self.queue.peek(block=True, timeout=0.1) except Empty: pass
class SubNameBrute(object): def __init__(self, *params): self.domain, self.options, self.process_num, self.dns_servers, self.next_subs, \ self.scan_count, self.found_count, self.queue_size_array, tmp_dir = params self.dns_count = len(self.dns_servers) self.scan_count_local = 0 self.found_count_local = 0 self.resolvers = [dns.resolver.Resolver(configure=False) for _ in range(self.options.threads)] for r in self.resolvers: r.lifetime = 4 r.timeout = 10.0 self.queue = PriorityQueue() self.priority = 0 self.ip_dict = {} self.found_subs = set() self.timeout_subs = {} self.count_time = time.time() self.outfile = open('%s/%s_part_%s.txt' % (tmp_dir, self.domain, self.process_num), 'w') self.normal_names_set = set() self.load_sub_names() self.lock = RLock() self.threads_status = ['1'] * self.options.threads def load_sub_names(self): normal_lines = [] wildcard_lines = [] wildcard_set = set() regex_list = [] lines = set() with open(self.options.file) as inFile: for line in inFile.readlines(): sub = line.strip() if not sub or sub in lines: continue lines.add(sub) brace_count = sub.count('{') if brace_count > 0: wildcard_lines.append((brace_count, sub)) sub = sub.replace('{alphnum}', '[a-z0-9]') sub = sub.replace('{alpha}', '[a-z]') sub = sub.replace('{num}', '[0-9]') if sub not in wildcard_set: wildcard_set.add(sub) regex_list.append('^' + sub + '$') else: normal_lines.append(sub) self.normal_names_set.add(sub) if regex_list: pattern = '|'.join(regex_list) _regex = re.compile(pattern) for line in normal_lines: if _regex.search(line): normal_lines.remove(line) for _ in normal_lines[self.process_num::self.options.process]: self.queue.put((0, _)) # priority set to 0 for _ in wildcard_lines[self.process_num::self.options.process]: self.queue.put(_) def scan(self, j): self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]] + self.dns_servers while True: try: if time.time() - self.count_time > 1.0: self.lock.acquire() self.scan_count.value += self.scan_count_local self.scan_count_local = 0 self.queue_size_array[self.process_num] = self.queue.qsize() if self.found_count_local: self.found_count.value += self.found_count_local self.found_count_local = 0 self.count_time = time.time() self.lock.release() brace_count, sub = self.queue.get_nowait() self.threads_status[j] = '1' if brace_count > 0: brace_count -= 1 if sub.find('{next_sub}') >= 0: for _ in self.next_subs: self.queue.put((0, sub.replace('{next_sub}', _))) if sub.find('{alphnum}') >= 0: for _ in 'abcdefghijklmnopqrstuvwxyz0123456789': self.queue.put((brace_count, sub.replace('{alphnum}', _, 1))) elif sub.find('{alpha}') >= 0: for _ in 'abcdefghijklmnopqrstuvwxyz': self.queue.put((brace_count, sub.replace('{alpha}', _, 1))) elif sub.find('{num}') >= 0: for _ in '0123456789': self.queue.put((brace_count, sub.replace('{num}', _, 1))) continue except gevent.queue.Empty as e: self.threads_status[j] = '0' gevent.sleep(0.5) if '1' not in self.threads_status: break else: continue try: if sub in self.found_subs: continue self.scan_count_local += 1 cur_domain = sub + '.' + self.domain answers = self.resolvers[j].query(cur_domain) if answers: self.found_subs.add(sub) ips = ', '.join(sorted([answer.address for answer in answers])) if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0', '0.0.0.1']: continue if self.options.i and is_intranet(answers[0].address): continue try: self.scan_count_local += 1 answers = self.resolvers[j].query(cur_domain, 'cname') cname = answers[0].target.to_unicode().rstrip('.') if cname.endswith(self.domain) and cname not in self.found_subs: cname_sub = cname[:len(cname) - len(self.domain) - 1] # new sub if cname_sub not in self.normal_names_set: self.found_subs.add(cname) self.queue.put((0, cname_sub)) except Exception as e: pass first_level_sub = sub.split('.')[-1] max_found = 20 if self.options.w: first_level_sub = '' max_found = 3 if (first_level_sub, ips) not in self.ip_dict: self.ip_dict[(first_level_sub, ips)] = 1 else: self.ip_dict[(first_level_sub, ips)] += 1 if self.ip_dict[(first_level_sub, ips)] > max_found: continue self.found_count_local += 1 self.outfile.write(cur_domain.ljust(30) + '\t' + ips + '\n') self.outfile.flush() try: self.scan_count_local += 1 self.resolvers[j].query('lijiejie-test-not-existed.' + cur_domain) except (dns.resolver.NXDOMAIN, ) as e: # dns.resolver.NoAnswer if self.queue.qsize() < 50000: for _ in self.next_subs: self.queue.put((0, _ + '.' + sub)) else: self.queue.put((1, '{next_sub}.' + sub)) except Exception as e: pass except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e: pass except dns.resolver.NoNameservers as e: self.queue.put((0, sub)) # Retry except dns.exception.Timeout as e: self.timeout_subs[sub] = self.timeout_subs.get(sub, 0) + 1 if self.timeout_subs[sub] <= 1: self.queue.put((0, sub)) # Retry except Exception as e: import traceback traceback.print_exc() with open('errors.log', 'a') as errFile: errFile.write('[%s] %s\n' % (type(e), str(e))) def run(self): threads = [gevent.spawn(self.scan, i) for i in range(self.options.threads)] gevent.joinall(threads)
class SubNameBrute: def __init__(self, target, options, process_num, dns_servers, cdns,next_subs, scan_count, found_count, queue_size_list, tmp_dir): self.target = target.strip() self.options = options self.process_num = process_num self.dns_servers = dns_servers self.cdns = cdns self.dns_count = len(dns_servers) self.next_subs = next_subs self.scan_count = scan_count self.scan_count_local = 0 self.found_count = found_count self.found_count_local = 0 self.queue_size_list = queue_size_list self.resolvers = [dns.resolver.Resolver(configure=False) for _ in range(options.threads)] for _r in self.resolvers: _r.lifetime = _r.timeout = 6.0 self.queue = PriorityQueue() self.item_index = 0 self.priority = 0 self._load_sub_names() self.ip_dict = {} self.found_subs = set() self.ex_resolver = dns.resolver.Resolver(configure=False) self.ex_resolver.nameservers = dns_servers self.local_time = time.time() self.outfile = open('%s/%s_part_%s.txt' % (tmp_dir, target, process_num), 'w') def _load_sub_names(self): if self.options.full_scan and self.options.file == 'subnames.txt': _file = 'dict/subnames_full.txt' else: if os.path.exists(self.options.file): _file = self.options.file elif os.path.exists('dict/%s' % self.options.file): _file = 'dict/%s' % self.options.file else: print_msg('[ERROR] Names file not found: %s' % self.options.file) exit(-1) normal_lines = [] wildcard_lines = [] wildcard_list = [] regex_list = [] lines = set() with open(_file) as f: for line in f.xreadlines(): sub = line.strip() if not sub or sub in lines: continue lines.add(sub) if sub.find('{alphnum}') >= 0 or sub.find('{alpha}') >= 0 or sub.find('{num}') >= 0: wildcard_lines.append(sub) sub = sub.replace('{alphnum}', '[a-z0-9]') sub = sub.replace('{alpha}', '[a-z]') sub = sub.replace('{num}', '[0-9]') if sub not in wildcard_list: wildcard_list.append(sub) regex_list.append('^' + sub + '$') else: normal_lines.append(sub) if regex_list: pattern = '|'.join(regex_list) _regex = re.compile(pattern) for line in normal_lines[:]: if _regex.search(line): normal_lines.remove(line) for item in normal_lines[self.process_num::self.options.process]: self.priority += 1 self.queue.put((self.priority, item)) for item in wildcard_lines[self.process_num::self.options.process]: self.queue.put((88888888, item)) def put_item(self, item): num = item.count('{alphnum}') + item.count('{alpha}') + item.count('{num}') if num == 0: self.priority += 1 self.queue.put((self.priority, item)) else: self.queue.put((self.priority + num * 10000000, item)) def check_cdn(self, cname): ''' bTrue = True bFound = False i = 0 while bTrue: cdn = self.cdns[i] i += 1 if (cdn in cname) or (i == len(self.cdns)): if (cdn in cname): bFound = True bTrue = False return bFound ''' for cdn in self.cdns: if cdn in cname: return True return False def _scan(self, j): self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]] while not self.queue.empty(): try: item = self.queue.get(timeout=3.0)[1] self.scan_count_local += 1 if time.time() - self.local_time > 3.0: self.scan_count.value += self.scan_count_local self.scan_count_local = 0 self.queue_size_list[self.process_num] = self.queue.qsize() except Exception as e: break try: if item.find('{alphnum}') >= 0: for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789': self.put_item(item.replace('{alphnum}', _letter, 1)) continue elif item.find('{alpha}') >= 0: for _letter in 'abcdefghijklmnopqrstuvwxyz': self.put_item(item.replace('{alpha}', _letter, 1)) continue elif item.find('{num}') >= 0: for _letter in '0123456789': self.put_item(item.replace('{num}', _letter, 1)) continue elif item.find('{next_sub}') >= 0: for _ in self.next_subs: self.queue.put((0, item.replace('{next_sub}', _, 1))) continue else: sub = item if sub in self.found_subs: continue cur_sub_domain = sub + '.' + self.target _sub = sub.split('.')[-1] try: answers = self.resolvers[j].query(cur_sub_domain) except dns.resolver.NoAnswer, e: answers = self.ex_resolver.query(cur_sub_domain) if answers: ans = self.resolvers[j].query(cur_sub_domain,'cname') cname = ans[0].target.to_unicode().rstrip('.') if self.check_cdn(cname): continue self.found_subs.add(sub) ips = ', '.join(sorted([answer.address for answer in answers])) if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0']: continue if self.options.i and is_intranet(answers[0].address): continue try: self.scan_count_local += 1 answers = self.resolvers[j].query(cur_sub_domain, 'cname') cname = answers[0].target.to_unicode().rstrip('.') if cname.endswith(self.target) and cname not in self.found_subs: self.found_subs.add(cname) cname_sub = cname[:len(cname) - len(self.target) - 1] # new sub self.queue.put((0, cname_sub)) except: pass if (_sub, ips) not in self.ip_dict: self.ip_dict[(_sub, ips)] = 1 else: self.ip_dict[(_sub, ips)] += 1 if self.ip_dict[(_sub, ips)] > 30: continue self.found_count_local += 1 if time.time() - self.local_time > 3.0: self.found_count.value += self.found_count_local self.found_count_local = 0 self.queue_size_list[self.process_num] = self.queue.qsize() self.local_time = time.time() msg = cur_sub_domain.ljust(30) + ips # print_msg(msg, line_feed=True) self.outfile.write(cur_sub_domain.ljust(30) + '\t' + ips + '\n') self.outfile.flush() try: self.resolvers[j].query('lijiejietest.' + cur_sub_domain) except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e: self.queue.put((999999999, '{next_sub}.' + sub)) except: pass except (dns.resolver.NXDOMAIN, dns.name.EmptyLabel) as e: pass except (dns.resolver.NoNameservers, dns.resolver.NoAnswer, dns.exception.Timeout) as e: pass
class SubNameBrute: def __init__(self, target, options, process_num, dns_servers, next_subs, scan_count, found_count, queue_size_list, tmp_dir): self.target = target.strip() self.options = options self.process_num = process_num self.dns_servers = dns_servers self.dns_count = len(dns_servers) self.next_subs = next_subs self.scan_count = scan_count self.scan_count_local = 0 self.found_count = found_count self.found_count_local = 0 self.queue_size_list = queue_size_list self.resolvers = [dns.resolver.Resolver(configure=False) for _ in range(options.threads)] for _r in self.resolvers: _r.lifetime = _r.timeout = 6.0 self.queue = PriorityQueue() self.item_index = 0 self.priority = 0 self._load_sub_names() self.ip_dict = {} self.found_subs = set() self.ex_resolver = dns.resolver.Resolver(configure=False) self.ex_resolver.nameservers = dns_servers self.local_time = time.time() self.outfile = open('%s/%s_part_%s.txt' % (tmp_dir, target, process_num), 'w') def _load_sub_names(self): if self.options.full_scan and self.options.file == 'subnames.txt': _file = 'dict/subnames_full.txt' else: if os.path.exists(self.options.file): _file = self.options.file elif os.path.exists('dict/%s' % self.options.file): _file = 'dict/%s' % self.options.file else: print_msg('[ERROR] Names file not found: %s' % self.options.file) exit(-1) normal_lines = [] wildcard_lines = [] wildcard_list = [] regex_list = [] lines = set() with open(_file) as f: for line in f.xreadlines(): sub = line.strip() if not sub or sub in lines: continue lines.add(sub) if sub.find('{alphnum}') >= 0 or sub.find('{alpha}') >= 0 or sub.find('{num}') >= 0: wildcard_lines.append(sub) sub = sub.replace('{alphnum}', '[a-z0-9]') sub = sub.replace('{alpha}', '[a-z]') sub = sub.replace('{num}', '[0-9]') if sub not in wildcard_list: wildcard_list.append(sub) regex_list.append('^' + sub + '$') else: normal_lines.append(sub) if regex_list: pattern = '|'.join(regex_list) _regex = re.compile(pattern) for line in normal_lines[:]: if _regex.search(line): normal_lines.remove(line) for item in normal_lines[self.process_num::self.options.process]: self.priority += 1 self.queue.put((self.priority, item)) for item in wildcard_lines[self.process_num::self.options.process]: self.queue.put((88888888, item)) def put_item(self, item): num = item.count('{alphnum}') + item.count('{alpha}') + item.count('{num}') if num == 0: self.priority += 1 self.queue.put((self.priority, item)) else: self.queue.put((self.priority + num * 10000000, item)) def _scan(self, j): self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]] while not self.queue.empty(): try: item = self.queue.get(timeout=3.0)[1] self.scan_count_local += 1 if time.time() - self.local_time > 3.0: self.scan_count.value += self.scan_count_local self.scan_count_local = 0 self.queue_size_list[self.process_num] = self.queue.qsize() except Exception as e: break try: if item.find('{alphnum}') >= 0: for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789': self.put_item(item.replace('{alphnum}', _letter, 1)) continue elif item.find('{alpha}') >= 0: for _letter in 'abcdefghijklmnopqrstuvwxyz': self.put_item(item.replace('{alpha}', _letter, 1)) continue elif item.find('{num}') >= 0: for _letter in '0123456789': self.put_item(item.replace('{num}', _letter, 1)) continue elif item.find('{next_sub}') >= 0: for _ in self.next_subs: self.queue.put((0, item.replace('{next_sub}', _, 1))) continue else: sub = item if sub in self.found_subs: continue cur_sub_domain = sub + '.' + self.target _sub = sub.split('.')[-1] try: answers = self.resolvers[j].query(cur_sub_domain) except dns.resolver.NoAnswer, e: answers = self.ex_resolver.query(cur_sub_domain) if answers: self.found_subs.add(sub) ips = ', '.join(sorted([answer.address for answer in answers])) if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0']: continue if self.options.i and is_intranet(answers[0].address): continue try: self.scan_count_local += 1 answers = self.resolvers[j].query(cur_sub_domain, 'cname') cname = answers[0].target.to_unicode().rstrip('.') if cname.endswith(self.target) and cname not in self.found_subs: self.found_subs.add(cname) cname_sub = cname[:len(cname) - len(self.target) - 1] # new sub self.queue.put((0, cname_sub)) except: pass if (_sub, ips) not in self.ip_dict: self.ip_dict[(_sub, ips)] = 1 else: self.ip_dict[(_sub, ips)] += 1 if self.ip_dict[(_sub, ips)] > 30: continue self.found_count_local += 1 if time.time() - self.local_time > 3.0: self.found_count.value += self.found_count_local self.found_count_local = 0 self.queue_size_list[self.process_num] = self.queue.qsize() self.local_time = time.time() msg = cur_sub_domain.ljust(30) + ips # print_msg(msg, line_feed=True) self.outfile.write(cur_sub_domain.ljust(30) + '\t' + ips + '\n') self.outfile.flush() try: self.resolvers[j].query('lijiejietest.' + cur_sub_domain) except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e: self.queue.put((999999999, '{next_sub}.' + sub)) except: pass except (dns.resolver.NXDOMAIN, dns.name.EmptyLabel) as e: pass except (dns.resolver.NoNameservers, dns.resolver.NoAnswer, dns.exception.Timeout) as e: pass