def test_priority():
refund_queue = PriorityQueue()
receipt = TransferReceipt(sender=123, amount=1, identifier=123, received_timestamp=1)
# higher priority
refund1 = Refund(receipt, priority=1, claim_fee=False)
# lower priority
refund2 = Refund(receipt, priority=5, claim_fee=False)
assert refund1 > refund2
refund_queue.put(refund1)
refund_queue.put(refund2)
received_first = refund_queue.get()
received_second = refund_queue.get()
assert received_first == refund2
assert received_second == refund1
def test_add_to_retry_queue(self, mocked_logger):
retry_items_queue = PriorityQueue()
worker = AgreementWorker(config_dict=self.worker_config,
retry_resource_items_queue=retry_items_queue)
resource_item = {'id': uuid.uuid4().hex}
priority = 1000
self.assertEqual(retry_items_queue.qsize(), 0)
# Add to retry_resource_items_queue
worker.add_to_retry_queue(resource_item, priority=priority)
self.assertEqual(retry_items_queue.qsize(), 1)
priority, retry_resource_item = retry_items_queue.get()
self.assertEqual((priority, retry_resource_item),
(1001, resource_item))
resource_item = {'id': 0}
# Add to retry_resource_items_queue with status_code '429'
worker.add_to_retry_queue(resource_item, priority, status_code=429)
self.assertEqual(retry_items_queue.qsize(), 1)
priority, retry_resource_item = retry_items_queue.get()
self.assertEqual((priority, retry_resource_item),
(1001, resource_item))
priority = 1002
worker.add_to_retry_queue(resource_item, priority=priority)
sleep(worker.config['retry_default_timeout'] * 2)
self.assertEqual(retry_items_queue.qsize(), 1)
priority, retry_resource_item = retry_items_queue.get()
self.assertEqual((priority, retry_resource_item),
(1003, resource_item))
worker.add_to_retry_queue(resource_item, priority=priority)
self.assertEqual(retry_items_queue.qsize(), 0)
mocked_logger.critical.assert_called_once_with(
'Tender {} reached limit retries count {} and droped from '
'retry_queue.'.format(resource_item['id'],
worker.config['retries_count']),
extra={
'MESSAGE_ID': 'dropped_documents',
'JOURNAL_TENDER_ID': resource_item['id']
})
del worker
def test_add_to_retry_queue(self, mocked_logger):
retry_items_queue = PriorityQueue()
worker = ResourceItemWorker(
config_dict=self.worker_config,
retry_resource_items_queue=retry_items_queue)
resource_item_id = uuid.uuid4().hex
priority = 1000
self.assertEqual(retry_items_queue.qsize(), 0)
# Add to retry_resource_items_queue
worker.add_to_retry_queue(resource_item_id, priority=priority)
# sleep(worker.config['retry_default_timeout'] * 0)
self.assertEqual(retry_items_queue.qsize(), 1)
priority, retry_resource_item_id = retry_items_queue.get()
self.assertEqual(priority, 1001)
self.assertEqual(retry_resource_item_id, resource_item_id)
# Add to retry_resource_items_queue with status_code '429'
worker.add_to_retry_queue(resource_item_id, priority, status_code=429)
self.assertEqual(retry_items_queue.qsize(), 1)
priority, retry_resource_item_id = retry_items_queue.get()
self.assertEqual(priority, 1001)
self.assertEqual(retry_resource_item_id, resource_item_id)
priority = 1002
worker.add_to_retry_queue(resource_item_id, priority=priority)
sleep(worker.config['retry_default_timeout'] * 2)
self.assertEqual(retry_items_queue.qsize(), 1)
priority, retry_resource_item_id = retry_items_queue.get()
self.assertEqual(priority, 1003)
self.assertEqual(retry_resource_item_id, resource_item_id)
worker.add_to_retry_queue(resource_item_id, priority=priority)
self.assertEqual(retry_items_queue.qsize(), 0)
mocked_logger.critical.assert_called_once_with(
'Tender {} reached limit retries count {} and droped from '
'retry_queue.'.format(resource_item_id,
worker.config['retries_count']),
extra={'MESSAGE_ID': 'dropped_documents'})
del worker
class HttpTest(object):
def __init__(self,host,keyword,ips,timeout):
self.threads = 100
self.queue = PriorityQueue()
self.host = host
self.keyword = keyword
self.result = []
for ip in ips:
self.queue.put(ip)
self.num = self.queue.qsize()
self.i = 0
self.success = 0
self.timeout = timeout
self.filename = os.path.join(rootPath,"result",host + ".log")
self.outfile = open(self.filename, 'w')
def _scan(self,j):
while not self.queue.empty():
try:
item = self.queue.get(timeout=3.0)
if config.HTTPS_Support:
host, domain, port = item, self.host , 443
else:
host, domain, port = item, self.host , 80
html = httpServer((host, domain, port),self.timeout)
if html is not None and self.keyword in html:
self.outfile.write(item + '\n')
self.outfile.flush()
self.success += 1
except:
pass
finally:
self.i += 1
msg = '[*] %s found, %s scanned , %s groups left'%(self.success,self.i,self.num - self.i)
print_msg(msg)
time.sleep(1.0)
def run(self):
threads = [gevent.spawn(self._scan, i) for i in range(self.threads)]
gevent.joinall(threads)
msg = '[+] All Done. Success:%d Saved in:%s'%(self.success,self.filename)
print_msg(msg, line_feed=True)
class AutoHack():
def __init__(self, zoomeye_results, threads_num):
self.threads_num = threads_num
self.targets = PriorityQueue()
self.zoomeye_results = zoomeye_results
self.result = []
for zoomeye_result in zoomeye_results:
self.targets.put(zoomeye_result)
self.total = self.targets.qsize()
self.pbar = tqdm(total=self.total,ascii=True)
def check(self):
while self.targets.qsize() > 0:
target = self.targets.get().strip()
try:
self.pbar.update(1)
result = exp.exp(target)
if result:
self.result.append(result)
except Exception as e:
#print(e)
pass
def run(self):
threads = [gevent.spawn(self.check) for i in range(self.threads_num)]
try:
gevent.joinall(threads)
except KeyboardInterrupt as e:
print ('[WARNING] User aborted')
for res in self.result:
print (res)
self.pbar.close()
print ("Hack it!")
for res in self.result:
print (res)
print("Found ",len(self.result))
print ("End!")
class _Channel(object):
def __init__(self, from_fsm, to_fsm, tracer, queue=None):
if queue is None:
self.queue = PriorityQueue()
else:
self.queue = queue
self.from_fsm = from_fsm
self.to_fsm = to_fsm
self.tracer = tracer
def put(self, priority_order_item):
priority, order, item = priority_order_item
self.tracer.send_trace_message(
messages.ChannelTrace(
self.tracer.trace_order_seq(),
self.from_fsm.fsm_id if self.from_fsm else None,
self.to_fsm.fsm_id if self.to_fsm else None, item.name))
self.queue.put(priority_order_item)
def get(self, block=True, timeout=None):
return self.queue.get(block, timeout)
receive = get
class Actor(Greenlet):
"""Simple implementation of the Actor pattern
"""
def __init__(self):
self.inbox = PriorityQueue()
self._handlers = {ShutdownRequest: self.receive_shutdown}
Greenlet.__init__(self)
def receive(self, msg):
"""Dispatch a received message to the appropriate type handler
"""
#log.debug("Received a message: " + repr(msg))
cls = msg.__class__
if cls in self._handlers.keys():
self._handlers[cls](msg)
else:
raise NotImplemented()
def receive_shutdown(self, msg):
self.running = False
def send(self, msg, priority=50):
"""Place a message into the actor's inbox
"""
self.inbox.put((priority, msg))
def _run(self):
"""Run the Actor in a blocking event loop
"""
self.running = True
while self.running:
prio, msg = self.inbox.get()
self.receive(msg)
del msg
class ZMQEventChannel(object):
def __init__(self, fsm_registry, connector_registry, configuration):
self.fsm_registry = fsm_registry
self.connector_registry = connector_registry
self.context = zmq.Context.instance()
self.socket = self.context.socket(zmq.ROUTER)
if 'bind_port' in configuration:
self.socket_port = configuration.get('bind_port')
self.socket.bind('tcp://{0}:{1}'.format(configuration.get('bind_address', '127.0.0.1'),
self.socket_port))
else:
self.socket_port = self.socket.bind_to_random_port('tcp://{0}'.format(configuration.get('bind_address', '127.0.0.1')))
logger.info('starting zmq_thread')
self.zmq_thread = gevent.spawn(self.receive_external_messages)
self.inbox_thread = gevent.spawn(self.receive_internal_messages)
self.inbox = PriorityQueue()
self.message_id_seq = count(0)
self.client_id_seq = count(0)
self.clients = dict()
def receive_internal_messages(self):
while True:
gevent.sleep(0.1)
logger.info("Waiting for messages")
priority, order, message = self.inbox.get()
message_type = message.name
logger.info('Received %s', message_type)
if 'client_id' in message.data and message.data['client_id'] in self.clients:
#Unicast
logger.info("Unicasting message to %s aka %r", message.data['client_id'], self.clients[message.data['client_id']])
msg = [self.clients[message.data['client_id']]]
msg.extend(messages.serialize(message))
self.socket.send_multipart(msg)
else:
#Broadcast
logger.info("Broadcasting message to all listening clients")
for client_id in list(self.clients.values()):
msg = [client_id]
msg.extend(messages.serialize(message))
self.socket.send_multipart(msg)
def receive_external_messages(self):
while True:
to_fsm_id = None
from_fsm_id = None
zmq_client_id = None
logger.info('waiting on recv_multipart')
message = self.socket.recv_multipart()
logger.info(repr(message))
zmq_client_id = message.pop(0)
client_id = str(next(self.client_id_seq))
self.clients[client_id] = zmq_client_id
try:
msg_type = message.pop(0).decode()
msg_data = yaml.safe_load(message.pop(0).decode())
if b'Listening' in message:
msg_data['data']['client_id'] = client_id
logger.info(repr(msg_type))
logger.info(repr(msg_data))
except Exception as e:
self.socket.send_multipart([zmq_client_id, b'Error'])
logger.error(str(e))
continue
if not isinstance(msg_type, str):
self.socket.send_multipart([zmq_client_id, 'Element 1 should be str was {}'.format(type(msg_type)).encode()])
logger.error([zmq_client_id, 'Element 1 should be str was {}'.format(type(msg_type)).encode()])
continue
if not isinstance(msg_data, dict):
self.socket.send_multipart([zmq_client_id, 'Element 2 should be a dict was {}'.format(type(msg_data)).encode()])
logger.error([zmq_client_id, 'Element 2 should be a dict was {}'.format(type(msg_data)).encode()])
continue
to_fsm_id = msg_data.get('to_fsm_id', None)
from_fsm_id = msg_data.get('from_fsm_id', None)
if not from_fsm_id:
from_fsm_id = 'zmq'
if to_fsm_id in self.fsm_registry:
logger.info('Sending to FSM {} from {}'.format(to_fsm_id, from_fsm_id))
self.fsm_registry[to_fsm_id].inbox.put((1,
next(self.fsm_registry[to_fsm_id].message_id_seq),
messages.Event(from_fsm_id,
to_fsm_id,
msg_data['name'],
msg_data['data'])))
logger.info('Processed')
self.socket.send_multipart([zmq_client_id, b'Processed'])
else:
logger.info('Not processed')
self.socket.send_multipart([zmq_client_id, b'Not processed'])
gevent.sleep(0)
class FactoryPool(object):
def __init__(self, factory, maxsize=200, timeout=60):
self.factory = factory
self.maxsize = maxsize
self.timeout = timeout
self.clients = PriorityQueue(maxsize)
# If there is a maxsize, prime the queue with empty slots.
if maxsize is not None:
for _ in xrange(maxsize):
self.clients.put(EMPTY_SLOT)
@contextlib.contextmanager
def reserve(self):
"""Context-manager to obtain a Client object from the pool."""
ts, client = self._checkout_connection()
try:
yield client
finally:
self._checkin_connection(ts, client)
def _checkout_connection(self):
# If there's no maxsize, no need to block waiting for a connection.
blocking = self.maxsize is not None
# Loop until we get a non-stale connection, or we create a new one.
while True:
try:
ts, client = self.clients.get(blocking)
except Empty:
# No maxsize and no free connections, create a new one.
# XXX TODO: we should be using a monotonic clock here.
# see http://www.python.org/dev/peps/pep-0418/
now = int(time.time())
return now, self.factory()
else:
now = int(time.time())
# If we got an empty slot placeholder, create a new connection.
if client is None:
return now, self.factory()
# If the connection is not stale, go ahead and use it.
if ts + self.timeout > now:
return ts, client
# Otherwise, the connection is stale.
# Close it, push an empty slot onto the queue, and retry.
if hasattr(client, 'disconnect'):
client.disconnect()
self.clients.put(EMPTY_SLOT)
continue
def _checkin_connection(self, ts, client):
"""Return a connection to the pool."""
if hasattr(client, '_closed') and client._closed:
self.clients.put(EMPTY_SLOT)
return
# If the connection is now stale, don't return it to the pool.
# Push an empty slot instead so that it will be refreshed when needed.
now = int(time.time())
if ts + self.timeout > now:
self.clients.put((ts, client))
else:
if self.maxsize is not None:
self.clients.put(EMPTY_SLOT)
class PriorityBlockingQueue(object):
"""
带优先级的阻塞队列。
优先级数字越小，优先级越高。
插入元素：
* put: 向队列尾部插入一个元素，如果该队列已满，则一直阻塞。
* offer: 向队列尾部插入一个元素，插入成功返回True。插入失败返回False。
获取元素：
* poll: 获取并移除队列的头元素，若队列为空，则返回null。
* take: 获取并移除队列的头元素，若队列为空，则一直阻塞。
* peek：获取但不移除队列的头元素，若队列为空，则返回null
队列状态状态：
* qsize：获取队列中当前元素数量
* maxsize：获取队列的最大容量
"""
def __init__(self, maxsize: int = None):
"""
init
:param maxsize: 队列的最大容量
"""
self.__queue = PriorityQueue(maxsize=maxsize)
def put(self, item, priority: int = 200) -> None:
"""
向队列尾部插入一个元素，如果该队列已满，则一直阻塞。
:param item:
:param priority: 优先级
:return:
"""
while True:
try:
self.__queue.put(PriorityEntry(priority, item))
break
except Exception as e:
logger.debug("put data failed error -> {0}".format(e))
time.sleep(0.5)
def offer(self, item, priority: int = 200) -> bool:
"""
向队列尾部插入一个元素，插入成功返回True。插入失败返回False。
:param item: 元素
:param priority: 优先级
:return:
"""
try:
self.__queue.put(PriorityEntry(priority, item), block=False)
return True
except Exception as e:
logger.debug("offer data failed error -> {0}".format(e))
return False
def poll(self):
"""
获取并移除队列的头元素，若队列为空，则返回null。
:return:
"""
try:
return self.__queue.get(block=False).data
except Exception as e:
logger.debug("poll data failed error -> {0}".format(e))
return None
def take(self):
"""
获取并移除队列的头元素，若队列为空，则一直阻塞。
:return:
"""
while True:
try:
return self.__queue.get().data
except Exception as e:
logger.debug("take data failed error -> {0}".format(e))
time.sleep(0.5)
def peek(self):
"""
获取但不移除队列的头元素，若队列为空，则返回null
:return:
"""
try:
return self.__queue.peek(block=False).data
except Exception as e:
logger.debug("peek data failed error -> {0}".format(e))
return None
def qsize(self) -> int:
"""
获取队列中当前元素数量
:return:
"""
return self.__queue.qsize()
def maxsize(self) -> int:
"""
获取队列的最大容量
:return:
"""
return self.__queue.maxsize
# -*- coding: utf-8 -*- """ 优先级队列测试 """ from gevent.queue import PriorityQueue tasks = PriorityQueue() tasks.put((3, '3')) tasks.put((5, '5')) tasks.put((1, '1')) print tasks.get() print tasks.get() print tasks.get()
class FSMController(object):
def __init__(self, name, fsm_id, states, initial_state, tracer,
channel_tracer, fsm_registry, fsm_id_seq, inventory,
play_header, outputs):
self.shutting_down = False
self.is_shutdown = False
self.fsm_registry = fsm_registry
self.name = name
self.fsm_id = fsm_id
self.tracer = tracer
self.channel_tracer = channel_tracer
self.state = initial_state
self.states = states
self.inbox = PriorityQueue()
self.message_buffer = Queue()
self.self_channel = Channel(self, self, tracer, self.inbox)
self.worker = AnsibleTaskWorker(tracer, next(fsm_id_seq), inventory,
play_header)
self.worker_output_queue = Queue()
self.worker.controller.outboxes['output'] = self.worker_output_queue
self.worker.queue.put(Inventory(0, inventory))
self.outboxes = dict(default=None)
self.last_event = NULL_EVENT
self.task_id_seq = count(0)
self.failure_count = 0
if outputs:
self.outboxes.update({name: None for name in outputs})
self.thread = gevent.spawn(self.receive_messages)
def enter(self):
self.state.exec_handler(self, 'enter', NULL_EVENT)
def change_state(self, state, handling_message_type):
if self.state:
self.state.exec_handler(self, 'exit', NULL_EVENT)
if settings.instrumented:
self.tracer.send_trace_message(
messages.FSMTrace(self.tracer.trace_order_seq(), self.name,
self.fsm_id, self.state.name, state.name,
handling_message_type))
self.state = state
if self.state:
self.state.exec_handler(self, 'enter', NULL_EVENT)
def handle_message(self, message_type, message):
self.state.exec_handler(self, message_type, message)
def shutdown(self):
self.shutting_down = True
if self.is_shutdown:
return
self.worker.queue.put(ShutdownRequested())
for _ in range(10):
gevent.sleep(1)
worker_message = self.worker_output_queue.get()
if isinstance(worker_message, ShutdownComplete):
break
self.is_shutdown = True
def receive_messages(self):
while not self.shutting_down:
gevent.sleep(0.1)
logger.info("Waiting for messages")
priority, order, message = self.inbox.get()
if self.shutting_down:
logger.info("Ignoring message due to shutdown")
break
message_type = message.name
if message_type == 'Shutdown':
logger.info("Shutting down")
self.shutdown()
break
elif message_type == 'ChangeState' and self.state.name != message.data[
'current_state']:
logger.info(
"Ignoring ChangeState message because the current state does not match"
)
elif message_type == 'ChangeState' and self.state.name == message.data[
'current_state']:
logger.info("Changing state")
self.change_state(self.states[message.data['next_state']],
message.data['handling_message_type'])
else:
logger.info("Handling message {}".format(message_type))
self.handle_message(message_type, message)
class SubNameBrute:
def __init__(self, target, options):
# 设置优先级
self.queue = PriorityQueue()
self.priority = 0
# 根据参数进行基本设置
self.target = target.strip()
self.options = options
self.ignore_intranet = options.get('ignore_intranet')
# 是否用大字典
if self.options.get('subnames_full'):
outfile_name+='_sfull'
if self.options.get('next_sub_full'):
outfile_name += '_nfull'
# 根据主域名确定结果文件名称
outfile_name = options.get('file') if options.get('file') else(target)
self.fname = 'results/'+outfile_name+'.txt'
self.outfile = open('results/'+outfile_name+'.txt', 'wb')
self.outfile_ips = open('results/'+outfile_name+'_ip.txt', 'w')
# 设置dns解析器 （根据预设的线程数量初始化dns resolver）
# QUESTION: configure = False还是不太明白 为什么要不以/etc/resolv.conf的常规常规配置？？
self.resolvers = [dns.resolver.Resolver(configure=False) for _ in range(options.get('threads'))]
for _ in self.resolvers:
'''
dns.resolver.Resolver: http://www.dnspython.org/docs/1.14.0/dns.resolver.Resolver-class.html
dns.resolver.Resolver.lifetime: The total number of seconds to spend trying to get an answer to the question.
dns.resolver.Resolver.timeout: The number of seconds to wait for a response from a server, before timing out.
'''
# QUESTION:lifetime 与 timeout 什么区别？
_.lifetime = _.timeout = 10.0
# 加载dns服务器列表
self._load_dns_servers()
# self.ex_resolver是备用的在出现except时使用的dns_resolver
self.ex_resolver = dns.resolver.Resolver(configure=False)
self.ex_resolver.nameservers = self.dns_servers
self.logfile = open('results/'+target+'_log.txt','a')
#set subdomain dct set
self._load_next_sub()
self._load_sub_names()
#set init paras
self.start_time = time.time()
self.scan_count = 0
self.found_count = 0 # 已验证过存在子域名的前缀
self.STOP_ME = False
self.ip_dict = {}
self.found_subs = set()
def _load_dns_servers(self):
"""
功能：导入可用的名称服务器 (init初始化时执行)
:return:
"""
print '[+] Validate DNS servers ...'
self.dns_servers = []
pool = Pool(30)
for server in open('dns_servers.txt').xreadlines():#xreadlines返回一个生成器
server = server.strip()
if server:
# apply_async 并行
pool.apply_async(self._test_server, (server,))#apply_async(func[, args[, kwds[, callback]]]) 它是非阻塞
pool.join()#主进程阻塞，等待子进程的退出
self.dns_count = len(self.dns_servers)
sys.stdout.write('\n')
print '[+] Found %s available DNS Servers in total' % self.dns_count
if self.dns_count == 0:
print '[ERROR] No DNS Servers available.'
sys.exit(-1)
def _test_server(self, server):
'''
功能：检测dns服务器是否可用（_load_dns_servers()在加载dns列表时会探测）
检测思路：1.已存在域名可成功解析出ip;
2.不存在的域名解析则会出错.
:param server:nameserver
:return: 无
'''
resolver = dns.resolver.Resolver()
resolver.lifetime = resolver.timeout = 10.0
try:
resolver.nameservers = [server]
existed_domain = 'public-dns-a.baidu.com'
corrs_ip = '180.76.76.76'
answers = resolver.query(existed_domain)
if answers[0].address != corrs_ip:
raise Exception('incorrect DNS response')
try:
non_existed_domain = 'test.bad.dns.lijiejie.com'
resolver.query(non_existed_domain)
print '[+] Bad DNS Server found %s' % server
except:
self.dns_servers.append(server)
print '[+] Check DNS Server %s < OK > Found %s' % (server.ljust(16), len(self.dns_servers))
except:
print '[+] Check DNS Server %s <Fail> Found %s' % (server.ljust(16), len(self.dns_servers))
def _get_filename(self,option,is_full):
'''
功能：构造要打开字典文件的目录
param: option: 字典的类型 subnames / next_sub
param: is_full: 决定使用大字典还是小字典
return: _file: 当前要加载字典的路径
'''
has_newdct = self.options.get('new_dct')
if has_newdct:
try:
# 有新字典文件名，则加载新的字典
next_sub,subnames = has_newdct.split(',')
except Exception:
print '[ERROR] Names file not exists: %s' % has_newdct
exit(-1)
else:
# 若新字典名next_sub,subnames加载异常，则打开原来的字典
self.new_filenames = {
'next_sub':'dict/'+next_sub,
'subnames':'dict/'+subnames
}
filename = self.new_filenames.get(option)
if os.path.exists(filename):
_file = filename
else:
print '[ERROR] Names file not exists: %s' % filename
exit(-1)
elif is_full:
_file = 'dict/'+option+'_full.txt'
else:
_file = 'dict/'+option+'.txt'
return _file
def _load_sub_names(self):
print '[+] Load sub names ...'
is_full = self.options.get('subnames_full')
# _file是完整的路径名
_file = self._get_filename('subnames',is_full)
normal_lines = []
wildcard_list = []
regex_list = []
lines = set()
with open(_file) as f:
wildcard_lines = []
for line in f.xreadlines():
sub = line.strip()
print 'sub:' + sub
if not sub or sub in lines:
continue
lines.add(sub)
# 通配符
# QUESTION：但实际的sub文件中都没有通配符？？？？
if sub.find('{alphnum}') >= 0 or sub.find('{alpha}') >= 0 or sub.find('{num}') >= 0:
# 如果存在某个通配符，则先将其加入到wildcard_lines
wildcard_lines.append(sub)
sub = sub.replace('{alphnum}', '[a-z0-9]')
sub = sub.replace('{alpha}', '[a-z]')
sub = sub.replace('{num}', '[0-9]')
print 'sun2: ' + sub
if sub not in wildcard_list:
# QUESTION：为什么替换通配符后还要加入到wildcard_list？？
wildcard_list.append(sub)
regex_list.append('^' + sub + '$')
else:
# 不存在通配符的加入到normal_lines
normal_lines.append(sub)
pattern = '|'.join(regex_list)
if pattern:
_regex = re.compile(pattern)
if _regex:
for line in normal_lines[:]:
if _regex.search(line):
normal_lines.remove(line)
for item in normal_lines:
# QUESTION: 为什么遍历时每次要令priority自增？
self.priority = self.priority+1
self.queue.put((self.priority, item))
for item in wildcard_lines:
# QUESTION： wildcard_lines中元素含有通配符所以优先级低？？？（大数对应低优先级）
self.queue.put((88888888, item))
def _load_next_sub(self):
"""
枚举一、二位子域并添加已存子域
:return:
"""
self.next_subs = []
_set = set()
is_full = self.options.get('next_sub_full')
# _file是nett_sub完整的路径
_file = self._get_filename('next_sub',is_full)
with open(_file) as f:
for line in f:
sub = line.strip()
if sub and sub not in self.next_subs:
# 利用{alphnum}等通配符组合新的子串
# QUESTION:但原文件中的其他子串没有用？
tmp_set = {sub} # 相当于tep_set = set(sub)
while len(tmp_set) > 0:
item = tmp_set.pop()
# print 'item: ' + item
if item.find('{alphnum}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789':
# for _letter in 'ab89':
# 如果是{alphnum}{alphnum}，则将'abcdefghijklmnopqrstuvwxyz0123456789' 两两组合的结果加入了tmp_set
tt = item.replace('{alphnum}', _letter, 1)
tmp_set.add(tt)
elif item.find('{alpha}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz':
tmp_set.add(item.replace('{alpha}', _letter, 1))
elif item.find('{num}') >= 0:
for _letter in '0123456789':
tmp_set.add(item.replace('{num}', _letter, 1))
elif item not in _set:
# 当所有的{alphnum}等通配符都被replace完后，将被加入到_set / self.next_subs
# 原文件中不包括通配符的子串直接加入了_set,也加入了self.next_subs
_set.add(item)
self.next_subs.append(item)
@staticmethod
# 判断是否是内网ip
def is_intranet(ip):
ret = ip.split('.')
if len(ret) != 4:
return True
if ret[0] == '10':
return True
if ret[0] == '172' and 16 <= int(ret[1]) <= 32:
return True
if ret[0] == '192' and ret[1] == '168':
return True
return False
def put_item(self, item):
# 向待检测前缀队列self.queue中添加新的子域名前缀
num = item.count('{alphnum}') + item.count('{alpha}') + item.count('{num}')
if num == 0:
self.priority += 1
self.queue.put((self.priority, item))
else:
# 存在通配符则将优先级设为低级
self.queue.put((self.priority + num * 10000000, item))
def _universal_parsing(self,sub,ips):
# 统计数量，与泛解析有关
_sub = sub.split('.')[-1]
# (_sub,ips)前缀与该前缀构成的子域名所得ip QUESTION：这和泛解析什么关系？？？
'''
a.b.baidu.com 与 a.baidu.com ，它们的_sub都是'a.',当他们解析到相同的A记录时，
则有可能其他_sub同为'a.'(最左侧一级为a.)的子域名也会解析到同样的ip，存在泛解析
'''
if (_sub, ips) not in self.ip_dict:
self.ip_dict[(_sub, ips)] = 1
else:
self.ip_dict[(_sub, ips)] += 1
# 计数：一组ips被多少个sub解析到 （如果一组ips被多组sub解析到，则可能是泛解析）
if ips not in self.ip_dict:
self.ip_dict[ips] = 1
else:
self.ip_dict[ips] += 1
return True if self.ip_dict[(_sub, ips)] > 3 or self.ip_dict[ips] > 6 else(False)
def _validate_subdomain(self,j,sub):
'''
功能：验证子域名是否存在
'''
# 构造新的子域名
subdmname = sub + '.' + self.target
try:
answers = self.resolvers[j].query(subdmname)
except dns.resolver.NoAnswer:
try:
# 出现异常则用备用dns解析器解析
answers = self.ex_resolver.query(subdmname)
except dns.resolver.NoAnswer:
# 如果2次都出现异常，则返回False
return False
if answers:
# 如果得到响应，则将该前缀加入到self.found_subs
# QUESTION： 验证说明不存在的不用单独存下来吗
self.found_subs.add(sub)
# 得到A记录集合
ips = ', '.join(sorted([answer.address for answer in answers]))
print ips
self.cur_ips = ips
# QUESTION: 只有一个ip且ip是一下其中之一的情况
if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0']:
return False
#除去内网域名
# self.ignore_instanet表示是否要进行内网ip过滤
# SubNameBrute.is_intranet(answers[0].address) 是实际进行是否是内外ip的测算
# QUESTION: 为什么只对answers[0]中的ip进行测试？？？
if self.ignore_intranet and SubNameBrute.is_intranet(answers[0].address):
return False
# 泛解析
if self._universal_parsing(sub, ips):
return False
else:
return False
return True
def _scan_cname(self,j,subdmname):
'''
功能：检测子域名的cname是否是新的子域名，是否可以得到新的前缀
'''
try:
self.scan_count += 1
# subdmname是已经验证有效的子域名，现获取其cname
answers = self.resolvers[j].query(subdmname, 'cname')
cname = answers[0].target.to_unicode().rstrip('.')
# cname.endswith(self.target)判断cname是不是子域名
if cname.endswith(self.target) and cname not in self.found_subs:
# 将是子域名的cname加入到self.found_subs
self.found_subs.add(cname)
# 假设cname是'www.a.shifen.com',target是'shifen.com',,则cname_sub是'www.a'
# 当cname是子域时，将i其前缀再次加入队列，当此前缀在不同级上时，可能构成新的子域
cname_sub = cname[:len(cname) - len(self.target) - 1] # new sub
self.queue.put((0, cname_sub))
except:
pass
def _scan(self, j):
# 具体运行的核心函数
self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]]
while not self.queue.empty():
try:
# 从队列中获得一个前缀
item = self.queue.get(timeout=1.0)[1]
self.scan_count += 1
except:
break
try:
# 根据_load_sub_names代码，会有包含通配符的sub被加入了queue，因此这里要进行处理
if item.find('{alphnum}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789':
self.put_item(item.replace('{alphnum}', _letter, 1))
continue
elif item.find('{alpha}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz':
self.put_item(item.replace('{alpha}', _letter, 1))
continue
elif item.find('{num}') >= 0:
for _letter in '0123456789':
self.put_item(item.replace('{num}', _letter, 1))
continue
elif item.find('{next_sub}') >= 0:
for _ in self.next_subs:
self.queue.put((0, item.replace('{next_sub}', _, 1)))
continue
else:
sub = item
# 如果是已经验证过的，则不进行验证处理
if sub in self.found_subs:
continue
if self._validate_subdomain(j,sub):
cur_sub_domain = sub+'.'+self.target
self._scan_cname(j,cur_sub_domain) # 检测子域名的cname中是否包含子域名
self.found_count += 1
# QUESTON
self.outfile.write(cur_sub_domain+'\n')
'''
关于flush与write：http://blog.csdn.net/fenfeiqinjian/article/details/49444973
一般的文件流操作都包含缓冲机制，write方法并不直接将数据写入文件，而是先写入内存中特定的缓冲区。
flush方法是用来刷新缓冲区的，即将缓冲区中的数据立刻写入文件，同时清空缓冲区
正常情况下缓冲区满时，操作系统会自动将缓冲数据写入到文件中。
至于close方法，原理是内部先调用flush方法来刷新缓冲区，再执行关闭操作，这样即使缓冲区数据未满也能保证数据的完整性。
如果进程意外退出或正常退出时而未执行文件的close方法，缓冲区中的内容将会丢失
'''
self.outfile.flush()
self.outfile_ips.write(self.cur_ips+'\n')
self.outfile_ips.flush()
print cur_sub_domain
# '{next_sub}.' + sub 目的在于给当前前缀再增加前缀，以构成多级域名
self.queue.put((999999999, '{next_sub}.' + sub))
except (dns.resolver.NXDOMAIN, dns.name.EmptyLabel):
pass
except (dns.resolver.NoNameservers, dns.resolver.NoAnswer, dns.exception.Timeout):
pass
except Exception:
pass
print "scan_count=%s,found_count=%s"%(self.scan_count,self.found_count)
def run(self):
# i用来标识是第几个写成，同时在协程中用来选择dns resolver
threads = [gevent.spawn(self._scan, i) for i in range(self.options.get('threads'))]
try:
gevent.joinall(threads)
except KeyboardInterrupt:
print '[WARNING] User aborted.'
self.end_time = time.time()
s = (self.end_time-self.start_time)
m = ((self.end_time - self.start_time)/60)
h = ((self.end_time - self.start_time) / 3600)
self.logfile.write(self.fname+'\n')
result = "scan_count=%s,found_count=%s"%(self.scan_count,self.found_count)
self.logfile.write(result+'\n')
time_consuming = "time-consuming:%d seconds"%s
print time_consuming
self.logfile.write(time_consuming+'\n')
time_consuming = "time-consuming:%d minutes" % m
print time_consuming
self.logfile.write(time_consuming+'\n')
time_consuming = "time-consuming:%d hours" % h
print time_consuming
self.logfile.write(time_consuming+'\n')
# 统计结果文件中各级域名的数量（self.fname是输出结果的文件名）
ocount, bcount, tcount, fcount = self.get_distribution(self.fname)
subdomain_count = '二级域名数量: %d' % ocount
print subdomain_count
self.logfile.write(subdomain_count+'\n')
subdomain_count = '三级域名数量: %d' % bcount
print subdomain_count
self.logfile.write(subdomain_count+'\n')
subdomain_count = '四级域名数量: %d' % tcount
print subdomain_count
self.logfile.write(subdomain_count+'\n')
subdomain_count = '五级域名数量: %d' % fcount
print subdomain_count
self.logfile.write(subdomain_count+'\n')
self.outfile.flush()
self.outfile.close()
self.outfile_ips.flush()
self.outfile_ips.close()
def get_distribution(self,filename):
'''
功能：统计结果文件filename中二级、三级和四级域名的数量
'''
with open(filename, 'rb') as f:
subdomains = [line.strip() for line in f.readlines()]
ocount = bcount = tcount = fcount = 0
for domain in subdomains:
if domain.count('.') == 2:
ocount += 1
elif domain.count('.') == 3:
bcount += 1
elif domain.count('.') == 4:
tcount += 1
else:
fcount += 1
return ocount, bcount, tcount, fcount
class TestResourceFilters(unittest.TestCase):
db = {}
conf = CONFIG
@patch('openprocurement.bridge.frameworkagreement.filters.INFINITY')
@patch('openprocurement.bridge.frameworkagreement.filters.logger')
def test_CFAUAFilter(self, logger, infinity):
self.input_queue = PriorityQueue()
self.filtered_queue = PriorityQueue()
resource = self.conf['resource'][:-1]
filter = CFAUAFilter(self.conf, self.input_queue, self.filtered_queue,
self.db)
mock_calls = [call.info('Init Close Framework Agreement Filter.')]
self.assertEqual(logger.mock_calls, mock_calls)
extra = {
'MESSAGE_ID': 'SKIPPED',
'JOURNAL_{}_ID'.format(resource.upper()): 'test_id'
}
infinity.__nonzero__.side_effect = [True, False]
filter._run()
doc = {'id': 'test_id', 'dateModified': '1970-01-01'}
self.input_queue.put((None, doc))
self.db['test_id'] = '1970-01-01'
infinity.__nonzero__.side_effect = [True, False]
filter._run()
mock_calls.append(
call.info(
'{} test_id not modified from last check. Skipping'.format(
resource.title()),
extra=extra))
self.assertEqual(logger.mock_calls, mock_calls)
doc['procurementMethodType'] = 'test'
doc['dateModified'] = '1970-01-02'
self.input_queue.put((None, doc))
infinity.__nonzero__.side_effect = [True, False]
filter._run()
mock_calls.append(
call.info('Skipping test {} test_id'.format(resource),
extra=extra))
self.assertEqual(logger.mock_calls, mock_calls)
filter.procurement_method_types = ('test')
doc['status'] = 'test_status'
self.input_queue.put((None, doc))
infinity.__nonzero__.side_effect = [True, False]
filter._run()
mock_calls.append(
call.info('Skipping test {} test_status test_id'.format(resource),
extra=extra))
self.assertEqual(logger.mock_calls, mock_calls)
filter.statuses = ('test_status')
filter.lot_status = 'test_status'
doc['lots'] = [{'status': 'spam_status'}]
self.input_queue.put((None, doc))
infinity.__nonzero__.side_effect = [True, False]
filter._run()
mock_calls.append(
call.info(
'Skipping multilot {} test_id in status test_status'.format(
resource),
extra=extra))
self.assertEqual(logger.mock_calls, mock_calls)
del doc['lots']
self.input_queue.put((None, doc))
infinity.__nonzero__.side_effect = [True, False]
filter._run()
mock_calls.append(
call.debug('Put to filtered queue {} test_id'.format(resource)))
self.assertEqual(logger.mock_calls, mock_calls)
priority, resource = self.filtered_queue.get()
self.assertEqual(priority, None)
self.assertEqual(resource, doc)
class Scanner:
def __init__(self):
self.start_time = time.time()
self.queue = PriorityQueue()
self.history = []
self.total_count = 0
self.scan_count = 0
self._load_target()
self.outfile = open("log.log", 'w')
self.console_width = getTerminalSize()[0] - 2
def _print_msg(self, _msg=None, _found_msg=False):
if _msg is None:
msg = '%s TotalCount| %s Scanned in %.2f seconds' % (
self.total_count,self.total_count - self.queue.qsize(), time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) + msg)
else:
sys.stdout.write('\r' + _msg + ' ' * (self.console_width - len(_msg)) + '\n')
self.outfile.write(_msg + '\n')
self.outfile.flush()
if _found_msg:
msg = '%s TotalCount| %s Scanned in %.2f seconds' % (
self.total_count,self.total_count - self.queue.qsize(), time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) + msg)
sys.stdout.flush()
def _load_target(self):
print '[+] Read targets ...'
target_file = raw_input("Target File :")
with open(target_file) as f:
for line in f.xreadlines():
target = line.strip()
self.queue.put(target)
print "TotalCount is %d" % self.queue.qsize()
self.total_count = self.queue.qsize()
print "Now scanning ..."
def _scan(self,case):
while not self.queue.empty():
target = self.queue.get()
if case == "1":
self.vulnCheck(target)
if case == "2":
self.s2_045(target)
if case == "3":
self.headers(target)
if case == "4":
self.weakfile(target)
if case == "5":
self.portscan_c(target)
#####################################################################
# #
# Vuln poc by:xi4okv QQ:48011203 #
# #
#####################################################################
def vulnCheck(self,target):
if ":2375" in target:
try:
res = requests.head("http://" + str(target) + "/containers/json",timeout=2)
if res.headers['Content-Type'] == 'application/json':
self._print_msg(target + "==> docker api Vuln",True)
else:
self._print_msg()
except:
self._print_msg()
self._print_msg()
if ":27017" in target:
ip,port = target.split(":")
try:
socket.setdefaulttimeout(3)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((ip, int(port)))
data = binascii.a2b_hex("3a000000a741000000000000d40700000000000061646d696e2e24636d640000000000ffffffff130000001069736d6173746572000100000000")
s.send(data)
result = s.recv(1024)
if "ismaster" in result:
getlog_data = binascii.a2b_hex("480000000200000000000000d40700000000000061646d696e2e24636d6400000000000100000021000000026765744c6f670010000000737461727475705761726e696e67730000")
s.send(getlog_data)
result = s.recv(1024)
if "totalLinesWritten" in result:
self._print_msg(target + "==> mongodb Vuln",True)
except Exception, e:
pass
if ":6379" in target:
ip,port = target.split(":")
try:
socket.setdefaulttimeout(3)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((ip, int(port)))
s.send("INFO\r\n")
result = s.recv(1024)
if "redis_version" in result:
self._print_msg(target + "==> redis Vuln",True)
elif "Authentication" in result:
for pass_ in ['123456','redis','pass','password']:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((ip, int(port)))
s.send("AUTH %s\r\n" % (pass_))
result = s.recv(1024)
if '+OK' in result:
self._print_msg(target + "==> redis pass Vuln :" + pass_,True)
except Exception, e:
pass
class SyncClient:
idle = idle
backward_class = BackwardWorker
forward_class = ForwardWorker
def __init__(self,
host_url,
resource,
auth=None,
params={},
headers=None,
retrievers_params=DEFAULT_RETRIEVERS_PARAMS,
adaptive=False,
with_priority=False):
LOGGER.info(f'Init SyncClient for resource {resource}')
self.host = host_url
self.auth = auth
self.resource = resource
self.adaptive = adaptive
self.headers = headers
self.params = params
self.retrievers_params = retrievers_params
self.queue = PriorityQueue(maxsize=retrievers_params['queue_size'])
def init_clients(self):
self.backward_client = ResourceClient(self.host, self.resource,
self.params, self.auth,
self.headers)
self.forward_client = ResourceClient(self.host, self.resource,
self.params, self.auth,
self.headers)
def handle_response_data(self, data):
for resource_item in data:
self.queue.put(PrioritizedItem(1, resource_item))
def worker_watcher(self):
while True:
if time() - self.heartbeat > DEFAULT_FORWARD_HEARTBEAT:
self.restart_sync()
LOGGER.warning(
'Restart sync, reason: Last response from workers greater than 15 min ago.'
)
sleep(300)
def start_sync(self):
LOGGER.info('Start sync...')
data = self.backward_client.get_resource_items(self.params)
self.handle_response_data(data[f'{self.resource}s'])
forward_params = deepcopy(self.params)
forward_params.update({
k: v[0]
for k, v in parse.parse_qs(parse.urlparse(
data.links.prev).query).items()
})
backward_params = deepcopy(self.params)
backward_params.update({
k: v[0]
for k, v in parse.parse_qs(parse.urlparse(
data.links.next).query).items()
})
self.forward_worker = self.forward_class(sync_client=self,
client=self.forward_client,
params=forward_params)
self.backward_worker = self.backward_class(sync_client=self,
client=self.backward_client,
params=backward_params)
self.workers = [self.forward_worker, self.backward_worker]
for worker in self.workers:
worker.start()
self.heartbeat = time()
self.watcher = spawn(self.worker_watcher)
def restart_sync(self):
"""
Restart retrieving from OCDS API.
"""
LOGGER.info('Restart workers')
for worker in self.workers:
worker.kill()
self.watcher.kill()
self.init_clients()
self.start_sync()
def get_resource_items(self):
self.init_clients()
self.start_sync()
while True:
if self.forward_worker.check() or self.backward_worker.check():
self.restart_sync()
while not self.queue.empty():
LOGGER.debug(f'Sync queue size: {self.queue.qsize()}',
extra={'SYNC_QUEUE_SIZE': self.queue.qsize()})
LOGGER.debug('Yield resource item',
extra={'MESSAGE_ID': 'sync_yield'})
item = self.queue.get()
yield item.data
LOGGER.debug(f'Sync queue size: {self.queue.qsize()}',
extra={'SYNC_QUEUE_SIZE': self.queue.qsize()})
try:
self.queue.peek(block=True, timeout=0.1)
except Empty:
pass
class Track:
def __init__(self, options, target):
self.start_time = time.time()
self.dns_servers = []
self.options = options
self.target = target
self.resolvers = [dns.resolver.Resolver(configure=False) for _ in range(self.options.threads)]
self.all_dns = dns.resolver.Resolver(configure=False)
self.all_dns.nameservers = self.dns_servers
detect_target.detect_target(self.target, True)
self.dns_servers = dns_server.load_dns_server()
self.dns_server_count = len(self.dns_servers)
self.queue = PriorityQueue()
self.priority = 0
self.scan_count = self.confirm_count = 0
self.confirm_subdomain = set()
self.cdn = ''
self.load_status = {}
self.load_status = load_subdomain_name.load_sub_name(self.options, self.target, self.priority, self.queue)
self.priority = self.load_status['priority']
self.queue = self.load_status['queue']
self.print_header_info()
self.outfile = self.outfile_descriptor()
def outfile_descriptor(self):
if self.options.outfile:
outfile_name = self.options.outfile + '.txt'
else:
outfile_name = self.target + '.txt'
f = open(outfile_name, 'w')
return f
def print_header_info(self):
print 'Domain Name\t\t\t\tIp Adress\t\tStatus\tCDN\tServer'
print '------------\t\t\t\t----------\t\t------\t------\t---------'
def subdomain_track(self, serial):
dns_serial_number = serial % self.dns_server_count
self.resolvers[serial].nameservers = [self.dns_servers[dns_serial_number]]
while not self.queue.empty():
try:
subdomain_name = self.queue.get(timeout=1)[1]
msg = '%.1fs | Found %d \r' % (time.time()-self.start_time, self.confirm_count)
sys.stdout.write(msg)
sys.stdout.flush()
self.scan_count += 1
except traceback:
traceback.print_exc()
sys.exit(1)
try:
if subdomain_name in self.confirm_subdomain:
continue
if not self.options.crt:
subdomian_url = subdomain_name + '.' + self.target
else:
subdomian_url = subdomain_name
try:
dns_responses = self.resolvers[serial].query(subdomian_url)
except dns.resolver.NoAnswer, e:
dns_responses = self.all_dns.query(subdomian_url)
if len(dns_responses) > 1:
self.cdn = 'Yes'
else:
self.cdn = 'Unknown'
server_responses = detect_target.detect_target(subdomian_url)
if dns_responses and server_responses:
self.confirm_subdomain.add(subdomain_name)
self.confirm_count += 1
ip = dns_responses[0].address
if server_responses.get('Server'):
server_info = server_responses['Server']
else:
server_info = 'Unknown'
status_code = server_responses['status_code']
print '%s\t\t%s\t\t%s\t%s\t%s' % (subdomian_url.ljust(30), ip, status_code, self.cdn, server_info)
msg = '%s\t\t%s\t\t%s\t%s\t%s' % (subdomian_url.ljust(30), ip, status_code, self.cdn, server_info)
self.outfile.write(msg)
self.outfile.flush()
except (dns.resolver.NoAnswer, dns.resolver.NoNameservers, dns.exception.Timeout, dns.resolver.NXDOMAIN) as e:
pass
except Exception as e:
traceback.print_exc()
sys.exit(1)
class SubNameBrute(object):
def __init__(self, *params):
(
self.domain,
self.options,
self.process_num,
self.dns_servers,
self.next_subs,
self.scan_count,
self.found_count,
self.queue_size_array,
tmp_dir,
) = params
self.dns_count = len(self.dns_servers)
self.scan_count_local = 0
self.found_count_local = 0
self.resolvers = [
dns.resolver.Resolver(configure=False)
for _ in range(self.options.threads)
]
for r in self.resolvers:
r.lifetime = r.timeout = 10.0
self.queue = PriorityQueue()
self.priority = 0
self.ip_dict = {}
self.found_subs = set()
self.timeout_subs = {}
self.count_time = time.time()
self.outfile = open(
"%s/%s_part_%s.txt" % (tmp_dir, self.domain, self.process_num),
"w")
self.normal_names_set = set()
self.load_sub_names()
self.lock = RLock()
def load_sub_names(self):
normal_lines = []
wildcard_lines = []
wildcard_set = set()
regex_list = []
lines = set()
with open(self.options.file) as inFile:
for line in inFile.readlines():
sub = line.strip()
if not sub or sub in lines:
continue
lines.add(sub)
brace_count = sub.count("{")
if brace_count > 0:
wildcard_lines.append((brace_count, sub))
sub = sub.replace("{alphnum}", "[a-z0-9]")
sub = sub.replace("{alpha}", "[a-z]")
sub = sub.replace("{num}", "[0-9]")
if sub not in wildcard_set:
wildcard_set.add(sub)
regex_list.append("^" + sub + "$")
else:
normal_lines.append(sub)
self.normal_names_set.add(sub)
if regex_list:
pattern = "|".join(regex_list)
_regex = re.compile(pattern)
for line in normal_lines:
if _regex.search(line):
normal_lines.remove(line)
for _ in normal_lines[self.process_num::self.options.process]:
self.queue.put((0, _)) # priority set to 0
for _ in wildcard_lines[self.process_num::self.options.process]:
self.queue.put(_)
def scan(self, j):
self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]
] + self.dns_servers
while True:
try:
self.lock.acquire()
if time.time() - self.count_time > 1.0:
self.scan_count.value += self.scan_count_local
self.scan_count_local = 0
self.queue_size_array[
self.process_num] = self.queue.qsize()
if self.found_count_local:
self.found_count.value += self.found_count_local
self.found_count_local = 0
self.count_time = time.time()
self.lock.release()
brace_count, sub = self.queue.get(timeout=3.0)
if brace_count > 0:
brace_count -= 1
if sub.find("{next_sub}") >= 0:
for _ in self.next_subs:
self.queue.put((0, sub.replace("{next_sub}", _)))
if sub.find("{alphnum}") >= 0:
for _ in "abcdefghijklmnopqrstuvwxyz0123456789":
self.queue.put(
(brace_count, sub.replace("{alphnum}", _, 1)))
elif sub.find("{alpha}") >= 0:
for _ in "abcdefghijklmnopqrstuvwxyz":
self.queue.put(
(brace_count, sub.replace("{alpha}", _, 1)))
elif sub.find("{num}") >= 0:
for _ in "0123456789":
self.queue.put(
(brace_count, sub.replace("{num}", _, 1)))
continue
except gevent.queue.Empty as e:
break
try:
if sub in self.found_subs:
continue
self.scan_count_local += 1
cur_domain = sub + "." + self.domain
answers = self.resolvers[j].query(cur_domain)
if answers:
self.found_subs.add(sub)
ips = ", ".join(
sorted([answer.address for answer in answers]))
if ips in ["1.1.1.1", "127.0.0.1", "0.0.0.0", "0.0.0.1"]:
continue
if self.options.i and is_intranet(answers[0].address):
continue
try:
self.scan_count_local += 1
answers = self.resolvers[j].query(cur_domain, "cname")
cname = answers[0].target.to_unicode().rstrip(".")
if cname.endswith(
self.domain) and cname not in self.found_subs:
cname_sub = cname[:len(cname) - len(self.domain) -
1] # new sub
if cname_sub not in self.normal_names_set:
self.found_subs.add(cname)
self.queue.put((0, cname_sub))
except Exception as e:
pass
first_level_sub = sub.split(".")[-1]
if (first_level_sub, ips) not in self.ip_dict:
self.ip_dict[(first_level_sub, ips)] = 1
else:
self.ip_dict[(first_level_sub, ips)] += 1
if self.ip_dict[(first_level_sub, ips)] > 30:
continue
self.found_count_local += 1
self.outfile.write(
cur_domain.ljust(30) + "\t" + ips + "\n")
self.outfile.flush()
try:
self.scan_count_local += 1
self.resolvers[j].query("test-not-existed." +
cur_domain)
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e:
if self.queue.qsize() < 10000:
for _ in self.next_subs:
self.queue.put((0, _ + "." + sub))
else:
self.queue.put((1, "{next_sub}." + sub))
except Exception as e:
pass
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e:
pass
except dns.resolver.NoNameservers as e:
self.queue.put((0, sub)) # Retry
except dns.exception.Timeout as e:
self.timeout_subs[sub] = self.timeout_subs.get(sub, 0) + 1
if self.timeout_subs[sub] <= 2:
self.queue.put((0, sub)) # Retry
except Exception as e:
import traceback
traceback.print_exc()
with open("errors.log", "a") as errFile:
errFile.write("[%s] %s\n" % (type(e), str(e)))
def run(self):
threads = [
gevent.spawn(self.scan, i) for i in range(self.options.threads)
]
gevent.joinall(threads)
class SubNameBrute:
def __init__(self, target, args, process_num, dns_servers, next_subs,
scan_count, found_count, queue_size_list, tmp_dir):
self.target = target.strip()
self.args = args
self.process_num = process_num
self.dns_servers = dns_servers
self.dns_count = len(dns_servers)
self.next_subs = next_subs
self.scan_count = scan_count
self.scan_count_local = 0
self.found_count = found_count
self.found_count_local = 0
self.queue_size_list = queue_size_list
self.resolvers = [
dns.resolver.Resolver(configure=False) for _ in range(args.threads)
]
for _r in self.resolvers:
_r.lifetime = _r.timeout = 6.0
self.queue = PriorityQueue()
self.item_index = 0
self.priority = 0
self._load_sub_names()
self.ip_dict = {}
self.found_subs = set()
self.ex_resolver = dns.resolver.Resolver(configure=False)
self.ex_resolver.nameservers = dns_servers
self.local_time = time.time()
self.outfile = open(
'%s/%s_part_%s.txt' % (tmp_dir, target, process_num), 'w')
self.outfile_html = open('tmp/%s_html_%s.txt' % (target, process_num),
'w')
def _load_sub_names(self):
if self.args.full_scan and self.args.file == 'subnames.txt':
_file = 'Dict/subnames_full.txt'
else:
if os.path.exists(self.args.file):
_file = self.args.file
elif os.path.exists('Dict/%s' % self.args.file):
_file = 'Dict/%s' % self.args.file
else:
print_msg('[ERROR] Names file not found: %s' % self.args.file)
exit(-1)
normal_lines = []
lines = set()
with open(_file) as f:
for line in f.xreadlines():
sub = line.strip()
if not sub or sub in lines:
continue
lines.add(sub)
normal_lines.append(sub)
for item in normal_lines[self.process_num::self.args.process]:
self.priority += 1
self.queue.put((self.priority, item))
def _scan(self, j):
self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]]
while not self.queue.empty():
try:
item = self.queue.get()[1]
self.scan_count_local += 1
if time.time() - self.local_time > 3.0:
self.scan_count.value += self.scan_count_local
self.scan_count_local = 0
self.queue_size_list[self.process_num] = self.queue.qsize()
except Exception as e:
break
try:
if item.find('{next_sub}') >= 0:
for _ in self.next_subs:
self.queue.put((0, item.replace('{next_sub}', _, 1)))
continue
else:
sub = item
if sub in self.found_subs:
continue
cur_sub_domain = sub + '.' + self.target
_sub = sub.split('.')[-1]
try:
answers = self.resolvers[j].query(cur_sub_domain)
except dns.resolver.NoAnswer, e:
answers = self.ex_resolver.query(cur_sub_domain)
if answers:
self.found_subs.add(sub)
ips = ', '.join(
sorted([answer.address for answer in answers]))
if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0']:
continue
if self.args.intranet and is_intranet(answers[0].address):
continue
try:
self.scan_count_local += 1
answers = self.resolvers[j].query(
cur_sub_domain, 'cname')
cname = answers[0].target.to_unicode().rstrip('.')
if cname.endswith(
self.target) and cname not in self.found_subs:
self.found_subs.add(cname)
cname_sub = cname[:len(cname) - len(self.target) -
1]
self.queue.put((0, cname_sub))
except:
pass
if (_sub, ips) not in self.ip_dict:
self.ip_dict[(_sub, ips)] = 1
else:
self.ip_dict[(_sub, ips)] += 1
if self.ip_dict[(_sub, ips)] > 30:
continue
self.found_count_local += 1
if time.time() - self.local_time > 3.0:
self.found_count.value += self.found_count_local
self.found_count_local = 0
self.queue_size_list[
self.process_num] = self.queue.qsize()
self.local_time = time.time()
self.outfile_html.write(
PrintHtml.Sub_html_print(cur_sub_domain, ips))
self.outfile_html.flush()
self.outfile.write(
cur_sub_domain.ljust(30) + '\t' + ips + '\n')
self.outfile.flush()
try:
self.resolvers[j].query('myzxcghelloha.' +
cur_sub_domain)
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e:
self.queue.put((999999999, '{next_sub}.' + sub))
except:
pass
except (dns.resolver.NXDOMAIN, dns.name.EmptyLabel) as e:
pass
except (dns.resolver.NoNameservers, dns.resolver.NoAnswer,
dns.exception.Timeout) as e:
pass
class TestStatuslistFilter(unittest.TestCase):
def setUp(self):
self.conf = CONFIG
self.db = {}
self.input_queue = PriorityQueue()
self.filtered_queue = PriorityQueue()
self.filter = StatuslistFilter(self.conf, self.input_queue,
self.filtered_queue, self.db)
@patch('openprocurement.bridge.rbot.filters.INFINITY')
def test_filter_ok(self, infinity):
infinity.__nonzero__.side_effect = [True, False]
doc = {
'id': 'test_id',
'dateModified': '1970-01-02',
'procurementMethodType': 'dgf',
'status': 'active.tendering',
'documents': [{
'documentType': 'contractProforma'
}]
}
self.input_queue.put((None, deepcopy(doc)))
self.filter._run()
self.assertEqual(len(self.filtered_queue), 1)
filtered_doc = self.filtered_queue.get(block=False)
self.assertEqual(doc, filtered_doc[1])
@patch('openprocurement.bridge.rbot.filters.INFINITY')
def test_filter_not_modified(self, infinity):
# Not changed dateModified
infinity.__nonzero__.side_effect = [True, False]
doc = {
'id': 'test_id',
'dateModified': '1970-01-01',
'procurementMethodType': 'dgf',
'status': 'status1',
'documents': [{
'documentType': 'contractProforma'
}],
}
self.input_queue.put((None, deepcopy(doc)))
self.db['test_id'] = '1970-01-01'
self.filter._run()
self.assertEqual(len(self.filtered_queue), 0)
self.db.pop('test_id')
@patch('openprocurement.bridge.rbot.filters.INFINITY')
def test_filter_by_status(self, infinity):
# Wrong tender status
infinity.__nonzero__.side_effect = [True, False]
doc = {
'id': 'test_id',
'dateModified': '1970-01-01',
'procurementMethodType': 'dgf',
'status': 'status3',
'documents': [{
'documentType': 'contractProforma'
}],
}
self.input_queue.put((None, deepcopy(doc)))
self.filter._run()
self.assertEqual(len(self.filtered_queue), 0)
class AWSLogs(object):
ACTIVE = 1
EXHAUSTED = 2
WATCH_SLEEP = 2
def __init__(self, **kwargs):
self.connection_cls = kwargs.get('connection_cls', AWSConnection)
self.aws_region = kwargs.get('aws_region')
self.aws_access_key_id = kwargs.get('aws_access_key_id')
self.aws_secret_access_key = kwargs.get('aws_secret_access_key')
self.log_group_name = kwargs.get('log_group_name')
self.log_stream_name = kwargs.get('log_stream_name')
self.watch = kwargs.get('watch')
self.color_enabled = kwargs.get('color_enabled')
self.output_stream_enabled = kwargs.get('output_stream_enabled')
self.output_group_enabled = kwargs.get('output_group_enabled')
self.start = self.parse_datetime(kwargs.get('start'))
self.end = self.parse_datetime(kwargs.get('end'))
self.pool_size = max(kwargs.get('pool_size', 0), 10)
self.max_group_length = 0
self.max_stream_length = 0
self.publishers = []
self.events_queue = Queue()
self.raw_events_queue = PriorityQueue()
self.publishers_queue = PriorityQueue()
self.publishers = []
self.stream_status = {}
self.stream_max_timestamp = {}
self.connection = self.connection_cls(
self.aws_region,
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key)
def _get_streams_from_patterns(self, log_group_pattern,
log_stream_pattern):
"""Returns pairs of group, stream matching ``log_group_pattern`` and
``log_stream_pattern``."""
for group in self._get_groups_from_pattern(log_group_pattern):
for stream in self._get_streams_from_pattern(
group, log_stream_pattern):
yield group, stream
def _get_groups_from_pattern(self, pattern):
"""Returns groups matching ``pattern``."""
pattern = '.*' if pattern == 'ALL' else pattern
reg = re.compile('^{0}'.format(pattern))
for group in self.get_groups():
if re.match(reg, group):
yield group
def _get_streams_from_pattern(self, group, pattern):
"""Returns streams in ``group`` matching ``pattern``."""
pattern = '.*' if pattern == 'ALL' else pattern
reg = re.compile('^{0}'.format(pattern))
for stream in self.get_streams(group):
if re.match(reg, stream):
yield stream
def _publisher_queue_consumer(self):
"""Consume ``publishers_queue`` api calls, run them and publish log
events to ``raw_events_queue``. If ``nextForwardToken`` is present
register a new api call into ``publishers_queue`` using as weight
the timestamp of the latest event."""
while True:
try:
_, (log_group_name, log_stream_name,
next_token) = self.publishers_queue.get(block=False)
except Empty:
if self.watch:
gevent.sleep(self.WATCH_SLEEP)
continue
else:
break
response = self.connection.get_log_events(
next_token=next_token,
log_group_name=log_group_name,
log_stream_name=log_stream_name,
start_time=self.start,
end_time=self.end,
start_from_head=True)
if not len(response['events']):
self.stream_status[(log_group_name,
log_stream_name)] = self.EXHAUSTED
continue
self.stream_status[(log_group_name, log_stream_name)] = self.ACTIVE
for event in response['events']:
event['group'] = log_group_name
event['stream'] = log_stream_name
self.raw_events_queue.put((event['timestamp'], event))
self.stream_max_timestamp[(
log_group_name, log_stream_name)] = event['timestamp']
if 'nextForwardToken' in response:
self.publishers_queue.put((response['events'][-1]['timestamp'],
(log_group_name, log_stream_name,
response['nextForwardToken'])))
def _get_min_timestamp(self):
"""Return the minimum timestamp available across all active streams."""
pending = [
self.stream_max_timestamp[k]
for k, v in self.stream_status.iteritems() if v != self.EXHAUSTED
]
return min(pending) if pending else None
def _get_all_streams_exhausted(self):
"""Return if all streams are exhausted."""
return all(
(s == self.EXHAUSTED for s in self.stream_status.itervalues()))
def _raw_events_queue_consumer(self):
"""Consume events from ``raw_events_queue`` if all active streams
have already publish events up to the ``_get_min_timestamp`` and
register them in order into ``events_queue``."""
while True:
if self._get_all_streams_exhausted(
) and self.raw_events_queue.empty():
if self.watch:
gevent.sleep(self.WATCH_SLEEP)
continue
self.events_queue.put(NO_MORE_EVENTS)
break
try:
timestamp, line = self.raw_events_queue.peek(timeout=1)
except Empty:
continue
min_timestamp = self._get_min_timestamp()
if min_timestamp and min_timestamp < timestamp:
gevent.sleep(0.3)
continue
timestamp, line = self.raw_events_queue.get()
output = [line['message']]
if self.output_stream_enabled:
output.insert(
0,
self.color(
line['stream'].ljust(self.max_stream_length, ' '),
'cyan'))
if self.output_group_enabled:
output.insert(
0,
self.color(line['group'].ljust(self.max_group_length, ' '),
'green'))
self.events_queue.put("{0}\n".format(' '.join(output)))
def _events_consumer(self):
"""Print events from ``events_queue`` as soon as they are available."""
while True:
event = self.events_queue.get(True)
if event == NO_MORE_EVENTS:
break
sys.stdout.write(event)
sys.stdout.flush()
def list_logs(self):
self.register_publishers()
pool = Pool(size=self.pool_size)
pool.spawn(self._raw_events_queue_consumer)
pool.spawn(self._events_consumer)
if self.watch:
pool.spawn(self.register_publishers_periodically)
for i in xrange(self.pool_size):
pool.spawn(self._publisher_queue_consumer)
pool.join()
def register_publishers(self):
"""Register publishers into ``publishers_queue``."""
for group, stream in self._get_streams_from_patterns(
self.log_group_name, self.log_stream_name):
if (group, stream) in self.publishers:
continue
self.publishers.append((group, stream))
self.max_group_length = max(self.max_group_length, len(group))
self.max_stream_length = max(self.max_stream_length, len(stream))
self.publishers_queue.put((0, (group, stream, None)))
self.stream_status[(group, stream)] = self.ACTIVE
self.stream_max_timestamp[(group, stream)] = -1
def register_publishers_periodically(self):
while True:
self.register_publishers()
gevent.sleep(2)
def list_groups(self):
"""Lists available CloudWatch logs groups"""
for group in self.get_groups():
print group
def list_streams(self, *args, **kwargs):
"""Lists available CloudWatch logs streams in ``log_group_name``."""
for stream in self.get_streams(*args, **kwargs):
print stream
def get_groups(self):
"""Returns available CloudWatch logs groups"""
next_token = None
while True:
response = self.connection.describe_log_groups(
next_token=next_token)
for group in response.get('logGroups', []):
yield group['logGroupName']
if 'nextToken' in response:
next_token = response['nextToken']
else:
break
def get_streams(self, log_group_name=None):
"""Returns available CloudWatch logs streams in ``log_group_name``."""
log_group_name = log_group_name or self.log_group_name
next_token = None
window_start = self.start or 0
window_end = self.end or sys.maxint
while True:
response = self.connection.describe_log_streams(
log_group_name=log_group_name, next_token=next_token)
for stream in response.get('logStreams', []):
if max(stream['firstEventTimestamp'], window_start) <= \
min(stream['lastEventTimestamp'], window_end):
yield stream['logStreamName']
if 'nextToken' in response:
next_token = response['nextToken']
else:
break
def color(self, text, color):
"""Returns coloured version of ``text`` if ``color_enabled``."""
if self.color_enabled:
return colored(text, color)
return text
def parse_datetime(self, datetime_text):
"""Parse ``datetime_text`` into a ``datetime``."""
if not datetime_text:
return None
ago_match = re.match(
r'(\d+)\s?(m|minute|minutes|h|hour|hours|d|day|days|w|weeks|weeks)(?: ago)?',
datetime_text)
if ago_match:
amount, unit = ago_match.groups()
amount = int(amount)
unit = {'m': 60, 'h': 3600, 'd': 86400, 'w': 604800}[unit[0]]
date = datetime.now() + timedelta(seconds=unit * amount * -1)
else:
try:
date = parse(datetime_text)
except ValueError:
raise exceptions.UnknownDateError(datetime_text)
return int(date.strftime("%s")) * 1000
class AWSLogs(object):
ACTIVE = 1
EXHAUSTED = 2
WATCH_SLEEP = 2
def __init__(self, **kwargs):
self.connection_cls = kwargs.get('connection_cls', AWSConnection)
self.aws_region = kwargs.get('aws_region')
self.aws_access_key_id = kwargs.get('aws_access_key_id')
self.aws_secret_access_key = kwargs.get('aws_secret_access_key')
self.log_group_name = kwargs.get('log_group_name')
self.log_stream_name = kwargs.get('log_stream_name')
self.watch = kwargs.get('watch')
self.color_enabled = kwargs.get('color_enabled')
self.output_stream_enabled = kwargs.get('output_stream_enabled')
self.output_group_enabled = kwargs.get('output_group_enabled')
self.start = self.parse_datetime(kwargs.get('start'))
self.end = self.parse_datetime(kwargs.get('end'))
self.pool_size = max(kwargs.get('pool_size', 0), 10)
self.max_group_length = 0
self.max_stream_length = 0
self.publishers = []
self.events_queue = Queue()
self.raw_events_queue = PriorityQueue()
self.publishers_queue = PriorityQueue()
self.publishers = []
self.stream_status = {}
self.stream_max_timestamp = {}
self.connection = self.connection_cls(
self.aws_region,
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key
)
def _get_streams_from_patterns(self, log_group_pattern, log_stream_pattern):
"""Returns pairs of group, stream matching ``log_group_pattern`` and
``log_stream_pattern``."""
for group in self._get_groups_from_pattern(log_group_pattern):
for stream in self._get_streams_from_pattern(group,
log_stream_pattern):
yield group, stream
def _get_groups_from_pattern(self, pattern):
"""Returns groups matching ``pattern``."""
pattern = '.*' if pattern == 'ALL' else pattern
reg = re.compile('^{0}'.format(pattern))
for group in self.get_groups():
if re.match(reg, group):
yield group
def _get_streams_from_pattern(self, group, pattern):
"""Returns streams in ``group`` matching ``pattern``."""
pattern = '.*' if pattern == 'ALL' else pattern
reg = re.compile('^{0}'.format(pattern))
for stream in self.get_streams(group):
if re.match(reg, stream):
yield stream
def _publisher_queue_consumer(self):
"""Consume ``publishers_queue`` api calls, run them and publish log
events to ``raw_events_queue``. If ``nextForwardToken`` is present
register a new api call into ``publishers_queue`` using as weight
the timestamp of the latest event."""
while True:
try:
_, (log_group_name, log_stream_name, next_token) = self.publishers_queue.get(block=False)
except Empty:
if self.watch:
gevent.sleep(self.WATCH_SLEEP)
else:
break
response = self.connection.get_log_events(
next_token=next_token,
log_group_name=log_group_name,
log_stream_name=log_stream_name,
start_time=self.start,
end_time=self.end,
start_from_head=True
)
if not len(response['events']):
self.stream_status[(log_group_name, log_stream_name)] = self.EXHAUSTED
continue
self.stream_status[(log_group_name, log_stream_name)] = self.ACTIVE
for event in response['events']:
event['group'] = log_group_name
event['stream'] = log_stream_name
self.raw_events_queue.put((event['timestamp'], event))
self.stream_max_timestamp[(log_group_name, log_stream_name)] = event['timestamp']
if 'nextForwardToken' in response:
self.publishers_queue.put(
(response['events'][-1]['timestamp'],
(log_group_name, log_stream_name, response['nextForwardToken']))
)
def _get_min_timestamp(self):
"""Return the minimum timestamp available across all active streams."""
pending = [self.stream_max_timestamp[k] for k, v in self.stream_status.iteritems() if v != self.EXHAUSTED]
return min(pending) if pending else None
def _get_all_streams_exhausted(self):
"""Return if all streams are exhausted."""
return all((s == self.EXHAUSTED for s in self.stream_status.itervalues()))
def _raw_events_queue_consumer(self):
"""Consume events from ``raw_events_queue`` if all active streams
have already publish events up to the ``_get_min_timestamp`` and
register them in order into ``events_queue``."""
while True:
if self._get_all_streams_exhausted() and self.raw_events_queue.empty():
if self.watch:
gevent.sleep(self.WATCH_SLEEP)
continue
self.events_queue.put(NO_MORE_EVENTS)
break
try:
timestamp, line = self.raw_events_queue.peek(timeout=1)
except Empty:
continue
min_timestamp = self._get_min_timestamp()
if min_timestamp and min_timestamp < timestamp:
gevent.sleep(0.3)
continue
timestamp, line = self.raw_events_queue.get()
output = [line['message']]
if self.output_stream_enabled:
output.insert(
0,
self.color(
line['stream'].ljust(self.max_stream_length, ' '),
'cyan'
)
)
if self.output_group_enabled:
output.insert(
0,
self.color(
line['group'].ljust(self.max_group_length, ' '),
'green'
)
)
self.events_queue.put("{0}\n".format(' '.join(output)))
def _events_consumer(self):
"""Print events from ``events_queue`` as soon as they are available."""
while True:
event = self.events_queue.get(True)
if event == NO_MORE_EVENTS:
break
sys.stdout.write(event)
sys.stdout.flush()
def list_logs(self):
self.register_publishers()
pool = Pool(size=self.pool_size)
pool.spawn(self._raw_events_queue_consumer)
pool.spawn(self._events_consumer)
if self.watch:
pool.spawn(self.register_publishers_periodically)
for i in xrange(self.pool_size):
pool.spawn(self._publisher_queue_consumer)
pool.join()
def register_publishers(self):
"""Register publishers into ``publishers_queue``."""
for group, stream in self._get_streams_from_patterns(self.log_group_name, self.log_stream_name):
if (group, stream) in self.publishers:
continue
self.publishers.append((group, stream))
self.max_group_length = max(self.max_group_length, len(group))
self.max_stream_length = max(self.max_stream_length, len(stream))
self.publishers_queue.put((0, (group, stream, None)))
self.stream_status[(group, stream)] = self.ACTIVE
self.stream_max_timestamp[(group, stream)] = -1
def register_publishers_periodically(self):
while True:
self.register_publishers()
gevent.sleep(2)
def list_groups(self):
"""Lists available CloudWatch logs groups"""
for group in self.get_groups():
print group
def list_streams(self, *args, **kwargs):
"""Lists available CloudWatch logs streams in ``log_group_name``."""
for stream in self.get_streams(*args, **kwargs):
print stream
def get_groups(self):
"""Returns available CloudWatch logs groups"""
next_token = None
while True:
response = self.connection.describe_log_groups(next_token=next_token)
for group in response.get('logGroups', []):
yield group['logGroupName']
if 'nextToken' in response:
next_token = response['nextToken']
else:
break
def get_streams(self, log_group_name=None):
"""Returns available CloudWatch logs streams in ``log_group_name``."""
log_group_name = log_group_name or self.log_group_name
next_token = None
while True:
response = self.connection.describe_log_streams(
log_group_name=log_group_name,
next_token=next_token
)
for stream in response.get('logStreams', []):
yield stream['logStreamName']
if 'nextToken' in response:
next_token = response['nextToken']
else:
break
def color(self, text, color):
"""Returns coloured version of ``text`` if ``color_enabled``."""
if self.color_enabled:
return colored(text, color)
return text
def parse_datetime(self, datetime_text):
"""Parse ``datetime_text`` into a ``datetime``."""
if not datetime_text:
return None
ago_match = re.match(r'(\d+)\s?(m|minute|minutes|h|hour|hours|d|day|days|w|weeks|weeks)(?: ago)?', datetime_text)
if ago_match:
amount, unit = ago_match.groups()
amount = int(amount)
unit = {'m': 60, 'h': 3600, 'd': 86400, 'w': 604800}[unit[0]]
date = datetime.now() + timedelta(seconds=unit * amount * -1)
else:
try:
date = parse(datetime_text)
except ValueError:
raise exceptions.UnknownDateError(datetime_text)
return int(date.strftime("%s")) * 1000
class FactoryPool(object):
def __init__(self, factory, maxsize=200, timeout=60):
self.factory = factory
self.maxsize = maxsize
self.timeout = timeout
self.clients = PriorityQueue(maxsize)
# If there is a maxsize, prime the queue with empty slots.
if maxsize is not None:
for _ in xrange(maxsize):
self.clients.put(EMPTY_SLOT)
@contextlib.contextmanager
def reserve(self):
"""Context-manager to obtain a Client object from the pool."""
ts, client = self._checkout_connection()
try:
yield client
finally:
self._checkin_connection(ts, client)
def _checkout_connection(self):
# If there's no maxsize, no need to block waiting for a connection.
blocking = self.maxsize is not None
# Loop until we get a non-stale connection, or we create a new one.
while True:
try:
ts, client = self.clients.get(blocking)
except Empty:
# No maxsize and no free connections, create a new one.
# XXX TODO: we should be using a monotonic clock here.
# see http://www.python.org/dev/peps/pep-0418/
now = int(time.time())
return now, self.factory()
else:
now = int(time.time())
# If we got an empty slot placeholder, create a new connection.
if client is None:
return now, self.factory()
# If the connection is not stale, go ahead and use it.
if ts + self.timeout > now:
return ts, client
# Otherwise, the connection is stale.
# Close it, push an empty slot onto the queue, and retry.
if hasattr(client, 'disconnect'):
client.disconnect()
self.clients.put(EMPTY_SLOT)
continue
def _checkin_connection(self, ts, client):
"""Return a connection to the pool."""
if hasattr(client, '_closed') and client._closed:
self.clients.put(EMPTY_SLOT)
return
# If the connection is now stale, don't return it to the pool.
# Push an empty slot instead so that it will be refreshed when needed.
now = int(time.time())
if ts + self.timeout > now:
self.clients.put((ts, client))
else:
if self.maxsize is not None:
self.clients.put(EMPTY_SLOT)
class Scanner:
def __init__(self):
self.start_time = time.time()
self.queue = PriorityQueue()
self.history = []
self.total_count = 0
self.scan_count = 0
self._load_target()
self.outfile = open("log.log", 'w')
self.console_width = getTerminalSize()[0] - 2
def _print_msg(self, _msg=None, _found_msg=False):
if _msg is None:
msg = '%s TotalCount| %s Scanned in %.2f seconds' % (
self.total_count, self.total_count - self.queue.qsize(),
time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) +
msg)
else:
sys.stdout.write('\r' + _msg + ' ' *
(self.console_width - len(_msg)) + '\n')
self.outfile.write(_msg + '\n')
self.outfile.flush()
if _found_msg:
msg = '%s TotalCount| %s Scanned in %.2f seconds' % (
self.total_count, self.total_count - self.queue.qsize(),
time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) +
msg)
sys.stdout.flush()
def _load_target(self):
print('[+] Read targets ...')
target_file = raw_input("Target File :")
with open(target_file) as f:
for line in f.xreadlines():
target = line.strip()
self.queue.put(target)
print("TotalCount is %d" % self.queue.qsize())
self.total_count = self.queue.qsize()
print("Now scanning ...")
def _scan(self, case):
while not self.queue.empty():
target = self.queue.get()
if case == "1":
self.vulnCheck(target)
if case == "2":
self.s2_045(target)
if case == "3":
self.headers(target)
if case == "4":
self.weakfile(target)
if case == "5":
self.portscan_c(target)
def vulnCheck(self, target):
if ":2375" in target:
try:
res = requests.head("http://" + str(target) +
"/containers/json",
timeout=2)
if res.headers['Content-Type'] == 'application/json':
self._print_msg(target + "==> docker api Vuln", True)
else:
self._print_msg()
except:
self._print_msg()
self._print_msg()
if ":27017" in target:
ip, port = target.split(":")
try:
socket.setdefaulttimeout(3)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((ip, int(port)))
data = binascii.a2b_hex(
"3a000000a741000000000000d40700000000000061646d696e2e24636d640000000000ffffffff130000001069736d6173746572000100000000"
)
s.send(data)
result = s.recv(1024)
if "ismaster" in result:
getlog_data = binascii.a2b_hex(
"480000000200000000000000d40700000000000061646d696e2e24636d6400000000000100000021000000026765744c6f670010000000737461727475705761726e696e67730000"
)
s.send(getlog_data)
result = s.recv(1024)
if "totalLinesWritten" in result:
self._print_msg(target + "==> mongodb Vuln", True)
except Exception as e:
pass
if ":6379" in target:
ip, port = target.split(":")
try:
socket.setdefaulttimeout(3)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((ip, int(port)))
s.send("INFO\r\n")
result = s.recv(1024)
if "redis_version" in result:
self._print_msg(target + "==> redis Vuln", True)
elif "Authentication" in result:
for pass_ in ['123456', 'redis', 'pass', 'password']:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((ip, int(port)))
s.send("AUTH %s\r\n" % (pass_))
result = s.recv(1024)
if '+OK' in result:
self._print_msg(
target + "==> redis pass Vuln :" + pass_,
True)
except Exception as e:
pass
if ":11211" in target:
ip, port = target.split(":")
try:
socket.setdefaulttimeout(3)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((ip, int(port)))
s.send("stats\r\n")
result = s.recv(1024)
if "STAT version" in result:
self._print_msg(target + "==> memcache Vuln", True)
except Exception as e:
pass
if ":9200" in target:
try:
res = requests.head("http://" + str(target) +
"/_rvier/_search",
timeout=2)
if res.status_code == 200:
self._print_msg(target + "==> Elasticsearch Vuln", True)
else:
self._print_msg()
except:
self._print_msg()
self._print_msg()
def headers(self, target):
try:
res = requests.head("http://" + str(target), timeout=1)
self._print_msg(target + "==>" + str(res.headers), True)
except:
self._print_msg()
self._print_msg()
def s2_045(self, target):
try:
data = {"image": " "}
headers = {
"User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36",
"Content-Type":
"%{#context['com.opensymphony.xwork2.dispatcher.HttpServletResponse'].addHeader('vul','s2-045')}.multtargetart/form-data"
}
req = requests.post("http://" + str(target),
data=data,
headers=headers)
if req.headers["vul"] == "s2-045":
self._print_msg(target + "==>" + "S2-045 Vuln", True)
except:
self._print_msg()
self._print_msg()
def weakfile(self, target):
weaklist = ["robots.txt", "/i.php", "/phpinfo.php"]
for weakfile in weaklist:
try:
res = requests.head("http://" + str(target) + weakfile,
timeout=1)
if res.status_code == 200:
if ("User-agent" in res.content) or ("phpinfo"
in res.content):
self._print_msg("http://" + target + weakfile, True)
except:
self._print_msg()
self._print_msg()
def portscan_c(self, target):
import socket
ip = socket.gethostbyname(target)
ports = [1433, 2375, 3306, 6379, 9200, 11211, 27017]
ip = ip.split(".")
ipc = ip[0] + "." + ip[1] + "." + ip[2] + "."
if ipc in self.history:
return
else:
self.history.append(ipc)
for port in ports:
for i in range(255):
try:
cs = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
cs.settimeout(float(2.5))
address = (ipc + str(i), int(port))
status = cs.connect_ex((address))
if status == 0:
self._print_msg(ipc + str(i) + ":" + str(port), True)
except Exception as e:
pass
finally:
cs.close()
self._print_msg()
def run(self, case):
threads = [gevent.spawn(self._scan, case) for i in xrange(1000)]
class TestResourceFilters(unittest.TestCase):
db = {}
conf = CONFIG
@patch('openprocurement.bridge.basic.filters.INFINITY')
@patch('openprocurement.bridge.basic.filters.logger')
def test_JMESPathFilter(self, logger, infinity):
self.input_queue = PriorityQueue()
self.filtered_queue = PriorityQueue()
resource = self.conf['resource'][:-1]
jmes_filter = JMESPathFilter(self.conf, self.input_queue, self.filtered_queue, self.db)
mock_calls = [call.info('Init Close Framework Agreement JMESPath Filter.')]
self.assertEqual(logger.mock_calls, mock_calls)
extra = {'MESSAGE_ID': 'SKIPPED', 'JOURNAL_{}_ID'.format(resource.upper()): 'test_id'}
infinity.__nonzero__.side_effect = [True, False]
jmes_filter._run()
doc = {
'id': 'test_id',
'dateModified': '1970-01-01',
'status': 'draft.pending'
}
self.input_queue.put((None, doc))
self.db['test_id'] = '1970-01-01'
infinity.__nonzero__.side_effect = [True, False]
jmes_filter._run()
mock_calls.append(
call.info('{} test_id not modified from last check. Skipping'.format(resource.title()),
extra=extra)
)
self.assertEqual(logger.mock_calls, mock_calls)
# no filters
doc['dateModified'] = '1970-01-02'
self.input_queue.put((None, doc))
infinity.__nonzero__.side_effect = [True, False]
jmes_filter._run()
mock_calls.append(
call.debug('Put to filtered queue {} test_id {}'.format(resource, doc['status']))
)
self.assertEqual(logger.mock_calls, mock_calls)
priority, filtered_doc = self.filtered_queue.get()
self.assertIsNone(priority)
self.assertEqual(filtered_doc, doc)
# not found
jmes_filter.filters = [jmespath.compile("contains([`test_status`], status)")]
doc['status'] = 'spam_status'
self.input_queue.put((None, doc))
infinity.__nonzero__.side_effect = [True, False]
jmes_filter._run()
mock_calls.append(
call.info('Skip {} test_id'.format(resource),
extra=extra)
)
# has found
doc['status'] = 'test_status'
self.input_queue.put((None, doc))
infinity.__nonzero__.side_effect = [True, False]
jmes_filter._run()
mock_calls.append(
call.debug('Put to filtered queue {} test_id {}'.format(resource, doc['status']))
)
self.assertEqual(logger.mock_calls, mock_calls)
priority, filtered_doc = self.filtered_queue.get()
self.assertIsNone(priority)
self.assertEqual(filtered_doc, doc)
class TestContractProformaFilter(unittest.TestCase):
conf = CONFIG
db = {}
@patch('openprocurement.bridge.templatesregistry.filters.INFINITY')
def test_init(self, infinity):
self.input_queue = PriorityQueue()
self.filtered_queue = PriorityQueue()
filter = ContractProformaFilter(self.conf, self.input_queue,
self.filtered_queue, self.db)
# Valid tender filtering
infinity.__nonzero__.side_effect = [True, False]
doc = {
'id': 'test_id',
'dateModified': '1970-01-02',
'procurementMethodType': 'dgf',
'status': 'status1',
'documents': [{
'documentType': 'contractProforma'
}]
}
self.input_queue.put((None, deepcopy(doc)))
filter._run()
self.assertEqual(len(self.filtered_queue), 1)
filtered_doc = self.filtered_queue.get(block=False)
self.assertEqual(doc, filtered_doc[1])
# Not changed dateModified
infinity.__nonzero__.side_effect = [True, False]
doc = {
'id': 'test_id',
'dateModified': '1970-01-01',
'procurementMethodType': 'dgf',
'status': 'status1',
'documents': [{
'documentType': 'contractProforma'
}],
}
self.input_queue.put((None, deepcopy(doc)))
self.db['test_id'] = '1970-01-01'
filter._run()
self.assertEqual(len(self.filtered_queue), 0)
self.db.pop('test_id')
# No contractProforma doc
infinity.__nonzero__.side_effect = [True, False]
doc = {
'id': 'test_id',
'dateModified': '1970-01-01',
'procurementMethodType': 'dgf',
'status': 'status1',
'documents': [{
'documentType': 'notContractProforma'
}],
}
filter._run()
self.assertEqual(len(self.filtered_queue), 0)
# Wrong tender status
infinity.__nonzero__.side_effect = [True, False]
doc = {
'id': 'test_id',
'dateModified': '1970-01-01',
'procurementMethodType': 'dgf',
'status': 'status3',
'documents': [{
'documentType': 'contractProforma'
}],
}
self.input_queue.put((None, deepcopy(doc)))
filter._run()
self.assertEqual(len(self.filtered_queue), 0)
# Wrong procurementMethodType without appropriate status
infinity.__nonzero__.side_effect = [True, False]
doc = {
'id': 'test_id',
'dateModified': '1970-01-01',
'procurementMethodType': 'other_pmt',
'status': 'status1',
'documents': [{
'documentType': 'contractProforma'
}],
}
self.input_queue.put((None, deepcopy(doc)))
filter._run()
self.assertEqual(len(self.filtered_queue), 0)
# Wrong procurementMethodType with appropriate status
infinity.__nonzero__.side_effect = [True, False]
doc = {
'id': 'test_id',
'dateModified': '1970-01-01',
'procurementMethodType': 'other_pmt',
'status': 'status2',
'documents': [{
'documentType': 'contractProforma'
}],
}
self.input_queue.put((None, deepcopy(doc)))
filter._run()
self.assertEqual(len(self.filtered_queue), 1)
class EdgeDataBridge(object):
"""Edge Bridge"""
def __init__(self, config):
super(EdgeDataBridge, self).__init__()
self.config = config
self.workers_config = {}
self.bridge_id = uuid.uuid4().hex
self.api_host = self.config_get('resources_api_server')
self.api_version = self.config_get('resources_api_version')
self.retrievers_params = self.config_get('retrievers_params')
# Check up_wait_sleep
up_wait_sleep = self.retrievers_params.get('up_wait_sleep')
if up_wait_sleep is not None and up_wait_sleep < 30:
raise DataBridgeConfigError('Invalid \'up_wait_sleep\' in '
'\'retrievers_params\'. Value must be '
'grater than 30.')
# Workers settings
for key in WORKER_CONFIG:
self.workers_config[key] = (self.config_get(key)
or WORKER_CONFIG[key])
# Init config
for key in DEFAULTS:
setattr(self, key, self.config_get(key) or DEFAULTS[key])
# Pools
self.workers_pool = gevent.pool.Pool(self.workers_max)
self.retry_workers_pool = gevent.pool.Pool(self.retry_workers_max)
self.filter_workers_pool = gevent.pool.Pool(self.filter_workers_count)
# Queues
if self.input_queue_size == -1:
self.input_queue = PriorityQueue()
else:
self.input_queue = PriorityQueue(self.input_queue_size)
if self.resource_items_queue_size == -1:
self.resource_items_queue = PriorityQueue()
else:
self.resource_items_queue = PriorityQueue(
self.resource_items_queue_size)
self.api_clients_queue = Queue()
# self.retry_api_clients_queue = Queue()
if self.retry_resource_items_queue_size == -1:
self.retry_resource_items_queue = PriorityQueue()
else:
self.retry_resource_items_queue = PriorityQueue(
self.retry_resource_items_queue_size)
if self.api_host != '' and self.api_host is not None:
api_host = urlparse(self.api_host)
if api_host.scheme == '' and api_host.netloc == '':
raise DataBridgeConfigError(
'Invalid \'tenders_api_server\' url.')
else:
raise DataBridgeConfigError('In config dictionary empty or missing'
' \'tenders_api_server\'')
self.db = prepare_couchdb(self.couch_url, self.db_name, logger)
db_url = self.couch_url + '/' + self.db_name
prepare_couchdb_views(db_url, self.workers_config['resource'], logger)
self.server = Server(self.couch_url,
session=Session(retry_delays=range(10)))
self.view_path = '_design/{}/_view/by_dateModified'.format(
self.workers_config['resource'])
extra_params = {
'mode': self.retrieve_mode,
'limit': self.resource_items_limit
}
self.feeder = ResourceFeeder(host=self.api_host,
version=self.api_version,
key='',
resource=self.workers_config['resource'],
extra_params=extra_params,
retrievers_params=self.retrievers_params,
adaptive=True,
with_priority=True)
self.api_clients_info = {}
def config_get(self, name):
try:
return self.config.get('main').get(name)
except AttributeError:
raise DataBridgeConfigError('In config dictionary missed section'
' \'main\'')
def create_api_client(self):
client_user_agent = self.user_agent + '/' + self.bridge_id
timeout = 0.1
while 1:
try:
api_client = APIClient(
host_url=self.api_host,
user_agent=client_user_agent,
api_version=self.api_version,
key='',
resource=self.workers_config['resource'])
client_id = uuid.uuid4().hex
logger.info('Started api_client {}'.format(
api_client.session.headers['User-Agent']),
extra={'MESSAGE_ID': 'create_api_clients'})
api_client_dict = {
'id': client_id,
'client': api_client,
'request_interval': 0,
'not_actual_count': 0
}
self.api_clients_info[api_client_dict['id']] = {
'drop_cookies': False,
'request_durations': {},
'request_interval': 0,
'avg_duration': 0
}
self.api_clients_queue.put(api_client_dict)
break
except RequestFailed as e:
logger.error(
'Failed start api_client with status code {}'.format(
e.status_code),
extra={'MESSAGE_ID': 'exceptions'})
timeout = timeout * 2
logger.info(
'create_api_client will be sleep {} sec.'.format(timeout))
sleep(timeout)
except Exception as e:
logger.error('Failed start api client with error: {}'.format(
e.message),
extra={'MESSAGE_ID': 'exceptions'})
timeout = timeout * 2
logger.info(
'create_api_client will be sleep {} sec.'.format(timeout))
sleep(timeout)
def fill_api_clients_queue(self):
while self.api_clients_queue.qsize() < self.workers_min:
self.create_api_client()
def fill_input_queue(self):
for resource_item in self.feeder.get_resource_items():
self.input_queue.put(resource_item)
logger.debug('Add to temp queue from sync: {} {} {}'.format(
self.workers_config['resource'][:-1], resource_item[1]['id'],
resource_item[1]['dateModified']),
extra={
'MESSAGE_ID': 'received_from_sync',
'TEMP_QUEUE_SIZE': self.input_queue.qsize()
})
def send_bulk(self, input_dict, priority_cache):
sleep_before_retry = 2
for i in xrange(0, 3):
try:
logger.debug('Send check bulk: {}'.format(len(input_dict)),
extra={'CHECK_BULK_LEN': len(input_dict)})
start = time()
rows = self.db.view(self.view_path, keys=input_dict.values())
end = time() - start
logger.debug('Duration bulk check: {} sec.'.format(end),
extra={'CHECK_BULK_DURATION': end * 1000})
resp_dict = {k.id: k.key for k in rows}
break
except (IncompleteRead, Exception) as e:
logger.error('Error while send bulk {}'.format(e.message),
extra={'MESSAGE_ID': 'exceptions'})
if i == 2:
raise e
sleep(sleep_before_retry)
sleep_before_retry *= 2
for item_id, date_modified in input_dict.items():
if item_id in resp_dict and date_modified == resp_dict[item_id]:
logger.debug('Skipped {} {}: In db exist newest.'.format(
self.workers_config['resource'][:-1], item_id),
extra={'MESSAGE_ID': 'skipped'})
elif ((1, item_id) not in self.resource_items_queue.queue
and (1000, item_id) not in self.resource_items_queue.queue):
self.resource_items_queue.put(
(priority_cache[item_id], item_id))
logger.debug(
'Put to main queue {}: {}'.format(
self.workers_config['resource'][:-1], item_id),
extra={'MESSAGE_ID': 'add_to_resource_items_queue'})
else:
logger.debug(
'Skipped {} {}: In queue exist with same id'.format(
self.workers_config['resource'][:-1], item_id),
extra={'MESSAGE_ID': 'skipped'})
def fill_resource_items_queue(self):
start_time = datetime.now()
input_dict = {}
priority_cache = {}
while True:
# Get resource_item from temp queue
if not self.input_queue.empty():
priority, resource_item = self.input_queue.get()
else:
timeout = self.bulk_query_interval -\
(datetime.now() - start_time).total_seconds()
if timeout > self.bulk_query_interval:
timeout = self.bulk_query_interval
try:
priority, resource_item = self.input_queue.get(
timeout=timeout)
except Empty:
resource_item = None
# Add resource_item to bulk
if resource_item is not None:
logger.debug('Add to input_dict {}'.format(
resource_item['id']))
input_dict[resource_item['id']] = resource_item['dateModified']
priority_cache[resource_item['id']] = priority
if (len(input_dict) >= self.bulk_query_limit
or (datetime.now() - start_time).total_seconds() >=
self.bulk_query_interval):
if len(input_dict) > 0:
self.send_bulk(input_dict, priority_cache)
input_dict = {}
priority_cache = {}
start_time = datetime.now()
def resource_items_filter(self, r_id, r_date_modified):
try:
local_document = self.db.get(r_id)
if local_document:
if local_document['dateModified'] < r_date_modified:
return True
else:
return False
else:
return True
except Exception as e:
logger.error(
'Filter error: Error while getting {} {} from couchdb: '
'{}'.format(self.workers_config['resource'][:-1], r_id,
e.message),
extra={'MESSAGE_ID': 'exceptions'})
return True
def _get_average_requests_duration(self):
req_durations = []
delta = timedelta(seconds=self.perfomance_window)
current_date = datetime.now() - delta
for cid, info in self.api_clients_info.items():
if len(info['request_durations']) > 0:
if min(info['request_durations'].keys()) <= current_date:
info['grown'] = True
avg = round(
sum(info['request_durations'].values()) * 1.0 /
len(info['request_durations']), 3)
req_durations.append(avg)
info['avg_duration'] = avg
if len(req_durations) > 0:
return round(sum(req_durations) / len(req_durations),
3), req_durations
else:
return 0, req_durations
# TODO: Add logic for restart sync if last response grater than some values
# and no active tasks specific for resource
def queues_controller(self):
while True:
if (self.workers_pool.free_count() > 0
and (self.resource_items_queue.qsize() >
((float(self.resource_items_queue_size) / 100) *
self.workers_inc_threshold))):
self.create_api_client()
w = ResourceItemWorker.spawn(self.api_clients_queue,
self.resource_items_queue,
self.db, self.workers_config,
self.retry_resource_items_queue,
self.api_clients_info)
self.workers_pool.add(w)
logger.info('Queue controller: Create main queue worker.')
elif (self.resource_items_queue.qsize() <
((float(self.resource_items_queue_size) / 100) *
self.workers_dec_threshold)):
if len(self.workers_pool) > self.workers_min:
wi = self.workers_pool.greenlets.pop()
wi.shutdown()
api_client_dict = self.api_clients_queue.get()
del self.api_clients_info[api_client_dict['id']]
logger.info('Queue controller: Kill main queue worker.')
filled_resource_items_queue = round(
self.resource_items_queue.qsize() /
(float(self.resource_items_queue_size) / 100), 2)
logger.info('Resource items queue filled on {} %'.format(
filled_resource_items_queue))
filled_retry_resource_items_queue \
= round(self.retry_resource_items_queue.qsize() / float(
self.retry_resource_items_queue_size) / 100, 2)
logger.info('Retry resource items queue filled on {} %'.format(
filled_retry_resource_items_queue))
sleep(self.queues_controller_timeout)
def gevent_watcher(self):
self.perfomance_watcher()
for t in self.server.tasks():
if (t['type'] == 'indexer' and t['database'] == self.db_name
and t.get('design_document', None) == '_design/{}'.format(
self.workers_config['resource'])):
logger.info(
'Watcher: Waiting for end of view indexing. Current'
' progress: {} %'.format(t['progress']))
# Check fill threads
input_threads = 1
if self.input_queue_filler.ready():
input_threads = 0
logger.error('Temp queue filler error: {}'.format(
self.input_queue_filler.exception.message),
extra={'MESSAGE_ID': 'exception'})
self.input_queue_filler = spawn(self.fill_input_queue)
logger.info('Input threads {}'.format(input_threads),
extra={'INPUT_THREADS': input_threads})
fill_threads = 1
if self.filler.ready():
fill_threads = 0
logger.error('Fill thread error: {}'.format(
self.filler.exception.message),
extra={'MESSAGE_ID': 'exception'})
self.filler = spawn(self.fill_resource_items_queue)
logger.info('Filter threads {}'.format(fill_threads),
extra={'FILTER_THREADS': fill_threads})
main_threads = self.workers_max - self.workers_pool.free_count()
logger.info('Main threads {}'.format(main_threads),
extra={'MAIN_THREADS': main_threads})
if len(self.workers_pool) < self.workers_min:
for i in xrange(0, (self.workers_min - len(self.workers_pool))):
w = ResourceItemWorker.spawn(self.api_clients_queue,
self.resource_items_queue,
self.db, self.workers_config,
self.retry_resource_items_queue,
self.api_clients_info)
self.workers_pool.add(w)
logger.info('Watcher: Create main queue worker.')
self.create_api_client()
retry_threads = self.retry_workers_max -\
self.retry_workers_pool.free_count()
logger.info('Retry threads {}'.format(retry_threads),
extra={'RETRY_THREADS': retry_threads})
if len(self.retry_workers_pool) < self.retry_workers_min:
for i in xrange(
0, self.retry_workers_min - len(self.retry_workers_pool)):
self.create_api_client()
w = ResourceItemWorker.spawn(self.api_clients_queue,
self.retry_resource_items_queue,
self.db, self.workers_config,
self.retry_resource_items_queue,
self.api_clients_info)
self.retry_workers_pool.add(w)
logger.info('Watcher: Create retry queue worker.')
# Log queues size and API clients count
main_queue_size = self.resource_items_queue.qsize()
logger.info('Resource items queue size {}'.format(main_queue_size),
extra={'MAIN_QUEUE_SIZE': main_queue_size})
retry_queue_size = self.retry_resource_items_queue.qsize()
logger.info(
'Resource items retry queue size {}'.format(retry_queue_size),
extra={'RETRY_QUEUE_SIZE': retry_queue_size})
api_clients_count = len(self.api_clients_info)
logger.info('API Clients count: {}'.format(api_clients_count),
extra={'API_CLIENTS': api_clients_count})
def _calculate_st_dev(self, values):
if len(values) > 0:
avg = sum(values) * 1.0 / len(values)
variance = map(lambda x: (x - avg)**2, values)
avg_variance = sum(variance) * 1.0 / len(variance)
st_dev = math.sqrt(avg_variance)
return round(st_dev, 3)
else:
return 0
def _mark_bad_clients(self, dev):
# Mark bad api clients
for cid, info in self.api_clients_info.items():
if info.get('grown', False) and info['avg_duration'] > dev:
info['drop_cookies'] = True
logger.debug('Perfomance watcher: Mark client {} as bad, avg.'
' request_duration is {} sec.'.format(
cid, info['avg_duration']),
extra={'MESSAGE_ID': 'marked_as_bad'})
elif info['avg_duration'] < dev and info['request_interval'] > 0:
info['drop_cookies'] = True
logger.debug('Perfomance watcher: Mark client {} as bad,'
' request_interval is {} sec.'.format(
cid, info['request_interval']),
extra={'MESSAGE_ID': 'marked_as_bad'})
def perfomance_watcher(self):
avg_duration, values = self._get_average_requests_duration()
for _, info in self.api_clients_info.items():
delta = timedelta(seconds=self.perfomance_window +
self.watch_interval)
current_date = datetime.now() - delta
delete_list = []
for key in info['request_durations']:
if key < current_date:
delete_list.append(key)
for k in delete_list:
del info['request_durations'][k]
delete_list = []
st_dev = self._calculate_st_dev(values)
if len(values) > 0:
min_avg = min(values) * 1000
max_avg = max(values) * 1000
else:
max_avg = 0
min_avg = 0
dev = round(st_dev + avg_duration, 3)
logger.info('Perfomance watcher:\nREQUESTS_STDEV - {} sec.\n'
'REQUESTS_DEV - {} ms.\nREQUESTS_MIN_AVG - {} ms.\n'
'REQUESTS_MAX_AVG - {} ms.\nREQUESTS_AVG - {} sec.'.format(
round(st_dev, 3), dev, min_avg, max_avg, avg_duration),
extra={
'REQUESTS_DEV': dev * 1000,
'REQUESTS_MIN_AVG': min_avg,
'REQUESTS_MAX_AVG': max_avg,
'REQUESTS_AVG': avg_duration * 1000
})
self._mark_bad_clients(dev)
def run(self):
logger.info('Start Edge Bridge',
extra={'MESSAGE_ID': 'edge_bridge_start_bridge'})
logger.info('Start data sync...',
extra={'MESSAGE_ID': 'edge_bridge__data_sync'})
self.input_queue_filler = spawn(self.fill_input_queue)
self.filler = spawn(self.fill_resource_items_queue)
spawn(self.queues_controller)
while True:
self.gevent_watcher()
sleep(self.watch_interval)
class SubNameBrute:
def __init__(self, target, options):
self.start_time = time.time()
self.target = target.strip()
self.options = options
self.ignore_intranet = options.i
self.scan_count = self.found_count = 0
self.console_width = getTerminalSize()[0] - 2
self.resolvers = [dns.resolver.Resolver(configure=False) for _ in range(options.threads)]
for _ in self.resolvers:
_.lifetime = _.timeout = 10.0
self.print_count = 0
self.STOP_ME = False
self._load_dns_servers()
self._load_next_sub()
self.queue = PriorityQueue()
self.priority = 0
self._load_sub_names()
if options.output:
outfile = options.output
else:
_name = os.path.basename(self.options.file).replace('subnames', '')
if _name != '.txt':
_name = '_' + _name
outfile = target + _name if not options.full_scan else target + '_full' + _name
self.outfile = open(outfile, 'w')
self.ip_dict = {}
self.found_subs = set()
self.ex_resolver = dns.resolver.Resolver(configure=False)
self.ex_resolver.nameservers = self.dns_servers
def _load_dns_servers(self):
print '[+] Validate DNS servers ...'
self.dns_servers = []
pool = Pool(30)
for server in open('dict/dns_servers.txt').xreadlines():
server = server.strip()
if server:
pool.apply_async(self._test_server, (server,))
pool.join()
self.dns_count = len(self.dns_servers)
sys.stdout.write('\n')
print '[+] Found %s available DNS Servers in total' % self.dns_count
if self.dns_count == 0:
print '[ERROR] No DNS Servers available.'
sys.exit(-1)
def _test_server(self, server):
resolver = dns.resolver.Resolver()
resolver.lifetime = resolver.timeout = 10.0
try:
resolver.nameservers = [server]
answers = resolver.query('public-dns-a.baidu.com') # test lookup a existed domain
if answers[0].address != '180.76.76.76':
raise Exception('incorrect DNS response')
try:
resolver.query('test.bad.dns.lijiejie.com') # Non-existed domain test
with open('bad_dns_servers.txt', 'a') as f:
f.write(server + '\n')
self._print_msg('[+] Bad DNS Server found %s' % server)
except:
self.dns_servers.append(server)
self._print_msg('[+] Check DNS Server %s < OK > Found %s' % (server.ljust(16), len(self.dns_servers)))
except:
self._print_msg('[+] Check DNS Server %s <Fail> Found %s' % (server.ljust(16), len(self.dns_servers)))
def _load_sub_names(self):
self._print_msg('[+] Load sub names ...')
if self.options.full_scan and self.options.file == 'subnames.txt':
_file = 'dict/subnames_full.txt'
else:
if os.path.exists(self.options.file):
_file = self.options.file
elif os.path.exists('dict/%s' % self.options.file):
_file = 'dict/%s' % self.options.file
else:
self._print_msg('[ERROR] Names file not exists: %s' % self.options.file)
exit(-1)
normal_lines = []
wildcard_lines = []
wildcard_list = []
regex_list = []
lines = set()
with open(_file) as f:
for line in f.xreadlines():
sub = line.strip()
if not sub or sub in lines:
continue
lines.add(sub)
if sub.find('{alphnum}') >= 0 or sub.find('{alpha}') >= 0 or sub.find('{num}') >= 0:
wildcard_lines.append(sub)
sub = sub.replace('{alphnum}', '[a-z0-9]')
sub = sub.replace('{alpha}', '[a-z]')
sub = sub.replace('{num}', '[0-9]')
if sub not in wildcard_list:
wildcard_list.append(sub)
regex_list.append('^' + sub + '$')
else:
normal_lines.append(sub)
pattern = '|'.join(regex_list)
if pattern:
_regex = re.compile(pattern)
if _regex:
for line in normal_lines[:]:
if _regex.search(line):
normal_lines.remove(line)
for item in normal_lines:
self.priority += 1
self.queue.put((self.priority, item))
for item in wildcard_lines:
self.queue.put((88888888, item))
def _load_next_sub(self):
self._print_msg('[+] Load next level subs ...')
self.next_subs = []
_set = set()
_file = 'dict/next_sub.txt' if not self.options.full_scan else 'dict/next_sub_full.txt'
with open(_file) as f:
for line in f:
sub = line.strip()
if sub and sub not in self.next_subs:
tmp_set = {sub}
while len(tmp_set) > 0:
item = tmp_set.pop()
if item.find('{alphnum}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789':
tmp_set.add(item.replace('{alphnum}', _letter, 1))
elif item.find('{alpha}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz':
tmp_set.add(item.replace('{alpha}', _letter, 1))
elif item.find('{num}') >= 0:
for _letter in '0123456789':
tmp_set.add(item.replace('{num}', _letter, 1))
elif item not in _set:
_set.add(item)
self.next_subs.append(item)
def _print_msg(self, _msg=None, _found_msg=False):
if _msg is None:
self.print_count += 1
if self.print_count < 100:
return
self.print_count = 0
msg = '%s Found| %s Groups| %s scanned in %.1f seconds' % (
self.found_count, self.queue.qsize(), self.scan_count, time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) + msg)
elif _msg.startswith('[+] Check DNS Server'):
sys.stdout.write('\r' + _msg + ' ' * (self.console_width - len(_msg)))
else:
sys.stdout.write('\r' + _msg + ' ' * (self.console_width - len(_msg)) + '\n')
if _found_msg:
msg = '%s Found| %s Groups| %s scanned in %.1f seconds' % (
self.found_count, self.queue.qsize(), self.scan_count, time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) + msg)
sys.stdout.flush()
@staticmethod
def is_intranet(ip):
ret = ip.split('.')
if len(ret) != 4:
return True
if ret[0] == '10':
return True
if ret[0] == '172' and 16 <= int(ret[1]) <= 32:
return True
if ret[0] == '192' and ret[1] == '168':
return True
return False
def put_item(self, item):
num = item.count('{alphnum}') + item.count('{alpha}') + item.count('{num}')
if num == 0:
self.priority += 1
self.queue.put((self.priority, item))
else:
self.queue.put((self.priority + num * 10000000, item))
def _scan(self, j):
self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]]
while not self.queue.empty():
try:
item = self.queue.get(timeout=1.0)[1]
self.scan_count += 1
except:
break
self._print_msg()
try:
if item.find('{alphnum}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789':
self.put_item(item.replace('{alphnum}', _letter, 1))
continue
elif item.find('{alpha}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz':
self.put_item(item.replace('{alpha}', _letter, 1))
continue
elif item.find('{num}') >= 0:
for _letter in '0123456789':
self.put_item(item.replace('{num}', _letter, 1))
continue
elif item.find('{next_sub}') >= 0:
for _ in self.next_subs:
self.queue.put((0, item.replace('{next_sub}', _, 1)))
continue
else:
sub = item
if sub in self.found_subs:
continue
cur_sub_domain = sub + '.' + self.target
_sub = sub.split('.')[-1]
try:
answers = self.resolvers[j].query(cur_sub_domain)
except dns.resolver.NoAnswer, e:
answers = self.ex_resolver.query(cur_sub_domain)
if answers:
self.found_subs.add(sub)
ips = ', '.join(sorted([answer.address for answer in answers]))
if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0']:
continue
if self.ignore_intranet and SubNameBrute.is_intranet(answers[0].address):
continue
try:
self.scan_count += 1
answers = self.resolvers[j].query(cur_sub_domain, 'cname')
cname = answers[0].target.to_unicode().rstrip('.')
if cname.endswith(self.target) and cname not in self.found_subs:
self.found_subs.add(cname)
cname_sub = cname[:len(cname) - len(self.target) - 1] # new sub
self.queue.put((0, cname_sub))
except:
pass
if (_sub, ips) not in self.ip_dict:
self.ip_dict[(_sub, ips)] = 1
else:
self.ip_dict[(_sub, ips)] += 1
if ips not in self.ip_dict:
self.ip_dict[ips] = 1
else:
self.ip_dict[ips] += 1
if self.ip_dict[(_sub, ips)] > 3 or self.ip_dict[ips] > 6:
continue
self.found_count += 1
msg = cur_sub_domain.ljust(30) + ips
self._print_msg(msg, _found_msg=True)
self._print_msg()
self.outfile.write(cur_sub_domain.ljust(30) + '\t' + ips + '\n')
self.outfile.flush()
try:
self.resolvers[j].query('lijiejietest.' + cur_sub_domain)
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e:
self.queue.put((999999999, '{next_sub}.' + sub))
except:
pass
except (dns.resolver.NXDOMAIN, dns.name.EmptyLabel) as e:
pass
except (dns.resolver.NoNameservers, dns.resolver.NoAnswer, dns.exception.Timeout) as e:
pass
class SubNameBrute:
def __init__(self, target, options):
# save result to list
self.subdomains = list()
self.start_time = time.time()
self.target = target.strip()
self.options = options
self.ignore_intranet = options.i
self.scan_count = self.found_count = 0
self.console_width = getTerminalSize()[0] - 2
self.resolvers = [dns.resolver.Resolver(configure=False) for _ in range(options.threads)]
for _ in self.resolvers:
_.lifetime = _.timeout = 10.0
self.STOP_ME = False
self._load_dns_servers()
self._load_next_sub()
self.queue = PriorityQueue()
self.priority = 0
self._load_sub_names()
self.ip_dict = {}
self.found_subs = set()
self.ex_resolver = dns.resolver.Resolver(configure=False)
self.ex_resolver.nameservers = self.dns_servers
def _load_dns_servers(self):
self.dns_servers = []
pool = Pool(30)
for server in open('dict/dns_servers.txt').xreadlines():
server = server.strip()
if server:
pool.apply_async(self._test_server, (server,))
pool.join()
self.dns_count = len(self.dns_servers)
sys.stdout.write('\n')
if self.dns_count == 0:
sys.exit(-1)
def _test_server(self, server):
resolver = dns.resolver.Resolver()
resolver.lifetime = resolver.timeout = 10.0
try:
resolver.nameservers = [server]
answers = resolver.query('public-dns-a.baidu.com') # test lookup a existed domain
if answers[0].address != '180.76.76.76':
raise Exception('incorrect DNS response')
try:
resolver.query('test.bad.dns.lijiejie.com') # Non-existed domain test
with open('bad_dns_servers.txt', 'a') as f:
f.write(server + '\n')
except:
self.dns_servers.append(server)
except:
pass
def _load_sub_names(self):
if self.options.full_scan and self.options.file == 'subnames.txt':
_file = 'dict/subnames_full.txt'
else:
if os.path.exists(self.options.file):
_file = self.options.file
elif os.path.exists('dict/%s' % self.options.file):
_file = 'dict/%s' % self.options.file
else:
exit(-1)
normal_lines = []
wildcard_lines = []
wildcard_list = []
regex_list = []
lines = set()
with open(_file) as f:
for line in f.xreadlines():
sub = line.strip()
if not sub or sub in lines:
continue
lines.add(sub)
if sub.find('{alphnum}') >= 0 or sub.find('{alpha}') >= 0 or sub.find('{num}') >= 0:
wildcard_lines.append(sub)
sub = sub.replace('{alphnum}', '[a-z0-9]')
sub = sub.replace('{alpha}', '[a-z]')
sub = sub.replace('{num}', '[0-9]')
if sub not in wildcard_list:
wildcard_list.append(sub)
regex_list.append('^' + sub + '$')
else:
normal_lines.append(sub)
pattern = '|'.join(regex_list)
if pattern:
_regex = re.compile(pattern)
if _regex:
for line in normal_lines[:]:
if _regex.search(line):
normal_lines.remove(line)
for item in normal_lines:
self.priority += 1
self.queue.put((self.priority, item))
for item in wildcard_lines:
self.queue.put((88888888, item))
def _load_next_sub(self):
self.next_subs = []
_set = set()
_file = 'dict/next_sub.txt' if not self.options.full_scan else 'dict/next_sub_full.txt'
with open(_file) as f:
for line in f:
sub = line.strip()
if sub and sub not in self.next_subs:
tmp_set = {sub}
while len(tmp_set) > 0:
item = tmp_set.pop()
if item.find('{alphnum}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789':
tmp_set.add(item.replace('{alphnum}', _letter, 1))
elif item.find('{alpha}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz':
tmp_set.add(item.replace('{alpha}', _letter, 1))
elif item.find('{num}') >= 0:
for _letter in '0123456789':
tmp_set.add(item.replace('{num}', _letter, 1))
elif item not in _set:
_set.add(item)
self.next_subs.append(item)
@staticmethod
def is_intranet(ip):
ret = ip.split('.')
if len(ret) != 4:
return True
if ret[0] == '10':
return True
if ret[0] == '172' and 16 <= int(ret[1]) <= 32:
return True
if ret[0] == '192' and ret[1] == '168':
return True
return False
def put_item(self, item):
num = item.count('{alphnum}') + item.count('{alpha}') + item.count('{num}')
if num == 0:
self.priority += 1
self.queue.put((self.priority, item))
else:
self.queue.put((self.priority + num * 10000000, item))
def _scan(self, j):
self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]]
while not self.queue.empty():
try:
item = self.queue.get(timeout=1.0)[1]
self.scan_count += 1
except:
break
try:
if item.find('{alphnum}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789':
self.put_item(item.replace('{alphnum}', _letter, 1))
continue
elif item.find('{alpha}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz':
self.put_item(item.replace('{alpha}', _letter, 1))
continue
elif item.find('{num}') >= 0:
for _letter in '0123456789':
self.put_item(item.replace('{num}', _letter, 1))
continue
elif item.find('{next_sub}') >= 0:
for _ in self.next_subs:
self.queue.put((0, item.replace('{next_sub}', _, 1)))
continue
else:
sub = item
if sub in self.found_subs:
continue
cur_sub_domain = sub + '.' + self.target
_sub = sub.split('.')[-1]
try:
answers = self.resolvers[j].query(cur_sub_domain)
except dns.resolver.NoAnswer, e:
answers = self.ex_resolver.query(cur_sub_domain)
if answers:
self.found_subs.add(sub)
ips = ', '.join(sorted([answer.address for answer in answers]))
if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0']:
continue
if self.ignore_intranet and SubNameBrute.is_intranet(answers[0].address):
continue
try:
self.scan_count += 1
answers = self.resolvers[j].query(cur_sub_domain, 'cname')
cname = answers[0].target.to_unicode().rstrip('.')
if cname.endswith(self.target) and cname not in self.found_subs:
self.found_subs.add(cname)
cname_sub = cname[:len(cname) - len(self.target) - 1] # new sub
self.queue.put((0, cname_sub))
except:
pass
if (_sub, ips) not in self.ip_dict:
self.ip_dict[(_sub, ips)] = 1
else:
self.ip_dict[(_sub, ips)] += 1
if ips not in self.ip_dict:
self.ip_dict[ips] = 1
else:
self.ip_dict[ips] += 1
if self.ip_dict[(_sub, ips)] > 3 or self.ip_dict[ips] > 6:
continue
self.found_count += 1
msg = cur_sub_domain.ljust(30) + ips
# save result to list
self.subdomains.append([cur_sub_domain,ips])
try:
self.resolvers[j].query('lijiejietest.' + cur_sub_domain)
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e:
self.queue.put((999999999, '{next_sub}.' + sub))
except:
pass
except (dns.resolver.NXDOMAIN, dns.name.EmptyLabel) as e:
pass
except (dns.resolver.NoNameservers, dns.resolver.NoAnswer, dns.exception.Timeout) as e:
pass
class SubNameBrute:
def __init__(self, target, options):
self.start_time = time.time()
self.target = target.strip()
self.options = options
self.ignore_intranet = options.i
self.scan_count = self.found_count = 0
self.console_width = getTerminalSize()[0] - 2
self.resolvers = [
dns.resolver.Resolver(configure=False)
for _ in range(options.threads)
]
for _ in self.resolvers:
_.lifetime = _.timeout = 10.0
self.print_count = 0
self.STOP_ME = False
self._load_dns_servers()
self._load_next_sub()
self.queue = PriorityQueue()
self.priority = 0
self._load_sub_names()
if options.output:
outfile = options.output
else:
_name = os.path.basename(self.options.file).replace('subnames', '')
if _name != '.txt':
_name = '_' + _name
outfile = target + _name if not options.full_scan else target + '_full' + _name
self.outfile = open(outfile, 'w')
self.ip_dict = {}
self.found_subs = set()
self.ex_resolver = dns.resolver.Resolver(configure=False)
self.ex_resolver.nameservers = self.dns_servers
self.result_lines = []
self.result_domains = []
self.result_ips = []
def _load_dns_servers(self):
print '[+] Validate DNS servers ...'
self.dns_servers = []
pool = Pool(30)
# change file location
for server in open('lijiejie/' + 'dict/dns_servers.txt').xreadlines():
server = server.strip()
if server:
pool.apply_async(self._test_server, (server, ))
pool.join()
self.dns_count = len(self.dns_servers)
sys.stdout.write('\n')
print '[+] Found %s available DNS Servers in total' % self.dns_count
if self.dns_count == 0:
print '[ERROR] No DNS Servers available.'
sys.exit(-1)
def _test_server(self, server):
resolver = dns.resolver.Resolver()
resolver.lifetime = resolver.timeout = 10.0
try:
resolver.nameservers = [server]
answers = resolver.query(
'public-dns-a.baidu.com') # test lookup a existed domain
if answers[0].address != '180.76.76.76':
raise Exception('incorrect DNS response')
try:
resolver.query(
'test.bad.dns.lijiejie.com') # Non-existed domain test
with open('bad_dns_servers.txt', 'a') as f:
f.write(server + '\n')
self._print_msg('[+] Bad DNS Server found %s' % server)
except:
self.dns_servers.append(server)
self._print_msg('[+] Check DNS Server %s < OK > Found %s' %
(server.ljust(16), len(self.dns_servers)))
except:
self._print_msg('[+] Check DNS Server %s <Fail> Found %s' %
(server.ljust(16), len(self.dns_servers)))
def _load_sub_names(self):
self._print_msg('[+] Load sub names ...')
if self.options.full_scan and self.options.file == 'subnames.txt':
_file = 'lijiejie/' + 'dict/subnames_full.txt'
else:
if os.path.exists(self.options.file):
_file = self.options.file
elif os.path.exists('lijiejie/' + 'dict/%s' % self.options.file):
_file = 'lijiejie/' + 'dict/%s' % self.options.file
else:
self._print_msg('[ERROR] Names file not exists: %s' %
self.options.file)
exit(-1)
if self.options.debug:
_file = 'lijiejie/' + 'dict/debug.txt'
if not os.path.exists(_file):
self._print_msg('[ERROR] Names file not exists: %s' %
self.options.file)
exit(-1)
normal_lines = []
wildcard_lines = []
wildcard_list = []
regex_list = []
lines = set()
with open(_file) as f:
for line in f.xreadlines():
sub = line.strip()
if not sub or sub in lines:
continue
lines.add(sub)
if sub.find('{alphnum}') >= 0 or sub.find(
'{alpha}') >= 0 or sub.find('{num}') >= 0:
wildcard_lines.append(sub)
sub = sub.replace('{alphnum}', '[a-z0-9]')
sub = sub.replace('{alpha}', '[a-z]')
sub = sub.replace('{num}', '[0-9]')
if sub not in wildcard_list:
wildcard_list.append(sub)
regex_list.append('^' + sub + '$')
else:
normal_lines.append(sub)
pattern = '|'.join(regex_list)
if pattern:
_regex = re.compile(pattern)
if _regex:
for line in normal_lines[:]:
if _regex.search(line):
normal_lines.remove(line)
for item in normal_lines:
self.priority += 1
self.queue.put((self.priority, item))
for item in wildcard_lines:
self.queue.put((88888888, item))
def _load_next_sub(self):
self._print_msg('[+] Load next level subs ...')
self.next_subs = []
_set = set()
_file = 'lijiejie/' + 'dict/next_sub.txt' if not self.options.full_scan else 'dict/next_sub_full.txt'
with open(_file) as f:
for line in f:
sub = line.strip()
if sub and sub not in self.next_subs:
tmp_set = {sub}
while len(tmp_set) > 0:
item = tmp_set.pop()
if item.find('{alphnum}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789':
tmp_set.add(
item.replace('{alphnum}', _letter, 1))
elif item.find('{alpha}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz':
tmp_set.add(item.replace(
'{alpha}', _letter, 1))
elif item.find('{num}') >= 0:
for _letter in '0123456789':
tmp_set.add(item.replace('{num}', _letter, 1))
elif item not in _set:
_set.add(item)
self.next_subs.append(item)
def _print_msg(self, _msg=None, _found_msg=False):
if _msg is None:
self.print_count += 1
if self.print_count < 100:
return
self.print_count = 0
msg = '%s Found| %s Groups| %s scanned in %.1f seconds' % (
self.found_count, self.queue.qsize(), self.scan_count,
time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) +
msg)
elif _msg.startswith('[+] Check DNS Server'):
sys.stdout.write('\r' + _msg + ' ' *
(self.console_width - len(_msg)))
else:
sys.stdout.write('\r' + _msg + ' ' *
(self.console_width - len(_msg)) + '\n')
if _found_msg:
msg = '%s Found| %s Groups| %s scanned in %.1f seconds' % (
self.found_count, self.queue.qsize(), self.scan_count,
time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) +
msg)
sys.stdout.flush()
@staticmethod
def is_intranet(ip):
ret = ip.split('.')
if len(ret) != 4:
return True
if ret[0] == '10':
return True
if ret[0] == '172' and 16 <= int(ret[1]) <= 32:
return True
if ret[0] == '192' and ret[1] == '168':
return True
return False
def put_item(self, item):
num = item.count('{alphnum}') + item.count('{alpha}') + item.count(
'{num}')
if num == 0:
self.priority += 1
self.queue.put((self.priority, item))
else:
self.queue.put((self.priority + num * 10000000, item))
def _scan(self, j):
self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]]
while not self.queue.empty():
try:
item = self.queue.get(timeout=1.0)[1]
self.scan_count += 1
except:
break
self._print_msg()
try:
if item.find('{alphnum}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789':
self.put_item(item.replace('{alphnum}', _letter, 1))
continue
elif item.find('{alpha}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz':
self.put_item(item.replace('{alpha}', _letter, 1))
continue
elif item.find('{num}') >= 0:
for _letter in '0123456789':
self.put_item(item.replace('{num}', _letter, 1))
continue
elif item.find('{next_sub}') >= 0:
for _ in self.next_subs:
self.queue.put((0, item.replace('{next_sub}', _, 1)))
continue
else:
sub = item
if sub in self.found_subs:
continue
cur_sub_domain = sub + '.' + self.target
_sub = sub.split('.')[-1]
try:
answers = self.resolvers[j].query(cur_sub_domain)
except dns.resolver.NoAnswer, e:
answers = self.ex_resolver.query(cur_sub_domain)
if answers:
self.found_subs.add(sub)
ips = ', '.join(
sorted([answer.address for answer in answers]))
if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0']:
continue
if self.ignore_intranet and SubNameBrute.is_intranet(
answers[0].address):
continue
try:
self.scan_count += 1
answers = self.resolvers[j].query(
cur_sub_domain, 'cname')
cname = answers[0].target.to_unicode().rstrip('.')
if cname.endswith(
self.target) and cname not in self.found_subs:
self.found_subs.add(cname)
cname_sub = cname[:len(cname) - len(self.target) -
1] # new sub
self.queue.put((0, cname_sub))
except:
pass
if (_sub, ips) not in self.ip_dict:
self.ip_dict[(_sub, ips)] = 1
else:
self.ip_dict[(_sub, ips)] += 1
if ips not in self.ip_dict:
self.ip_dict[ips] = 1
else:
self.ip_dict[ips] += 1
if self.ip_dict[(_sub, ips)] > 3 or self.ip_dict[ips] > 6:
continue
self.found_count += 1
msg = cur_sub_domain.ljust(30) + ips
self._print_msg(msg, _found_msg=True)
self._print_msg()
# TODO: close write file
self.outfile.write(
cur_sub_domain.ljust(30) + '\t' + ips + '\n')
self.outfile.flush()
self.result_lines.append(
cur_sub_domain.ljust(30) + '\t' + ips)
self.result_domains.append(cur_sub_domain)
self.result_ips.extend(ips.split(", "))
try:
self.resolvers[j].query('lijiejietest.' +
cur_sub_domain)
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e:
self.queue.put((999999999, '{next_sub}.' + sub))
except:
pass
except (dns.resolver.NXDOMAIN, dns.name.EmptyLabel) as e:
pass
except (dns.resolver.NoNameservers, dns.resolver.NoAnswer,
dns.exception.Timeout) as e:
pass
class JobSpider:
def __init__(self, start_requests):
self.start_request = start_requests
self.domain = tldextract.extract(self.start_request.url).domain
self.request_queue = PriorityQueue()
self.result = {
start_requests.url: 0,
}
self.gl_list = []
self.stop_flag = False
def start(self, number):
resp = requests.get(self.start_request.url)
if resp.status_code != 200:
raise Exception('HTTPError<%d>' % resp.status_code)
self.request_queue.put((0, self.start_request))
for i in range(number):
gl = gevent.spawn(self.downloader)
self.gl_list.append(gl)
gl.start()
def stop(self):
self.stop_flag = True
def join(self):
return gevent.joinall(self.gl_list)
def downloader(self):
a_re = re.compile(
r'''<a.+?href=(['"])([^>\s]+)\1.*?>([\S\s]+?)<\/a>''',
re.IGNORECASE)
while not self.request_queue.empty():
if self.stop_flag: break
prio, request = self.request_queue.get()
headers = {'User-Agent': choice(random_ua)}
try:
resp = requests.get(request.url, headers=headers)
except Exception as e:
continue
encoding = chardet.detect(resp.content)['encoding']
html_text = resp.content.decode(
encoding) if encoding is not None else resp.text
self.result[request.url] += calc_text_weight(html_text)
if self.result[request.url] >= 100:
self.stop()
break
if request.depth == max_depth:
continue
matches = a_re.findall(html_text)
for each_a in matches:
href = each_a[1]
name = each_a[2]
if href.startswith('javascript'): continue
if href.startswith('/'): href = request.url + href
if href.startswith('http'):
new_request = Request(href, request.depth + 1)
self.result[href] = calc_name_url_weight(name, href)
if tldextract.extract(href).domain == self.domain:
self.request_queue.put(
(-self.result[href], new_request))
elif self.result[href] >= 80:
self.request_queue.put(
(-self.result[href], new_request))
class SubNameBrute:
"""
receive commandline args and do some initialization work
"""
def __init__(self, target, options):
self.start_time = time.time()
self.target = target.strip()
self.options = options
self.scan_count = self.found_count = 0
self.console_width = os.get_terminal_size()[0] - 2
# create dns resolver pool ~ workers
self.resolvers = [
dns.resolver.Resolver(configure=False)
for _ in range(options.threads)
]
for resolver in self.resolvers:
resolver.lifetime = resolver.timeout = 10.0
self.print_count = 0
self.STOP_ME = False
# load dns servers and check whether these dns servers works fine ?
self._load_dns_servers()
# load sub names
self.subs = [] # subs in file
self.goodsubs = [] # checks ok for further exploitation
self._load_subname('dict/subnames.txt', self.subs)
# load sub.sub names
self.subsubs = []
self._load_subname('dict/next_sub.txt', self.subsubs)
# results will save to target.txt
global path
path = os.path.join("results", target)
if not os.path.exists(path):
os.makedirs(path)
self.outfile = open('%s/%s.txt' % (path, target), 'w')
self.ip_dict = set() #
self.found_sub = set()
# task queue
self.queue = PriorityQueue()
for sub in self.subs:
self.queue.put(sub)
"""
Load DNS Servers(ip saved in file), and check whether the DNS servers works fine
"""
def _load_dns_servers(self):
print('[*] Validate DNS servers ...')
self.dns_servers = []
# create a process pool for checking DNS servers, the number is your processors(cores) * 2, just change it!
processors = cpu_count() * 2
pool = Pool(processors)
# read dns ips and check one by one
for server in open('dict/dns_servers.txt').readlines():
server = server.strip()
if server:
pool.apply_async(self._test_server, (server, ))
pool.join() # waiting for process finish
self.dns_count = len(self.dns_servers)
sys.stdout.write('\n')
dns_info = '[+] Found {} available DNS Servers in total'.format(
self.dns_count)
print(dns_info)
if self.dns_count == 0:
print('[ERROR] No DNS Servers available.')
sys.exit(-1)
"""
test these dns servers whether works fine
"""
def _test_server(self, server):
# create a dns resolver and set timeout
resolver = dns.resolver.Resolver()
resolver.lifetime = resolver.timeout = 10.0
try:
resolver.nameservers = [server]
answers = resolver.query('public-dns-a.baidu.com')
if answers[0].address != '180.76.76.76':
raise Exception('incorrect DNS response')
self.dns_servers.append(server)
except:
self._print_msg('[-] Check DNS Server %s <Fail> Found %s' %
(server.ljust(16), len(self.dns_servers)))
self._print_msg('[+] Check DNS Server %s < OK > Found %s' %
(server.ljust(16), len(self.dns_servers)))
"""
load sub names in dict/*.txt, one function would be enough
file for read, subname_list for saving sub names
"""
def _load_subname(self, file, subname_list):
self._print_msg('[*] Load sub names ...')
with open(file) as f:
for line in f:
sub = line.strip()
if sub and sub not in subname_list:
tmp_set = {sub}
"""
in case of the sub names which contains the following expression
and replace them {alphnum}, {alpha}, {num} with character and num
"""
while len(tmp_set) > 0:
item = tmp_set.pop()
if item.find('{alphnum}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789':
tmp_set.add(
item.replace('{alphnum}', _letter, 1))
elif item.find('{alpha}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz':
tmp_set.add(item.replace(
'{alpha}', _letter, 1))
elif item.find('{num}') >= 0:
for _letter in '0123456789':
tmp_set.add(item.replace('{num}', _letter, 1))
elif item not in subname_list:
subname_list.append(item)
"""
for better presentation of brute force results, not really matters ...
"""
def _print_msg(self, _msg=None, _found_msg=False):
if _msg is None:
self.print_count += 1
if self.print_count < 100:
return
self.print_count = 0
msg = '%s Found| %s Groups| %s scanned in %.1f seconds' % (
self.found_count, self.queue.qsize(), self.scan_count,
time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) +
msg)
elif _msg.startswith('[+] Check DNS Server'):
sys.stdout.write('\r' + _msg + ' ' *
(self.console_width - len(_msg)))
else:
sys.stdout.write('\r' + _msg + ' ' *
(self.console_width - len(_msg)) + '\n')
if _found_msg:
msg = '%s Found| %s Groups| %s scanned in %.1f seconds' % (
self.found_count, self.queue.qsize(), self.scan_count,
time.time() - self.start_time)
sys.stdout.write('\r' + ' ' * (self.console_width - len(msg)) +
msg)
sys.stdout.flush()
def _print_domain(self, msg):
console_width = os.get_terminal_size()[0]
msg = '\r' + msg + ' ' * (console_width - len(msg))
# msg = '\033[0;31;47m%s{}\033[0m'.format(msg)
sys.stdout.write(msg)
def _print_progress(self):
"""
显示扫描进度,显示更美观
"""
msg = '\033[0;31;47m%s\033[0m found | %s remaining | %s scanned in %.2f seconds' % \
(self.found_count, self.queue.qsize(), self.scan_count, time.time()- self.start_time)
console_width = os.get_terminal_size()[0]
out = '\r' + ' ' * int((console_width - len(msg)) / 2) + msg
sys.stdout.write(out)
"""
important : assign task to resolvers
"""
def _scan(self, j):
self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]]
while not self.queue.empty():
sub = self.queue.get(timeout=1.0)
self.scan_count += 1
try:
cur_sub_domain = sub + '.' + self.target
answers = self.resolvers[j].query(cur_sub_domain)
except:
continue
if answers:
ips = ', '.join(sorted([answer.address for answer in answers]))
# exclude : intranet or kept addresses
if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0', '0.0.0.1']:
continue
if SubNameBrute.is_intranet(answers[0].address):
continue
self.found_sub.add(cur_sub_domain)
for answer in answers:
self.ip_dict.add(answer.address)
if sub not in self.goodsubs:
self.goodsubs.append(sub)
self.found_count += 1
ip_info = '{} \t {}'.format(cur_sub_domain, ips)
# print(ip_info)
self.outfile.write(cur_sub_domain + '\t' + ips + '\n')
self._print_domain(ip_info)
sys.stdout.flush()
self._print_progress()
sys.stdout.flush()
@staticmethod
def is_intranet(ip):
ret = ip.split('.')
if len(ret) != 4:
return True
if ret[0] == '10':
return True
if ret[0] == '172' and 16 <= int(ret[1]) <= 32:
return True
if ret[0] == '192' and ret[1] == '168':
return True
return False
"""
assign task to threads ...
"""
def run(self):
threads = [
gevent.spawn(self._scan, i) for i in range(self.options.threads)
]
print('[*] Initializing %d threads' % self.options.threads)
try:
gevent.joinall(threads)
except KeyboardInterrupt as e:
msg = '[WARNING] User aborted.'
sys.stdout.write('\r' + msg + ' ' *
(self.console_width - len(msg)) + '\n\r')
sys.stdout.flush()
class TaskList:
"""
Task list if a FIFO queue of tasks
"""
def __init__(self, service):
self.service = service
self._queue = PriorityQueue()
# done keeps the tasks that have been extracted from the queue
# so we can inspect them later
# keep the done task on disk, not in memory.
# now we use the filesystem, but we could plug any key-value stor or database behind
# check TaskStorageBase to see the interface your storage needs to have
# to be used to store tasks
# self._done = TaskStorageFile(self)
self._done = TaskStorageSqlite(self)
# pointer to current task
self._current = None
self._current_mu = Semaphore()
@property
def current(self):
with self._current_mu:
return self._current
@current.setter
def current(self, value):
with self._current_mu:
self._current = value
def __del__(self):
if self._done:
self._done.close()
def get(self):
"""
pop out a task from the task list
this call is blocking when the task list is empty
"""
_, task = self._queue.get()
self.current = task
nr_task_waiting.labels(service_guid=self.service.guid).dec()
return task
def put(self, task, priority=PRIORITY_NORMAL):
"""
append task to the task list
"""
if not isinstance(task, Task):
raise ValueError(
"task should be an instance of the Task class not %s" %
type(task))
task._priority = priority
nr_task_waiting.labels(service_guid=self.service.guid).inc()
self._queue.put((priority, task))
def done(self, task):
"""
notify that a task is done
"""
if task._priority != PRIORITY_SYSTEM:
self.current = None
self._done.add(task)
def empty(self):
"""
return True if the task list is empty, False otherwise
"""
return self._queue.empty()
def clear(self):
"""
clear emtpy the task list from all its tasks
"""
try:
while not self.empty():
self._queue.get_nowait()
except gevent.queue.Empty:
return
def list_tasks(self, all=False):
"""
@param all: if True, also return the task that have been executed
if False only return the task waiting in the task list
returns all the task that are currently in the task list
"""
tasks = [x[1] for x in self._queue.queue]
if all:
tasks.extend(self._done.list())
if self.current and self.current.state == TASK_STATE_RUNNING:
# also return the current running
# task as part of the task list
tasks.insert(0, self.current)
return tasks
def get_task_by_guid(self, guid):
"""
return a task from the list by it's guid
"""
# FIXME: this is really inefficient
def find_task(guid, l):
for task in l:
if task.guid == guid:
return task
raise TaskNotFoundError()
# check if it's not the current running task
if self.current and self.current.guid == guid:
return self.current
# search in waiting tasks
try:
task = find_task(guid, [x[1] for x in self._queue.queue])
return task
except TaskNotFoundError:
pass
# search in done task
# this will raise TaskNotFoundError if can't find the task
return self._done.get(guid)
def save(self, path):
"""
serialize the task list to disk
@param path: file path where to serialize the task list
"""
def serialize_task(task):
return {
"guid": task.guid,
"action_name": task.action_name,
"args": task._args,
"state": task.state,
"eco": json.loads(task.eco.json) if task.eco else None,
"created": task.created,
}
output = []
for task in self.list_tasks(all=False):
output.append(serialize_task(task))
j.data.serializer.yaml.dump(path, output)
def load(self, path):
"""
load a task list that have been serialized with save method
@param path: file path where the task list is serialized
@param service: the service object to which this task list belongs
"""
if not os.path.exists(path):
return
data = j.data.serializer.yaml.load(path)
for task in data:
if task['state'] in [TASK_STATE_NEW, TASK_STATE_RUNNING]:
self.put(_instantiate_task(task, self.service))
else:
# None supported state, just skip it
continue
class SubNameBrute:
def __init__(self, target, options, process_num, dns_servers, cdns,next_subs,
scan_count, found_count, queue_size_list, tmp_dir):
self.target = target.strip()
self.options = options
self.process_num = process_num
self.dns_servers = dns_servers
self.cdns = cdns
self.dns_count = len(dns_servers)
self.next_subs = next_subs
self.scan_count = scan_count
self.scan_count_local = 0
self.found_count = found_count
self.found_count_local = 0
self.queue_size_list = queue_size_list
self.resolvers = [dns.resolver.Resolver(configure=False) for _ in range(options.threads)]
for _r in self.resolvers:
_r.lifetime = _r.timeout = 6.0
self.queue = PriorityQueue()
self.item_index = 0
self.priority = 0
self._load_sub_names()
self.ip_dict = {}
self.found_subs = set()
self.ex_resolver = dns.resolver.Resolver(configure=False)
self.ex_resolver.nameservers = dns_servers
self.local_time = time.time()
self.outfile = open('%s/%s_part_%s.txt' % (tmp_dir, target, process_num), 'w')
def _load_sub_names(self):
if self.options.full_scan and self.options.file == 'subnames.txt':
_file = 'dict/subnames_full.txt'
else:
if os.path.exists(self.options.file):
_file = self.options.file
elif os.path.exists('dict/%s' % self.options.file):
_file = 'dict/%s' % self.options.file
else:
print_msg('[ERROR] Names file not found: %s' % self.options.file)
exit(-1)
normal_lines = []
wildcard_lines = []
wildcard_list = []
regex_list = []
lines = set()
with open(_file) as f:
for line in f.xreadlines():
sub = line.strip()
if not sub or sub in lines:
continue
lines.add(sub)
if sub.find('{alphnum}') >= 0 or sub.find('{alpha}') >= 0 or sub.find('{num}') >= 0:
wildcard_lines.append(sub)
sub = sub.replace('{alphnum}', '[a-z0-9]')
sub = sub.replace('{alpha}', '[a-z]')
sub = sub.replace('{num}', '[0-9]')
if sub not in wildcard_list:
wildcard_list.append(sub)
regex_list.append('^' + sub + '$')
else:
normal_lines.append(sub)
if regex_list:
pattern = '|'.join(regex_list)
_regex = re.compile(pattern)
for line in normal_lines[:]:
if _regex.search(line):
normal_lines.remove(line)
for item in normal_lines[self.process_num::self.options.process]:
self.priority += 1
self.queue.put((self.priority, item))
for item in wildcard_lines[self.process_num::self.options.process]:
self.queue.put((88888888, item))
def put_item(self, item):
num = item.count('{alphnum}') + item.count('{alpha}') + item.count('{num}')
if num == 0:
self.priority += 1
self.queue.put((self.priority, item))
else:
self.queue.put((self.priority + num * 10000000, item))
def check_cdn(self, cname):
'''
bTrue = True
bFound = False
i = 0
while bTrue:
cdn = self.cdns[i]
i += 1
if (cdn in cname) or (i == len(self.cdns)):
if (cdn in cname): bFound = True
bTrue = False
return bFound
'''
for cdn in self.cdns:
if cdn in cname:
return True
return False
def _scan(self, j):
self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]]
while not self.queue.empty():
try:
item = self.queue.get(timeout=3.0)[1]
self.scan_count_local += 1
if time.time() - self.local_time > 3.0:
self.scan_count.value += self.scan_count_local
self.scan_count_local = 0
self.queue_size_list[self.process_num] = self.queue.qsize()
except Exception as e:
break
try:
if item.find('{alphnum}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789':
self.put_item(item.replace('{alphnum}', _letter, 1))
continue
elif item.find('{alpha}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz':
self.put_item(item.replace('{alpha}', _letter, 1))
continue
elif item.find('{num}') >= 0:
for _letter in '0123456789':
self.put_item(item.replace('{num}', _letter, 1))
continue
elif item.find('{next_sub}') >= 0:
for _ in self.next_subs:
self.queue.put((0, item.replace('{next_sub}', _, 1)))
continue
else:
sub = item
if sub in self.found_subs:
continue
cur_sub_domain = sub + '.' + self.target
_sub = sub.split('.')[-1]
try:
answers = self.resolvers[j].query(cur_sub_domain)
except dns.resolver.NoAnswer, e:
answers = self.ex_resolver.query(cur_sub_domain)
if answers:
ans = self.resolvers[j].query(cur_sub_domain,'cname')
cname = ans[0].target.to_unicode().rstrip('.')
if self.check_cdn(cname):
continue
self.found_subs.add(sub)
ips = ', '.join(sorted([answer.address for answer in answers]))
if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0']:
continue
if self.options.i and is_intranet(answers[0].address):
continue
try:
self.scan_count_local += 1
answers = self.resolvers[j].query(cur_sub_domain, 'cname')
cname = answers[0].target.to_unicode().rstrip('.')
if cname.endswith(self.target) and cname not in self.found_subs:
self.found_subs.add(cname)
cname_sub = cname[:len(cname) - len(self.target) - 1] # new sub
self.queue.put((0, cname_sub))
except:
pass
if (_sub, ips) not in self.ip_dict:
self.ip_dict[(_sub, ips)] = 1
else:
self.ip_dict[(_sub, ips)] += 1
if self.ip_dict[(_sub, ips)] > 30:
continue
self.found_count_local += 1
if time.time() - self.local_time > 3.0:
self.found_count.value += self.found_count_local
self.found_count_local = 0
self.queue_size_list[self.process_num] = self.queue.qsize()
self.local_time = time.time()
msg = cur_sub_domain.ljust(30) + ips
# print_msg(msg, line_feed=True)
self.outfile.write(cur_sub_domain.ljust(30) + '\t' + ips + '\n')
self.outfile.flush()
try:
self.resolvers[j].query('lijiejietest.' + cur_sub_domain)
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e:
self.queue.put((999999999, '{next_sub}.' + sub))
except:
pass
except (dns.resolver.NXDOMAIN, dns.name.EmptyLabel) as e:
pass
except (dns.resolver.NoNameservers, dns.resolver.NoAnswer, dns.exception.Timeout) as e:
pass
class SubNameBrute:
def __init__(self, target, options, process_num, dns_servers, next_subs,
scan_count, found_count, queue_size_list, tmp_dir):
self.target = target.strip()
self.options = options
self.process_num = process_num
self.dns_servers = dns_servers
self.dns_count = len(dns_servers)
self.next_subs = next_subs
self.scan_count = scan_count
self.scan_count_local = 0
self.found_count = found_count
self.found_count_local = 0
self.queue_size_list = queue_size_list
self.resolvers = [dns.resolver.Resolver(configure=False) for _ in range(options.threads)]
for _r in self.resolvers:
_r.lifetime = _r.timeout = 6.0
self.queue = PriorityQueue()
self.item_index = 0
self.priority = 0
self._load_sub_names()
self.ip_dict = {}
self.found_subs = set()
self.ex_resolver = dns.resolver.Resolver(configure=False)
self.ex_resolver.nameservers = dns_servers
self.local_time = time.time()
self.outfile = open('%s/%s_part_%s.txt' % (tmp_dir, target, process_num), 'w')
def _load_sub_names(self):
if self.options.full_scan and self.options.file == 'subnames.txt':
_file = 'dict/subnames_full.txt'
else:
if os.path.exists(self.options.file):
_file = self.options.file
elif os.path.exists('dict/%s' % self.options.file):
_file = 'dict/%s' % self.options.file
else:
print_msg('[ERROR] Names file not found: %s' % self.options.file)
exit(-1)
normal_lines = []
wildcard_lines = []
wildcard_list = []
regex_list = []
lines = set()
with open(_file) as f:
for line in f.xreadlines():
sub = line.strip()
if not sub or sub in lines:
continue
lines.add(sub)
if sub.find('{alphnum}') >= 0 or sub.find('{alpha}') >= 0 or sub.find('{num}') >= 0:
wildcard_lines.append(sub)
sub = sub.replace('{alphnum}', '[a-z0-9]')
sub = sub.replace('{alpha}', '[a-z]')
sub = sub.replace('{num}', '[0-9]')
if sub not in wildcard_list:
wildcard_list.append(sub)
regex_list.append('^' + sub + '$')
else:
normal_lines.append(sub)
if regex_list:
pattern = '|'.join(regex_list)
_regex = re.compile(pattern)
for line in normal_lines[:]:
if _regex.search(line):
normal_lines.remove(line)
for item in normal_lines[self.process_num::self.options.process]:
self.priority += 1
self.queue.put((self.priority, item))
for item in wildcard_lines[self.process_num::self.options.process]:
self.queue.put((88888888, item))
def put_item(self, item):
num = item.count('{alphnum}') + item.count('{alpha}') + item.count('{num}')
if num == 0:
self.priority += 1
self.queue.put((self.priority, item))
else:
self.queue.put((self.priority + num * 10000000, item))
def _scan(self, j):
self.resolvers[j].nameservers = [self.dns_servers[j % self.dns_count]]
while not self.queue.empty():
try:
item = self.queue.get(timeout=3.0)[1]
self.scan_count_local += 1
if time.time() - self.local_time > 3.0:
self.scan_count.value += self.scan_count_local
self.scan_count_local = 0
self.queue_size_list[self.process_num] = self.queue.qsize()
except Exception as e:
break
try:
if item.find('{alphnum}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz0123456789':
self.put_item(item.replace('{alphnum}', _letter, 1))
continue
elif item.find('{alpha}') >= 0:
for _letter in 'abcdefghijklmnopqrstuvwxyz':
self.put_item(item.replace('{alpha}', _letter, 1))
continue
elif item.find('{num}') >= 0:
for _letter in '0123456789':
self.put_item(item.replace('{num}', _letter, 1))
continue
elif item.find('{next_sub}') >= 0:
for _ in self.next_subs:
self.queue.put((0, item.replace('{next_sub}', _, 1)))
continue
else:
sub = item
if sub in self.found_subs:
continue
cur_sub_domain = sub + '.' + self.target
_sub = sub.split('.')[-1]
try:
answers = self.resolvers[j].query(cur_sub_domain)
except dns.resolver.NoAnswer, e:
answers = self.ex_resolver.query(cur_sub_domain)
if answers:
self.found_subs.add(sub)
ips = ', '.join(sorted([answer.address for answer in answers]))
if ips in ['1.1.1.1', '127.0.0.1', '0.0.0.0']:
continue
if self.options.i and is_intranet(answers[0].address):
continue
try:
self.scan_count_local += 1
answers = self.resolvers[j].query(cur_sub_domain, 'cname')
cname = answers[0].target.to_unicode().rstrip('.')
if cname.endswith(self.target) and cname not in self.found_subs:
self.found_subs.add(cname)
cname_sub = cname[:len(cname) - len(self.target) - 1] # new sub
self.queue.put((0, cname_sub))
except:
pass
if (_sub, ips) not in self.ip_dict:
self.ip_dict[(_sub, ips)] = 1
else:
self.ip_dict[(_sub, ips)] += 1
if self.ip_dict[(_sub, ips)] > 30:
continue
self.found_count_local += 1
if time.time() - self.local_time > 3.0:
self.found_count.value += self.found_count_local
self.found_count_local = 0
self.queue_size_list[self.process_num] = self.queue.qsize()
self.local_time = time.time()
msg = cur_sub_domain.ljust(30) + ips
# print_msg(msg, line_feed=True)
self.outfile.write(cur_sub_domain.ljust(30) + '\t' + ips + '\n')
self.outfile.flush()
try:
self.resolvers[j].query('lijiejietest.' + cur_sub_domain)
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer) as e:
self.queue.put((999999999, '{next_sub}.' + sub))
except:
pass
except (dns.resolver.NXDOMAIN, dns.name.EmptyLabel) as e:
pass
except (dns.resolver.NoNameservers, dns.resolver.NoAnswer, dns.exception.Timeout) as e:
pass
