def __check_targets(self):
self.wp = WorkerPool()
title_regex = re.compile("<title>(.*?)<\/title>", re.DOTALL | re.M)
for key in ['root_domain', 'ip', 'domain']:
for item in result[key]:
self.wp.add_job(
self.__load_targets,
result[key][item],
title_regex
)
self.wp.run()
class subdomain_brute(Plugin):
def __init__(self):
super(subdomain_brute, self).__init__('subdomain_brute')
def start(self, domain, domain_type, level):
super(subdomain_brute, self).start(domain, domain_type, level)
subdomain = self._random_subdomain(domain)
if self._check_wildcard(subdomain):
self.result = {
'root_domain': [],
'ip': [],
'domain': [subdomain]
}
else:
result = []
self.wp = WorkerPool()
for pre in PREFIXS:
subdomain = pre + '.' + domain
self.wp.add_job(self._check_is_subdomain, subdomain, result)
self.wp.run()
self.result = {
'root_domain': [],
'ip': [],
'domain': result
}
super(subdomain_brute, self).end()
return self.result
def _check_wildcard(self, subdomain):
try:
socket.gethostbyname(subdomain)
return True
except:
return False
def _random_subdomain(self, domain):
chars = string.letters + string.digits
length = randint(5, 15)
random_str = ''.join([choice(chars) for _ in range(length)])
subdomain = random_str + '.' + domain
return subdomain
def _check_is_subdomain(self, subdomain, result):
try:
socket.gethostbyname(subdomain)
except:
pass
else:
result.append(subdomain)
class AliveCheck(object):
def __init__(self):
self.exit_flag = False
self.req = api.request
self.count_max = 0
def __check_targets(self):
self.wp = WorkerPool()
title_regex = re.compile("<title>(.*?)<\/title>", re.DOTALL | re.M)
for key in ['root_domain', 'ip', 'domain']:
for item in result[key]:
self.count_max += 1
self.wp.add_job(
self.__load_targets,
result[key][item],
title_regex
)
self.bar = Bar('AliveCheck', max=self.count_max)
self.wp.run()
self.bar.finish()
def __load_targets(self, target, title_regex):
try:
req = self.req.request(
'GET', 'http://' + target['domain'], timeout=5)
except:
pass
else:
target['status_code'] = req.status_code
if req.status_code == 200:
content = req.content
title_match = title_regex.search(content)
target['title'] = title_match.group(1) if title_match else 'failed'
finally:
self.bar.next()
def __init_targets(self):
for key in ['root_domain', 'ip', 'domain']:
for item in result[key]:
result[key][item]['status_code'] = 'unkonwn'
result[key][item]['title'] = 'unkonwn'
def exit(self):
self.exit_flag = True
def start(self):
self.__init_targets()
self.__check_targets()
def start(self, domain, domain_type, level):
super(subdomain_brute, self).start(domain, domain_type, level)
subdomain = self._random_subdomain(domain)
if self._check_wildcard(subdomain):
self.result = {'root_domain': [], 'ip': [], 'domain': [subdomain]}
else:
result = []
self.wp = WorkerPool()
for pre in PREFIXS:
subdomain = pre + '.' + domain
self.wp.add_job(self._check_is_subdomain, subdomain, result)
self.wp.run()
self.result = {'root_domain': [], 'ip': [], 'domain': result}
super(subdomain_brute, self).end()
return self.result
class AliveCheck(object):
def __init__(self):
self.exit_flag = False
self.req = api.request
def __check_targets(self):
self.wp = WorkerPool()
title_regex = re.compile("<title>(.*?)<\/title>", re.DOTALL | re.M)
for key in ['root_domain', 'ip', 'domain']:
for item in result[key]:
self.wp.add_job(
self.__load_targets,
result[key][item],
title_regex
)
self.wp.run()
def __load_targets(self, target, title_regex):
try:
req = self.req.request(
'GET', 'http://' + target['domain'], timeout=5)
except:
pass
else:
target['status_code'] = req.status_code
if req.status_code == 200:
content = req.content
title_match = title_regex.search(content)
target['title'] = title_match.group(1) if title_match else 'failed'
def __init_targets(self):
for key in ['root_domain', 'ip', 'domain']:
for item in result[key]:
result[key][item]['status_code'] = 'unkonwn'
result[key][item]['title'] = 'unkonwn'
def exit(self):
self.exit_flag = True
def start(self):
self.__init_targets()
self.__check_targets()
class subdomain_brute(Plugin):
def __init__(self):
super(subdomain_brute, self).__init__('subdomain_brute')
def start(self, domain, domain_type, level):
super(subdomain_brute, self).start(domain, domain_type, level)
subdomain = self._random_subdomain(domain)
if self._check_wildcard(subdomain):
self.result = {'root_domain': [], 'ip': [], 'domain': [subdomain]}
else:
result = []
self.wp = WorkerPool()
for pre in PREFIXS:
subdomain = pre + '.' + domain
self.wp.add_job(self._check_is_subdomain, subdomain, result)
self.wp.run()
self.result = {'root_domain': [], 'ip': [], 'domain': result}
super(subdomain_brute, self).end()
return self.result
def _check_wildcard(self, subdomain):
try:
socket.gethostbyname(subdomain)
return True
except:
return False
def _random_subdomain(self, domain):
chars = string.letters + string.digits
length = randint(5, 15)
random_str = ''.join([choice(chars) for _ in range(length)])
subdomain = random_str + '.' + domain
return subdomain
def _check_is_subdomain(self, subdomain, result):
try:
socket.gethostbyname(subdomain)
except:
pass
else:
result.append(subdomain)
def __check_targets(self):
self.wp = WorkerPool()
title_regex = re.compile("<title>(.*?)<\/title>", re.DOTALL | re.M)
for key in ['root_domain', 'ip', 'domain']:
for item in result[key]:
self.count_max += 1
self.wp.add_job(
self.__load_targets,
result[key][item],
title_regex
)
self.bar = Bar('AliveCheck', max=self.count_max)
self.wp.run()
self.bar.finish()
def start(self, domain, domain_type, level):
super(subdomain_brute, self).start(domain, domain_type, level)
subdomain = self._random_subdomain(domain)
if self._check_wildcard(subdomain):
self.result = {
'root_domain': [],
'ip': [],
'domain': [subdomain]
}
else:
result = []
self.wp = WorkerPool()
for pre in PREFIXS:
subdomain = pre + '.' + domain
self.wp.add_job(self._check_is_subdomain, subdomain, result)
self.wp.run()
self.result = {
'root_domain': [],
'ip': [],
'domain': result
}
super(subdomain_brute, self).end()
return self.result
def __init__(self):
self.exit_flag = False
self.wp = WorkerPool()
self.plugin_path = conf.settings.PLUGINS_PATH
class PluginController(object):
def __init__(self):
self.exit_flag = False
self.wp = WorkerPool()
self.plugin_path = conf.settings.PLUGINS_PATH
def plugin_init(self):
"""
初始化插件
"""
plugin_list = os.listdir(self.plugin_path)
for plugin in plugin_list:
plugin_config_path = os.path.join(self.plugin_path, plugin,
'config.yaml')
if os.path.exists(plugin_config_path):
with open(plugin_config_path) as f:
try:
plugin_config = yaml.load(f)
except Exception:
logger.exception('load %s\'s config fail!' % plugin)
else:
if plugin_config['enable']:
conf.plugins[plugin] = plugin_config
self.__register_plugin(plugin)
def __register_plugin(self, plugin):
"""
注册插件
"""
kb.plugins[plugin] = {}
kb.plugins[plugin]['name'] = plugin
try:
_import_path = '.'.join(
conf.settings.PLUGINS_OPPOSITE_PATH.split(os.path.sep))
plugin_path = '%s.%s.work' % (_import_path, plugin)
_plugin = __import__(plugin_path, fromlist='*') # 动态加载函数
kb.plugins[plugin]['handle'] = _plugin
except Exception:
logger.exception('register plugin %s failed!' % plugin)
else:
self.__classify_plugin(plugin)
def __classify_plugin(self, plugin):
"""
归类插件
"""
inputs = conf.plugins[plugin]['input']
for inp in inputs:
if inp in conf.settings.ALLOW_INPUTS:
conf.reg_plugins[inp].add(plugin)
def start(self):
while not self.exit_flag:
try:
target = self.wp.target_queue.get(timeout=1)
except gevent.queue.Empty:
pass
else:
self.__add_job_by_type(target)
def __add_job_by_type(self, target):
domain_type = target.get('domain_type')
parent_module = target.pop('parent_module')
onerepeat = conf.plugins.get(parent_module, {}).get('onerepeat')
domain = target.get('domain')
if domain_type in conf.settings.ALLOW_INPUTS:
for plugin in conf.reg_plugins[domain_type]:
if onerepeat and parent_module == plugin:
continue
_plugin = getattr(kb.plugins[plugin]['handle'], plugin)()
self.wp.add_job(getattr(_plugin, 'start'), **target)
self.__init_plugin_progress(plugin, domain)
def __init_plugin_progress(self, plugin, domain):
key = '%s_%s' % (plugin, domain)
kb.progress[key] = {'status': 'wait', 'start_time': 0, 'end_time': 0}
def run_job(self):
self.wp.run()
def exit(self):
self.exit_flag = True
def __init__(self):
self.exit_flag = False
self.wp = WorkerPool()
self.plugin_path = conf.settings.PLUGINS_PATH
class PluginController(object):
def __init__(self):
self.exit_flag = False
self.wp = WorkerPool()
self.plugin_path = conf.settings.PLUGINS_PATH
def plugin_init(self):
"""
初始化插件
"""
plugin_list = os.listdir(self.plugin_path)
for plugin in plugin_list:
plugin_config_path = os.path.join(
self.plugin_path, plugin, 'config.yaml'
)
if os.path.exists(plugin_config_path):
with open(plugin_config_path) as f:
try:
plugin_config = yaml.load(f)
except Exception:
logger.exception('load %s\'s config fail!' % plugin)
else:
if plugin_config['enable']:
conf.plugins[plugin] = plugin_config
self.__register_plugin(plugin)
def __register_plugin(self, plugin):
"""
注册插件
"""
kb.plugins[plugin] = {}
kb.plugins[plugin]['name'] = plugin
try:
_import_path = '.'.join(
conf.settings.PLUGINS_OPPOSITE_PATH.split(os.path.sep)
)
plugin_path = '%s.%s.work' % (_import_path, plugin)
_plugin = __import__(plugin_path, fromlist='*') # 动态加载函数
kb.plugins[plugin]['handle'] = _plugin
except Exception:
logger.exception('register plugin %s failed!' % plugin)
else:
self.__classify_plugin(plugin)
def __classify_plugin(self, plugin):
"""
归类插件
"""
inputs = conf.plugins[plugin]['input']
for inp in inputs:
if inp in conf.settings.ALLOW_INPUTS:
conf.reg_plugins[inp].add(plugin)
def start(self):
while not self.exit_flag:
try:
target = self.wp.target_queue.get(timeout=1)
except gevent.queue.Empty:
pass
else:
self.__add_job_by_type(target)
def __add_job_by_type(self, target):
domain_type = target.get('domain_type')
parent_module = target.pop('parent_module')
onerepeat = conf.plugins.get(parent_module, {}).get('onerepeat')
domain = target.get('domain')
if domain_type in conf.settings.ALLOW_INPUTS:
for plugin in conf.reg_plugins[domain_type]:
if onerepeat and parent_module == plugin:
continue
_plugin = getattr(kb.plugins[plugin]['handle'], plugin)()
self.wp.add_job(getattr(_plugin, 'start'), **target)
self.__init_plugin_progress(plugin, domain)
def __init_plugin_progress(self, plugin, domain):
key = '%s_%s' % (plugin, domain)
kb.progress[key] = {'status':'wait', 'start_time':0, 'end_time':0}
def run_job(self):
self.wp.run()
def exit(self):
self.exit_flag = True
class PluginController(object):
def __init__(self):
self.exit_flag = False
self.wp = WorkerPool()
self.plugin_path = paths.PLUGINS_PATH
@classmethod
def get_available_plugins(cls):
"""
返回plugins目录下所有enable为true的plugin名称
"""
plugin_list = os.listdir(paths.PLUGINS_PATH)
for plugin in plugin_list:
plugin_config_path = os.path.join(paths.PLUGINS_PATH, plugin,
'config.yaml')
if os.path.exists(plugin_config_path):
with open(plugin_config_path) as f:
try:
plugin_config = yaml.load(f)
except Exception:
api.logger.exception('load %s\'s config fail!' %
plugin)
else:
if plugin_config['enable']:
# 在conf中载入plugin的配置信息
conf.plugins_available[plugin] = plugin_config
return conf.plugins_available
def plugin_init(self, plugins_sepcific=None):
"""
初始化插件
"""
plugins_available = conf.plugins_available.keys()
if plugins_sepcific:
for plugin in plugins_sepcific:
if plugin in plugins_available:
self.__load_plugin(plugin)
else:
api.logger.exception('plugin: %s NOT found!' % plugin)
else:
for plugin in plugins_available:
self.__load_plugin(plugin)
def __load_plugin(self, plugin):
"""
载入名为plugin的插件. plugin的存在性和config的enable合法性由调用者保证
"""
# 指向在conf.plugins_availavle中的信息, 不再重新读入配置文件
conf.plugins_load[plugin] = conf.plugins_available[plugin]
self.__register_plugin(plugin)
def __register_plugin(self, plugin):
"""
注册插件
"""
kb.plugins[plugin] = {}
kb.plugins[plugin]['name'] = plugin
try:
_import_path = '.'.join(
paths.PLUGINS_OPPOSITE_PATH.split(os.path.sep))
plugin_path = '%s.%s.work' % (_import_path, plugin)
_plugin = __import__(plugin_path, fromlist='*') # 动态加载函数
kb.plugins[plugin]['handle'] = _plugin
except Exception:
api.logger.exception('register plugin %s failed!' % plugin)
else:
self.__classify_plugin(plugin)
def __classify_plugin(self, plugin):
"""
归类插件
"""
inputs = conf.plugins_load[plugin]['input']
for inp in inputs:
if inp in settings.ALLOW_INPUTS:
conf.reg_plugins[inp].add(plugin)
def start(self):
while not self.exit_flag:
try:
target = self.wp.target_queue.get(timeout=1)
except gevent.queue.Empty:
pass
else:
self.__add_job_by_type(target)
def __add_job_by_type(self, target):
domain_type = target.get('domain_type')
parent_module = target.pop('parent_module')
onerepeat = conf.plugins_load.get(parent_module, {}).get('onerepeat')
domain = target.get('domain')
if domain_type in settings.ALLOW_INPUTS:
for plugin in conf.reg_plugins[domain_type]:
if onerepeat and parent_module == plugin:
continue
_plugin = getattr(kb.plugins[plugin]['handle'], plugin)()
self.wp.add_job(getattr(_plugin, 'start'), **target)
self.__init_plugin_progress(plugin, domain)
def __init_plugin_progress(self, plugin, domain):
key = '%s_%s' % (plugin, domain)
kb.progress[key] = {'status': 'wait', 'start_time': 0, 'end_time': 0}
def run_job(self):
self.wp.run()
def exit(self):
self.exit_flag = True
class PluginController(object):
def __init__(self):
self.exit_flag = False
self.wp = WorkerPool()
self.plugin_path = paths.PLUGINS_PATH
@classmethod
def get_available_plugins(cls):
"""
返回plugins目录下所有enable为true的plugin名称
"""
plugin_list = os.listdir(paths.PLUGINS_PATH)
for plugin in plugin_list:
plugin_config_path = os.path.join(
paths.PLUGINS_PATH, plugin, 'config.yaml'
)
if os.path.exists(plugin_config_path):
with open(plugin_config_path) as f:
try:
plugin_config = yaml.load(f)
except Exception:
api.logger.exception('load %s\'s config fail!' % plugin)
else:
if plugin_config['enable']:
# 在conf中载入plugin的配置信息
conf.plugins_available[plugin] = plugin_config
return conf.plugins_available
def plugin_init(self, plugins_sepcific=None):
"""
初始化插件
"""
plugins_available = conf.plugins_available.keys()
if plugins_sepcific:
for plugin in plugins_sepcific:
if plugin in plugins_available:
self.__load_plugin(plugin)
else:
api.logger.exception('plugin: %s NOT found!' % plugin)
else:
for plugin in plugins_available:
self.__load_plugin(plugin)
def __load_plugin(self, plugin):
"""
载入名为plugin的插件. plugin的存在性和config的enable合法性由调用者保证
"""
# 指向在conf.plugins_availavle中的信息, 不再重新读入配置文件
conf.plugins_load[plugin] = conf.plugins_available[plugin]
self.__register_plugin(plugin)
def __register_plugin(self, plugin):
"""
注册插件
"""
kb.plugins[plugin] = {}
kb.plugins[plugin]['name'] = plugin
try:
_import_path = '.'.join(
paths.PLUGINS_OPPOSITE_PATH.split(os.path.sep)
)
plugin_path = '%s.%s.work' % (_import_path, plugin)
_plugin = __import__(plugin_path, fromlist='*') # 动态加载函数
kb.plugins[plugin]['handle'] = _plugin
except Exception:
api.logger.exception('register plugin %s failed!' % plugin)
else:
self.__classify_plugin(plugin)
def __classify_plugin(self, plugin):
"""
归类插件
"""
inputs = conf.plugins_load[plugin]['input']
for inp in inputs:
if inp in settings.ALLOW_INPUTS:
conf.reg_plugins[inp].add(plugin)
def start(self):
while not self.exit_flag:
try:
target = self.wp.target_queue.get(timeout=1)
except gevent.queue.Empty:
pass
else:
self.__add_job_by_type(target)
def __add_job_by_type(self, target):
domain_type = target.get('domain_type')
parent_module = target.pop('parent_module')
onerepeat = conf.plugins_load.get(parent_module, {}).get('onerepeat')
domain = target.get('domain')
if domain_type in settings.ALLOW_INPUTS:
for plugin in conf.reg_plugins[domain_type]:
if onerepeat and parent_module == plugin:
continue
_plugin = getattr(kb.plugins[plugin]['handle'], plugin)()
self.wp.add_job(getattr(_plugin, 'start'), **target)
self.__init_plugin_progress(plugin, domain)
def __init_plugin_progress(self, plugin, domain):
key = '%s_%s' % (plugin, domain)
kb.progress[key] = {'status': 'wait', 'start_time': 0, 'end_time': 0}
def run_job(self):
self.wp.run()
def exit(self):
self.exit_flag = True
