def __check_targets(self): self.wp = WorkerPool() title_regex = re.compile("<title>(.*?)<\/title>", re.DOTALL | re.M) for key in ['root_domain', 'ip', 'domain']: for item in result[key]: self.wp.add_job( self.__load_targets, result[key][item], title_regex ) self.wp.run()
class subdomain_brute(Plugin): def __init__(self): super(subdomain_brute, self).__init__('subdomain_brute') def start(self, domain, domain_type, level): super(subdomain_brute, self).start(domain, domain_type, level) subdomain = self._random_subdomain(domain) if self._check_wildcard(subdomain): self.result = { 'root_domain': [], 'ip': [], 'domain': [subdomain] } else: result = [] self.wp = WorkerPool() for pre in PREFIXS: subdomain = pre + '.' + domain self.wp.add_job(self._check_is_subdomain, subdomain, result) self.wp.run() self.result = { 'root_domain': [], 'ip': [], 'domain': result } super(subdomain_brute, self).end() return self.result def _check_wildcard(self, subdomain): try: socket.gethostbyname(subdomain) return True except: return False def _random_subdomain(self, domain): chars = string.letters + string.digits length = randint(5, 15) random_str = ''.join([choice(chars) for _ in range(length)]) subdomain = random_str + '.' + domain return subdomain def _check_is_subdomain(self, subdomain, result): try: socket.gethostbyname(subdomain) except: pass else: result.append(subdomain)
class AliveCheck(object): def __init__(self): self.exit_flag = False self.req = api.request self.count_max = 0 def __check_targets(self): self.wp = WorkerPool() title_regex = re.compile("<title>(.*?)<\/title>", re.DOTALL | re.M) for key in ['root_domain', 'ip', 'domain']: for item in result[key]: self.count_max += 1 self.wp.add_job( self.__load_targets, result[key][item], title_regex ) self.bar = Bar('AliveCheck', max=self.count_max) self.wp.run() self.bar.finish() def __load_targets(self, target, title_regex): try: req = self.req.request( 'GET', 'http://' + target['domain'], timeout=5) except: pass else: target['status_code'] = req.status_code if req.status_code == 200: content = req.content title_match = title_regex.search(content) target['title'] = title_match.group(1) if title_match else 'failed' finally: self.bar.next() def __init_targets(self): for key in ['root_domain', 'ip', 'domain']: for item in result[key]: result[key][item]['status_code'] = 'unkonwn' result[key][item]['title'] = 'unkonwn' def exit(self): self.exit_flag = True def start(self): self.__init_targets() self.__check_targets()
def start(self, domain, domain_type, level): super(subdomain_brute, self).start(domain, domain_type, level) subdomain = self._random_subdomain(domain) if self._check_wildcard(subdomain): self.result = {'root_domain': [], 'ip': [], 'domain': [subdomain]} else: result = [] self.wp = WorkerPool() for pre in PREFIXS: subdomain = pre + '.' + domain self.wp.add_job(self._check_is_subdomain, subdomain, result) self.wp.run() self.result = {'root_domain': [], 'ip': [], 'domain': result} super(subdomain_brute, self).end() return self.result
class AliveCheck(object): def __init__(self): self.exit_flag = False self.req = api.request def __check_targets(self): self.wp = WorkerPool() title_regex = re.compile("<title>(.*?)<\/title>", re.DOTALL | re.M) for key in ['root_domain', 'ip', 'domain']: for item in result[key]: self.wp.add_job( self.__load_targets, result[key][item], title_regex ) self.wp.run() def __load_targets(self, target, title_regex): try: req = self.req.request( 'GET', 'http://' + target['domain'], timeout=5) except: pass else: target['status_code'] = req.status_code if req.status_code == 200: content = req.content title_match = title_regex.search(content) target['title'] = title_match.group(1) if title_match else 'failed' def __init_targets(self): for key in ['root_domain', 'ip', 'domain']: for item in result[key]: result[key][item]['status_code'] = 'unkonwn' result[key][item]['title'] = 'unkonwn' def exit(self): self.exit_flag = True def start(self): self.__init_targets() self.__check_targets()
class subdomain_brute(Plugin): def __init__(self): super(subdomain_brute, self).__init__('subdomain_brute') def start(self, domain, domain_type, level): super(subdomain_brute, self).start(domain, domain_type, level) subdomain = self._random_subdomain(domain) if self._check_wildcard(subdomain): self.result = {'root_domain': [], 'ip': [], 'domain': [subdomain]} else: result = [] self.wp = WorkerPool() for pre in PREFIXS: subdomain = pre + '.' + domain self.wp.add_job(self._check_is_subdomain, subdomain, result) self.wp.run() self.result = {'root_domain': [], 'ip': [], 'domain': result} super(subdomain_brute, self).end() return self.result def _check_wildcard(self, subdomain): try: socket.gethostbyname(subdomain) return True except: return False def _random_subdomain(self, domain): chars = string.letters + string.digits length = randint(5, 15) random_str = ''.join([choice(chars) for _ in range(length)]) subdomain = random_str + '.' + domain return subdomain def _check_is_subdomain(self, subdomain, result): try: socket.gethostbyname(subdomain) except: pass else: result.append(subdomain)
def __check_targets(self): self.wp = WorkerPool() title_regex = re.compile("<title>(.*?)<\/title>", re.DOTALL | re.M) for key in ['root_domain', 'ip', 'domain']: for item in result[key]: self.count_max += 1 self.wp.add_job( self.__load_targets, result[key][item], title_regex ) self.bar = Bar('AliveCheck', max=self.count_max) self.wp.run() self.bar.finish()
def start(self, domain, domain_type, level): super(subdomain_brute, self).start(domain, domain_type, level) subdomain = self._random_subdomain(domain) if self._check_wildcard(subdomain): self.result = { 'root_domain': [], 'ip': [], 'domain': [subdomain] } else: result = [] self.wp = WorkerPool() for pre in PREFIXS: subdomain = pre + '.' + domain self.wp.add_job(self._check_is_subdomain, subdomain, result) self.wp.run() self.result = { 'root_domain': [], 'ip': [], 'domain': result } super(subdomain_brute, self).end() return self.result
def __init__(self): self.exit_flag = False self.wp = WorkerPool() self.plugin_path = conf.settings.PLUGINS_PATH
class PluginController(object): def __init__(self): self.exit_flag = False self.wp = WorkerPool() self.plugin_path = conf.settings.PLUGINS_PATH def plugin_init(self): """ 初始化插件 """ plugin_list = os.listdir(self.plugin_path) for plugin in plugin_list: plugin_config_path = os.path.join(self.plugin_path, plugin, 'config.yaml') if os.path.exists(plugin_config_path): with open(plugin_config_path) as f: try: plugin_config = yaml.load(f) except Exception: logger.exception('load %s\'s config fail!' % plugin) else: if plugin_config['enable']: conf.plugins[plugin] = plugin_config self.__register_plugin(plugin) def __register_plugin(self, plugin): """ 注册插件 """ kb.plugins[plugin] = {} kb.plugins[plugin]['name'] = plugin try: _import_path = '.'.join( conf.settings.PLUGINS_OPPOSITE_PATH.split(os.path.sep)) plugin_path = '%s.%s.work' % (_import_path, plugin) _plugin = __import__(plugin_path, fromlist='*') # 动态加载函数 kb.plugins[plugin]['handle'] = _plugin except Exception: logger.exception('register plugin %s failed!' % plugin) else: self.__classify_plugin(plugin) def __classify_plugin(self, plugin): """ 归类插件 """ inputs = conf.plugins[plugin]['input'] for inp in inputs: if inp in conf.settings.ALLOW_INPUTS: conf.reg_plugins[inp].add(plugin) def start(self): while not self.exit_flag: try: target = self.wp.target_queue.get(timeout=1) except gevent.queue.Empty: pass else: self.__add_job_by_type(target) def __add_job_by_type(self, target): domain_type = target.get('domain_type') parent_module = target.pop('parent_module') onerepeat = conf.plugins.get(parent_module, {}).get('onerepeat') domain = target.get('domain') if domain_type in conf.settings.ALLOW_INPUTS: for plugin in conf.reg_plugins[domain_type]: if onerepeat and parent_module == plugin: continue _plugin = getattr(kb.plugins[plugin]['handle'], plugin)() self.wp.add_job(getattr(_plugin, 'start'), **target) self.__init_plugin_progress(plugin, domain) def __init_plugin_progress(self, plugin, domain): key = '%s_%s' % (plugin, domain) kb.progress[key] = {'status': 'wait', 'start_time': 0, 'end_time': 0} def run_job(self): self.wp.run() def exit(self): self.exit_flag = True
def __init__(self): self.exit_flag = False self.wp = WorkerPool() self.plugin_path = conf.settings.PLUGINS_PATH
class PluginController(object): def __init__(self): self.exit_flag = False self.wp = WorkerPool() self.plugin_path = conf.settings.PLUGINS_PATH def plugin_init(self): """ 初始化插件 """ plugin_list = os.listdir(self.plugin_path) for plugin in plugin_list: plugin_config_path = os.path.join( self.plugin_path, plugin, 'config.yaml' ) if os.path.exists(plugin_config_path): with open(plugin_config_path) as f: try: plugin_config = yaml.load(f) except Exception: logger.exception('load %s\'s config fail!' % plugin) else: if plugin_config['enable']: conf.plugins[plugin] = plugin_config self.__register_plugin(plugin) def __register_plugin(self, plugin): """ 注册插件 """ kb.plugins[plugin] = {} kb.plugins[plugin]['name'] = plugin try: _import_path = '.'.join( conf.settings.PLUGINS_OPPOSITE_PATH.split(os.path.sep) ) plugin_path = '%s.%s.work' % (_import_path, plugin) _plugin = __import__(plugin_path, fromlist='*') # 动态加载函数 kb.plugins[plugin]['handle'] = _plugin except Exception: logger.exception('register plugin %s failed!' % plugin) else: self.__classify_plugin(plugin) def __classify_plugin(self, plugin): """ 归类插件 """ inputs = conf.plugins[plugin]['input'] for inp in inputs: if inp in conf.settings.ALLOW_INPUTS: conf.reg_plugins[inp].add(plugin) def start(self): while not self.exit_flag: try: target = self.wp.target_queue.get(timeout=1) except gevent.queue.Empty: pass else: self.__add_job_by_type(target) def __add_job_by_type(self, target): domain_type = target.get('domain_type') parent_module = target.pop('parent_module') onerepeat = conf.plugins.get(parent_module, {}).get('onerepeat') domain = target.get('domain') if domain_type in conf.settings.ALLOW_INPUTS: for plugin in conf.reg_plugins[domain_type]: if onerepeat and parent_module == plugin: continue _plugin = getattr(kb.plugins[plugin]['handle'], plugin)() self.wp.add_job(getattr(_plugin, 'start'), **target) self.__init_plugin_progress(plugin, domain) def __init_plugin_progress(self, plugin, domain): key = '%s_%s' % (plugin, domain) kb.progress[key] = {'status':'wait', 'start_time':0, 'end_time':0} def run_job(self): self.wp.run() def exit(self): self.exit_flag = True
class PluginController(object): def __init__(self): self.exit_flag = False self.wp = WorkerPool() self.plugin_path = paths.PLUGINS_PATH @classmethod def get_available_plugins(cls): """ 返回plugins目录下所有enable为true的plugin名称 """ plugin_list = os.listdir(paths.PLUGINS_PATH) for plugin in plugin_list: plugin_config_path = os.path.join(paths.PLUGINS_PATH, plugin, 'config.yaml') if os.path.exists(plugin_config_path): with open(plugin_config_path) as f: try: plugin_config = yaml.load(f) except Exception: api.logger.exception('load %s\'s config fail!' % plugin) else: if plugin_config['enable']: # 在conf中载入plugin的配置信息 conf.plugins_available[plugin] = plugin_config return conf.plugins_available def plugin_init(self, plugins_sepcific=None): """ 初始化插件 """ plugins_available = conf.plugins_available.keys() if plugins_sepcific: for plugin in plugins_sepcific: if plugin in plugins_available: self.__load_plugin(plugin) else: api.logger.exception('plugin: %s NOT found!' % plugin) else: for plugin in plugins_available: self.__load_plugin(plugin) def __load_plugin(self, plugin): """ 载入名为plugin的插件. plugin的存在性和config的enable合法性由调用者保证 """ # 指向在conf.plugins_availavle中的信息, 不再重新读入配置文件 conf.plugins_load[plugin] = conf.plugins_available[plugin] self.__register_plugin(plugin) def __register_plugin(self, plugin): """ 注册插件 """ kb.plugins[plugin] = {} kb.plugins[plugin]['name'] = plugin try: _import_path = '.'.join( paths.PLUGINS_OPPOSITE_PATH.split(os.path.sep)) plugin_path = '%s.%s.work' % (_import_path, plugin) _plugin = __import__(plugin_path, fromlist='*') # 动态加载函数 kb.plugins[plugin]['handle'] = _plugin except Exception: api.logger.exception('register plugin %s failed!' % plugin) else: self.__classify_plugin(plugin) def __classify_plugin(self, plugin): """ 归类插件 """ inputs = conf.plugins_load[plugin]['input'] for inp in inputs: if inp in settings.ALLOW_INPUTS: conf.reg_plugins[inp].add(plugin) def start(self): while not self.exit_flag: try: target = self.wp.target_queue.get(timeout=1) except gevent.queue.Empty: pass else: self.__add_job_by_type(target) def __add_job_by_type(self, target): domain_type = target.get('domain_type') parent_module = target.pop('parent_module') onerepeat = conf.plugins_load.get(parent_module, {}).get('onerepeat') domain = target.get('domain') if domain_type in settings.ALLOW_INPUTS: for plugin in conf.reg_plugins[domain_type]: if onerepeat and parent_module == plugin: continue _plugin = getattr(kb.plugins[plugin]['handle'], plugin)() self.wp.add_job(getattr(_plugin, 'start'), **target) self.__init_plugin_progress(plugin, domain) def __init_plugin_progress(self, plugin, domain): key = '%s_%s' % (plugin, domain) kb.progress[key] = {'status': 'wait', 'start_time': 0, 'end_time': 0} def run_job(self): self.wp.run() def exit(self): self.exit_flag = True
class PluginController(object): def __init__(self): self.exit_flag = False self.wp = WorkerPool() self.plugin_path = paths.PLUGINS_PATH @classmethod def get_available_plugins(cls): """ 返回plugins目录下所有enable为true的plugin名称 """ plugin_list = os.listdir(paths.PLUGINS_PATH) for plugin in plugin_list: plugin_config_path = os.path.join( paths.PLUGINS_PATH, plugin, 'config.yaml' ) if os.path.exists(plugin_config_path): with open(plugin_config_path) as f: try: plugin_config = yaml.load(f) except Exception: api.logger.exception('load %s\'s config fail!' % plugin) else: if plugin_config['enable']: # 在conf中载入plugin的配置信息 conf.plugins_available[plugin] = plugin_config return conf.plugins_available def plugin_init(self, plugins_sepcific=None): """ 初始化插件 """ plugins_available = conf.plugins_available.keys() if plugins_sepcific: for plugin in plugins_sepcific: if plugin in plugins_available: self.__load_plugin(plugin) else: api.logger.exception('plugin: %s NOT found!' % plugin) else: for plugin in plugins_available: self.__load_plugin(plugin) def __load_plugin(self, plugin): """ 载入名为plugin的插件. plugin的存在性和config的enable合法性由调用者保证 """ # 指向在conf.plugins_availavle中的信息, 不再重新读入配置文件 conf.plugins_load[plugin] = conf.plugins_available[plugin] self.__register_plugin(plugin) def __register_plugin(self, plugin): """ 注册插件 """ kb.plugins[plugin] = {} kb.plugins[plugin]['name'] = plugin try: _import_path = '.'.join( paths.PLUGINS_OPPOSITE_PATH.split(os.path.sep) ) plugin_path = '%s.%s.work' % (_import_path, plugin) _plugin = __import__(plugin_path, fromlist='*') # 动态加载函数 kb.plugins[plugin]['handle'] = _plugin except Exception: api.logger.exception('register plugin %s failed!' % plugin) else: self.__classify_plugin(plugin) def __classify_plugin(self, plugin): """ 归类插件 """ inputs = conf.plugins_load[plugin]['input'] for inp in inputs: if inp in settings.ALLOW_INPUTS: conf.reg_plugins[inp].add(plugin) def start(self): while not self.exit_flag: try: target = self.wp.target_queue.get(timeout=1) except gevent.queue.Empty: pass else: self.__add_job_by_type(target) def __add_job_by_type(self, target): domain_type = target.get('domain_type') parent_module = target.pop('parent_module') onerepeat = conf.plugins_load.get(parent_module, {}).get('onerepeat') domain = target.get('domain') if domain_type in settings.ALLOW_INPUTS: for plugin in conf.reg_plugins[domain_type]: if onerepeat and parent_module == plugin: continue _plugin = getattr(kb.plugins[plugin]['handle'], plugin)() self.wp.add_job(getattr(_plugin, 'start'), **target) self.__init_plugin_progress(plugin, domain) def __init_plugin_progress(self, plugin, domain): key = '%s_%s' % (plugin, domain) kb.progress[key] = {'status': 'wait', 'start_time': 0, 'end_time': 0} def run_job(self): self.wp.run() def exit(self): self.exit_flag = True