def start(args): global output_file global output_format global domain global plugin_controller domain = args.target domain_type = get_domain_type(domain) if domain_type in settings.ALLOW_INPUTS: domain = Domain.url_format(domain) # 初始化日志 log_level = get_log_level(args.log_level) init_logger(log_file_path=args.log_file, log_level=log_level) logger.info('system init...') # 初始化配置 conf.settings = settings conf.max_level = args.max_level output_file = args.output_file output_format = args.output_format # 初始化爬虫 proxy_list = get_proxy_list_by_file(args.proxy_file) api.request = Req(args.timeout, proxy_list, args.verify_proxy) plugin_controller = PluginController() plugin_controller.plugin_init() logger.info('Loaded plugins: %s' % ','.join(conf.plugins.keys())) # 绑定信号事件 signal.signal(signal.SIGUSR1, on_signal) signal.signal(signal.SIGTERM, on_signal) signal.signal(signal.SIGINT, on_signal) logger.info('start target...') # 首个目标 first_target = { 'result': {'root_domain': [], 'domain': [], 'ip': []}, 'module': '', 'level': 0, 'parent_domain': '' } first_target['result'][domain_type].append(domain) plugin_controller.wp.result.put(first_target) # 开启任务监控 logger.info('start task monitor and plugin...') kwargs = {'pc': plugin_controller} monitor = threading.Thread(target=task_monitor, kwargs=kwargs) monitor.start() # 开启插件执行 plugin_controller.start() complate() else: logger.error( 'Please input a target in the correct' ' format(domain/root_domain/ip)!' )
def start(args): global output_file global output_format global domain global plugin_controller domain = args.target domain_type = get_domain_type(domain) if domain_type in settings.ALLOW_INPUTS: domain = Domain.url_format(domain) # 初始化日志 log_level = get_log_level(args.log_level) init_logger(log_file_path=args.log_file, log_level=log_level) logger.info('system init...') # 初始化配置 conf.settings = settings conf.max_level = args.max_level output_file = args.output_file output_format = args.output_format # 初始化爬虫 proxy_list = get_proxy_list_by_file(args.proxy_file) api.request = Req(args.timeout, proxy_list, args.verify_proxy) plugin_controller = PluginController() plugin_controller.plugin_init() logger.info('Loaded plugins: %s' % ','.join(conf.plugins.keys())) # 绑定信号事件 signal.signal(signal.SIGUSR1, on_signal) signal.signal(signal.SIGTERM, on_signal) signal.signal(signal.SIGINT, on_signal) logger.info('start target...') # 首个目标 first_target = { 'result': { 'root_domain': [], 'domain': [], 'ip': [] }, 'module': '', 'level': 0, 'parent_domain': '' } first_target['result'][domain_type].append(domain) plugin_controller.wp.result.put(first_target) # 开启任务监控 logger.info('start task monitor and plugin...') kwargs = {'pc': plugin_controller} monitor = threading.Thread(target=task_monitor, kwargs=kwargs) monitor.start() # 开启插件执行 plugin_controller.start() complate() else: logger.error('Please input a target in the correct' ' format(domain/root_domain/ip)!')
def parseCmdOptions(): ''' 解析命令行参数 ''' parser = argparse.ArgumentParser( usage=USAGE, formatter_class=argparse.RawTextHelpFormatter, add_help=False ) parser.add_argument( '-h', '--help', action='help', help='Show this help message and exit' ) parser.add_argument( '-V', '--version', action='version', version=VERSION_INFO ) parser.add_argument( '-t', '--target', dest='target', required=True, help=_format_help('Target domain/rootdomain/ip') ) available_plugins = PluginController.get_available_plugins().keys() parser.add_argument( '-p', '--plugins', metavar='plugin', dest='plugins_specific', nargs='+', default=None, help=_format_help([ 'Specify the plugins', 'avaliable: ' + '\n'.join([' '.join(available_plugins[i:i+3]) for i in range(0, len(available_plugins), 4)]) ]) ) parser.add_argument( '-m', '--max_level', dest='max_level', default=4, type=int, help=_format_help('Max level to get domain/ip/rootdomain') ) parser.add_argument( '--pool_size', dest='pool_size', type=int, default=500, help=_format_help('Max number of Thread pool size') ) parser.add_argument( '-o', '--output_file', dest='output_file', default=None, help=_format_help('File to ouput result') ) parser.add_argument( '--format', dest='output_format', default='csv', help=_format_help([ 'The format to output result,', 'default list:', '/'.join(Output.get_output_formats()) ]) ) # Log parser.add_argument( '--log_file', dest='log_file', default=None, help=_format_help('Log file') ) loglevel_choices = { 1: 'DEBUG', 2: 'INFO', 3: 'WARNING', 4: 'ERROR', } parser.add_argument( '--log_level', dest='log_level', type=int, default=1, choices=loglevel_choices, help=_format_help('Log level of output to file', loglevel_choices) ) # Proxy parser.add_argument( '--proxy_file', dest='proxy_file', default=None, help=_format_help([ 'Proxy file, one line one proxy, each line format:', 'schem,proxy url,', 'eg:http,http://1.1.1.1:123', ]) ) parser.add_argument( '--verify_proxy', dest='verify_proxy', default=False, action='store_true', help=_format_help('If verify the proxy list') ) parser.add_argument( '--timeout', dest='timeout', type=int, default=10, help=_format_help('Request timeout') ) parser.add_argument( '--alive_check', dest='alive_check', default=False, action='store_true', help=_format_help('Check alive and accessible status of domain') ) args = parser.parse_args() return args.__dict__
def start(): conf.domain = options.target domain_type = get_domain_type(conf.domain) if domain_type in settings.ALLOW_INPUTS: conf.domain = Domain.url_format(conf.domain) # 初始化日志 log_level = get_log_level(options.log_level) init_logger(log_file_path=options.log_file, log_level=log_level) api.logger.info('System init at %s' % time.strftime("%X")) # 初始化配置 conf.settings = settings # 初始化爬虫 proxy_list = get_proxy_list_by_file(options.proxy_file) api.request = Req(options.timeout, proxy_list, options.verify_proxy) conf.plugin_controller = PluginController() conf.plugin_controller.plugin_init(options.plugins_specific) api.logger.info('Loaded plugins: %s' % ', '.join(conf.plugins_load.keys())) # 绑定信号事件 if hasattr(signal, 'SIGUSR1'): signal.signal(signal.SIGUSR1, on_signal) signal.signal(signal.SIGTERM, on_signal) signal.signal(signal.SIGINT, on_signal) api.logger.info('Start target [%s]' % conf.domain) # 首个目标 first_target = { 'result': { 'root_domain': [], 'domain': [], 'ip': [] }, 'module': '', 'level': 0, 'parent_domain': '' } first_target['result'][domain_type].append(conf.domain) conf.plugin_controller.wp.result.put(first_target) # 开启任务监控 api.logger.info('Start task monitor and plugin...') kwargs = {'pc': conf.plugin_controller} monitor = threading.Thread(target=task_monitor, kwargs=kwargs) monitor.start() # 开启插件执行 conf.plugin_controller.start() if options.alive_check: alivecheck = AliveCheck() print '\n' api.logger.info('Init alive check...') alivecheck.start() api.logger.info('Alive check completed') complete() else: api.logger.error('Please input a target in the correct' ' format(domain/root_domain/ip)!')
def parse(args=None): parser = argparse.ArgumentParser( usage=USAGE, formatter_class=argparse.RawTextHelpFormatter, add_help=False) parser.add_argument('-h', '--help', action='help', help='Show this help message and exit') parser.add_argument('-V', '--version', action='version', version=VERSION_INFO) parser.add_argument('-t', '--target', dest='target', required=True, help=_format_help('Target domain/rootdomain/ip')) available_plugins = PluginController.get_available_plugins().keys() parser.add_argument('-p', '--plugins', metavar='plugin', dest='plugins_specific', nargs='+', default=None, help=_format_help([ 'Specify the plugins', 'avaliable: ' + '\n'.join([ ' '.join(available_plugins[i:i + 4]) for i in range(0, len(available_plugins), 4) ]) ])) parser.add_argument( '-m', '--max_level', dest='max_level', default=4, type=int, help=_format_help('Max level to get domain/ip/rootdomain')) parser.add_argument('-o', '--output_file', dest='output_file', help=_format_help('File to ouput result')) parser.add_argument('--format', dest='output_format', default='csv', help=_format_help([ 'The format to output result,', 'default list:', '/'.join(Output.get_output_formats()) ])) parser.add_argument('--log_file', dest='log_file', help=_format_help('Log file')) loglevel_choices = { 1: 'DEBUG', 2: 'INFO', 3: 'WARNING', 4: 'ERROR', } parser.add_argument('--log_level', dest='log_level', type=int, default=1, choices=loglevel_choices, help=_format_help('Log level of output to file', loglevel_choices)) parser.add_argument( '--proxy_file', dest='proxy_file', help=_format_help([ 'Proxy file, one line one proxy, each line format:', 'schem,proxy url,', 'eg:http,http://1.1.1.1:123', ])) parser.add_argument('--verify_proxy', dest='verify_proxy', action='store_true', default=False, help=_format_help('If verify the proxy list')) parser.add_argument('--timeout', dest='timeout', type=int, default=10, help=_format_help('Request timeout')) parser.add_argument( '--alive_check', action='store_true', help=_format_help('Check alive and accessible status of domain')) args = parser.parse_args(args) return args