def add_task_and_save(pc, one_result):
level = one_result.get('level', -1) + 1
if level <= conf.max_level:
if level > kb.status.level:
kb.status.level = level
print_task_status(True)
module = one_result.get('module')
for task_type in one_result.get('result', {}).keys():
for domain in one_result.get('result', {}).get(task_type, []):
domain = Domain.url_format(domain)
if task_type == 'ip' and is_intra_ip(domain):
continue
if domain not in result[task_type]:
target = {
'level': level,
'domain_type': task_type,
'domain': domain,
'parent_module': module
}
pc.wp.target_queue.put(target)
save_result(one_result, domain, task_type)
print_task_status()
def start(args):
global output_file
global output_format
global domain
global plugin_controller
domain = args.target
domain_type = get_domain_type(domain)
if domain_type in settings.ALLOW_INPUTS:
domain = Domain.url_format(domain)
# 初始化日志
log_level = get_log_level(args.log_level)
init_logger(log_file_path=args.log_file, log_level=log_level)
logger.info('system init...')
# 初始化配置
conf.settings = settings
conf.max_level = args.max_level
output_file = args.output_file
output_format = args.output_format
# 初始化爬虫
proxy_list = get_proxy_list_by_file(args.proxy_file)
api.request = Req(args.timeout, proxy_list, args.verify_proxy)
plugin_controller = PluginController()
plugin_controller.plugin_init()
logger.info('Loaded plugins: %s' % ','.join(conf.plugins.keys()))
# 绑定信号事件
signal.signal(signal.SIGUSR1, on_signal)
signal.signal(signal.SIGTERM, on_signal)
signal.signal(signal.SIGINT, on_signal)
logger.info('start target...')
# 首个目标
first_target = {
'result': {
'root_domain': [],
'domain': [],
'ip': []
},
'module': '',
'level': 0,
'parent_domain': ''
}
first_target['result'][domain_type].append(domain)
plugin_controller.wp.result.put(first_target)
# 开启任务监控
logger.info('start task monitor and plugin...')
kwargs = {'pc': plugin_controller}
monitor = threading.Thread(target=task_monitor, kwargs=kwargs)
monitor.start()
# 开启插件执行
plugin_controller.start()
complate()
else:
logger.error('Please input a target in the correct'
' format(domain/root_domain/ip)!')
def start():
conf.domain = options.target
domain_type = get_domain_type(conf.domain)
if domain_type in settings.ALLOW_INPUTS:
conf.domain = Domain.url_format(conf.domain)
# 初始化日志
log_level = get_log_level(options.log_level)
init_logger(log_file_path=options.log_file, log_level=log_level)
api.logger.info('System init at %s' % time.strftime("%X"))
# 初始化配置
conf.settings = settings
# 初始化爬虫
proxy_list = get_proxy_list_by_file(options.proxy_file)
api.request = Req(options.timeout, proxy_list, options.verify_proxy)
conf.plugin_controller = PluginController()
conf.plugin_controller.plugin_init(options.plugins_specific)
api.logger.info('Loaded plugins: %s' % ', '.join(conf.plugins_load.keys()))
# 绑定信号事件
if hasattr(signal, 'SIGUSR1'):
signal.signal(signal.SIGUSR1, on_signal)
signal.signal(signal.SIGTERM, on_signal)
signal.signal(signal.SIGINT, on_signal)
api.logger.info('Start target [%s]' % conf.domain)
# 首个目标
first_target = {
'result': {'root_domain': [], 'domain': [], 'ip': []},
'module': '',
'level': 0,
'parent_domain': ''
}
first_target['result'][domain_type].append(conf.domain)
conf.plugin_controller.wp.result.put(first_target)
# 开启任务监控
api.logger.info('Start task monitor and plugin...')
kwargs = {'pc': conf.plugin_controller}
monitor = threading.Thread(target=task_monitor, kwargs=kwargs)
monitor.start()
# 开启插件执行
conf.plugin_controller.start()
if options.alive_check:
alivecheck = AliveCheck()
print '\n'
api.logger.info('Start alive check...')
alivecheck.start()
api.logger.info('Alive check completed')
complete()
else:
api.logger.error(
'Please input a target in the correct'
' format(domain/root_domain/ip)!'
)
def start(args):
global output_file
global output_format
global domain
global plugin_controller
domain = args.target
domain_type = get_domain_type(domain)
if domain_type in settings.ALLOW_INPUTS:
domain = Domain.url_format(domain)
# 初始化日志
log_level = get_log_level(args.log_level)
init_logger(log_file_path=args.log_file, log_level=log_level)
logger.info('system init...')
# 初始化配置
conf.settings = settings
conf.max_level = args.max_level
output_file = args.output_file
output_format = args.output_format
# 初始化爬虫
proxy_list = get_proxy_list_by_file(args.proxy_file)
api.request = Req(args.timeout, proxy_list, args.verify_proxy)
plugin_controller = PluginController()
plugin_controller.plugin_init()
logger.info('Loaded plugins: %s' % ','.join(conf.plugins.keys()))
# 绑定信号事件
signal.signal(signal.SIGUSR1, on_signal)
signal.signal(signal.SIGTERM, on_signal)
signal.signal(signal.SIGINT, on_signal)
logger.info('start target...')
# 首个目标
first_target = {
'result': {'root_domain': [], 'domain': [], 'ip': []},
'module': '',
'level': 0,
'parent_domain': ''
}
first_target['result'][domain_type].append(domain)
plugin_controller.wp.result.put(first_target)
# 开启任务监控
logger.info('start task monitor and plugin...')
kwargs = {'pc': plugin_controller}
monitor = threading.Thread(target=task_monitor, kwargs=kwargs)
monitor.start()
# 开启插件执行
plugin_controller.start()
complate()
else:
logger.error(
'Please input a target in the correct'
' format(domain/root_domain/ip)!'
)
def get_domain_type(domain):
if is_ip(domain):
return 'ip'
elif is_url(domain):
domain = Domain.url_format(domain)
root_domain = Domain.get_root_domain(domain)
if root_domain == domain:
return 'root_domain'
else:
return 'domain'
else:
return False
def get_domain_type(domain):
if is_ip(domain):
return "ip"
elif is_url(domain):
domain = Domain.url_format(domain)
root_domain = Domain.get_root_domain(domain)
if root_domain == domain:
return "root_domain"
else:
return "domain"
else:
return False
def get_domain_type(domain):
if is_ip(domain):
return 'ip'
elif is_url(domain):
domain = Domain.url_format(domain)
root_domain = Domain.get_root_domain(domain)
if root_domain == domain:
return 'root_domain'
else:
return 'domain'
else:
return False
def start():
conf.domain = options.target
domain_type = get_domain_type(conf.domain)
if domain_type in settings.ALLOW_INPUTS:
conf.domain = Domain.url_format(conf.domain)
# 初始化日志
log_level = get_log_level(options.log_level)
init_logger(log_file_path=options.log_file, log_level=log_level)
api.logger.info('System init at %s' % time.strftime("%X"))
# 初始化配置
conf.settings = settings
# 初始化爬虫
proxy_list = get_proxy_list_by_file(options.proxy_file)
api.request = Req(options.timeout, proxy_list, options.verify_proxy)
conf.plugin_controller = PluginController()
conf.plugin_controller.plugin_init(options.plugins_specific)
api.logger.info('Loaded plugins: %s' %
', '.join(conf.plugins_load.keys()))
# 绑定信号事件
if hasattr(signal, 'SIGUSR1'):
signal.signal(signal.SIGUSR1, on_signal)
signal.signal(signal.SIGTERM, on_signal)
signal.signal(signal.SIGINT, on_signal)
api.logger.info('Start target [%s]' % conf.domain)
# 首个目标
first_target = {
'result': {
'root_domain': [],
'domain': [],
'ip': []
},
'module': '',
'level': 0,
'parent_domain': ''
}
first_target['result'][domain_type].append(conf.domain)
conf.plugin_controller.wp.result.put(first_target)
# 开启任务监控
api.logger.info('Start task monitor and plugin...')
kwargs = {'pc': conf.plugin_controller}
monitor = threading.Thread(target=task_monitor, kwargs=kwargs)
monitor.start()
# 开启插件执行
conf.plugin_controller.start()
if options.alive_check:
alivecheck = AliveCheck()
print '\n'
api.logger.info('Init alive check...')
alivecheck.start()
api.logger.info('Alive check completed')
complete()
else:
api.logger.error('Please input a target in the correct'
' format(domain/root_domain/ip)!')
