def _get_tasks_result_list(**kwargs):
results = kwargs.get('results', [])
all = []
success_num = 1
results_len = len(results)
while len(results) > 0:
for r_index, r in enumerate(results):
if r.ready():
try:
all.append(r.get(timeout=2, propagate=False))
except TimeoutError:
pass
success_num += 1
try:
results.pop(r_index)
except:
pass
else:
pass
print('\r' + _get_simulate_logger() +
'proxy_tasks._get_proxy: success_num: {}, rest_num: {}'.
format(success_num, results_len - success_num),
end='',
flush=True)
else:
pass
print('\r', end='', flush=True)
return all
def main():
global time_str, local_ip
_welcome()
print('Getting local ip...')
local_ip = get_local_external_network_ip()
assert local_ip != '', 'local_ip获取失败!'
print('[+] local ip: {}'.format(local_ip))
while True:
origin_proxy_data = list_remove_repeat_dict(
target=ip_pools_obj._get_all_ip_proxy(_k=proxy_list_key_name),
repeat_key='ip')
while len(origin_proxy_data) < MAX_PROXY_NUM:
print('\r' + _get_simulate_logger() +
'Ip Pools --->>> 已存在proxy_num(匿名度未知): {}'.format(
len(origin_proxy_data)),
end='',
flush=True)
get_proxy_process_data()
# 重置
origin_proxy_data = list_remove_repeat_dict(
target=ip_pools_obj._get_all_ip_proxy(_k=proxy_list_key_name),
repeat_key='ip')
else:
print()
lg.info('达标!休眠{}s...'.format(WAIT_TIME))
sleep(WAIT_TIME)
lg.info('Async Checking all_proxy(匿名度未知)...')
origin_proxy_data = list_remove_repeat_dict(
target=origin_proxy_data, repeat_key='ip')
check_all_proxy(origin_proxy_data,
redis_key_name=_key,
delete_score=90)
'''删除失效的, 时刻保持最新高匿可用proxy'''
high_origin_proxy_list = list_remove_repeat_dict(
target=ip_pools_obj._get_all_ip_proxy(
_k=high_proxy_list_key_name),
repeat_key='ip')
lg.info('Async Checking hign_proxy(高匿名)状态...')
check_all_proxy(high_origin_proxy_list,
redis_key_name=_h_key,
delete_score=MIN_SCORE)
def _get_tasks_result_list(resutls):
'''得到结果集'''
def write_hign_proxy_info_2_redis(one_proxy_info):
'''redis新写入高匿名ip'''
old_h_proxy_list = ip_pools_obj._get_all_ip_proxy(
_k=high_proxy_list_key_name)
old_ip_list = [i.get('ip') for i in old_h_proxy_list]
if one_proxy_info.get('ip') not in old_ip_list:
old_score = one_proxy_info.get('score')
one_proxy_info.update({ # 加分
'score': old_score + 5,
})
old_h_proxy_list.append(one_proxy_info)
old_h_proxy_list = serialize_obj_item_2_dict(
old_h_proxy_list) # 防止反序列化时, 提示无法识别ProxyItem
redis_cli.set(name=_h_key, value=dumps(old_h_proxy_list))
else:
pass
return None
all = []
success_num = 1
available_num = 0
results_len = len(resutls)
while len(resutls) > 0:
for r_index, r in enumerate(resutls):
proxy = r.get('proxy_info', {}).get('ip') + ':' + str(
r.get('proxy_info', {}).get('port'))
task_id = r.get('async_obj').id
status = r.get('async_obj').status
one_proxy_info = r.get('proxy_info', {})
# lg.info('task_id: {}, status: {}'.format(task_id, status))
if r.get('async_obj').ready():
async_res = False
try:
async_res = r.get('async_obj').get(
timeout=2, propagate=False
) # 抛出异常，但程序不会停止, r.get('async_obj').traceback 追踪完整异常
except TimeoutError:
pass
if async_res:
available_num += 1
# 高匿ip写入redis
write_hign_proxy_info_2_redis(one_proxy_info)
all.append({
'async_res': async_res,
'proxy_info': one_proxy_info,
})
# 动态输出, '\r'回到当前开头
print('\r' + _get_simulate_logger() +
'已检测ip: {}, 剩余: {}, 实际可用高匿个数: {}'.format(
success_num, results_len - success_num,
available_num),
end='',
flush=True)
success_num += 1
try:
resutls.pop(r_index)
except:
pass
else:
# lg.info('{} 未完成!'.format(proxy))
pass
else:
print()
# lg.info('所有异步结果完成!!')
print('\r', end='', flush=True)
return all