def update_chrome_pac_by_gatherproxy(self):
'''
从proxies.txt里面检测ip_port，最后更新chrome用的pac
:return:
'''
FileService.merge_proxy()
with open('file/proxy_file/proxies.txt', 'r') as fr:
try:
ip_port_list = fr.read().split('\n')
except Exception:
logger.error(traceback.format_exc())
return
finally:
fr.close()
q = queue.Queue()
ip_port_list = list(set(ip_port_list)) # 去重
for ip_with_port in ip_port_list:
ip_dict = {
'ip_with_port': ip_with_port,
'proxy_type': 'socks5',
}
q.put(ip_dict)
for i in range(20):
t = threading.Thread(
target=self.__get_useful_fanqiang_ip_gatherproxy_worker,
args=(q, ))
t.start()
q.join()
# 跑完，吧proxy文件删了
os.remove('file/proxy_file/proxies.txt')
def run(method_with_params_list):
'''
代理方法，需要在全局维护一个registest_list
:param method_with_params_list:
:return:
'''
def do_chain(method, params_list, current_step):
method_name = method.replace('()', '')
# 判断有没有这属性
if not hasattr(current_step, method_name):
raise Exception('{0}没有属性{1}'.format(str(current_step),
method_name))
if re.findall(r'\(\)', method): # 是方法
this_params = params_list.pop(0)
this_args = this_params['args']
this_kwargs = this_params['kwargs']
current_step = getattr(current_step, method_name)(*this_args,
**this_kwargs)
else:
current_step = getattr(current_step, method_name)
return current_step
method = method_with_params_list[0]
params_list = method_with_params_list[1]
try:
current_step = None
logger.info('-------------------处理 {0}，参数：{1}'.format(
method, params_list))
first_method = method.split('.')[0]
first_method_name = first_method.replace('()', '')
# 判断是否已经注册该方法/类
if not first_method_name in register_list:
raise Exception('类/方法%s没有注册' % first_method_name)
else:
first_registed_method_or_obj = register_list[first_method_name]
if re.findall(r'\(\)', first_method): # 是方法
this_params = params_list.pop(0)
this_args = this_params['args']
this_kwargs = this_params['kwargs']
current_step = first_registed_method_or_obj(
*this_args, **this_kwargs)
else: # 不是方法
current_step = first_registed_method_or_obj
for m in method.split('.')[1:]:
current_step = do_chain(m, params_list, current_step)
return current_step
except Exception as e:
logger.error('{0} -------------- {1}'.format(method,
traceback.format_exc()))
raise e
def get_client_error():
'''
获取有用的ip_port
:return:
'''
try:
data = request.args.to_dict()
account = data['account']
error = data['error']
logger.error('Client Error : {0}------{1}--------{2}'.format(
account, request.environ['REMOTE_ADDR'], error))
return Response(status=200)
except Exception:
logger.error(traceback.format_exc())
def modify_chrome_pac_file_and_push(self, ip_with_port):
'''
更新pac文件并提交（加锁）
:param ip_with_port:
:return:
'''
def modify_chrome_file(file_path, ip_with_port):
# 替换ip和port
new_text = ''
with open(file_path, 'r', encoding='utf-8') as fr:
try:
old_text = fr.read()
new_text = old_text.replace(
re.findall(
r'(?:SOCKS |SOCKS5 )(\d+\.\d+\.\d+\.\d+:\d+)',
old_text)[0], ip_with_port)
new_text = new_text.replace(
re.findall(
r'(?:SOCKS |SOCKS5 )(\d+\.\d+\.\d+\.\d+:\d+)',
old_text)[1], ip_with_port)
finally:
fr.close()
with open(file_path, 'w', encoding='utf-8') as fw:
try:
fw.write(new_text)
finally:
fw.close()
logger.debug("已更新文件 %s,ip_port为：%s" % (file_path, ip_with_port))
try:
io_lock.acquire()
global is_ok_at_least_one
if is_ok_at_least_one:
return
modify_chrome_file(file_path_chrome, ip_with_port)
modify_chrome_file(file_path_chrome_socks, ip_with_port)
Git.git_push(file_path_chrome)
Git.git_push(file_path_chrome_socks)
is_ok_at_least_one = True
return
except Exception:
logger.error(traceback.format_exc())
finally:
io_lock.release()
def get_pac():
# request_id = request.args['uuid']
try:
pac_type = request.args['type']
filename, pac = fanqiang_ip_service.get_dynamic_pac(pac_type)
response = make_response(pac)
response.headers['Content-Type'] = 'application/x-ns-proxy-autoconfig'
response.headers[
'Content-Disposition'] = 'attachment; filename={}'.format(
filename.encode().decode('latin-1'))
return response
except Exception as e:
logger.error("%s" % (traceback.format_exc()))
# return Response("服务器出错，请联系管理员，请求id是：%s" % request_id,status=500)
abort(500)
def add_to_useful(ip_port, useful_list):
try:
logger.debug('测试ip_port:{0}'.format(ip_port['ip_with_port']))
is_useful = pickle.loads(
fanqiang_service_client.run(
Transformer().FanqiangService().test_useful_fanqiang(
ip_port).done()).data)
if is_useful:
try:
lock.acquire()
logger.debug('{0}能用'.format(ip_port['ip_with_port']))
useful_list.append(ip_port)
finally:
lock.release()
except Exception:
logger.error(traceback.format_exc())
def get_ip_port():
'''
获取有用的ip_port
:return:
'''
try:
data = request.args.to_dict()
if not (data['account'] == 'CK_test'
and data['password'] == 'CK_test'):
abort(401)
ip_port_dict = fanqiang_ip_service.get_ip_port_dict_list(1)[0]
resp = Response(ip_port_dict['ip_with_port'],
mimetype='text/plain',
status=200)
return resp
except Exception:
logger.error(traceback.format_exc())
def save(ip_with_port, proxy_type):
try:
elite = FanqiangService.test_elite(ip_with_port, proxy_type)
except Exception as e:
logger.info(str(e))
return
try:
lock.acquire()
if elite:
db_client.run(Transformer().Fanqiang().save({
'proxy_type':
proxy_type,
'ip_with_port':
ip_with_port,
'time':
0.00,
'location':
FanqiangService.get_location(
ip_with_port.split(':')[0]),
'Elite':
elite
}).done())
else:
db_client.run(Transformer().Fanqiang().save({
'proxy_type':
proxy_type,
'ip_with_port':
ip_with_port,
'time':
0.00,
'location':
FanqiangService.get_location(
ip_with_port.split(':')[0])
}).done())
except Exception as e:
logger.error(e)
finally:
lock.release()
def save_all_from_gatherproxy_to_db(self):
'''
保存收集到的所有的gather_proxydoa数据库，不筛选
:return:
'''
self.merge_proxy()
with open('file/proxy_file/proxies.txt', 'r') as fr:
try:
ip_port_list = fr.read().split('\n')
except Exception:
logger.error(traceback.format_exc())
return
finally:
fr.close()
def save(ip_with_port, proxy_type):
try:
elite = FanqiangService.test_elite(ip_with_port, proxy_type)
except Exception as e:
logger.info(str(e))
return
try:
lock.acquire()
if elite:
db_client.run(Transformer().Fanqiang().save({
'proxy_type':
proxy_type,
'ip_with_port':
ip_with_port,
'time':
0.00,
'location':
FanqiangService.get_location(
ip_with_port.split(':')[0]),
'Elite':
elite
}).done())
else:
db_client.run(Transformer().Fanqiang().save({
'proxy_type':
proxy_type,
'ip_with_port':
ip_with_port,
'time':
0.00,
'location':
FanqiangService.get_location(
ip_with_port.split(':')[0])
}).done())
except Exception as e:
logger.error(e)
finally:
lock.release()
q = queue.Queue()
tf = thread_utils.ThreadFactory()
for i in range(20):
t = threading.Thread(target=tf.queue_threads_worker,
args=(q, save))
t.start()
for ip_with_port in ip_port_list:
q.put({'ip_with_port': ip_with_port, 'proxy_type': 'socks5'})
q.join()
tf.all_task_done = True
os.remove('file/proxy_file/proxies.txt')
def get_ip_port_to_pool():
'''
定时获取ip_port到备用池
:return:
'''
try:
lock = threading.Lock()
def add_to_useful(ip_port, useful_list):
try:
logger.debug('测试ip_port:{0}'.format(ip_port['ip_with_port']))
is_useful = pickle.loads(
fanqiang_service_client.run(
Transformer().FanqiangService().test_useful_fanqiang(
ip_port).done()).data)
if is_useful:
try:
lock.acquire()
logger.debug('{0}能用'.format(ip_port['ip_with_port']))
useful_list.append(ip_port)
finally:
lock.release()
except Exception:
logger.error(traceback.format_exc())
# 先读取存下来的pool
with open('file/bak_pool', 'rb') as fr:
try:
cache.ip_port_pool = pickle.load(fr)
logger.debug('从文件读到的数量为：{0}'.format(len(cache.ip_port_pool)))
except EOFError:
pass
except Exception:
logger.error(traceback.format_exc())
new_num = int(Conf.get('IP_PORT_POOL', 'new_num'))
while True:
useful_list = []
## 开20个线程筛掉之前没用的ip_port（由于xmlrpc不支持并发，所以有问题）
# q = queue.Queue()
# tf = thread_utils.ThreadFactory()
# for i in range(20):
# t = threading.Thread(target=tf.queue_threads_worker, args=(q, add_to_useful))
# t.start()
# tf.all_task_done = False
# for ip_port in cache.ip_port_pool:
# q.put({'ip_port': ip_port, 'useful_list': useful_list})
# q.join()
# tf.all_task_done = True
for ip_port in cache.ip_port_pool:
logger.debug('测试ip_port:{0}'.format(ip_port['ip_with_port']))
is_useful = pickle.loads(
fanqiang_service_client.run(
Transformer().FanqiangService().test_useful_fanqiang(
ip_port).done()).data)
if is_useful:
logger.debug('{0}能用'.format(ip_port['ip_with_port']))
useful_list.append(ip_port)
cache.ip_port_pool = useful_list
# 如果cache.ip_port_pool不及预期,获取能翻墙的ip_port
if len(cache.ip_port_pool) < int(
Conf.get('IP_PORT_POOL', 'bak_num')):
new_num += 1
logger.debug('pool数量{0}达不到要求的{1}'.format(
len(cache.ip_port_pool), Conf.get('IP_PORT_POOL',
'bak_num')))
ip_port_list = pickle.loads(
fanqiang_service_client.run(Transformer().FanqiangService(
).get_useful_fanqiang_ip_port_from_mongo(
new_num).done()).data)
cache.ip_port_pool.extend(ip_port_list)
# ------------去重------------
ip_port_map = {}
for ip_port_dict in cache.ip_port_pool:
ip_port_map[ip_port_dict['ip_with_port']] = ip_port_dict
new_ip_list = []
for ip_port_dict in ip_port_map.values():
new_ip_list.append(ip_port_dict)
cache.ip_port_pool = new_ip_list
# ------------去重------------
else:
new_num -= 1
# 把能用的写到文件里面
with open('file/bak_pool', 'wb') as fw:
try:
logger.debug('写到的数量为：{0}'.format(len(cache.ip_port_pool)))
pickle.dump(cache.ip_port_pool, fw)
except EOFError:
pass
except Exception:
logger.error(traceback.format_exc())
# 每分钟检查一次
time.sleep(60)
except Exception:
logger.error(traceback.format_exc())
logger.error('get_ip_port_to_pool 线程错误关闭')
get_ip_port_to_pool()
