def update_chrome_pac_by_gatherproxy(self): ''' 从proxies.txt里面检测ip_port,最后更新chrome用的pac :return: ''' FileService.merge_proxy() with open('file/proxy_file/proxies.txt', 'r') as fr: try: ip_port_list = fr.read().split('\n') except Exception: logger.error(traceback.format_exc()) return finally: fr.close() q = queue.Queue() ip_port_list = list(set(ip_port_list)) # 去重 for ip_with_port in ip_port_list: ip_dict = { 'ip_with_port': ip_with_port, 'proxy_type': 'socks5', } q.put(ip_dict) for i in range(20): t = threading.Thread( target=self.__get_useful_fanqiang_ip_gatherproxy_worker, args=(q, )) t.start() q.join() # 跑完,吧proxy文件删了 os.remove('file/proxy_file/proxies.txt')
def run(method_with_params_list): ''' 代理方法,需要在全局维护一个registest_list :param method_with_params_list: :return: ''' def do_chain(method, params_list, current_step): method_name = method.replace('()', '') # 判断有没有这属性 if not hasattr(current_step, method_name): raise Exception('{0}没有属性{1}'.format(str(current_step), method_name)) if re.findall(r'\(\)', method): # 是方法 this_params = params_list.pop(0) this_args = this_params['args'] this_kwargs = this_params['kwargs'] current_step = getattr(current_step, method_name)(*this_args, **this_kwargs) else: current_step = getattr(current_step, method_name) return current_step method = method_with_params_list[0] params_list = method_with_params_list[1] try: current_step = None logger.info('-------------------处理 {0},参数:{1}'.format( method, params_list)) first_method = method.split('.')[0] first_method_name = first_method.replace('()', '') # 判断是否已经注册该方法/类 if not first_method_name in register_list: raise Exception('类/方法%s没有注册' % first_method_name) else: first_registed_method_or_obj = register_list[first_method_name] if re.findall(r'\(\)', first_method): # 是方法 this_params = params_list.pop(0) this_args = this_params['args'] this_kwargs = this_params['kwargs'] current_step = first_registed_method_or_obj( *this_args, **this_kwargs) else: # 不是方法 current_step = first_registed_method_or_obj for m in method.split('.')[1:]: current_step = do_chain(m, params_list, current_step) return current_step except Exception as e: logger.error('{0} -------------- {1}'.format(method, traceback.format_exc())) raise e
def get_client_error(): ''' 获取有用的ip_port :return: ''' try: data = request.args.to_dict() account = data['account'] error = data['error'] logger.error('Client Error : {0}------{1}--------{2}'.format( account, request.environ['REMOTE_ADDR'], error)) return Response(status=200) except Exception: logger.error(traceback.format_exc())
def modify_chrome_pac_file_and_push(self, ip_with_port): ''' 更新pac文件并提交(加锁) :param ip_with_port: :return: ''' def modify_chrome_file(file_path, ip_with_port): # 替换ip和port new_text = '' with open(file_path, 'r', encoding='utf-8') as fr: try: old_text = fr.read() new_text = old_text.replace( re.findall( r'(?:SOCKS |SOCKS5 )(\d+\.\d+\.\d+\.\d+:\d+)', old_text)[0], ip_with_port) new_text = new_text.replace( re.findall( r'(?:SOCKS |SOCKS5 )(\d+\.\d+\.\d+\.\d+:\d+)', old_text)[1], ip_with_port) finally: fr.close() with open(file_path, 'w', encoding='utf-8') as fw: try: fw.write(new_text) finally: fw.close() logger.debug("已更新文件 %s,ip_port为:%s" % (file_path, ip_with_port)) try: io_lock.acquire() global is_ok_at_least_one if is_ok_at_least_one: return modify_chrome_file(file_path_chrome, ip_with_port) modify_chrome_file(file_path_chrome_socks, ip_with_port) Git.git_push(file_path_chrome) Git.git_push(file_path_chrome_socks) is_ok_at_least_one = True return except Exception: logger.error(traceback.format_exc()) finally: io_lock.release()
def get_pac(): # request_id = request.args['uuid'] try: pac_type = request.args['type'] filename, pac = fanqiang_ip_service.get_dynamic_pac(pac_type) response = make_response(pac) response.headers['Content-Type'] = 'application/x-ns-proxy-autoconfig' response.headers[ 'Content-Disposition'] = 'attachment; filename={}'.format( filename.encode().decode('latin-1')) return response except Exception as e: logger.error("%s" % (traceback.format_exc())) # return Response("服务器出错,请联系管理员,请求id是:%s" % request_id,status=500) abort(500)
def add_to_useful(ip_port, useful_list): try: logger.debug('测试ip_port:{0}'.format(ip_port['ip_with_port'])) is_useful = pickle.loads( fanqiang_service_client.run( Transformer().FanqiangService().test_useful_fanqiang( ip_port).done()).data) if is_useful: try: lock.acquire() logger.debug('{0}能用'.format(ip_port['ip_with_port'])) useful_list.append(ip_port) finally: lock.release() except Exception: logger.error(traceback.format_exc())
def get_ip_port(): ''' 获取有用的ip_port :return: ''' try: data = request.args.to_dict() if not (data['account'] == 'CK_test' and data['password'] == 'CK_test'): abort(401) ip_port_dict = fanqiang_ip_service.get_ip_port_dict_list(1)[0] resp = Response(ip_port_dict['ip_with_port'], mimetype='text/plain', status=200) return resp except Exception: logger.error(traceback.format_exc())
def save(ip_with_port, proxy_type): try: elite = FanqiangService.test_elite(ip_with_port, proxy_type) except Exception as e: logger.info(str(e)) return try: lock.acquire() if elite: db_client.run(Transformer().Fanqiang().save({ 'proxy_type': proxy_type, 'ip_with_port': ip_with_port, 'time': 0.00, 'location': FanqiangService.get_location( ip_with_port.split(':')[0]), 'Elite': elite }).done()) else: db_client.run(Transformer().Fanqiang().save({ 'proxy_type': proxy_type, 'ip_with_port': ip_with_port, 'time': 0.00, 'location': FanqiangService.get_location( ip_with_port.split(':')[0]) }).done()) except Exception as e: logger.error(e) finally: lock.release()
def save_all_from_gatherproxy_to_db(self): ''' 保存收集到的所有的gather_proxydoa数据库,不筛选 :return: ''' self.merge_proxy() with open('file/proxy_file/proxies.txt', 'r') as fr: try: ip_port_list = fr.read().split('\n') except Exception: logger.error(traceback.format_exc()) return finally: fr.close() def save(ip_with_port, proxy_type): try: elite = FanqiangService.test_elite(ip_with_port, proxy_type) except Exception as e: logger.info(str(e)) return try: lock.acquire() if elite: db_client.run(Transformer().Fanqiang().save({ 'proxy_type': proxy_type, 'ip_with_port': ip_with_port, 'time': 0.00, 'location': FanqiangService.get_location( ip_with_port.split(':')[0]), 'Elite': elite }).done()) else: db_client.run(Transformer().Fanqiang().save({ 'proxy_type': proxy_type, 'ip_with_port': ip_with_port, 'time': 0.00, 'location': FanqiangService.get_location( ip_with_port.split(':')[0]) }).done()) except Exception as e: logger.error(e) finally: lock.release() q = queue.Queue() tf = thread_utils.ThreadFactory() for i in range(20): t = threading.Thread(target=tf.queue_threads_worker, args=(q, save)) t.start() for ip_with_port in ip_port_list: q.put({'ip_with_port': ip_with_port, 'proxy_type': 'socks5'}) q.join() tf.all_task_done = True os.remove('file/proxy_file/proxies.txt')
def get_ip_port_to_pool(): ''' 定时获取ip_port到备用池 :return: ''' try: lock = threading.Lock() def add_to_useful(ip_port, useful_list): try: logger.debug('测试ip_port:{0}'.format(ip_port['ip_with_port'])) is_useful = pickle.loads( fanqiang_service_client.run( Transformer().FanqiangService().test_useful_fanqiang( ip_port).done()).data) if is_useful: try: lock.acquire() logger.debug('{0}能用'.format(ip_port['ip_with_port'])) useful_list.append(ip_port) finally: lock.release() except Exception: logger.error(traceback.format_exc()) # 先读取存下来的pool with open('file/bak_pool', 'rb') as fr: try: cache.ip_port_pool = pickle.load(fr) logger.debug('从文件读到的数量为:{0}'.format(len(cache.ip_port_pool))) except EOFError: pass except Exception: logger.error(traceback.format_exc()) new_num = int(Conf.get('IP_PORT_POOL', 'new_num')) while True: useful_list = [] ## 开20个线程筛掉之前没用的ip_port(由于xmlrpc不支持并发,所以有问题) # q = queue.Queue() # tf = thread_utils.ThreadFactory() # for i in range(20): # t = threading.Thread(target=tf.queue_threads_worker, args=(q, add_to_useful)) # t.start() # tf.all_task_done = False # for ip_port in cache.ip_port_pool: # q.put({'ip_port': ip_port, 'useful_list': useful_list}) # q.join() # tf.all_task_done = True for ip_port in cache.ip_port_pool: logger.debug('测试ip_port:{0}'.format(ip_port['ip_with_port'])) is_useful = pickle.loads( fanqiang_service_client.run( Transformer().FanqiangService().test_useful_fanqiang( ip_port).done()).data) if is_useful: logger.debug('{0}能用'.format(ip_port['ip_with_port'])) useful_list.append(ip_port) cache.ip_port_pool = useful_list # 如果cache.ip_port_pool不及预期,获取能翻墙的ip_port if len(cache.ip_port_pool) < int( Conf.get('IP_PORT_POOL', 'bak_num')): new_num += 1 logger.debug('pool数量{0}达不到要求的{1}'.format( len(cache.ip_port_pool), Conf.get('IP_PORT_POOL', 'bak_num'))) ip_port_list = pickle.loads( fanqiang_service_client.run(Transformer().FanqiangService( ).get_useful_fanqiang_ip_port_from_mongo( new_num).done()).data) cache.ip_port_pool.extend(ip_port_list) # ------------去重------------ ip_port_map = {} for ip_port_dict in cache.ip_port_pool: ip_port_map[ip_port_dict['ip_with_port']] = ip_port_dict new_ip_list = [] for ip_port_dict in ip_port_map.values(): new_ip_list.append(ip_port_dict) cache.ip_port_pool = new_ip_list # ------------去重------------ else: new_num -= 1 # 把能用的写到文件里面 with open('file/bak_pool', 'wb') as fw: try: logger.debug('写到的数量为:{0}'.format(len(cache.ip_port_pool))) pickle.dump(cache.ip_port_pool, fw) except EOFError: pass except Exception: logger.error(traceback.format_exc()) # 每分钟检查一次 time.sleep(60) except Exception: logger.error(traceback.format_exc()) logger.error('get_ip_port_to_pool 线程错误关闭') get_ip_port_to_pool()