def run(self): global new_checked_proxy_queue while True: try: if new_checked_proxy_queue.qsize() >= 1: checked_proxy_item = new_checked_proxy_queue.get() used_label = checked_proxy_item.get('used', False) if used_label: print('{} new add 高匿ip 1个'.format(self.name, )) else: continue sqlite3_cli = BaseSqlite3Cli(db_path=db_path) _insert_into_db( checked_proxy_list=[ checked_proxy_item, ], sqlite3_cli=sqlite3_cli, ) else: continue except Exception as e: print(e)
def _select_all_proxy_data_in_db(sqlite3_cli: BaseSqlite3Cli=None) -> list: """ 返回db中所有proxy item :return: """ cursor = sqlite3_cli._execute(sql_str=select_sql_str) db_old_data = cursor.fetchall() cursor.close() return db_old_data
def look_up_db_all_proxy_data() -> list: ''' 查看db中所有proxy :return: ''' sqlite_cli = BaseSqlite3Cli(db_path='proxy.db') sql_str = 'select * from proxy_obj_table' cursor = sqlite_cli._execute(sql_str=sql_str) all = cursor.fetchall() cursor.close() return all
def _delete_proxy_in_db(ip, sqlite3_cli: BaseSqlite3Cli=None,) -> bool: """ 删除db中的ip :param sqlite3_cli: :param ip: :return: """ cursor = sqlite3_cli._execute(sql_str=delete_sql_str, params=(ip,)) res = True if cursor.rowcount > 0 else False # print('[{}] {} delete {}'.format('+' if res else '-', ip, 'success' if res else 'error')) cursor.close() return res
def empty_db_proxy_data() -> None: ''' 清空db原有proxy数据 :return: ''' sqlite_cli = BaseSqlite3Cli(db_path='proxy.db') sql_str = 'delete from proxy_obj_table' cursor = sqlite_cli._execute(sql_str=sql_str) row_count = cursor.rowcount cursor.close() print('已影响行数: {}'.format(row_count)) return None
def _update_score_in_db(ip, score, check_time, sqlite3_cli: BaseSqlite3Cli=None,) -> bool: """ 更新db中的score :return: """ cursor = sqlite3_cli._execute(sql_str=update_sql_str, params=(score, check_time, ip,)) res = True if cursor.rowcount > 0 else False # print('[{}] {} update {} {}'.format('+' if res else '-', ip, score, check_time)) cursor.close() return res
def run(self): while True: try: sqlite3_cli = BaseSqlite3Cli(db_path=db_path) db_proxy_item_list = _select_all_proxy_data_in_db( sqlite3_cli=sqlite3_cli) # 删除过期ip delete_count = 0 activity_time = _get_ip_activity_time(id=tri_id) # print(activity_time) if isinstance(db_proxy_item_list, (list, tuple)): for proxy_item in db_proxy_item_list: ip = proxy_item[1] score = proxy_item[3] check_time = string_to_datetime(proxy_item[5]) now_time = get_shanghai_time() # 两个判断条件分开: 针对不同情况进行优先级判断 time_diff = datetime_to_timestamp( now_time) - datetime_to_timestamp(check_time) if time_diff < activity_time - 10: continue if score < MIN_SCORE: continue # print('now_time: {}, item_check_time: {}'.format( # now_time, # check_time, )) delete_res = _delete_proxy_in_db( ip=ip, sqlite3_cli=sqlite3_cli) if delete_res: delete_count += 1 else: pass else: pass if delete_count > 0: print('@@@ {}过期消费者 delete_ip_count: {}'.format( self.name, delete_count)) else: pass except Exception as e: print(e) finally: sleep(6.)
def __init__(self, tri_id, *params, **kwargs): AsyncCrawler.__init__( self, *params, **kwargs, ) self.concurrency = CONCURRENCY_NUM self.local_ip = '' # 本地ip值 self.checked_proxy_list = [] # 已被验证的proxy的list self.CHECKED_PROXY_SLEEP_TIME = CHECKED_PROXY_SLEEP_TIME self.sqlite3_cli = BaseSqlite3Cli(db_path='proxy.db') self.tri_id = tri_id # 三方的id self.concurrent_type = 0 proxy_checker_welcome_page()
def __init__(self, tri_id, *params, **kwargs): AsyncCrawler.__init__( self, *params, **kwargs, ) self.concurrency = CONCURRENCY_NUM self.local_ip = '' # 本地ip值 self.checked_proxy_list = [] # 已被验证的proxy的list self.score = INIT_SCORE self.CHECKED_PROXY_SLEEP_TIME = CHECKED_PROXY_SLEEP_TIME self.MIN_IP_POOl_NUM = MIN_IP_POOl_NUM self.MIN_SCORE = MIN_SCORE self._init_base_sql_str() self.sqlite3_cli = BaseSqlite3Cli(db_path='proxy.db') self.tri_id = tri_id # 三方的id
def create_proxy_obj_table() -> bool: ''' if not exist 则 create proxy_obj_table :return: ''' res = False sqlite_cli = BaseSqlite3Cli(db_path='proxy.db') sql_str_1 = 'select name from sqlite_master where type=? and name=? order by name' cursor = sqlite_cli._execute(sql_str=sql_str_1, params=('table', 'proxy_obj_table')) _ = cursor.fetchall() cursor.close() # print(_) if _ != []: print('proxy_obj_table表已存在!') else: print('创建proxy_obj_table...') sql_str = ''' create table proxy_obj_table ( id integer primary key autoincrement, ip varchar (100) not null unique, port int not null, score int not null, agency_agreement varchar (20) not null, check_time datetime not null ); ''' cursor = sqlite_cli._execute(sql_str=sql_str) cursor.close() cursor = sqlite_cli._execute(sql_str=sql_str_1, params=('table', 'proxy_obj_table')) _ = cursor.fetchall() cursor.close() # print(_) if _ != []: res = True return res
def _insert_into_db(checked_proxy_list: list, sqlite3_cli: BaseSqlite3Cli = None) -> None: """ 插入db :return: """ db_old_data = _select_all_proxy_data_in_db(sqlite3_cli=sqlite3_cli) db_ip_list = [i[1] for i in db_old_data] for item in checked_proxy_list: ip = item['ip'] if ip not in db_ip_list: params = _get_insert_params(item) # pprint(params) cursor = sqlite3_cli._execute( sql_str=insert_sql_str, params=params) res = True if cursor.rowcount > 0 else False cursor.close() # print('[{}] {}插入db中.'.format('+' if res else '-', item['ip'])) else: pass return
from os import getcwd from settings import ( SERVER_PORT, ) from json import dumps from pprint import pprint from fzutils.sql_utils import BaseSqlite3Cli try: from gevent.wsgi import WSGIServer # 高并发部署 except Exception as e: from gevent.pywsgi import WSGIServer app = Flask(__name__, root_path=getcwd()) sqlite3_cli = BaseSqlite3Cli(db_path='proxy.db') select_sql_str = 'select * from proxy_obj_table' @app.route('/', methods=['GET', 'POST']) def home(): return '欢迎来到 proxy checker 主页!' @app.route('/get_all', methods=['GET', 'POST']) def get_proxy_list(): ''' 获取代理的接口 :return: ''' res = []
from json import dumps from pprint import pprint from fzutils.sql_utils import BaseSqlite3Cli from fzutils.gevent_utils import gevent_monkey from fzutils.common_utils import get_random_int_number try: from gevent.wsgi import WSGIServer # 高并发部署 except Exception as e: from gevent.pywsgi import WSGIServer gevent_monkey.patch_all() app = Flask(__name__, root_path=getcwd()) sqlite3_cli0 = BaseSqlite3Cli(db_path='proxy.db') sqlite3_cli1 = BaseSqlite3Cli(db_path='proxy.db') sqlite3_cli2 = BaseSqlite3Cli(db_path='proxy.db') sqlite3_cli3 = BaseSqlite3Cli(db_path='proxy.db') sqlite3_cli4 = BaseSqlite3Cli(db_path='proxy.db') select_sql_str = 'select * from proxy_obj_table' @app.route('/', methods=['GET', 'POST']) def home(): return '欢迎来到 proxy checker 主页!' @app.route('/get_all', methods=['GET', 'POST']) def get_proxy_list():