def run(self):
global new_checked_proxy_queue
while True:
try:
if new_checked_proxy_queue.qsize() >= 1:
checked_proxy_item = new_checked_proxy_queue.get()
used_label = checked_proxy_item.get('used', False)
if used_label:
print('{} new add 高匿ip 1个'.format(self.name, ))
else:
continue
sqlite3_cli = BaseSqlite3Cli(db_path=db_path)
_insert_into_db(
checked_proxy_list=[
checked_proxy_item,
],
sqlite3_cli=sqlite3_cli,
)
else:
continue
except Exception as e:
print(e)
def _select_all_proxy_data_in_db(sqlite3_cli: BaseSqlite3Cli=None) -> list:
"""
返回db中所有proxy item
:return:
"""
cursor = sqlite3_cli._execute(sql_str=select_sql_str)
db_old_data = cursor.fetchall()
cursor.close()
return db_old_data
def look_up_db_all_proxy_data() -> list:
'''
查看db中所有proxy
:return:
'''
sqlite_cli = BaseSqlite3Cli(db_path='proxy.db')
sql_str = 'select * from proxy_obj_table'
cursor = sqlite_cli._execute(sql_str=sql_str)
all = cursor.fetchall()
cursor.close()
return all
def _delete_proxy_in_db(ip, sqlite3_cli: BaseSqlite3Cli=None,) -> bool:
"""
删除db中的ip
:param sqlite3_cli:
:param ip:
:return:
"""
cursor = sqlite3_cli._execute(sql_str=delete_sql_str, params=(ip,))
res = True if cursor.rowcount > 0 else False
# print('[{}] {} delete {}'.format('+' if res else '-', ip, 'success' if res else 'error'))
cursor.close()
return res
def empty_db_proxy_data() -> None:
'''
清空db原有proxy数据
:return:
'''
sqlite_cli = BaseSqlite3Cli(db_path='proxy.db')
sql_str = 'delete from proxy_obj_table'
cursor = sqlite_cli._execute(sql_str=sql_str)
row_count = cursor.rowcount
cursor.close()
print('已影响行数: {}'.format(row_count))
return None
def _update_score_in_db(ip,
score,
check_time,
sqlite3_cli: BaseSqlite3Cli=None,) -> bool:
"""
更新db中的score
:return:
"""
cursor = sqlite3_cli._execute(sql_str=update_sql_str, params=(score, check_time, ip,))
res = True if cursor.rowcount > 0 else False
# print('[{}] {} update {} {}'.format('+' if res else '-', ip, score, check_time))
cursor.close()
return res
def run(self):
while True:
try:
sqlite3_cli = BaseSqlite3Cli(db_path=db_path)
db_proxy_item_list = _select_all_proxy_data_in_db(
sqlite3_cli=sqlite3_cli)
# 删除过期ip
delete_count = 0
activity_time = _get_ip_activity_time(id=tri_id)
# print(activity_time)
if isinstance(db_proxy_item_list, (list, tuple)):
for proxy_item in db_proxy_item_list:
ip = proxy_item[1]
score = proxy_item[3]
check_time = string_to_datetime(proxy_item[5])
now_time = get_shanghai_time()
# 两个判断条件分开: 针对不同情况进行优先级判断
time_diff = datetime_to_timestamp(
now_time) - datetime_to_timestamp(check_time)
if time_diff < activity_time - 10:
continue
if score < MIN_SCORE:
continue
# print('now_time: {}, item_check_time: {}'.format(
# now_time,
# check_time, ))
delete_res = _delete_proxy_in_db(
ip=ip, sqlite3_cli=sqlite3_cli)
if delete_res:
delete_count += 1
else:
pass
else:
pass
if delete_count > 0:
print('@@@ {}过期消费者 delete_ip_count: {}'.format(
self.name, delete_count))
else:
pass
except Exception as e:
print(e)
finally:
sleep(6.)
def __init__(self, tri_id, *params, **kwargs):
AsyncCrawler.__init__(
self,
*params,
**kwargs,
)
self.concurrency = CONCURRENCY_NUM
self.local_ip = '' # 本地ip值
self.checked_proxy_list = [] # 已被验证的proxy的list
self.CHECKED_PROXY_SLEEP_TIME = CHECKED_PROXY_SLEEP_TIME
self.sqlite3_cli = BaseSqlite3Cli(db_path='proxy.db')
self.tri_id = tri_id # 三方的id
self.concurrent_type = 0
proxy_checker_welcome_page()
def __init__(self, tri_id, *params, **kwargs):
AsyncCrawler.__init__(
self,
*params,
**kwargs,
)
self.concurrency = CONCURRENCY_NUM
self.local_ip = '' # 本地ip值
self.checked_proxy_list = [] # 已被验证的proxy的list
self.score = INIT_SCORE
self.CHECKED_PROXY_SLEEP_TIME = CHECKED_PROXY_SLEEP_TIME
self.MIN_IP_POOl_NUM = MIN_IP_POOl_NUM
self.MIN_SCORE = MIN_SCORE
self._init_base_sql_str()
self.sqlite3_cli = BaseSqlite3Cli(db_path='proxy.db')
self.tri_id = tri_id # 三方的id
def create_proxy_obj_table() -> bool:
'''
if not exist 则 create proxy_obj_table
:return:
'''
res = False
sqlite_cli = BaseSqlite3Cli(db_path='proxy.db')
sql_str_1 = 'select name from sqlite_master where type=? and name=? order by name'
cursor = sqlite_cli._execute(sql_str=sql_str_1, params=('table', 'proxy_obj_table'))
_ = cursor.fetchall()
cursor.close()
# print(_)
if _ != []:
print('proxy_obj_table表已存在!')
else:
print('创建proxy_obj_table...')
sql_str = '''
create table proxy_obj_table (
id integer primary key autoincrement,
ip varchar (100) not null unique,
port int not null,
score int not null,
agency_agreement varchar (20) not null,
check_time datetime not null
);
'''
cursor = sqlite_cli._execute(sql_str=sql_str)
cursor.close()
cursor = sqlite_cli._execute(sql_str=sql_str_1, params=('table', 'proxy_obj_table'))
_ = cursor.fetchall()
cursor.close()
# print(_)
if _ != []:
res = True
return res
def _insert_into_db(checked_proxy_list: list, sqlite3_cli: BaseSqlite3Cli = None) -> None:
"""
插入db
:return:
"""
db_old_data = _select_all_proxy_data_in_db(sqlite3_cli=sqlite3_cli)
db_ip_list = [i[1] for i in db_old_data]
for item in checked_proxy_list:
ip = item['ip']
if ip not in db_ip_list:
params = _get_insert_params(item)
# pprint(params)
cursor = sqlite3_cli._execute(
sql_str=insert_sql_str,
params=params)
res = True if cursor.rowcount > 0 else False
cursor.close()
# print('[{}] {}插入db中.'.format('+' if res else '-', item['ip']))
else:
pass
return
from os import getcwd
from settings import (
SERVER_PORT, )
from json import dumps
from pprint import pprint
from fzutils.sql_utils import BaseSqlite3Cli
try:
from gevent.wsgi import WSGIServer # 高并发部署
except Exception as e:
from gevent.pywsgi import WSGIServer
app = Flask(__name__, root_path=getcwd())
sqlite3_cli = BaseSqlite3Cli(db_path='proxy.db')
select_sql_str = 'select * from proxy_obj_table'
@app.route('/', methods=['GET', 'POST'])
def home():
return '欢迎来到 proxy checker 主页!'
@app.route('/get_all', methods=['GET', 'POST'])
def get_proxy_list():
'''
获取代理的接口
:return:
'''
res = []
from json import dumps
from pprint import pprint
from fzutils.sql_utils import BaseSqlite3Cli
from fzutils.gevent_utils import gevent_monkey
from fzutils.common_utils import get_random_int_number
try:
from gevent.wsgi import WSGIServer # 高并发部署
except Exception as e:
from gevent.pywsgi import WSGIServer
gevent_monkey.patch_all()
app = Flask(__name__, root_path=getcwd())
sqlite3_cli0 = BaseSqlite3Cli(db_path='proxy.db')
sqlite3_cli1 = BaseSqlite3Cli(db_path='proxy.db')
sqlite3_cli2 = BaseSqlite3Cli(db_path='proxy.db')
sqlite3_cli3 = BaseSqlite3Cli(db_path='proxy.db')
sqlite3_cli4 = BaseSqlite3Cli(db_path='proxy.db')
select_sql_str = 'select * from proxy_obj_table'
@app.route('/', methods=['GET', 'POST'])
def home():
return '欢迎来到 proxy checker 主页!'
@app.route('/get_all', methods=['GET', 'POST'])
def get_proxy_list():