def process_res():
"""
验证ip有效性，并处理结果
:return:
"""
db = DBHelper()
ip_list = db.get_all_ips()
ip_list_copy = ip_list.copy()
ip_valid_list = []
threads = []
# 创建处理ip结果的线程
for i in range(50):
threads.append(VerifyIp(ip_list, ip_valid_list))
# 开始所有线程
for thread in threads:
thread.start()
# 等待所有线程完成
for thread in threads:
thread.join()
for ip in ip_list_copy:
if ip not in ip_valid_list:
db.delete_one_ip(ip)
print(ip_valid_list, len(ip_valid_list))
def check_user(username):
"""
@summary: 校验用户名和密码
@param username: 用户名
@return: True/False
"""
db = None
count = 0
try:
db = DBHelper()
sql = "select count(*) from user_list where username=%s"
data = db.query_one(sql, username)
count = data[0]
except Exception as e:
logger.debug('check user failure:' + str(e) + '. ' + traceback.format_exc())
raise
finally:
if db:
db.release()
return count == 1
def validate_user(username, password):
"""
@summary: 校验用户名和密码
@param username: 用户名
@param password: 密码
@return: True/False
"""
db = None
count = 0
try:
db = DBHelper()
sql = "select count(*) from user_list where username=%s and password=%s"
data = db.query_one(sql, (username, hashlib.sha256(str(password).encode('utf-8')).hexdigest()))
count = data[0]
except Exception as e:
logger.debug('validate user failure:' + str(e) + '. ' + traceback.format_exc())
raise
finally:
if db:
db.release()
return count == 1
def process_res(ip_list):
"""
验证ip有效性，并处理结果
:param res:
:return:
"""
ip_valid_list = []
threads = []
# 创建处理ip结果的线程
for i in range(50):
threads.append(VerifyIp(ip_list, ip_valid_list))
# 开始所有线程
for thread in threads:
thread.start()
# 等待所有线程完成
for thread in threads:
thread.join()
db = DBHelper()
db.insert_many_ips(ip_valid_list)
print(ip_valid_list)
# coding: utf-8
from utils.requests_kit import get_content_by_proxy
from utils.db_helper import DBHelper
from utils.logger import logger
url = 'https://www.cnblogs.com/time-read/p/9397615.html'
db = DBHelper()
headers = {'referer': 'https://www.cnblogs.com/'}
proxies_ip = db.get_all_ips()
url = 'https://www.nyloner.cn/checkip'
for i in proxies_ip:
headers.update({"X-Forwarded-For": "{}".format(i.split(':')[0])})
res = get_content_by_proxy(url, i, headers=headers)
if res:
print(i)
print(res.text)
print(res.status_code)
else:
print('failed')
from utils.db_helper import DBHelper
from items import PgcScrapyItem
def callback():
print()
print('error!')
print()
if __name__ == "__main__":
db = DBHelper(callback, '127.0.0.1', 'root', '', 'pgc')
# item = PgcScrapyItem(url='atest', v_title='', v_, v_actors=['a1', 'a2'], v_directors=[])
for url in ['1', '2', '3', '4', '1', '5']:
item = PgcScrapyItem(url=url,
v_title='v_title',
v_lang='v_lang',
v_tags=['asd'],
v_type='v_type',
v_actors=['a2', 'b2', 'uad'],
v_directors=['d1', 'd2', 'qdg'],
v_time=2000,
v_score=9.0,
v_area='v_area',
v_desc='v_desc')
db.handleVideo(item)
db.handlePeople(item)