def checking(self, sql):
try:
fetchall = MySql(DB_ADDRESS, DB_USER, DB_PASS, DB_DATABASE,
DB_CHARSET).query(sql)
if fetchall is None:
sql = 'SELECT * FROM httpbin WHERE alive=0 ORDER BY update_time'
fetchall = MySql(DB_ADDRESS, DB_USER, DB_PASS, DB_DATABASE,
DB_CHARSET).query(sql)
if fetchall is None:
time.sleep(10)
return
for each in fetchall:
each_id = each[0]
proxy = each[1] + ':' + each[2]
verify_count = int(each[12]) + 1
r = VerifyProxy().validate_proxy(proxy,
protocol='http',
timeout=3)
threadLock.acquire()
if isinstance(r, str):
# 失效
sql = 'SELECT leave_count FROM httpbin WHERE id={0}'.format(
each_id)
fetchone = MySql(DB_ADDRESS, DB_USER, DB_PASS, DB_DATABASE,
DB_CHARSET).query_one(sql)
alive = '0'
leave_count = int(fetchone[0]) - 1
if leave_count <= 0:
# 尝试20次之后，删除该条代理
sql = 'DELETE FROM httpbin WHERE id={0}'.format(
each_id)
Util.log_error(sql)
else:
# 更新
sql = "UPDATE httpbin SET verify_count={0}, leave_count={1}, alive={2} WHERE id={3}" \
.format(verify_count, leave_count, alive, each_id)
Util.log_info(sql)
MySql(DB_ADDRESS, DB_USER, DB_PASS, DB_DATABASE,
DB_CHARSET).execute(sql)
elif isinstance(r, dict):
if 'exception' not in r.keys():
alive = '1'
leave_count = 20
speed = r['timedelta']
# 暂不重复验证是否支持HTTPS
# result_https = VerifyProxy().validate_proxy(proxy, 'https', timeout=3)
sql = "UPDATE httpbin SET verify_count={0}, speed={1}, alive={2}, leave_count={3} WHERE id={4}" \
.format(verify_count, speed, alive, leave_count, each_id)
Util.log_info(sql)
MySql(DB_ADDRESS, DB_USER, DB_PASS, DB_DATABASE,
DB_CHARSET).execute(sql)
threadLock.release()
except Exception as e:
Util.log_error(e)
async def verify_and_save(proxy, source):
try:
r = VerifyProxy().validate_proxy(proxy, protocol='http', timeout=3)
if isinstance(r, str):
r = json.loads(r)
if isinstance(r, dict):
if 'exception' not in r.keys():
https = '0'
ip = proxy.split(':')[0]
port = proxy.split(':')[1]
speed = r['timedelta']
origin = r['origin']
if origin == ip:
# 高匿
anonymity = '2'
else:
# 透明
anonymity = '0'
country = VerifyProxy().country_proxy(ip)
result_https = VerifyProxy().validate_proxy(proxy,
'https',
timeout=3)
if not isinstance(result_https, str):
https = '1'
# 1. 查找是否已存在该IP和Port
sql = "SELECT * FROM httpbin WHERE ip = '{0}' AND port = '{1}'".format(
ip, port)
fetchone = MySql(DB_ADDRESS, DB_USER, DB_PASS, DB_DATABASE,
DB_CHARSET).query_one(sql)
if fetchone is None:
# 2. 插入数据库中没有的IP
sql = "INSERT INTO httpbin(ip, port, https, anonymity, country, speed, source, " \
"insert_time) VALUES ('{0}', '{1}', '{2}', '{3}', '{4}', '{5}', '{6}', '{7}')" \
.format(ip, port, https, anonymity, country, speed, source, Util.get_current_time())
Util.log_info('Save Proxy ' + proxy + ' From ' +
source)
MySql(DB_ADDRESS, DB_USER, DB_PASS, DB_DATABASE,
DB_CHARSET).execute(sql)
except Exception as e:
# Util.log_error(e)
pass