async def test_single_proxy(proxy):
'''Test single proxy
args:
redis: name of redisDB
proxy: proxy
return:
None
'''
redis = RedisClient()
conn = aiohttp.TCPConnector(verify_ssl=False)
async with aiohttp.ClientSession(connector=conn) as seesion:
try:
if isinstance(proxy, bytes):
proxy = proxy.decode('utf-8')
real_proxy = 'http://' + proxy
print('Testing ', proxy)
async with seesion.get(TEST_URL, proxy=real_proxy,
timeout=15) as response:
if response.status in VALID_STATUS_CODE:
redis.max(proxy)
print('Proxy ', proxy, ' is valid, set max score.')
else:
redis.decrease(proxy)
print('Response code is not valid. Proxy ', proxy,
' is invalid, decrease score.')
except:
redis.decrease(proxy)
print('Proxy response failed. Proxy ', proxy,
' is invalid, decrease score.')
def get_conn():
'''
Get redis client object
'''
if not hasattr(g, 'redis'):
g.redis = RedisClient()
return g.redis
def tester_run():
print(' Tester is running...')
redis = RedisClient()
count = redis.count()
proxy_batchs = []
for i in range(0, count, BATCH_TEST_SIZE):
start, end = i, min(i + BATCH_TEST_SIZE, count) - 1
proxy_batchs.append(redis.batch(start, end))
pool = ThreadPool()
pool.map(test_batch_proxies, proxy_batchs)
pool.close()
pool.join()
def __init__(self):
self.redis = RedisClient()
def __init__(self):
self.db = RedisClient()
def __init__(self):
self.redis = RedisClient()
self.crawler = Crawler()
def __init__(self, website='default'):
self.website = website
self.cookies_db = RedisClient('cookies', self.website)
self.accounts_db = RedisClient('accounts', self.website)
from RedisDB import RedisClient
conn = RedisClient()
def set(proxy):
result = conn.add(proxy)
print(proxy)
print('录入成功' if result else '录入失败')
def scan():
print('请输入代理, 输入exit退出读入')
while True:
proxy = input()
if proxy == 'exit':
break
set(proxy)
if __name__ == '__main__':
scan()
import time
from multiprocessing import Process
from api import app
from generator import GithubCookiesGenerator
from tester import GithubValidTester
from RedisDB import RedisClient
if __name__ == "__main__":
redis = RedisClient('accounts', 'github')
redis.set('jsrglc', 'liuchennuaa2010')
redis.set('js', 'liu')
print('Cookies 生成进程开始运行 ')
generator = GithubCookiesGenerator()
generator.run()
print('Cookies 生成完成 ')
print('Cookies 检测进程开始运行 ')
tester = GithubValidTester()
tester.run()
print('Cookies 检测完成 ')
app.run()