async def get_animequeue(self):
db.execute('SELECT a.ANIME_ID,a.ANIME_IMAGE FROM anime_home a', None)
records = db.fetchall()
animequeue = asyncio.Queue()
if records:
for r in records:
await animequeue.put(r)
self.animequeue = animequeue
if not os.path.exists(self.file_path):
print('文件夹', self.file_path, '不存在，重新建立')
os.makedirs(self.file_path)
return animequeue
def getanimeimgurlfrommysql(self):
try:
if not os.path.exists(self.file_path):
print ('文件夹',self.file_path,'不存在，重新建立')
os.makedirs(self.file_path)
db.execute('SELECT a.ANIME_ID,a.ANIME_IMAGE FROM anime_home a',None)
records = db.fetchall()
if records:
for r in records:
self.queue.put_nowait(r)
except Exception as e:
pass
else:
pass
finally:
pass
pass
def repair(self):
try:
db.execute(
'SELECT h.* FROM anime_home h LEFT JOIN anime_info i on (h.ANIME_ID = i.ANIME_ID) WHERE i.ANIME_ID IS NOT NULL',
None)
records = db.fetchall()
if records:
for index, r in enumerate(records):
db.execute(
'UPDATE anime_home a SET a.ANIME_INFO_DOWNLOAD_STATUS = 2 WHERE a.ID = %s'
% (r[0]), None)
db.execute(
'UPDATE anime_home h SET h.ANIME_INFO_DOWNLOAD_STATUS = 0 WHERE h.ANIME_INFO_DOWNLOAD_STATUS =1',
None)
except Exception as e:
db.rollback()
raise e
finally:
db.commit()
def pop(self):
try:
db.execute(
'SELECT a.ID,a.ANIME_LINE,a.ANIME_INFO_DOWNLOAD_STATUS FROM anime_home a WHERE a.ANIME_INFO_DOWNLOAD_STATUS = 0',
None)
records = db.fetchall()
db.execute(
'UPDATE anime_home a SET a.ANIME_INFO_DOWNLOAD_STATUS = 1 WHERE a.ANIME_INFO_DOWNLOAD_STATUS = 0',
None)
if records:
for r in records:
self.queue.put(r)
return self.queue
# else:
# self.repair()
# raise KeyError
except Exception as e:
db.rollback()
raise e
finally:
db.commit()
def testbythread(self):
threads=[]
db.execute('select * from proxyls',None);
proxyLs = db.fetchall()
start = time.clock()
for p in proxyLs:
self.queue.put(p)
for i in range(8):
thread = threading.Thread(target=self.checkproxy)
thread.start() #启动线程
threads.append(thread)
time.sleep(1)
#结束线程
for thread in threads:
thread.join()
#等待所有任务完成
self.queue.join()
db.commit()
db.close()
print(str(time.clock()-start) + "秒")
print(u'验证完毕')
def __init__(self):
db.execute('select * from proxyls', None)
self.proxyLs = db.fetchall()
self.user_agent_list = [ \
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1" \
"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11", \
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6", \
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6", \
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1", \
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5", \
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5", \
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", \
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", \
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", \
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", \
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", \
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3", \
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24", \
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",\
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
]