def complete(self, animeid):
"""这个函数是更新已完成的URL完成"""
try:
db.execute(
'UPDATE anime_home a SET a.ANIME_INFO_DOWNLOAD_STATUS = 2 where a.ANIME_ID = %s'
% (animeid), None)
except Exception as e:
db.rollback()
raise e
finally:
db.commit()
async def get_animequeue(self):
db.execute('SELECT a.ANIME_ID,a.ANIME_IMAGE FROM anime_home a', None)
records = db.fetchall()
animequeue = asyncio.Queue()
if records:
for r in records:
await animequeue.put(r)
self.animequeue = animequeue
if not os.path.exists(self.file_path):
print('文件夹', self.file_path, '不存在，重新建立')
os.makedirs(self.file_path)
return animequeue
def getData(self, url):
content = self.requestbyproxy(url)
dataJson = json.loads(content.text)
if dataJson['status'] == 200:
if len(dataJson['data']['page']['list']):
values_insert = []
for i in dataJson['data']['page']['list']:
anime_bid = int(i['bid'])
anime_name = i['title']
anime_cover = i['cover']
anime_vertical_cover = i['verticalCover']
anime_play_date = i['playDate']
anime_play_time = i['playTime']
anime_origin_time = i['originTime']
anime_play_site = i['playSite']
anime_origin_station = i['originStation']
anime_play_url = i['playUrl']
anime_play_episode = i['episode']
if db.execute(
'select 1 from anime_timetable a where a.ANIME_BID = %s limit 1',
(anime_bid)) == 0:
values_insert.append([
anime_bid, None, anime_name, anime_cover,
anime_vertical_cover, anime_play_date,
anime_play_time, anime_origin_time,
anime_play_site, anime_origin_station,
anime_play_url, anime_play_episode
])
else:
try:
db.execute(
"update anime_timetable t set t.ANIME_PLAY_DATE = '%s',t.ANIME_PLAY_EPISODE = '%s' where t.ANIME_BID = %d"
% (anime_play_date, anime_play_episode,
anime_bid), None)
except Exception as e:
db.rollback()
traceback.print_exc()
try:
if len(values_insert):
db.executemany(
'insert into anime_timetable(ANIME_BID,ANIME_ID,ANIME_NAME,ANIME_COVER,ANIME_VERTICAL_COVER,ANIME_PLAY_DATE,ANIME_PLAY_TIME,ANIME_ORIGIN_TIME,ANIME_PLAY_SITE,ANIME_ORIGIN_STATION,ANIME_PLAY_URL,ANIME_PLAY_EPISODE) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)',
values_insert)
except Exception as e:
db.rollback()
traceback.print_exc()
finally:
db.commit()
time.sleep(1)
else:
pass
else:
pass
def getanimeimgurlfrommysql(self):
try:
if not os.path.exists(self.file_path):
print ('文件夹',self.file_path,'不存在，重新建立')
os.makedirs(self.file_path)
db.execute('SELECT a.ANIME_ID,a.ANIME_IMAGE FROM anime_home a',None)
records = db.fetchall()
if records:
for r in records:
self.queue.put_nowait(r)
except Exception as e:
pass
else:
pass
finally:
pass
pass
def pop(self):
try:
db.execute(
'SELECT a.ID,a.ANIME_LINE,a.ANIME_INFO_DOWNLOAD_STATUS FROM anime_home a WHERE a.ANIME_INFO_DOWNLOAD_STATUS = 0',
None)
records = db.fetchall()
db.execute(
'UPDATE anime_home a SET a.ANIME_INFO_DOWNLOAD_STATUS = 1 WHERE a.ANIME_INFO_DOWNLOAD_STATUS = 0',
None)
if records:
for r in records:
self.queue.put(r)
return self.queue
# else:
# self.repair()
# raise KeyError
except Exception as e:
db.rollback()
raise e
finally:
db.commit()
def testbythread(self):
threads=[]
db.execute('select * from proxyls',None);
proxyLs = db.fetchall()
start = time.clock()
for p in proxyLs:
self.queue.put(p)
for i in range(8):
thread = threading.Thread(target=self.checkproxy)
thread.start() #启动线程
threads.append(thread)
time.sleep(1)
#结束线程
for thread in threads:
thread.join()
#等待所有任务完成
self.queue.join()
db.commit()
db.close()
print(str(time.clock()-start) + "秒")
print(u'验证完毕')
def __init__(self):
db.execute('select * from proxyls', None)
self.proxyLs = db.fetchall()
self.user_agent_list = [ \
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1" \
"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11", \
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6", \
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6", \
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1", \
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5", \
"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5", \
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", \
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", \
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", \
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", \
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", \
"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3", \
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24", \
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",\
"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
]
def repair(self):
try:
db.execute(
'SELECT h.* FROM anime_home h LEFT JOIN anime_info i on (h.ANIME_ID = i.ANIME_ID) WHERE i.ANIME_ID IS NOT NULL',
None)
records = db.fetchall()
if records:
for index, r in enumerate(records):
db.execute(
'UPDATE anime_home a SET a.ANIME_INFO_DOWNLOAD_STATUS = 2 WHERE a.ID = %s'
% (r[0]), None)
db.execute(
'UPDATE anime_home h SET h.ANIME_INFO_DOWNLOAD_STATUS = 0 WHERE h.ANIME_INFO_DOWNLOAD_STATUS =1',
None)
except Exception as e:
db.rollback()
raise e
finally:
db.commit()
def peek(self):
record = db.execute(
'SELECT ID FROM anime_home a WHERE a.ANIME_INFO_DOWNLOAD_STATUS != 2 LIMIT 1',
None)
return record
def __bool__(self):
record = db.execute(
'SELECT 1 FROM anime_home a WHERE a.ANIME_INFO_DOWNLOAD_STATUS != 2 LIMIT 1',
None)
return True if record else False
def testdb(self):
db.execute('delete from proxy where ID = %s' % (73),None)
db.commit()
db.close()
def delproxy(self,id):
try:
db.execute('delete from proxyls where ID = %s' % (id),None)
except:
db.rollback()
def writeproxy(self,id):
try:
db.execute('update proxyls p set p.PROXY_CHECK_TIME = %s where p.ID = %s' % (round(time.time()*1000),id),None)
except:
db.rollback()
