def complete(self, animeid): """这个函数是更新已完成的URL完成""" try: db.execute( 'UPDATE anime_home a SET a.ANIME_INFO_DOWNLOAD_STATUS = 2 where a.ANIME_ID = %s' % (animeid), None) except Exception as e: db.rollback() raise e finally: db.commit()
async def get_animequeue(self): db.execute('SELECT a.ANIME_ID,a.ANIME_IMAGE FROM anime_home a', None) records = db.fetchall() animequeue = asyncio.Queue() if records: for r in records: await animequeue.put(r) self.animequeue = animequeue if not os.path.exists(self.file_path): print('文件夹', self.file_path, '不存在,重新建立') os.makedirs(self.file_path) return animequeue
def getData(self, url): content = self.requestbyproxy(url) dataJson = json.loads(content.text) if dataJson['status'] == 200: if len(dataJson['data']['page']['list']): values_insert = [] for i in dataJson['data']['page']['list']: anime_bid = int(i['bid']) anime_name = i['title'] anime_cover = i['cover'] anime_vertical_cover = i['verticalCover'] anime_play_date = i['playDate'] anime_play_time = i['playTime'] anime_origin_time = i['originTime'] anime_play_site = i['playSite'] anime_origin_station = i['originStation'] anime_play_url = i['playUrl'] anime_play_episode = i['episode'] if db.execute( 'select 1 from anime_timetable a where a.ANIME_BID = %s limit 1', (anime_bid)) == 0: values_insert.append([ anime_bid, None, anime_name, anime_cover, anime_vertical_cover, anime_play_date, anime_play_time, anime_origin_time, anime_play_site, anime_origin_station, anime_play_url, anime_play_episode ]) else: try: db.execute( "update anime_timetable t set t.ANIME_PLAY_DATE = '%s',t.ANIME_PLAY_EPISODE = '%s' where t.ANIME_BID = %d" % (anime_play_date, anime_play_episode, anime_bid), None) except Exception as e: db.rollback() traceback.print_exc() try: if len(values_insert): db.executemany( 'insert into anime_timetable(ANIME_BID,ANIME_ID,ANIME_NAME,ANIME_COVER,ANIME_VERTICAL_COVER,ANIME_PLAY_DATE,ANIME_PLAY_TIME,ANIME_ORIGIN_TIME,ANIME_PLAY_SITE,ANIME_ORIGIN_STATION,ANIME_PLAY_URL,ANIME_PLAY_EPISODE) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)', values_insert) except Exception as e: db.rollback() traceback.print_exc() finally: db.commit() time.sleep(1) else: pass else: pass
def getanimeimgurlfrommysql(self): try: if not os.path.exists(self.file_path): print ('文件夹',self.file_path,'不存在,重新建立') os.makedirs(self.file_path) db.execute('SELECT a.ANIME_ID,a.ANIME_IMAGE FROM anime_home a',None) records = db.fetchall() if records: for r in records: self.queue.put_nowait(r) except Exception as e: pass else: pass finally: pass pass
def pop(self): try: db.execute( 'SELECT a.ID,a.ANIME_LINE,a.ANIME_INFO_DOWNLOAD_STATUS FROM anime_home a WHERE a.ANIME_INFO_DOWNLOAD_STATUS = 0', None) records = db.fetchall() db.execute( 'UPDATE anime_home a SET a.ANIME_INFO_DOWNLOAD_STATUS = 1 WHERE a.ANIME_INFO_DOWNLOAD_STATUS = 0', None) if records: for r in records: self.queue.put(r) return self.queue # else: # self.repair() # raise KeyError except Exception as e: db.rollback() raise e finally: db.commit()
def testbythread(self): threads=[] db.execute('select * from proxyls',None); proxyLs = db.fetchall() start = time.clock() for p in proxyLs: self.queue.put(p) for i in range(8): thread = threading.Thread(target=self.checkproxy) thread.start() #启动线程 threads.append(thread) time.sleep(1) #结束线程 for thread in threads: thread.join() #等待所有任务完成 self.queue.join() db.commit() db.close() print(str(time.clock()-start) + "秒") print(u'验证完毕')
def __init__(self): db.execute('select * from proxyls', None) self.proxyLs = db.fetchall() self.user_agent_list = [ \ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1" \ "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11", \ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6", \ "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6", \ "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1", \ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5", \ "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5", \ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \ "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \ "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", \ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", \ "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", \ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", \ "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", \ "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3", \ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24", \ "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",\ "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24" ]
def repair(self): try: db.execute( 'SELECT h.* FROM anime_home h LEFT JOIN anime_info i on (h.ANIME_ID = i.ANIME_ID) WHERE i.ANIME_ID IS NOT NULL', None) records = db.fetchall() if records: for index, r in enumerate(records): db.execute( 'UPDATE anime_home a SET a.ANIME_INFO_DOWNLOAD_STATUS = 2 WHERE a.ID = %s' % (r[0]), None) db.execute( 'UPDATE anime_home h SET h.ANIME_INFO_DOWNLOAD_STATUS = 0 WHERE h.ANIME_INFO_DOWNLOAD_STATUS =1', None) except Exception as e: db.rollback() raise e finally: db.commit()
def peek(self): record = db.execute( 'SELECT ID FROM anime_home a WHERE a.ANIME_INFO_DOWNLOAD_STATUS != 2 LIMIT 1', None) return record
def __bool__(self): record = db.execute( 'SELECT 1 FROM anime_home a WHERE a.ANIME_INFO_DOWNLOAD_STATUS != 2 LIMIT 1', None) return True if record else False
def testdb(self): db.execute('delete from proxy where ID = %s' % (73),None) db.commit() db.close()
def delproxy(self,id): try: db.execute('delete from proxyls where ID = %s' % (id),None) except: db.rollback()
def writeproxy(self,id): try: db.execute('update proxyls p set p.PROXY_CHECK_TIME = %s where p.ID = %s' % (round(time.time()*1000),id),None) except: db.rollback()