def getFiltedVideolid(self):
getvideolist = VideoList(datalist, SavePath)
datalist['exsistFile'] = SavePath['playlist_ListPath']
print('Count：' + str(len(Integrate.readFiltedPlayList())))
RunCount = 0
RequestCount = 0
RefreshCount = 0
BeforeRunCount = 0
exisitList = Integrate.getsuccessList()
FiltedPlayList = Integrate.readFiltedPlayList()
listcount = len(FiltedPlayList)
random.seed(datetime.utcnow())
random.shuffle(FiltedPlayList)
ProcessStart = time.time()
Tstart = time.time()
for key in FiltedPlayList:
RunCount += 1
if key not in exisitList:
try:
datalist['playlistId'] = key
getvideolist.getvideolist()
RequestCount += 1
except Exception as identifier:
Error = str(identifier)
if Error.find('404', 0, 20) != -1:
print('Error_404無法找到相關影片資訊')
elif Error.find('304', 0, 20) != -1:
print('Error_304')
elif Error.find('403', 0, 20) != -1:
# print('Error_403無法存取資料')
pass
else:
print(identifier)
else:
# print('IDexist')
pass
Tend = time.time()
if ((Tend - Tstart) > datalist['RefreshTime']):
exisitList = Integrate.getsuccessList()
Basic.runstatus(self, RunCount, RequestCount, Tstart, Tend,
RefreshCount, ProcessStart, BeforeRunCount,
listcount)
RequestCount = 0
RefreshCount = 0
BeforeRunCount = RunCount
Tstart = time.time()
else:
RefreshCount += 1
ProcessEnd = time.time()
print('GetVideoList_Done')
print('ProcessTimeCost : %.4f /s' % (ProcessEnd - ProcessStart))
def getchannelList(self):
datalist['exsistFile'] = SavePath['channel_ListPath']
exisitList = Integrate.getsuccessList()
getchannel = SubscriptList(datalist, SavePath)
channelList = Integrate.readchannelList()
random.shuffle(channelList)
RunCount = 0
RequestCount = 0
RefreshCount = 0
BeforeRunCount = 0
listcount = len(channelList)
Tstart = time.time()
ProcessStart = time.time()
for key in channelList:
RunCount += 1
try:
if key not in exisitList:
datalist['channelID'] = key
getchannel.getSubscriptList()
RequestCount += 1
else:
# print('IDexist')
pass
except Exception as identifier:
Error = str(identifier)
if Error.find('404', 0, 20) != -1:
print('Error_404未找到位置(檢查網路問題)')
elif Error.find('304', 0, 20) != -1:
print('Error_304')
elif Error.find('403', 0, 20) != -1:
# print('Error_403無法存取資料')
pass
else:
pass
# print(identifier)
finally:
pass
Tend = time.time()
if ((Tend - Tstart) > datalist['RefreshTime']):
exisitList = Integrate.getsuccessList()
Basic.runstatus(self, RunCount, RequestCount, Tstart, Tend,
RefreshCount, ProcessStart, BeforeRunCount,
listcount)
RequestCount = 0
RefreshCount = 0
BeforeRunCount = RunCount
Tstart = time.time()
else:
RefreshCount += 1
ProcessEnd = time.time()
print('GetChannel_List_Done')
print('ProcessTimeCost : %.4f' % (ProcessEnd - ProcessStart))
Integrate.TotalchannelList()
def getchannelInfo(self):
Integrate.TotalchannelList()
Info = ChannelInfo(datalist, SavePath)
datalist['exsistFile'] = SavePath['channel_InfoPath']
print('Count：' + str(len(Integrate.readchannelList())))
RunCount = 0
RequestCount = 0
RefreshCount = 0
BeforeRunCount = 0
exisitList = Integrate.getsuccessList()
channelList = Integrate.readchannelList()
listcount = len(channelList)
random.shuffle(channelList)
ProcessStart = time.time()
Tstart = time.time()
for key in channelList:
RunCount += 1
if key not in exisitList:
try:
datalist['id'] = key
Info.squent_getInfo()
RequestCount += 1
except Exception as identifier:
print(identifier)
else:
# print('IDexist')
pass
Tend = time.time()
if ((Tend - Tstart) > datalist['RefreshTime']):
exisitList = Integrate.getsuccessList()
Basic.runstatus(self, RunCount, RequestCount, Tstart, Tend,
RefreshCount, ProcessStart, BeforeRunCount,
listcount)
RequestCount = 0
RefreshCount = 0
BeforeRunCount = RunCount
Tstart = time.time()
else:
RefreshCount += 1
Integrate.TotalchannelList()
Integrate.TotalchannelInfo()
ProcessEnd = time.time()
print('GetChannel_Info_Done')
print('ProcessTimeCost : %.4f /s' % (ProcessEnd - ProcessStart))
def getFiltedCommentTop(self):
getcomment = CommentThreads(datalist, SavePath)
datalist['id'] = Integrate.readchannelVideoID()
datalist['exsistFile'] = SavePath['channelcomment_ListPath']
exisitList = Integrate.getsuccessList()
filtedvideoid = Integrate.readchannelVideoID()
random.seed(datetime.utcnow())
random.shuffle(filtedvideoid)
RunCount = 0
BeforeRunCount = 0
RequestCount = 0
RefreshCount = 0
videolistID = []
ProcessStart = time.time()
Tstart = time.time()
for temp in filtedvideoid:
for videoid in temp:
videolistID.append(videoid)
listcount = len(videolistID)
print('VideoCount:' + str(listcount))
for temp in videolistID:
if temp not in exisitList:
RunCount += 1
datalist['id'] = temp
getcomment.getcomment()
RequestCount += 1
else:
pass
# print('IDexsist')
Tend = time.time()
if ((Tend - Tstart) > datalist['RefreshTime']):
exisitList = Integrate.getsuccessList()
Basic.runstatus(self, RunCount, RequestCount, Tstart, Tend,
RefreshCount, ProcessStart, BeforeRunCount,
listcount)
RequestCount = 0
RefreshCount = 0
BeforeRunCount = RunCount
Tstart = time.time()
else:
RefreshCount += 1
ProcessEnd = time.time()
print('GetVideoComment_Done')
print('ProcessTimeCost : %.4f /s' % (ProcessEnd - ProcessStart))
def getFiltedVideolInfo(self):
getvideoinfo = VideoInfo(datalist, SavePath)
datalist['id'] = Integrate.readchannelVideoID()
datalist['exsistFile'] = SavePath['video_InfoPath']
videolist = datalist['id']
videolistID = []
RunCount = 0
BeforeRunCount = 0
RequestCount = 0
RefreshCount = 0
exisitList = Integrate.getsuccessList()
ProcessStart = time.time()
Tstart = time.time()
for temp in videolist:
for videoid in temp:
videolistID.append(videoid)
listcount = len(videolistID)
print('VideoCount:' + str(listcount))
random.seed(datetime.utcnow())
random.shuffle(videolistID)
for videoid in videolistID:
RunCount += 1
if videoid not in exisitList:
datalist['id'] = videoid
getvideoinfo.getVideoInfo()
RequestCount += 1
else:
pass
# print('IDexsist')
Tend = time.time()
if ((Tend - Tstart) > datalist['RefreshTime']):
exisitList = Integrate.getsuccessList()
Basic.runstatus(self, RunCount, RequestCount, Tstart, Tend,
RefreshCount, ProcessStart, BeforeRunCount,
listcount)
RequestCount = 0
RefreshCount = 0
BeforeRunCount = RunCount
Tstart = time.time()
else:
RefreshCount += 1
ProcessEnd = time.time()
print('GetVideoInfo_Done')
print('ProcessTimeCost : %.4f /s' % (ProcessEnd - ProcessStart))
def simpleRun(self):
Basic.firstRun(datalist)
Basic.deepRun(datalist)
print("simpleRun_Done")
def initialize_check(self):
Basic.checkPath(SavePath)