def test_add_video_default_isDownloaded_isUploaded_value(self):
video = VideoInfo()
video.url = 'http://testAddDefaultValue'
video.description = '__test__'
DownloadInfoService.addVideoInfo(video)
video2 = DownloadInfoService.getVideoInfo(video.url)
self.assertEqual(video2.isDownloaded, 0)
self.assertEqual(video2.isUploaded, 0)
def __createVideoInfo__(self):
video = VideoInfo('')
video.url = 'http://www.baidu1.com'
video.author = 'zw'
video.uploader = 'zw'
video.isDownloaded = 0
video.isUploaded = 0
video.description = 'This is test video description'
return video
def extractLinkWorker(pendingSearchLinks):
allSearchSet = set(pendingSearchLinks)
while len(pendingSearchLinks) > 0:
search_page_url = pendingSearchLinks.pop()
pyContent = PyQuery(search_page_url)
extractInfo = extractLink(pyContent)
videoLinks = extractInfo['videoLinks']
searchLinks = extractInfo['searchLinks']
if videoLinks is not None:
logger.info('[LinkExtractor] Add video links to db: [%s]', ' ,\n'.join(videoLinks))
# Add video links to db
for videoLink in videoLinks:
videoInfo = VideoInfo()
videoInfo.url = videoLink
try:
DownloadInfoService.addVideoInfo(videoInfo)
except DuplicateError:
logging.info("[LinkExtractor] URL [%s] has been already in DB.", videoInfo.url)
newSearchLinks = [];
if searchLinks is not None:
for newLink in searchLinks:
if newLink not in allSearchSet:
pendingSearchLinks.append(newLink)
newSearchLinks.append(newLink)
allSearchSet.add(newLink)
if len(newSearchLinks) != 0:
logger.debug( '[LinkExtractor] Extracted new search links: [{}]'.format(' ,\n'.join(newSearchLinks)))
else:
logger.debug('[LinkExtractor] search links is none for url:[{}]'.format(search_page_url))
logger.info('[LinkExtractor] Sleep 3 seconds to continue.')
time.sleep(3)
pass
logger.warn('[LinkExtractor] Link Extractor stopped')
