class Task:
def __init__(self, searchWord, type):
self.searchWord = searchWord
self.type = type
self.keys = set()
self.c = Crawler()
self.fsdb = DBConnector()
def start(self, depth, startURL=None):
print(int(depth))
print(startURL)
print(self.searchWord)
if startURL is None or startURL == '':
starturl = self.c.getGoogleVidYTUrl(self.searchWord)
self.c.crawel(int(depth), starturl, self.keys)
else:
self.c.crawel(int(depth), startURL, self.keys)
save_dict_to_setting = {'lastTaskUrl': self.c.nexturl}
self.fsdb.uploadDataToDoc(
"settings/" + self.type + "/keywords/" + self.searchWord,
save_dict_to_setting)
for key in self.keys:
print(key)
upload_via_key_strem(key, self.type)
def upload_via_key_strem(youtubeKeyStream,
videotype,
backup=True,
listener=None):
try:
url = "https://www.youtube.com/watch?v=" + youtubeKeyStream
youTubeVideoRef = YouTube(url)
data = {
"title": youTubeVideoRef.title,
"length": youTubeVideoRef.length / 60,
"publishDate": youTubeVideoRef.publish_date,
"ratedNum": 0,
"avgRating": 0,
"YTSK": youtubeKeyStream,
"type": videotype,
}
db = DBConnector()
db.uploadDataToDoc("videos/" + youtubeKeyStream, data)
if backup:
dbBack = BackUp()
dbBack.addKey(youtubeKeyStream, videotype)
if listener is not None:
listener(youtubeKeyStream + " video up loaded")
except Exception as e:
if listener is not None:
listener(e)
return
raise e