class Task:
def __init__(self, searchWord, type):
self.searchWord = searchWord
self.type = type
self.keys = set()
self.c = Crawler()
self.fsdb = DBConnector()
def start(self, depth, startURL=None):
print(int(depth))
print(startURL)
print(self.searchWord)
if startURL is None or startURL == '':
starturl = self.c.getGoogleVidYTUrl(self.searchWord)
self.c.crawel(int(depth), starturl, self.keys)
else:
self.c.crawel(int(depth), startURL, self.keys)
save_dict_to_setting = {'lastTaskUrl': self.c.nexturl}
self.fsdb.uploadDataToDoc(
"settings/" + self.type + "/keywords/" + self.searchWord,
save_dict_to_setting)
for key in self.keys:
print(key)
upload_via_key_strem(key, self.type)
def upload_via_key_strem(youtubeKeyStream,
videotype,
backup=True,
listener=None):
try:
url = "https://www.youtube.com/watch?v=" + youtubeKeyStream
youTubeVideoRef = YouTube(url)
data = {
"title": youTubeVideoRef.title,
"length": youTubeVideoRef.length / 60,
"publishDate": youTubeVideoRef.publish_date,
"ratedNum": 0,
"avgRating": 0,
"YTSK": youtubeKeyStream,
"type": videotype,
}
db = DBConnector()
db.uploadDataToDoc("videos/" + youtubeKeyStream, data)
if backup:
dbBack = BackUp()
dbBack.addKey(youtubeKeyStream, videotype)
if listener is not None:
listener(youtubeKeyStream + " video up loaded")
except Exception as e:
if listener is not None:
listener(e)
return
raise e
def delTypeInSettings(typeToDelete,typeComboBox1,typeComboBox2):
l = getTypeVideosList()
if typeToDelete in l:
db = DBConnector()
db.deleteDoc("settings/"+typeToDelete)
typeComboBox1["values"] = getTypeVideosList()
typeComboBox2["values"] = getTypeVideosList()
def clean_Op(log,*args):
log.set("start cleaning the DB")
try:
db = DBConnector()
vids = db.readCollaction("videos")
for v in vids:
if ivaluaetFanction(vids[v]['avgRating'],vids[v]['ratedNum'], args[0].get(), args[1].get()):
db.deleteDoc("videos/"+v)
except Exception as e:
print(e)
log.set(e.__str__())
return
log.set("FINISHED cleaning the DB")
def startWork(self):
for setting_type in self.settings:
keywords = DBConnector().readCollaction("settings/" +
setting_type + "/keywords")
for kw in keywords:
myTask = self.create_task(kw, setting_type)
Thread(target=myTask.start,
args=[DEPTH, keywords[kw]['lastTaskUrl']]).start()
def getTypesKewords():
try:
settings = DBConnector().readCollaction("settings")
dict_key_word = {}
for type_in_settings in settings:
dict_key_word[type_in_settings] = []
keywords = DBConnector().readCollaction("settings/"+type_in_settings+"/keywords")
for kw in keywords:
dict_key_word[type_in_settings].append(kw)
dict_key_word[type_in_settings] = makeCustomFormatString(dict_key_word[type_in_settings])
return dict_key_word
except Exception as e:
print(e.__str__())
def commitSettingButtonFunction(typeToAddOrChange, keywordListInEntry, typeComboBox1, typeComboBox2,logView):
db = DBConnector()
if typeToAddOrChange is None or typeToAddOrChange=='' or typeToAddOrChange.isspace():
logView.set("cant add or change type ")
return
if not strOnlydigAndAlpha(typeToAddOrChange):
logView.set("type can only contains chars and numbers ")
return
if keywordListInEntry is None or len(keywordListInEntry) == 0 or keywordListInEntry[0] == '':
logView.set("cant add type with empty key words ")
return
for w in keywordListInEntry:
if not strOnlydigAndAlpha(w):
logView.set("key words only contains chars and numbers ")
return
try:
task = threading.Thread(target=db.uploadDataToDoc, args=["settings/" + typeToAddOrChange, {"onlyForCreate": "a"}])
task.start()
task.join()
keywords_json_fb = db.readCollaction("settings/" + typeToAddOrChange + "/keywords")
keywords_fb = keywords_json_fb.keys()
# adding new key word for the type
for kw in keywordListInEntry:
if kw not in keywords_fb:
data = {
'lastTaskUrl': ''
}
db.uploadDocToCollection("settings/" + typeToAddOrChange + "/keywords", kw, data)
# removing thing that are not in the entry fro fb
for kw in keywords_fb:
if kw not in keywordListInEntry:
db.deleteDoc("settings/" + typeToAddOrChange + "/keywords/" + kw)
typeComboBox1["values"] = getTypeVideosList()
typeComboBox2["values"] = getTypeVideosList()
logView.set("changes commited")
except Exception as e:
logView.set("something want wrong" , e.__str__())
def __init__(self, searchWord, type):
self.searchWord = searchWord
self.type = type
self.keys = set()
self.c = Crawler()
self.fsdb = DBConnector()
def runSearchAndUpload(log):
log.set("start search and upload data")
settings = DBConnector().readCollaction("settings")
TaskFactory(settings).startWork()
def getTypeVideosList():
try:
db = DBConnector()
return list(db.readCollaction("settings").keys())
except:
return []
print("d1 =", d1)
# Textual month, day and year
d2 = today.strftime("%B %d, %Y")
print("d2 =", d2)
# mm/dd/y
d3 = today.strftime("%m/%d/%y")
print("d3 =", d3)
# Month abbreviation, day and year
d4 = today.strftime("%b-%d-%Y")
print("d4 =", d4)
'''
db = DBConnector()
#db.upload(r"C:\Users\User\Desktop\youtube\Beginners Back Exercises that Strengthen your Back-TGI5TFnY8Ck.mp4","llll")
#db.uploadDocToCollection("videos",doc="test1",data={"test": 1})
#db.uploadDataToDoc("videos/"+str(uuid.uuid4()),{"test": 1})
'''
def importBankfromFile(file_name):
bank = []
file = open(file_name,'r')
numOfLines = int(file.readline())
for i in range(numOfLines):
bank.append(file.readline().replace("\n", ""))
return Analyst(bank)
bloblist = []
blobs = storage.bucket(db.bucket_name).list_blobs()