class Task: def __init__(self, searchWord, type): self.searchWord = searchWord self.type = type self.keys = set() self.c = Crawler() self.fsdb = DBConnector() def start(self, depth, startURL=None): print(int(depth)) print(startURL) print(self.searchWord) if startURL is None or startURL == '': starturl = self.c.getGoogleVidYTUrl(self.searchWord) self.c.crawel(int(depth), starturl, self.keys) else: self.c.crawel(int(depth), startURL, self.keys) save_dict_to_setting = {'lastTaskUrl': self.c.nexturl} self.fsdb.uploadDataToDoc( "settings/" + self.type + "/keywords/" + self.searchWord, save_dict_to_setting) for key in self.keys: print(key) upload_via_key_strem(key, self.type)
def upload_via_key_strem(youtubeKeyStream, videotype, backup=True, listener=None): try: url = "https://www.youtube.com/watch?v=" + youtubeKeyStream youTubeVideoRef = YouTube(url) data = { "title": youTubeVideoRef.title, "length": youTubeVideoRef.length / 60, "publishDate": youTubeVideoRef.publish_date, "ratedNum": 0, "avgRating": 0, "YTSK": youtubeKeyStream, "type": videotype, } db = DBConnector() db.uploadDataToDoc("videos/" + youtubeKeyStream, data) if backup: dbBack = BackUp() dbBack.addKey(youtubeKeyStream, videotype) if listener is not None: listener(youtubeKeyStream + " video up loaded") except Exception as e: if listener is not None: listener(e) return raise e
def delTypeInSettings(typeToDelete,typeComboBox1,typeComboBox2): l = getTypeVideosList() if typeToDelete in l: db = DBConnector() db.deleteDoc("settings/"+typeToDelete) typeComboBox1["values"] = getTypeVideosList() typeComboBox2["values"] = getTypeVideosList()
def clean_Op(log,*args): log.set("start cleaning the DB") try: db = DBConnector() vids = db.readCollaction("videos") for v in vids: if ivaluaetFanction(vids[v]['avgRating'],vids[v]['ratedNum'], args[0].get(), args[1].get()): db.deleteDoc("videos/"+v) except Exception as e: print(e) log.set(e.__str__()) return log.set("FINISHED cleaning the DB")
def startWork(self): for setting_type in self.settings: keywords = DBConnector().readCollaction("settings/" + setting_type + "/keywords") for kw in keywords: myTask = self.create_task(kw, setting_type) Thread(target=myTask.start, args=[DEPTH, keywords[kw]['lastTaskUrl']]).start()
def getTypesKewords(): try: settings = DBConnector().readCollaction("settings") dict_key_word = {} for type_in_settings in settings: dict_key_word[type_in_settings] = [] keywords = DBConnector().readCollaction("settings/"+type_in_settings+"/keywords") for kw in keywords: dict_key_word[type_in_settings].append(kw) dict_key_word[type_in_settings] = makeCustomFormatString(dict_key_word[type_in_settings]) return dict_key_word except Exception as e: print(e.__str__())
def commitSettingButtonFunction(typeToAddOrChange, keywordListInEntry, typeComboBox1, typeComboBox2,logView): db = DBConnector() if typeToAddOrChange is None or typeToAddOrChange=='' or typeToAddOrChange.isspace(): logView.set("cant add or change type ") return if not strOnlydigAndAlpha(typeToAddOrChange): logView.set("type can only contains chars and numbers ") return if keywordListInEntry is None or len(keywordListInEntry) == 0 or keywordListInEntry[0] == '': logView.set("cant add type with empty key words ") return for w in keywordListInEntry: if not strOnlydigAndAlpha(w): logView.set("key words only contains chars and numbers ") return try: task = threading.Thread(target=db.uploadDataToDoc, args=["settings/" + typeToAddOrChange, {"onlyForCreate": "a"}]) task.start() task.join() keywords_json_fb = db.readCollaction("settings/" + typeToAddOrChange + "/keywords") keywords_fb = keywords_json_fb.keys() # adding new key word for the type for kw in keywordListInEntry: if kw not in keywords_fb: data = { 'lastTaskUrl': '' } db.uploadDocToCollection("settings/" + typeToAddOrChange + "/keywords", kw, data) # removing thing that are not in the entry fro fb for kw in keywords_fb: if kw not in keywordListInEntry: db.deleteDoc("settings/" + typeToAddOrChange + "/keywords/" + kw) typeComboBox1["values"] = getTypeVideosList() typeComboBox2["values"] = getTypeVideosList() logView.set("changes commited") except Exception as e: logView.set("something want wrong" , e.__str__())
def __init__(self, searchWord, type): self.searchWord = searchWord self.type = type self.keys = set() self.c = Crawler() self.fsdb = DBConnector()
def runSearchAndUpload(log): log.set("start search and upload data") settings = DBConnector().readCollaction("settings") TaskFactory(settings).startWork()
def getTypeVideosList(): try: db = DBConnector() return list(db.readCollaction("settings").keys()) except: return []
print("d1 =", d1) # Textual month, day and year d2 = today.strftime("%B %d, %Y") print("d2 =", d2) # mm/dd/y d3 = today.strftime("%m/%d/%y") print("d3 =", d3) # Month abbreviation, day and year d4 = today.strftime("%b-%d-%Y") print("d4 =", d4) ''' db = DBConnector() #db.upload(r"C:\Users\User\Desktop\youtube\Beginners Back Exercises that Strengthen your Back-TGI5TFnY8Ck.mp4","llll") #db.uploadDocToCollection("videos",doc="test1",data={"test": 1}) #db.uploadDataToDoc("videos/"+str(uuid.uuid4()),{"test": 1}) ''' def importBankfromFile(file_name): bank = [] file = open(file_name,'r') numOfLines = int(file.readline()) for i in range(numOfLines): bank.append(file.readline().replace("\n", "")) return Analyst(bank) bloblist = [] blobs = storage.bucket(db.bucket_name).list_blobs()