print("\nAdding to word list file: " + APP_LIST_FILE) cfile.append(APP_LIST_FILE, listWord) APP_DICT_ENTR_FILE = APP_DICT_DIR + cfile.cleanse_filename(APP_DICT_ENTR.format(WORD=listWord)) if os.path.isfile(APP_DICT_ENTR_FILE) and os.path.getsize(APP_DICT_ENTR_FILE) > 100: print("Definition file {} exists. Skipping.".format(APP_DICT_ENTR_FILE)) else: print("Creating definition file: " + APP_DICT_ENTR_FILE) cfile.write(APP_DICT_ENTR_FILE, wordEntry) APP_DICT_CLIP_FILE = APP_DICT_DIR + cfile.cleanse_filename(APP_DICT_CLIP.format(WORD=listWord)) if os.path.isfile(APP_DICT_CLIP_FILE) and os.path.getsize(APP_DICT_CLIP_FILE) > 100: print("Pronunciation file {} exists. Skipping.".format(APP_DICT_CLIP_FILE)) else: print("Creating pronunciation file: " + APP_DICT_CLIP_FILE) cfile.download(connectionPool, wordAudioURL, APP_DICT_CLIP_FILE) while True: nextButton = browser.find_element_by_id("nextButton") nextButton.click() print("Clicked Next button") wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, ".col-sm-9.col-md-9"))) time.sleep(APP_WAIT_DELAY) nextWordInfoElement = browser.find_element_by_xpath("//form[contains(@name,'moduleform')]") nextWordID = nextWordInfoElement.find_element_by_xpath("//*[contains(@id,'_id')]").get_property("value") if nextWordID == wordID: coutput.print_err("Page not refreshed. Retrying in {}s ...".format(APP_TIMEOUT_WAIT_DELAY)) time.sleep(APP_TIMEOUT_WAIT_DELAY)
def lookup_dictionary_by_word(self, word): _FUNC_NAME_ = "lookup_dictionary_by_word" DEBUG_VAR="self.wordList[0]" coutput.print_debug(SB_ERR_DEBUG, _FUNC_NAME_, "{0} :: {1}".format(DEBUG_VAR, type(self.wordList[0]))) coutput.print_debug(SB_ERR_DEBUG, _FUNC_NAME_, eval(DEBUG_VAR)) DEBUG_VAR="word" coutput.print_debug(SB_ERR_DEBUG, _FUNC_NAME_, "{0} :: {1}".format(DEBUG_VAR, type(word))) coutput.print_debug(SB_ERR_DEBUG, _FUNC_NAME_, eval(DEBUG_VAR)) self.activeWord = word.strip() DEBUG_VAR="self.activeWord" coutput.print_debug(SB_ERR_DEBUG, _FUNC_NAME_, "{0} :: {1}".format(DEBUG_VAR, type(self.activeWord))) coutput.print_debug(SB_ERR_DEBUG, _FUNC_NAME_, eval(DEBUG_VAR)) # Setup connection and error logging connectionPool = urllib3.PoolManager(10, headers=SB_USER_AGENT) errorFileName = SB_DATA_DIR + SB_ERR_LOG # Check offline for dictionary entry self.activeEntry = SB_EMPTY_STRING self.activeDefinition = [] overrideDefnFileName = SB_DICT_OVERRIDE_DIR + SB_DICT_OVERRIDE_DEFN.format(WORD=word).replace(" ", "_") offlineEntryFileName = SB_DICT_OFFLINE_DIR + SB_DICT_OFFLINE_ENTR.format(WORD=word).replace(" ", "_") # Check for dictionary definition override if os.path.isfile(overrideDefnFileName) and os.path.getsize(overrideDefnFileName) > 0: self.activeEntry = unicode("[Dictionary Definition Override]", 'utf-8') self.activeDefinition = cfile.read(overrideDefnFileName).splitlines() DEBUG_VAR="self.activeEntry" coutput.print_debug(SB_ERR_DEBUG, _FUNC_NAME_, "{0} :: {1}".format(DEBUG_VAR, type(self.activeEntry))) coutput.print_debug(SB_ERR_DEBUG, _FUNC_NAME_, eval(DEBUG_VAR)) # Check primary source for dictionary entry elif os.path.isfile(offlineEntryFileName) and os.path.getsize(offlineEntryFileName) > 100: coutput.print_debug(SB_ERR_DEBUG, _FUNC_NAME_, "offlineEntryFile size :: {0}".format(os.path.getsize(offlineEntryFileName))) self.activeEntry = cfile.read(offlineEntryFileName) self.activeDefinition = cdict.parse_word_definition(self.activeWord, self.activeEntry) DEBUG_VAR="self.activeEntry" coutput.print_debug(SB_ERR_DEBUG, _FUNC_NAME_, "{0} :: {1}".format(DEBUG_VAR, type(self.activeEntry))) coutput.print_debug(SB_ERR_DEBUG, _FUNC_NAME_, eval(DEBUG_VAR)) else: # Download dictionary entry self.activeEntry = cdict.get_dictionary_entry(connectionPool, self.activeWord) DEBUG_VAR="self.activeEntry" coutput.print_debug(SB_ERR_DEBUG, _FUNC_NAME_, "{0} :: {1}".format(DEBUG_VAR, type(self.activeEntry))) coutput.print_debug(SB_ERR_DEBUG, _FUNC_NAME_, eval(DEBUG_VAR)) # Save dictionary entry offline cfile.write(offlineEntryFileName, self.activeEntry) # Retrieve word definition self.activeDefinition = cdict.parse_word_definition(self.activeWord, self.activeEntry) if len(self.activeDefinition) == 0: # Log missing definition error errorText = unicode("ERROR:Missing Definition:{0}\n", 'utf-8') errorText = errorText.format(self.activeWord) cfile.append(errorFileName, errorText) # Check offline for word pronunciation self.activePronunciation = SB_EMPTY_STRING self.activePronunciationWord = SB_EMPTY_STRING overrideProncnFileName = SB_DICT_OVERRIDE_DIR + SB_DICT_OVERRIDE_CLIP.format(WORD=self.activeWord).replace(" ", "_") offlineProncnFileName = SB_DICT_OFFLINE_DIR + SB_DICT_OFFLINE_CLIP.format(WORD=self.activeWord).replace(" ", "_") # Check for dictionary pronunciation override if os.path.isfile(overrideProncnFileName) and os.path.getsize(overrideProncnFileName) > 0: self.activePronunciation = overrideProncnFileName self.activePronunciationWord = self.activeWord # Check primary source for dictionary entry and pronunciation elif os.path.isfile(offlineEntryFileName) and os.path.getsize(offlineEntryFileName) > 100 and os.path.isfile(offlineProncnFileName) and os.path.getsize(offlineProncnFileName) > 1000: coutput.print_debug(SB_ERR_DEBUG, _FUNC_NAME_, "offlineProncnFile size :: {0}".format(os.path.getsize(offlineProncnFileName))) self.activePronunciation = offlineProncnFileName # Retrieve pronunciation audio clip word form and filename [wordClipForm, wordClipURL] = cdict.parse_word_clip(self.activeWord, self.activeEntry) self.activePronunciationWord = wordClipForm else: # Retrieve pronunciation audio clip word form and filename [wordClipForm, wordClipURL] = cdict.parse_word_clip(self.activeWord, self.activeEntry) # Save pronunciation offline if wordClipURL == SB_EMPTY_STRING: # Log missing audio error errorText = unicode("ERROR:Missing Audio:{0}\n", 'utf-8') errorText = errorText.format(self.activeWord) cfile.append(errorFileName, errorText) else: # Download audio clip cfile.download(connectionPool, wordClipURL, offlineProncnFileName) self.activePronunciation = offlineProncnFileName self.activePronunciationWord = wordClipForm # Log audio mismatch error wordToken = re.sub('[^a-zA-Z]', SB_EMPTY_STRING, self.activeWord.lower()) pronunciationToken = re.sub('[^a-zA-Z]', SB_EMPTY_STRING, self.activePronunciationWord.lower()) if self.activePronunciation != SB_EMPTY_STRING and wordToken != pronunciationToken: errorText = unicode("ERROR:Audio Mismatch:{0}\n", 'utf-8') errorText = errorText.format(self.activeWord) cfile.append(errorFileName, errorText) # Close connection connectionPool.clear()
coutput.print_watcher(SDO_ERR_DEBUG, _FUNC_NAME_, 'overrideData') cfile.write(SDO_OVERRIDE_DEFN_FILE.format(WORD=word), overrideData) else: SDO_ERR_DEFN_MISSING = True coutput.print_color('yellow', "WARNING: Definition override not available") if SDO_ERR_AUDIO_REGEX_PATTERN.match(logValues[2]): coutput.print_watcher( SDO_ERR_DEBUG, _FUNC_NAME_, 'wordDictionary.simplified_word_entry.pronunciation') if wordDictionary.simplified_word_entry.pronunciation is not None and wordDictionary.simplified_word_entry.pronunciation.audio_url != "": print ">> Downloaded pronunciation override" cfile.download( connectionPool, wordDictionary.simplified_word_entry.pronunciation.audio_url, SDO_OVERRIDE_PRON_FILE.format(WORD=word)) else: SDO_ERR_CLIP_MISSING = True coutput.print_color( 'yellow', "WARNING: Pronunciation override not available") # Log errors errorText = unicode("ERROR:{0}:", 'utf-8').format(word) if SDO_ERR_DEFN_MISSING: errorText += unicode(">Definition Missing", 'utf-8') if SDO_ERR_CLIP_MISSING: errorText += unicode(">Audio Missing", 'utf-8') if errorText != unicode("ERROR:{0}:", 'utf-8').format(word): errorText += unicode("\n", 'utf-8')
print unicode("Word: {0}\t{1}", 'utf-8').format(word, logValues[2]) if SDO_ERR_DEFN_REGEX_PATTERN.match(logValues[2]): coutput.print_watcher(SB_ERR_DEBUG, _FUNC_NAME_, 'wordEntry[1]') if len(wordEntry[1]) > 0: print ">> Downloaded definition override" cfile.write(SDO_OVERRIDE_DEFN_FILE.format(WORD=word), coutput.multiline_text(wordEntry[1])) else: SDO_ERR_DEFN_MISSING = True coutput.print_color('yellow', "WARNING: Definition override not available") if SDO_ERR_AUDIO_REGEX_PATTERN.match(logValues[2]): coutput.print_watcher(SB_ERR_DEBUG, _FUNC_NAME_, 'wordEntry[4]') if wordEntry[4] != "": print ">> Downloaded pronunciation override" cfile.download(connectionPool, wordEntry[4], SDO_OVERRIDE_PRON_FILE.format(WORD=word)) else: SDO_ERR_CLIP_MISSING = True coutput.print_color('yellow', "WARNING: Pronunciation override not available") # Log errors errorText = unicode("ERROR:{0}:", 'utf-8').format(word) if SDO_ERR_DEFN_MISSING: errorText += unicode(">Definition Missing", 'utf-8') if SDO_ERR_CLIP_MISSING: errorText += unicode(">Audio Missing", 'utf-8') if errorText != unicode("ERROR:{0}:", 'utf-8').format(word): errorText += unicode("\n", 'utf-8') cfile.append(SDO_LOG_FILE, errorText)
connectionPool = urllib3.PoolManager(10, headers=SDO_USER_AGENT) wordList = cfile.read(SDO_CONF_FILE).splitlines() print u"Downloading overrides ..." for word in wordList: print u"Word: " + word wordEntry = cdict.fetch_dictionary_entry(connectionPool, word) if len(wordEntry[1]) > 0: cfile.write(SDO_OVERRIDE_DEFN_FILE.format(WORD=word), coutput.multiline_text(wordEntry[1])) else: coutput.print_color('yellow', "WARNING: Definition override not available") if wordEntry[4] != "": cfile.download(connectionPool, wordEntry[4], SDO_OVERRIDE_PRON_FILE.format(WORD=word)) else: coutput.print_color('yellow', "WARNING: Pronunciation override not available") connectionPool.clear() ######################################################################## # Debugging Commands ######################################################################## ''' cd $PROJ sudo python spellit_download_overrides.py '''