def test_word_extractor(self): extractor = WordExtractor() rawText = "What? Wait! Stop now. We-are-the-champs. & $%^&*()!@" expected = [ "What", "Wait", "Stop", "now", "We", "are", "the", "champs" ] actual = extractor.get_words_from_text(rawText) self.assertListEqual(actual, expected)
ERROR_RATE_LIMIT = "ERROR RATE LIMIT" ERROR_CLIENT_POOL ="ERROR CLIENT POOL" ERROR_UKNOWN = "ERROR UKNOWN" THROTTLE_SECONDS = 10 EXCEPTION_SLEEP_SECONDS = 60 ERROR_COUNT_SLEEP_SECONDS = 500 extractor = WordExtractor() fetcher = LeoFetcher() dao = MySqlDataSouce() parser = LeoParser() with open('resources/words.txt', 'r') as hall: data = hall.read() words = extractor.get_words_from_text(data) error_count = 0 for word in words : if error_count > 5 : print("Error count reached sleeping " + ERROR_COUNT_SLEEP_SECONDS+ " seconds") error_count = 0 time.sleep(ERROR_COUNT_SLEEP_SECONDS) print("Searching for word: " + word) id = dao.add_search_history(word, ENGLISH) if id <= 0 : print("Adding search history record errored with : " + id) continue try: result = fetcher.search_english_word(ENGLISH, word)