def get_sentence(word_input): found=False sentence="" for obj in google_json_object: google_word = clean_data_from_html(obj["WORD"]) if google_word == word_input: if obj["SENTENCE"] != "" and obj["SENTENCE"] != "ga_non" and obj["SENTENCE"] is not None and obj["SENTENCE"] != "": sentence = obj["SENTENCE"] found = True break if found: print("sentence from google data..:"+sentence) return sentence else: for obj in your_dictionary_json_object: your_dict_word=clean_data_from_html(obj["WORD"]) if your_dict_word == word_input: if obj["SENTENCE"] != "" and obj["SENTENCE"] != "ga_non" and obj["SENTENCE"] is not None and obj["SENTENCE"] != "": sentence = obj["SENTENCE"][0] found = True break if found: print("sentence from your dictionary.."+ sentence) return sentence else: print("sentence not found for word....: "+word_input) return ""
def fetch_word_details(): initialize_variables("OPEN") word = "Bedizen" find_id = random.randint(10000, 20000) final_type = get_type(word) final_meaning = get_meaning(word) final_sentence = clean_data_from_html(get_sentence(word)) final_synonyms = get_synonyms(word) final_antonyms = get_antonyms(word) final_attr1 = clean_data_from_html(get_attr1(word)) word_string = "" word_string = word_string+"{\"ID\":"+str(find_id)+"," word_string = word_string+"\"WORD\":\""+word+"\"," word_string = word_string + "\"TYPE\":\"" + final_type + "\"," word_string = word_string + "\"MEANING\":\"" + final_meaning + "\"," word_string = word_string + "\"SENTENCE\":\"" + final_sentence + "\"," word_string = word_string + "\"SYNONYMS\":\"" + final_synonyms + "\"," word_string = word_string + "\"ANTONYMS\":\"" + final_antonyms + "\"," word_string = word_string + "\"ATTR1\":\"" + final_attr1 + "\"," word_string = word_string + "\"LINK\":\"\"," word_string = word_string + "\"ATTR2\":\"\"}" print("final word string..:"+word_string) write_with_remove_redundancy(word_string,word) initialize_variables("CLOSE")
def json_creator_barrons(): print("creating barron 333 and 800 ..:") filein = open( "D:\Video Work Area\GRE WORDS APP\data\production\WordsList.json", "r") filein_obj = json.load(filein)["words"] barron800_words_in = open( "D:\Video Work Area\GRE WORDS APP\data\Barron800WordsList.txt", "r") barron800_json_out = open( "D:\Video Work Area\GRE WORDS APP\data\production\Barron800.json", "w") barron800_json_out.seek(0) barron800_json_out.truncate() barron333_words_in = open( "D:\Video Work Area\GRE WORDS APP\data\Barron333WordsList.txt", "r") barron333_json_out = open( "D:\Video Work Area\GRE WORDS APP\data\production\Barron333.json", "w") barron333_json_out.seek(0) barron333_json_out.truncate() barron800_final_string = "{\"BARRON800\":[" barron333_final_string = "{\"BARRON333\":[" for obj in filein_obj: word = clean_data_from_html(obj["WORD"]) barron800_words_in.seek(0) barron333_words_in.seek(0) wid = obj["ID"] for barron800_word in barron800_words_in: if word == clean_data_from_html(barron800_word): barron800_final_string = barron800_final_string + "{\"BARRONID\":" + str( wid) + "}," break for barron333_word in barron333_words_in: if word == clean_data_from_html(barron333_word): barron333_final_string = barron333_final_string + "{\"BARRONID\":" + str( wid) + "}," break barron333_final_string = barron333_final_string[:-1] barron800_final_string = barron800_final_string[:-1] barron333_final_string = barron333_final_string + "]}" barron800_final_string = barron800_final_string + "]}" print("generated barron 333 string..:" + barron333_final_string) print("generated barron 800 string..:" + barron800_final_string) barron333_json_out.write(barron333_final_string) barron800_json_out.write(barron800_final_string) barron333_json_out.flush() barron800_json_out.flush() barron800_json_out.close() barron333_json_out.close() barron800_words_in.close() barron333_words_in.close()
def write_with_remove_redundancy(word_string, word): file = open( r"D:\Video Work Area\GRE WORDS APP\data\scrapped\WordsList.json", "r+") redundant = False redundant_count = 1 all_obj = json.load(file)["words"] for obj in all_obj: if word == obj["WORD"] and obj["WORD"] != "": redundant = True if redundant_count == 1: print( "Word details was added in file.. updating values in temp word list file" ) obj["TYPE"] = get_type(word) obj["MEANING"] = get_meaning(word) obj["SENTENCE"] = clean_data_from_html(get_sentence(word)) obj["SYNONYMS"] = get_synonyms(word) obj["ANTONYMS"] = get_antonyms(word) obj["ATTR1"] = clean_data_from_html(get_attr1(word)) redundant_count = redundant_count + 1 else: print( "Found multiple entries in temp word details .. " "updating values to empty all data.....total redundancy..:" + str(redundant_count)) obj["WORD"] = "" obj["TYPE"] = "" obj["MEANING"] = "" obj["SENTENCE"] = "" obj["SYNONYMS"] = "" obj["ANTONYMS"] = "" obj["ATTR1"] = "" if redundant: file.seek(0) file.truncate() file.write("{\"words\":") json.dump(all_obj, file) file.write("}") file.flush() file.close() else: file.truncate((file.tell()) - 2) file.write(",") file.write(word_string) file.write("]}") file.flush() file.close()
def marriamwebsterScrap(site, word): marriamDicObj = {} marriamDicObj["WORD"] = word marriamwebsterFinalSite = site + word marriamDicObj["SITE"] = marriamwebsterFinalSite marriamwebsterContent = requests.get(marriamwebsterFinalSite, headers=header).text marriamSoup = BeautifulSoup(marriamwebsterContent, features="html.parser") # print(marriamSoup.prettify()) synm_comin = False antm_coming = False try: mariamSynmDiv = marriamSoup.find_all( "div", {"class": "thesaurus-synonyms-module-anchor"})[0] pTagAll = mariamSynmDiv.findAll('p') for pTag in pTagAll: if synm_comin: marriamDicObj["SYNONYMS"] = clean_data_from_html(pTag.text) synm_comin = False if clean_data_from_html(pTag.text) == "Synonyms": synm_comin = True if antm_coming: marriamDicObj["ANTONYMS"] = clean_data_from_html(pTag.text) antm_coming = False if clean_data_from_html(pTag.text) == "Antonyms": antm_coming = True except Exception as ex: marriamDicObj["SYNONYMS"] = "ga_non" marriamDicObj["SYNONYMS"] = "ga_non" if "SYNONYMS" not in marriamDicObj: marriamDicObj["SYNONYMS"] = "ga_non" if "ANTONYMS" not in marriamDicObj: marriamDicObj["ANTONYMS"] = "ga_non" try: citeexamples = marriamSoup.find_all("div", {"class": "in-sentences"})[0] citespan = citeexamples.findAll('span') citeexamplesList = [] for span in citespan: citeexamplesList.append(clean_data_from_html(span.text)) break marriamDicObj["SENTENCE"] = citeexamplesList except Exception as ex: marriamDicObj["SENTENCE"] = "ga_non" return marriamDicObj
def get_meaning(word_input): found = False meaning = "" for obj in vocab_json_object: vocab_word = clean_data_from_html(obj["WORD"]) if vocab_word == word_input: found = True meaning = obj["MEANING"] break if found: print("meaning found in vocab dictionary : " + meaning) return meaning else: dict_meaning = dictionary.meaning(word_input) if dict_meaning is not None: for key in dict_meaning: for m in dict_meaning.get(key): found = True #meaning = meaning+m+" OR " meaning = m break if found: # meaning=meaning[:-4] print("meaning found in pyDictionary..:" + meaning) return meaning else: print("error.. meaning not found for...." + word_input) return ""
def get_antonyms(word_input): antonyms = "" found = False for obj in google_json_object: google_word = clean_data_from_html(obj["WORD"]) if google_word == word_input: if obj["ANTONYMS"] != "" and obj["ANTONYMS"] != "ga_non" and obj["ANTONYMS"] is not None and obj["ANTONYMS"] != "": antonyms = obj["ANTONYMS"] found = True break if found: print("antonyms from google..:" + antonyms) return antonyms else: for obj in marrian_json_object: marrian_word = obj["WORD"] if marrian_word == word_input: if obj["ANTONYMS"] != "" and obj["ANTONYMS"] != "ga_non" and obj["ANTONYMS"] is not None and obj["ANTONYMS"] != "": antonyms = obj["ANTONYMS"] found = True break if found: print("antonyms from marriam dictionary..:" + antonyms) return antonyms else: print("Antonyms not found for ...:" + word_input+".... returning empty") return antonyms
def get_synonyms(word_input): synonyms = "" found = False for obj in google_json_object: google_word = clean_data_from_html(obj["WORD"]) if google_word == word_input: if obj["SYNONYMS"] != "" and obj["SYNONYMS"] != "ga_non" and obj[ "SYNONYMS"] is not None and obj["SYNONYMS"] != "": synonyms = obj["SYNONYMS"] found = True break if found: print("synonyms from google..:" + synonyms) return synonyms else: for obj in marrian_json_object: marrian_word = obj["WORD"] if marrian_word == word_input: if obj["SYNONYMS"] != "" and obj[ "SYNONYMS"] != "ga_non" and obj[ "SYNONYMS"] is not None and obj["SYNONYMS"] != "": synonyms = obj["SYNONYMS"] found = True break if found: print("synonyms from marriam dictionary..:" + synonyms) return synonyms else: for obj in vocab_json_object: vocab_word = obj["WORD"] if vocab_word == word_input: if obj["SYNONYMS"] != "" and obj[ "SYNONYMS"] != "ga_non" and obj[ "SYNONYMS"] is not None and obj["SYNONYMS"] != "": synonyms = obj["SYNONYMS"] found = True break if found: print("synonyms from vocab dictionary..:" + synonyms) return synonyms else: print("not found for ...:" + word_input) return synonyms
def collinScrap(site, word): collinDicObj = {} collinDicObj["WORD"] = word collingsDictionaryFinalURL = site + word collinDicObj["SITE"] = collingsDictionaryFinalURL collinDictionaryContent = requests.get(collingsDictionaryFinalURL, headers=header).text collingDictionarySoup = BeautifulSoup(collinDictionaryContent, features="html.parser") try: collingDescription = collingDictionarySoup.find_all( "div", {"class": "def"})[0].text collingDescription = clean_data_from_html(collingDescription) collinDicObj["ATTR1"] = collingDescription except Exception as ex: collinDicObj["ATTR1"] = "ga_non" return collinDicObj
def googleScrap(googleSite, word): googleDictionaryObj = {} googleDictionaryObj["WORD"] = clean_data_from_html(word) googleDefineFinalSite = googleSite + word #print("google final site:" + googleDefineFinalSite) googleDictionaryObj["SITE"] = googleDefineFinalSite try: page = requests.get(googleDefineFinalSite, headers=header).text googleDefineSoup = BeautifulSoup(page, features="html.parser") googleSentene = "" googleSentenceSkipFirst = True try: for div in googleDefineSoup.find_all( lambda tag: tag.name == 'div' and tag.get('class' ) == ['vk_gy']): if (googleSentenceSkipFirst): googleSentenceSkipFirst = False else: googleSentene += div.text break except (IndexError, ValueError): googleSentene += "ga_non" googleDictionaryObj["SENTENCE"] = clean_data_from_html(googleSentene) googleSynms = "" removeSymnText = True try: for td in googleDefineSoup.find_all( "table", {"class": "vk_tbl vk_gy"})[0].find_all('td'): if (removeSymnText): removeSymnText = False else: googleSynms += td.text break except (IndexError, ValueError): googleSynms += "ga_non" googleSynms = clean_data_from_html(googleSynms) googleSynms = googleSynms.split(";")[0] googleDictionaryObj["SYNONYMS"] = googleSynms #print(googleSynms) googleAntonm = "" removeAntonymText = True try: for td in googleDefineSoup.find_all( "table", {"class": "vk_tbl vk_gy"})[1].find_all('td'): if (removeAntonymText): removeAntonymText = False else: googleAntonm += td.text break except (IndexError, ValueError): googleAntonm += "ga_non" googleAntonm = clean_data_from_html(googleAntonm) googleDictionaryObj["ANTONYMS"] = googleAntonm #print("Google Antonyms......::::::::::::"+googleAntonm) #print(googleAntonm) except Exception as e: print( "google connection attempt failed..host not responded with request" ) #print(googleDefineSoup.prettify()) return googleDictionaryObj
def vocalbularySiteScrap(site, word): vocabularyDictionaryObj = {} vocabularyDictionaryObj["WORD"] = word vocabularySiteFinalUrl = site + word vocabularyDictionaryObj["SITE"] = vocabularySiteFinalUrl #print(vocabularySiteFinalUrl) try: vocabularySiteContent = urlopen(vocabularySiteFinalUrl) vocabularySiteSoup = BeautifulSoup(vocabularySiteContent, features="html.parser") countLine = 3 counter = 1 counter = 1 vocanSiteMeaning = vocabularySiteSoup.find( lambda tag: tag.name == 'h3' and tag.get('class' ) == ['definition']) for vocabMeaningSTR in vocanSiteMeaning: if (countLine == counter): #print(cleanDataFromHTML(vocabMeaningSTR)) vocabMeaningSTR = clean_data_from_html(vocabMeaningSTR) break else: counter = counter + 1 except Exception as ex: vocabMeaningSTR = "ga_non" vocabularyDictionaryObj["MEANING"] = vocabMeaningSTR try: VocabularySitedescription = vocabularySiteSoup.find( "meta", property="og:description") vocabularyDictionaryObj["ATTR1"] = ( VocabularySitedescription["content"] if VocabularySitedescription else "ga_non") #print(VocabularySitedescription["content"] if VocabularySitedescription else "ga_non") except Exception as ex: vocabularyDictionaryObj["ATTR1"] = "ga_non" try: vocabularySentence = vocabularySiteSoup.find("div", { "class": "example" }).get_text() vocabularyDictionaryObj["SENTENCE"] = clean_data_from_html( vocabularySentence if vocabularySentence else "ga_non") #print(vocabularyDictionaryObj["SENTENCE"]) except Exception as ex: vocabularyDictionaryObj["SENTENCE"] = "ga_non" vocabSynmsList = "" try: vocabularySynonyms = vocabularySiteSoup.findAll("a", {"class": "word"}) for element in vocabularySynonyms: vocabSynmsList += element.text + ", " vocabSynmsList = vocabSynmsList[:-2] vocabularyDictionaryObj["SYNONYMS"] = vocabSynmsList #print(vocabSynmsList) except Exception as ex: vocabularyDictionaryObj["SYNONYMS"] = "ga_non" #print(vocabularySiteSoup.prettify()) return vocabularyDictionaryObj
from gre_words_scripts.utils import init_files import time import random all_words = open("D:\Video Work Area\GRE WORDS APP\data\\all_words.txt", "r") words_list = open( "D:\Video Work Area\GRE WORDS APP\data\production\WordsList.json", "r") # init_files() # >>>>>>>>>>>> reset values most imp method <<<<<<<<<<<<<<<<< words_list_dict = json.load(words_list)["words"] count = 1 for word in all_words: word = clean_data_from_html(word) count = count + 1 exists = False for word_obj in words_list_dict: if word_obj["WORD"] == word: exists = True break if exists is False: print("\n\n\n >>>>>>>>>>>>>>>>>>>>>>>>>>> Processing ..: " + word + " ::" + str(count) + " Percent Done..:" + str((int((count / 801) * 100))) + "% <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n\n") # get word details and store them in scrapper folder