def company_data(companies: list): company_results = [] with open('data/raw_companies.json', 'w') as outfile: outfile.write('[') for company in companies: company_serp_params = { "q": company.lower(), "location": "New York, United States", "hl": "en", "gl": "us", "google_domain": "google.com" } company_query = GoogleSearchResults(company_serp_params) company_query.BACKEND = 'https://serpapi.com/search' company_query.SERP_API_KEY = config.API_KEY dictionary_results = company_query.get_dictionary() company_results.append(dictionary_results) print('completed for {}'.format(company)) time.sleep(5) json.dump(dictionary_results, outfile) outfile.write(',') outfile.write(']') return company_results
def test_get_dictionary(self): client = GoogleSearchResults({ "q": "Coffee", "location": "Austin,Texas" }) data = client.get_dictionary() self.assertIsNotNone(data.get('local_results'))
def test_get_html(self): client = GoogleSearchResults({ "q": "Coffee", "location": "Austin,Texas" }) data = client.get_html() self.assertGreater(len(data), 10)
def test_sample(self): query = GoogleSearchResults({ "q": "Coffee", "location": "Austin,Texas" }) results = query.get_json() self.assertEqual(results["local_results"][0]["title"], "Houndstooth Coffee")
def test_get_json(self): client = GoogleSearchResults({ "q": "Coffee", "location": "Austin,Texas" }) data = client.get_json() self.assertIsNotNone(data["local_results"][0]["title"]) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data)
def map_site_helper(url, link_list, count, iterations): if (count == iterations): return link_list new_url = url query = GoogleSearchResults({"q": url, "num": 20}) json_results = query.get_json() print json_results for item in json_results["organic_results"]: new_url += " -" + item['link'] link_list.append(item['link']) return map_site_helper(new_url, link_list, count + 1, iterations)
def _get_serpapi_results(params): try: query = GoogleSearchResults(params) # query.BACKEND = 'http://serpapi.com/search' query.SERP_API_KEY = config.API_KEY dictionary_results = query.get_dictionary() return dictionary_results except Exception as ex: with open('data/serpapi_logs.log', 'a') as f: f.writelines(ex) return None
def search(keyword): query = keyword params = { "playground": "true", "q": query, "hl": "en", "gl": "us", "google_domain": "google.com", "api_key": "demo", } engine_query = GoogleSearchResults(params) dictionary_results = engine_query.get_dictionary() return dictionary_results
def serpapi(query): params = { "q": query, "hl": "en", "gl": "us", "google_domain": "google.com", "api_key": "325864d0-8efc-11e9-b84e-393b768a5ed9", } client = GoogleSearchResults(params) results = client.get_dict() urllist = [x["link"] for x in results["organic_results"]] #extended_urllist = [ (x["link"], x["titel"]) for x in results["organic_results"]] #print(urllist) return urllist
def extractPageFromGoogle(companyName): params = { "q": f"{companyName} ticker symbol", "location": "New York, New York, United States", "hl": "en", "gl": "us", "google_domain": "google.com", "api_key": "ea8844b4b82278532aaf2ef9d64d254cf0db59150b7e3b8e7d09752eccdfffcb", "output": "html", "device": "desktop", } client = GoogleSearchResults(params) #results = [client.get_html(), client.get_json()] results = client.get_html() return results
def get_batch(pageNumber): params = { "q": "apple", "location": "Dallas", "hl": "en", "gl": "us", #"google_domain" : "google.com", "api_key": "demo", "tbm": "isch", "ijn": pageNumber, "source": "test" } # TODO Switch to production query = GoogleSearchResults(params) data = query.get_dictionary() print(data) # extract link links = [] for item in data['images_results']: print(item) links.append(item['original']) return links
def extractTickerFromGoogle(companyName): params = { "q": f" \"{companyName}\" + \"ticker symbol\" ", "location": "New York, New York, United States", "hl": "en", "gl": "us", "google_domain": "google.com", "api_key": "9b66566f114d4a876cb052f3ff8d5ef682bf76424fdab1928f45433f1cf5f06d", "output": "html", "device": "desktop", } client = GoogleSearchResults(params) results = client.get_html() try: found = re.search('<span class=\"HfMth\">(.+?)</span>', results).group(1) except AttributeError: # AAA, ZZZ not found in the original string found = '' return found
from lib.google_search_results import GoogleSearchResults f = open('recognizedtext.txt','r') queryText = f.read() f.close() query = GoogleSearchResults({'q':queryText}) json_results = query.get_json() json_obj1 = json_results['organic_results'] json_obj2 = json_obj1[1] snippet = json_obj2['snippet'] print(snippet) f =open('snippet.txt','w') f.write(snippet) f.close()
stopwords = [ 'what', 'who', 'when', 'where', 'is', 'a', 'at', 'is', 'which', 'the', 'of', 'these', 'this' ] #Test Location Answer querywords = qtext.split() resultwords = [word for word in querywords if word.lower() not in stopwords] qtext = ' '.join(resultwords) print(qtext) #print(qtext) #print(atext) answerlist = atext.split('\n') answerlist = list(filter(None, answerlist)) #print(answerlist) num = 50 answerscore = [0, 0, 0] query = GoogleSearchResults({"q": qtext + answerlist[2], "num": num}) json_results = query.get_json() #pprint(json_results) for i in range(1, num - 1): snippet = json_results['organic_results'][i]['snippet'] print(snippet) print('\n') for j in range(1, 3): if answerlist[j].lower() in snippet.lower(): answerscore[j] = answerscore[j] + 1 print(answerscore) print(time.time() - start_time)
def test_get_account(self): client = GoogleSearchResults({}) account = client.get_account() self.assertIsNotNone(account.get("account_id")) self.assertEqual(account.get("api_key"), GoogleSearchResults.SERP_API_KEY)
def test_get_search_archive(self): client = GoogleSearchResults({"q": "Coffee", "location": "Austin,Texas"}) search_result = client.get_dictionary() search_id = search_result.get("search_metadata").get("id") archived_search_result = GoogleSearchResults({}).get_search_archive(search_id, 'json') self.assertEqual(archived_search_result.get("search_metadata").get("id"), search_id)
with open('serp_urls.txt') as content: content = [line.rstrip('\n') for line in content] for line in content: errorType = '' params = { "q": str(line), "location": "Cleveland, Ohio, United States", "hl": "en", "gl": "us", "google_domain": "google.com", "api_key": api_key, } query = GoogleSearchResults(params) dictionary_results = query.get_dictionary() # print(dictionary_results['knowledge_graph']) try: name = dictionary_results['knowledge_graph']['title'] address = dictionary_results['knowledge_graph']['address'] except KeyError: errorType = 'Error' row = f'"{line}" ~ No listing exists OR is in local map pack. Perform manual search to verify.\n' print( f'{line} ~ No listing exists OR is in local map pack. Perform manual search to verify.' ) file.write(row) if errorType != 'Error':
def test_get_location(self): client = GoogleSearchResults({"q": None, "async": True}) location_list = client.get_location("Austin", 3) self.assertIsNotNone(location_list[0].get("id")) pp = pprint.PrettyPrinter(indent=2) pp.pprint(location_list)
search_list = _create_search_list(params) # loop combines all the possible combinations of key_word & modifier for (zipcode,key_word,modifier,device_type) in search_list: # TODO: delete [:1] after code is ready for zipcode,key_word,modifier,device_type in search_list[:1]: serp_params = { "q" : key_word + " " + modifier, "device" : device_type, "location" : "{}, New York, United States".format(str(zipcode)), "hl" : "en", "gl" : "us", "google_domain" : "google.com", "google_domain" : "google.com" } query = GoogleSearchResults(serp_params) query.BACKEND = 'http://serpapi.com/search' query.SERP_API_KEY = config.API_KEY dictionary_results = query.get_dictionary() # this will concentrate only on out 4 requirments results.append(dictionary_results) @@ -38,8 +42,8 @@ def _create_search_list(params: params): params: its a dict of search list """ # results = [] prod = product(params.zipcodes, params.key_words, params.modifiers, params.device_types) return [prod] prod = product(params["zipcodes"], params["key_words"], params["modifiers"], params["device_types"]) return [i for i in prod] # testing how the prodict results shows up