def company_data(companies: list):
company_results = []
with open('data/raw_companies.json', 'w') as outfile:
outfile.write('[')
for company in companies:
company_serp_params = {
"q": company.lower(),
"location": "New York, United States",
"hl": "en",
"gl": "us",
"google_domain": "google.com"
}
company_query = GoogleSearchResults(company_serp_params)
company_query.BACKEND = 'https://serpapi.com/search'
company_query.SERP_API_KEY = config.API_KEY
dictionary_results = company_query.get_dictionary()
company_results.append(dictionary_results)
print('completed for {}'.format(company))
time.sleep(5)
json.dump(dictionary_results, outfile)
outfile.write(',')
outfile.write(']')
return company_results
def test_get_dictionary(self):
client = GoogleSearchResults({
"q": "Coffee",
"location": "Austin,Texas"
})
data = client.get_dictionary()
self.assertIsNotNone(data.get('local_results'))
def test_get_html(self):
client = GoogleSearchResults({
"q": "Coffee",
"location": "Austin,Texas"
})
data = client.get_html()
self.assertGreater(len(data), 10)
def test_sample(self):
query = GoogleSearchResults({
"q": "Coffee",
"location": "Austin,Texas"
})
results = query.get_json()
self.assertEqual(results["local_results"][0]["title"],
"Houndstooth Coffee")
def test_get_json(self):
client = GoogleSearchResults({
"q": "Coffee",
"location": "Austin,Texas"
})
data = client.get_json()
self.assertIsNotNone(data["local_results"][0]["title"])
pp = pprint.PrettyPrinter(indent=2)
pp.pprint(data)
def map_site_helper(url, link_list, count, iterations):
if (count == iterations):
return link_list
new_url = url
query = GoogleSearchResults({"q": url, "num": 20})
json_results = query.get_json()
print json_results
for item in json_results["organic_results"]:
new_url += " -" + item['link']
link_list.append(item['link'])
return map_site_helper(new_url, link_list, count + 1, iterations)
def _get_serpapi_results(params):
try:
query = GoogleSearchResults(params)
# query.BACKEND = 'http://serpapi.com/search'
query.SERP_API_KEY = config.API_KEY
dictionary_results = query.get_dictionary()
return dictionary_results
except Exception as ex:
with open('data/serpapi_logs.log', 'a') as f:
f.writelines(ex)
return None
def search(keyword):
query = keyword
params = {
"playground": "true",
"q": query,
"hl": "en",
"gl": "us",
"google_domain": "google.com",
"api_key": "demo",
}
engine_query = GoogleSearchResults(params)
dictionary_results = engine_query.get_dictionary()
return dictionary_results
def serpapi(query):
params = {
"q": query,
"hl": "en",
"gl": "us",
"google_domain": "google.com",
"api_key": "325864d0-8efc-11e9-b84e-393b768a5ed9",
}
client = GoogleSearchResults(params)
results = client.get_dict()
urllist = [x["link"] for x in results["organic_results"]]
#extended_urllist = [ (x["link"], x["titel"]) for x in results["organic_results"]]
#print(urllist)
return urllist
def extractPageFromGoogle(companyName):
params = {
"q": f"{companyName} ticker symbol",
"location": "New York, New York, United States",
"hl": "en",
"gl": "us",
"google_domain": "google.com",
"api_key":
"ea8844b4b82278532aaf2ef9d64d254cf0db59150b7e3b8e7d09752eccdfffcb",
"output": "html",
"device": "desktop",
}
client = GoogleSearchResults(params)
#results = [client.get_html(), client.get_json()]
results = client.get_html()
return results
def get_batch(pageNumber):
params = {
"q": "apple",
"location": "Dallas",
"hl": "en",
"gl": "us",
#"google_domain" : "google.com",
"api_key": "demo",
"tbm": "isch",
"ijn": pageNumber,
"source": "test"
}
# TODO Switch to production
query = GoogleSearchResults(params)
data = query.get_dictionary()
print(data)
# extract link
links = []
for item in data['images_results']:
print(item)
links.append(item['original'])
return links
def extractTickerFromGoogle(companyName):
params = {
"q": f" \"{companyName}\" + \"ticker symbol\" ",
"location": "New York, New York, United States",
"hl": "en",
"gl": "us",
"google_domain": "google.com",
"api_key":
"9b66566f114d4a876cb052f3ff8d5ef682bf76424fdab1928f45433f1cf5f06d",
"output": "html",
"device": "desktop",
}
client = GoogleSearchResults(params)
results = client.get_html()
try:
found = re.search('<span class=\"HfMth\">(.+?)</span>',
results).group(1)
except AttributeError:
# AAA, ZZZ not found in the original string
found = ''
return found
from lib.google_search_results import GoogleSearchResults
f = open('recognizedtext.txt','r')
queryText = f.read()
f.close()
query = GoogleSearchResults({'q':queryText})
json_results = query.get_json()
json_obj1 = json_results['organic_results']
json_obj2 = json_obj1[1]
snippet = json_obj2['snippet']
print(snippet)
f =open('snippet.txt','w')
f.write(snippet)
f.close()
stopwords = [
'what', 'who', 'when', 'where', 'is', 'a', 'at', 'is', 'which', 'the',
'of', 'these', 'this'
] #Test Location Answer
querywords = qtext.split()
resultwords = [word for word in querywords if word.lower() not in stopwords]
qtext = ' '.join(resultwords)
print(qtext)
#print(qtext)
#print(atext)
answerlist = atext.split('\n')
answerlist = list(filter(None, answerlist))
#print(answerlist)
num = 50
answerscore = [0, 0, 0]
query = GoogleSearchResults({"q": qtext + answerlist[2], "num": num})
json_results = query.get_json()
#pprint(json_results)
for i in range(1, num - 1):
snippet = json_results['organic_results'][i]['snippet']
print(snippet)
print('\n')
for j in range(1, 3):
if answerlist[j].lower() in snippet.lower():
answerscore[j] = answerscore[j] + 1
print(answerscore)
print(time.time() - start_time)
def test_get_account(self):
client = GoogleSearchResults({})
account = client.get_account()
self.assertIsNotNone(account.get("account_id"))
self.assertEqual(account.get("api_key"),
GoogleSearchResults.SERP_API_KEY)
def test_get_search_archive(self):
client = GoogleSearchResults({"q": "Coffee", "location": "Austin,Texas"})
search_result = client.get_dictionary()
search_id = search_result.get("search_metadata").get("id")
archived_search_result = GoogleSearchResults({}).get_search_archive(search_id, 'json')
self.assertEqual(archived_search_result.get("search_metadata").get("id"), search_id)
with open('serp_urls.txt') as content:
content = [line.rstrip('\n') for line in content]
for line in content:
errorType = ''
params = {
"q": str(line),
"location": "Cleveland, Ohio, United States",
"hl": "en",
"gl": "us",
"google_domain": "google.com",
"api_key": api_key,
}
query = GoogleSearchResults(params)
dictionary_results = query.get_dictionary()
# print(dictionary_results['knowledge_graph'])
try:
name = dictionary_results['knowledge_graph']['title']
address = dictionary_results['knowledge_graph']['address']
except KeyError:
errorType = 'Error'
row = f'"{line}" ~ No listing exists OR is in local map pack. Perform manual search to verify.\n'
print(
f'{line} ~ No listing exists OR is in local map pack. Perform manual search to verify.'
)
file.write(row)
if errorType != 'Error':
def test_get_location(self):
client = GoogleSearchResults({"q": None, "async": True})
location_list = client.get_location("Austin", 3)
self.assertIsNotNone(location_list[0].get("id"))
pp = pprint.PrettyPrinter(indent=2)
pp.pprint(location_list)
search_list = _create_search_list(params)
# loop combines all the possible combinations of key_word & modifier
for (zipcode,key_word,modifier,device_type) in search_list:
# TODO: delete [:1] after code is ready
for zipcode,key_word,modifier,device_type in search_list[:1]:
serp_params = {
"q" : key_word + " " + modifier,
"device" : device_type,
"location" : "{}, New York, United States".format(str(zipcode)),
"hl" : "en",
"gl" : "us",
"google_domain" : "google.com",
"google_domain" : "google.com"
}
query = GoogleSearchResults(serp_params)
query.BACKEND = 'http://serpapi.com/search'
query.SERP_API_KEY = config.API_KEY
dictionary_results = query.get_dictionary()
# this will concentrate only on out 4 requirments
results.append(dictionary_results)
@@ -38,8 +42,8 @@ def _create_search_list(params: params):
params: its a dict of search list
"""
# results = []
prod = product(params.zipcodes, params.key_words, params.modifiers, params.device_types)
return [prod]
prod = product(params["zipcodes"], params["key_words"], params["modifiers"], params["device_types"])
return [i for i in prod]
# testing how the prodict results shows up