def post(self, request):
title = 'Scrap Data'
serpwow = GoogleSearchResults("6F46BE7319E84E65978F88651E180FC2")
keyword = {
"q": request.POST.get("keyword"),
"location": request.POST.get("country")
}
result = serpwow.get_json(keyword)
keyword_search = json.dumps(result, indent=2, sort_keys=True)
array = json.loads(keyword_search)
keyword_find = array
# keyword_find = json2html.convert(keyword_search)
country = CountryForm()
messages.success(request, 'Your data is here...')
context = {
'title': title,
'keyword_find': keyword_find,
'country': country
}
return render(request, 'admin/scraping/scrap_data.html', context)
def populate_search_results():
query = input("WRITE YOUR SEARCH QUERY:")
search_params = {"q": query, "location": "Austin, Texas"}
# search_engines = ["yandex", "google"]
search_engines = ["yandex", "google", "bing"]
searcher = GoogleSearchResults(MY_SERPWOW_API_KEY)
common_result = [] # 2d
for engine in search_engines:
print("*" * 300)
print("" + engine + " RESULT")
print("*" * 300)
data = searcher.get_json({**search_params, **{"engine": engine}})
result = data["organic_results"]
# json_formatted_str = json.dumps(result, indent=2)
# print(json_formatted_str)
mapped_entities = object_decoder(result, engine)
[obj.pretty_print() for obj in mapped_entities]
common_result.append(mapped_entities)
return common_result
def locate_url(query, location):
serpwow = GoogleSearchResults("6C6E8E58B43740A0BB943B8AD68D53C4")
# set up a dict for the search parameters
params = {"q": query, "location": location}
# retrieve the search results as JSON
result = serpwow.get_json(params)
# pretty-print the result
return (json.dumps(result, indent=2, sort_keys=True))
def __init__(self):
# ================= setup wolfram alpha ==============
self.wolfram_app_id = 'G5WRTA-882W6ATLGU'
self.wolframalpha_client = wolframalpha.Client(self.wolfram_app_id)
# ================= setup google api =================
self.google_api_key = "AIzaSyB4J9XxLeCeKdVm9Vs0fcihWhaQBV7jpjI"
self.google_cse_id = "3499ed51db04db22e"
self.google_service = build("customsearch",
"v1",
developerKey=self.google_api_key)
self.serpwow = GoogleSearchResults(self.google_api_key)
self.object_template = {
"head": {
"vars": ["v"]
},
"results": {
"bindings": []
}
}
from serpwow.google_search_results import GoogleSearchResults
import json
import os
from os.path import join, dirname
from dotenv import load_dotenv
dotenv_path = join(dirname(__file__), '.env')
load_dotenv(dotenv_path)
SERP_TOK = os.environ.get("SERP_TOKEN")
serpwow = GoogleSearchResults(SERP_TOK)
#Using SERPWOW library to search google history
def search_google(params):
result = serpwow.get_json(params)
links_result = ""
for i in range(0, (len(result["organic_results"]))):
links_result += result["organic_results"][i]["link"] + "\n"
links_result += "\n\nFriendly Reminder, only " + str(
result["request_info"]
["credits_remaining"]) + " searches are remaining."
return (links_result)
print('error')
if __name__ == '__main__':
# search_name = 'fidget kid spinner toys'
# links = get_links(search_name)
# save_images(links, search_name)
os.chdir('training_images')
##//////////////////////////////////////////////////////////
## /////////////// Collecting the images //////////////////
##//////////////////////////////////////////////////////////
# create the serpwow object, passing in our API key
serpwow = GoogleSearchResults("D91EDA24")
# search_names=['yield sign france .jpg',
# 'yield sign usa .jpg',
# 'yield sign uk .jpg',
# 'yield sign .jpg',
# 'canada yield sign .jpg',
# 'american yield sign .jpg',
# 'yield sign canada .jpg',
# 'canadian yield signs .jpg',
# 'panneaux cédez le passage .jpg'
# ]
# search_names=['stop sign france ',
# 'stop sign usa ',
# 'stop sign uk ',
# 'stop sign ',
# 'canada stop sign ',
# from serpwow.google_search_results import GoogleSearchResultsimport
from serpwow.google_search_results import GoogleSearchResults
import geocoder
import json
serpwow = GoogleSearchResults("7491F4F9F2204722BD6A2BEEFAB0158E")
ipad = geocoder.ip('me')
def searchhosp(cityname):
q = "Hospitals near " + cityname
location = cityname + "," + str(ipad[0])
params = {"q": q, "type": location}
# retreive the google sewarc results
result_json = serpwow.get_json(params)
print(result_json)
names = result_json['local_results']
print(names)
return names
# ----------------------------------------------------------------
# BlackBess
# ----------------------------------------------------------------
from serpwow.google_search_results import GoogleSearchResults
import json
import pandas as pd
import csv
datos = [] # para guardar las variables
#API key
serpwow = GoogleSearchResults("D7D59D53562A42AEBE6FA07653381B48")
# Parametros de busqueda
params = {
"q": "pizza", # Que buscar?
"gl": "ec", # Pais
"hl": "es", # Idioma
"location": "Quito,Pichincha,Ecuador",
"google_domain": "google.com.ec",
"search_type": "places" # Google places
}
# Resultados en formato csv
result = serpwow.get_csv(params)
# Impresion
# print(json.dumps(result, indent=2, sort_keys=True))
print(result)
# Guardar datos en Excel
from serpwow.google_search_results import GoogleSearchResults
import stanza
from joblib import load
import langdetect
absDir = os.path.dirname(os.path.abspath(__file__))
clf_file_name = './classifier.joblib'
demo_data_file_name = './demo_data.json'
key_file_name = './key.pem'
cert_file_name = './cert.pem'
clf_file = os.path.join(absDir, clf_file_name)
demo_data_file = os.path.join(absDir, demo_data_file_name)
key_file = os.path.join(absDir, key_file_name)
cert_file = os.path.join(absDir, cert_file_name)
serpwow = GoogleSearchResults("FA2C2241D0A346A28A4A3A6E180C09A4")
nlp = stanza.Pipeline('uk', processors='tokenize,pos,lemma,depparse')
clf = load(clf_file)
''' NLP '''
def parse_features(feats_string):
res = {}
feats = feats_string.split('|')
for feat in feats:
k, v = feat.split('=')
res[k] = v
return res
def find_dates(string, is_future=False):
import itertools
import seaborn as sns
from collections import Counter
import operator
import requests
import json
import pandas
import csv
from bs4 import BeautifulSoup
from serpwow.google_search_results import GoogleSearchResults
"""## API Cajones"""
# This API KEY cames from this site https://app.serpwow.com/playground
serpwow = GoogleSearchResults("7563AA721C454E038A3146CCF1AA949A") #This APIKEY is not valid anymore as I already used the 100 free requests.
queries = [
'emuca cajones ebay',
'emuca cajones ferreteria.es',
'emuca cajones amazon',
'emuca cajones ferreterias industriales',
'emuca cajones manommano.es',
'emuca cajones tuandco.com',
'emuca cajones masferreteria',
'emuca cajones valomix',
]
results = []
class WolframGoogle:
def __init__(self):
# ================= setup wolfram alpha ==============
self.wolfram_app_id = 'G5WRTA-882W6ATLGU'
self.wolframalpha_client = wolframalpha.Client(self.wolfram_app_id)
# ================= setup google api =================
self.google_api_key = "AIzaSyB4J9XxLeCeKdVm9Vs0fcihWhaQBV7jpjI"
self.google_cse_id = "3499ed51db04db22e"
self.google_service = build("customsearch",
"v1",
developerKey=self.google_api_key)
self.serpwow = GoogleSearchResults(self.google_api_key)
self.object_template = {
"head": {
"vars": ["v"]
},
"results": {
"bindings": []
}
}
# res = client.query('stability of FMOC amide + zinc')
def get_result_from_google(self, question):
URL = f"https://google.com/search?q={question}"
driver = webdriver.Firefox()
driver.get(URL)
html = driver.find_element_by_tag_name('html').get_attribute(
'innerHTML')
# print(html)
soup = BeautifulSoup(html, 'html.parser')
try:
div_result = soup.find_all('div', class_='ifM9O')[0]
driver.quit()
return str(div_result)
except:
return 'Google failed to provide an answer'
def get_result_from_google_directly(self, question):
# query = question.replace(' ', '+')
URL = f"https://google.com/search?q={question}"
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36"
MOBILE_USER_AGENT = "Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, " \
"like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36 "
headers = {"user-agent": USER_AGENT}
resp = requests.get(URL, headers=headers)
print(resp.status_code)
if resp.status_code == 200:
soup = BeautifulSoup(resp.content, "html.parser")
all_images = soup.find_all('g-img')[0]
print('====== all images =====')
print(all_images)
print('-----------')
print(all_images.attrs)
blk_result = soup.find_all('div', class_='g mnr-c g-blk')[0]
print('-------- this is the blk ----------')
print(blk_result)
print('===================================')
images = blk_result.find_all('g-img')
rst = ''
for img in images:
print(img)
print('-----')
print(img.text)
print('============')
return rst
# print('--------- here is what you want --------')
# pprint(blk_result)
# print('----------------------------------------')
#
#
# div_result = soup.find_all('div', id='rso')[0]
# children_divs = div_result.findChildren()
# first_child = children_divs[0]
# first_result = first_child.findChildren()[0]
# headings = first_result.find_all('div', role='heading')
# valid_results = []
# for head in headings:
# if 'People also search for' in str(head) or ('View 10+' in str(head)):
# pass
# else:
# valid_results.append(head.text)
# result_div = '<br/>'.join(valid_results)
# print('====== result div ==========')
# print(result_div)
#
# return result_div
# return str(blk_result)
else:
# This indicates that Google does not give you a direct answer...
return None
def get_result_from_serpwow(self, question):
params = {"q": question}
result = self.serpwow.get_json(params)
pprint(result)
def get_result_from_google_official(self, question):
print('----- question received for google --------', question)
res = self.google_service.cse().list(q=question.replace(
'add_sign', '+'),
cx=self.google_cse_id).execute()
pprint(res)
# for item in res['items']:
# pprint(item)
def get_result_from_wolfram(self, question):
print('----- question received --------', question)
question = question.replace('add_sign', ' + ')
res = self.wolframalpha_client.query(question)
bindings = ''
counter = 0
print('------------ wolfram alpha result -----------')
try:
for pod in res.pods:
counter = counter + 1
for sub in pod.subpods:
print('------------ sub ------------')
print(sub)
try:
text = sub.plaintext
if text is None:
try:
text = sub.img
except:
print('The attempt to get image failed')
except:
print('the attempt to get text failed')
if counter <= 2 and (text is not None):
bindings = bindings + '<br/>' + text
except KeyError:
print('Wolfram alpha failed to provide an answer')
return 'Wolfram alpha failed to provide an answer'
except AttributeError:
print('Wolfram alpha failed to provide an answer')
return 'Wolfram alpha failed to provide an answer'
except TypeError:
print('Wolfram alpha failed to provide an answer')
return 'Wolfram alpha failed to provide an answer'
return bindings.replace('"', '')
from serpwow.google_search_results import GoogleSearchResults
import json
import pickle
if __name__ == "__main__":
# create the serpwow object, passing in our API key
ai_api = "request_online"
serpwow = GoogleSearchResults(ai_api)
# set up a dict for the search parameters
params = {"q": "covid", "search_type": "trends"}
# retrieve the search results as JSON
result = serpwow.get_json(params)
pickle.dump(result, open('covid_trends.pkl', 'wb'))
import geocoder
from serpwow.google_search_results import GoogleSearchResults
import json
ipad = geocoder.ip('me')
serpwow = GoogleSearchResults("883CB6B9970E4AE1858E3381DF8F0674")
def hospital_search(place):
q = "Good hospitals near " + place
l = place + "," + str(ipad[0])
params = {
"q": q,
"location": l,
"hl": "en",
}
# retrieve the Google search results as JSON
result_json = serpwow.get_json(params)
names = result_json['local_results']
return names
def pharmacy_search(place):
q = "Good Medical shops near " + place
l = place + "," + str(ipad[0])
params = {
"q": q,
"location": l,
"hl": "en",
}
import itertools
import seaborn as sns
from collections import Counter
import operator
import requests
import json
import pandas
import csv
from bs4 import BeautifulSoup
from serpwow.google_search_results import GoogleSearchResults
"""## API Cajones"""
# This API KEY cames from this site https://app.serpwow.com/playground
serpwow = GoogleSearchResults("9E083A81FB0B470EB5C83270280C9998") #This APIKEY is not valid anymore as I already used the 100 free requests.
queries = [
'cajones ebay',
'cajones ferreteria.es',
'cajones amazon',
'cajones ferreterias industriales',
'cajones manommano.es',
'cajones tuandco.com',
'cajones masferreteria',
'cajones valomix',
]
results = []
stap = stap + 1
x=ser.readline().strip('\r\n') #Scanner 1 input uitlezen
x1=ser2.readline().strip('\r\n') #Scanner 2 input uitlezen
#Scanner 1
if len(x) == 0:
#Er wordt niks gescand
print ( "Scanner 1: Scan iets in")
barcodeScanned = False
else:
barcodeScanned = True
print ( "In: ", x)
authPort.write("s2\n")
#Image Search
# create the serpwow object, passing in our API key
serpwow = GoogleSearchResults("4F87B9EDF89C4E7F8E5648AF0D257DAC")
# set up a dict for the search parameters
params = {
"q" : x,
"search_type" : "images"
}
mime = magic.Magic(mime=True)
# retrieve the search results as JSON
result = serpwow.get_json(params)
# define name for file name
try:
name = result['image_results'][0]['title']
# get imageurl value of json result
imgurl = result['image_results'][0]['image']
channel.queue_declare(queue=CONFIG["RABBITMQ"]["QUEUE_SEARCH"], durable=True)
logging.info('Connected RabbitMQ Success!')
# 鏈接mysql
logging.info('Connecting MySQL Server...')
db = pymysql.connect(
host=CONFIG["Database"]["host"],
user=CONFIG["Database"]["user"],
passwd=CONFIG["Database"]["passwd"],
db=CONFIG["Database"]["dbname"],
charset=CONFIG["Database"]["charset"],
cursorclass=pymysql.cursors.DictCursor)
logging.info('Connected MySQL Server Success!')
cursor = db.cursor()
serpwow = GoogleSearchResults(CONFIG["GSR_API_KEY"])
searchResultLimit = 50
TEXT_LIMIT = 150
def insert_into_search(search_string: str):
sql = "SELECT `SearchId` FROM `search` WHERE `SearchString`=%s ORDER BY `SearchId` DESC"
# 需要先執行sql語句
if cursor.execute(sql, (search_string)):
result = cursor.fetchone()
# 通過下標取出值即可
print('已有相同搜尋,對應的id是：', result['SearchId'])
return result['SearchId']
else:
print('沒有對應的搜尋，新增中。。。。')
insert_color = "INSERT INTO `search` (`SearchString`) VALUES (%s)"
# # data = client.get_json()
#
# # client = YahooSearchResults({"p": "Coffee"})
# # data = client.get_json()
# #
# # json_formatted_str = json.dumps(data, indent=2)
# # print(json_formatted_str)
import json
from serpwow.google_search_results import GoogleSearchResults
search_params = {"q": "wow", "location": "Austin, Texas"}
search_engines = ["yandex", "google"]
# search_engines = ["yandex", "google", "bing", "naver", "amazon"]
searcher = GoogleSearchResults("53B07FA3EE984B17AFE7300A43EA4FB6")
for engine in search_engines:
print("*" * 300)
print("" + engine + " RESULT")
print("*" * 300)
data = searcher.get_json({**search_params, **{"engine": engine}})
result = data["organic_results"]
json_formatted_str = json.dumps(result, indent=2)
print(json_formatted_str)
def serpwow_request(self, keywords: str, engine: str) -> dict:
"""
"""
serpwow = GoogleSearchResults(self.api_key)
params = {"q": keywords, "engine": engine}
return serpwow.get_json(params)
from sklearn.preprocessing import OneHotEncoder
import timeit
import itertools
import seaborn as sns
from collections import Counter
import operator
import requests
import json
import pandas
import csv
from bs4 import BeautifulSoup
from serpwow.google_search_results import GoogleSearchResults
# This API KEY cames from this site https://app.serpwow.com/playground
serpwow = GoogleSearchResults("2A086E3D970C450C9ED9A369982ADFEA") #This APIKEY is not valid anymore as I already used the 100 free requests.
queries = [
'emuca cajones ebay',
'emuca cajones ferreteria.es',
'emuca cajones amazon',
'emuca cajones ferreterias industriales',
'emuca cajones manommano.es',
'emuca cajones tuandco.com',
'emuca cajones masferreteria',
'emuca cajones valomix',
'emuca bisagras ebay',
'emuca bisagras ferreteria.es',
'emuca bisagras amazon',
'emuca bisagras ferreterias industriales',
from serpwow.google_search_results import GoogleSearchResults
import json
import mysql.connector
import csv
import os
SERPWOW_API_KEY = os.environ.get("SERPWOW_API_KEY")
# create the serpwow object, passing in our API key
serpwow = GoogleSearchResults(SERPWOW_API_KEY)
# set up a dict for the search parameters
categories = [
'Cafe', 'Grocery Store', 'Bank', 'Liquor Store', 'Gas Station', 'Pharmacy',
'Restaurant', 'Hospital', 'Retail Store'
]
# mydb = mysql.connector.connect(
# host="localhost",
# user="******",
# passwd=""
# )
# mycursor = mydb.cursor()
# mycursor.execute("CREATE DATABASE IF NOT EXISTS BUSINESS_DB")
# mydb = mysql.connector.connect(host = "localhost", user = '******', passwd = '', database='business_db', port="3306")
# cursor = mydb.cursor()
# def save_mysql(tablename, data): cursor.execute("CREATE TABLE IF NOT EXISTS" + " `"+tablename+"`(`name` TEXT,
# `address` TEXT,`latitude` TEXT,`longitude` TEXT)") insertstring = "INSERT INTO"+ " " + "`"+ tablename + "`(`name`,
# `address`,`latitude`,`longitude`) VALUES (%s,%s,%s,%s)"