def getTranslate(place_id):
Key = Keygen()
translate = {}
languages = [
"uk", "ru", "ca", "lv", "en", "pl", "de", "fr", "it", "es", "ro", "nl",
"el", "cs", "pt", "hu", "sv", "bg", "sr", "da", "fi", "sk", "sl", "hr",
"lt"
]
for lang in languages:
keyAPI = Key.get_key_geocode()
if not keyAPI:
sys.exit()
cnf = {'googlemaps': {'geocoding': {'key': keyAPI}}}
config.set(cnf)
spider = Spider(loader_factory=LoaderFactory,
gmap_parser=MapFactory.spain,
wiki_parser=True,
doc_factory=doc_factory,
config=config,
use_cache=True)
spider.gmap_loader._language = lang
objects = spider.get_gmap_place_id(place_id)
gmap = {}
if objects:
gmap = objects[0].get_document()
translate[lang] = {
'name': gmap.get('name'),
}
return translate
def by_place_id(list_places, city_id):
config = Config('./config/config.yml')
Key = Keygen()
keyAPI = Key.get_key_geocode()
if not keyAPI:
sys.exit()
cnf = {'googlemaps': {'geocoding': {'key': keyAPI}}}
config.set(cnf)
language = 'uk'
doc_factory = DocFactory(config.get('mongodb'))
spider = Spider(loader_factory=LoaderFactory,
gmap_parser=MapFactory.spain,
doc_factory=doc_factory,
language=language,
config=config,
use_cache=True)
for loc in list_places:
objects = spider.get_gmap_place_id(loc)
gmap = {}
try:
gmap = objects[0].get_document()
gmap["city_id"] = city_id
exixts = db.ukraine_city_sublocal.find({
"city_id": city_id,
"code": gmap['code']
}).count()
if exixts < 1:
gmap.pop('_id', None)
db.ukraine_city_sublocal.save(gmap)
print(gmap)
except Exception as e:
print(str(e))
def get_place(lat, lng):
datalist = list()
Key = Keygen()
keyAPI = Key.get_key_place()
if not keyAPI:
sys.exit()
url = 'https://maps.googleapis.com/maps/api/place/radarsearch/json?location=' + str(
lat) + ',' + str(lng) + '&radius=20000&type=neighborhood&key=' + str(
keyAPI) + ''
response = requests.get(url)
data = response.json()
result = data['results']
# print(data['results'])
for x in result:
# print (x['place_id'])
datalist.append(x['place_id'])
# print ("====================")
url = 'https://maps.googleapis.com/maps/api/place/radarsearch/json?location=' + str(
lat) + ',' + str(lng) + '&radius=20000&type=sublocality&key=' + str(
keyAPI) + ''
response = requests.get(url)
data = response.json()
result = data['results']
# print(data['results'])
for x in result:
# print (x['place_id'])
datalist.append(x['place_id'])
# print ("#########################################")
return datalist
def getTranslate(place_id):
Key = Keygen()
translate = {}
languages = [
"uk", "ru", "ca", "lv", "en", "pl", "de", "fr", "it", "es", "ro", "nl",
"el", "cs", "pt", "hu", "sv", "bg", "sr", "da", "fi", "sk", "sl", "hr",
"lt"
]
for lang in languages:
keyAPI = Key.get_key_geocode()
if not keyAPI:
sys.exit()
cnf = {'googlemaps': {'geocoding': {'key': keyAPI}}}
config.set(cnf)
objects = spider.get_gmap_place_id(place_id)
gmap = {}
if objects:
gmap = objects[0].get_document()
translate[lang] = {
'name': gmap.get('name'),
}
return translate
def getDistance(lat1,lon1,lat2,lon2): Key = Keygen() url = 'https://maps.googleapis.com/maps/api/distancematrix/json?units=imperial&origins='+str(lat1)+','+str(lon1)+'&destinations='+str(lat2)+','+str(lon2)+'&key='+str(Key.get_key_distance())+'' print(url) response = requests.get(url) data = response.json() try: resp = data['rows'][0]['elements'][0]['distance']['value'] / 1000 except Exception as e: resp = 0 print (round(resp, 2)) return round(resp, 2)
def gmap_by_place_id(place_id):
Key = Keygen()
keyAPI = Key.get_key_geocode()
if not keyAPI:
sys.exit()
cnf = {'googlemaps':{'geocoding':{'key': keyAPI}}}
config.set(cnf)
spider.gmap_loader._language = language
objects = spider.get_gmap_place_id(place_id)
gmap = {}
if objects:
gmap = objects[0].get_document()
gmap.update(language=spider.gmap_loader._language)
return gmap
def getTranslate(address):
Key = Keygen()
translate = {}
languages = ["uk","ru","ca", "lv", "en" ,"pl" ,"de" "fr" , "it", "es", "ro", "nl", "el" "cs", "pt", "hu" , "sv", "bg", "sr", "da", "fi", "sk", "sl", "hr", "lt"]
for lang in languages:
Key.get_key_geocode()
cnf = {'geocoding':{'key': Key.get_key_geocode()}}
config.set(cnf)
spider.gmap_loader._language = lang
objects = spider.get_gmap_address(address)
gmap = {}
if objects:
gmap = objects[0].get_document()
translate[lang] = {'name': gmap.get('name'),
'short_name': gmap.get('short_name'),
'admin_hierarchy_translate': gmap.get('admin_hierarchy')}
return translate
def getPlace_id(lat, lon, gmap_name, gmap_type):
Key = Keygen()
url = 'https://maps.googleapis.com/maps/api/geocode/json?latlng=' + str(
lat) + ',' + str(
lon) + '&result_type=' + gmap_type + '&language=es&key=' + str(
Key.get_key_geocode()) + ''
response = requests.get(url)
data = response.json()
results = data['results']
i = 0
for item in results:
if item['address_components'][0]['long_name'] == gmap_name:
print(item['address_components'][0]['long_name'], gmap_name)
return results[i]['place_id']
i += 1
return None
def getByPlace(adress, mytype, Name_w_Article):
Key = Keygen()
keyAPI = Key.get_key_place()
if not keyAPI:
sys.exit()
url = 'https://maps.googleapis.com/maps/api/place/autocomplete/json?input='+str(adress)+'&types=geocode&language=es&key='+str(keyAPI)+''
types = {"Municipio": ["administrative_area_level_4"],
"Entidad colectiva" : ["administrative_area_level_5", "neighborhood"],
"Otras entidades": ["locality", "neighborhood"],
"Capital de municipio":["locality"],
"Entidad singular": ["locality"]}
print(url)
response = requests.get(url)
data = response.json()
print (types[mytype])
for objects in data['predictions']:
if objects['types'][0] in types[mytype] :
if Name_w_Article.lower().lstrip().strip() == objects['structured_formatting']['main_text'].lower().lstrip().strip():
print (objects['place_id'], objects['structured_formatting']['main_text'])
return objects['place_id']
return False
def getDistance(adress):
Key = Keygen()
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0 Mozilla/5.0 (Macintosh; Intel Mac OS X x.y; rv:42.0) Gecko/20100101 Firefox/42.0.',
'From': '*****@*****.**' # This is another valid field
}
url = 'https://nominatim.openstreetmap.org/search?q='+adress+'&format=json&polygon=1&addressdetails=1'
print(url)
proxy = {'https':'https://66.82.123.234:8080'}
response = requests.get(url, headers=headers, proxies=proxy, timeout=10)
print (response)
data = response.json()
return data
import hashlib
from lib.logger.File import File as FileLog
from argparse import ArgumentParser
import sys
import json
import requests
import pymongo
from bson.json_util import dumps
from lib.keygen.gmap_keygen import Keygen
# from lib.parser.wiki.Spain import Spain as ParserSpain
country = 'Spain'
config = Config('./config/config.yml')
mongo_config = config.get('mongodb')
conn = pymongo.MongoClient(mongo_config['host'], mongo_config['port'])
Key = Keygen()
cnf = {'geocoding': {'key': Key.get_key_geocode()}}
config.set(cnf)
db = conn.location
coll = db.sinoplik_romania
# print(config.get('googlemaps').get('geocoding').get('key'))
doc_factory = DocFactory(config.get('mongodb'))
# try:
# skiprows = sys.argv[1]
# except Exception as e:
# skiprows = 0
# df = pd.read_csv('./data/spain/Spain_notDublicate.csv', skiprows=int(skiprows), low_memory=False)
loader = Loader.loader_with_mongodb(config.get('mongodb'))
headers = {'User-Agent': 'Mozilla/5.0'}
language = 'ro'
"Entidad colectiva": "administrative_area_level_5",
"Otras entidades": "locality",
"Capital de municipio": "locality",
"Entidad singular": "locality"
}
if doc['12_gmap_type'] == types[doc['25_SNIG_TIPO']]:
print("YESSS")
else:
print("Noooooooo")
# types = {"Municipio": "administrative_area_level_4",
# "Entidad colectiva" : "administrative_area_level_5",
# "Otras entidades": "locality",
# "Capital de municipio": "locality",
# "Entidad singular": "locality" }
# place = get_place_ids_by_address_for_type
Key = Keygen()
keyAPI = Key.get_key_place()
if not keyAPI:
sys.exit()
cnf = {'googlemaps': {'geocoding': {'key': keyAPI}}}
config.set(cnf)
print(keyAPI)
spider.gmap_loader._language = language
print(doc['08_INE_Name_w_Article'], types[doc['25_SNIG_TIPO']])
dta = spider.get_place_ids_by_address_for_type(
doc['08_INE_Name_w_Article'] + ', España',
types[doc['25_SNIG_TIPO']])
# gmap = {}
# if objects:
# gmap = objects[0].get_document()
if len(dta) > 0:
