def getTranslate(place_id): Key = Keygen() translate = {} languages = [ "uk", "ru", "ca", "lv", "en", "pl", "de", "fr", "it", "es", "ro", "nl", "el", "cs", "pt", "hu", "sv", "bg", "sr", "da", "fi", "sk", "sl", "hr", "lt" ] for lang in languages: keyAPI = Key.get_key_geocode() if not keyAPI: sys.exit() cnf = {'googlemaps': {'geocoding': {'key': keyAPI}}} config.set(cnf) spider = Spider(loader_factory=LoaderFactory, gmap_parser=MapFactory.spain, wiki_parser=True, doc_factory=doc_factory, config=config, use_cache=True) spider.gmap_loader._language = lang objects = spider.get_gmap_place_id(place_id) gmap = {} if objects: gmap = objects[0].get_document() translate[lang] = { 'name': gmap.get('name'), } return translate
def by_place_id(list_places, city_id): config = Config('./config/config.yml') Key = Keygen() keyAPI = Key.get_key_geocode() if not keyAPI: sys.exit() cnf = {'googlemaps': {'geocoding': {'key': keyAPI}}} config.set(cnf) language = 'uk' doc_factory = DocFactory(config.get('mongodb')) spider = Spider(loader_factory=LoaderFactory, gmap_parser=MapFactory.spain, doc_factory=doc_factory, language=language, config=config, use_cache=True) for loc in list_places: objects = spider.get_gmap_place_id(loc) gmap = {} try: gmap = objects[0].get_document() gmap["city_id"] = city_id exixts = db.ukraine_city_sublocal.find({ "city_id": city_id, "code": gmap['code'] }).count() if exixts < 1: gmap.pop('_id', None) db.ukraine_city_sublocal.save(gmap) print(gmap) except Exception as e: print(str(e))
def get_place(lat, lng): datalist = list() Key = Keygen() keyAPI = Key.get_key_place() if not keyAPI: sys.exit() url = 'https://maps.googleapis.com/maps/api/place/radarsearch/json?location=' + str( lat) + ',' + str(lng) + '&radius=20000&type=neighborhood&key=' + str( keyAPI) + '' response = requests.get(url) data = response.json() result = data['results'] # print(data['results']) for x in result: # print (x['place_id']) datalist.append(x['place_id']) # print ("====================") url = 'https://maps.googleapis.com/maps/api/place/radarsearch/json?location=' + str( lat) + ',' + str(lng) + '&radius=20000&type=sublocality&key=' + str( keyAPI) + '' response = requests.get(url) data = response.json() result = data['results'] # print(data['results']) for x in result: # print (x['place_id']) datalist.append(x['place_id']) # print ("#########################################") return datalist
def getTranslate(place_id): Key = Keygen() translate = {} languages = [ "uk", "ru", "ca", "lv", "en", "pl", "de", "fr", "it", "es", "ro", "nl", "el", "cs", "pt", "hu", "sv", "bg", "sr", "da", "fi", "sk", "sl", "hr", "lt" ] for lang in languages: keyAPI = Key.get_key_geocode() if not keyAPI: sys.exit() cnf = {'googlemaps': {'geocoding': {'key': keyAPI}}} config.set(cnf) objects = spider.get_gmap_place_id(place_id) gmap = {} if objects: gmap = objects[0].get_document() translate[lang] = { 'name': gmap.get('name'), } return translate
def getDistance(lat1,lon1,lat2,lon2): Key = Keygen() url = 'https://maps.googleapis.com/maps/api/distancematrix/json?units=imperial&origins='+str(lat1)+','+str(lon1)+'&destinations='+str(lat2)+','+str(lon2)+'&key='+str(Key.get_key_distance())+'' print(url) response = requests.get(url) data = response.json() try: resp = data['rows'][0]['elements'][0]['distance']['value'] / 1000 except Exception as e: resp = 0 print (round(resp, 2)) return round(resp, 2)
def gmap_by_place_id(place_id): Key = Keygen() keyAPI = Key.get_key_geocode() if not keyAPI: sys.exit() cnf = {'googlemaps':{'geocoding':{'key': keyAPI}}} config.set(cnf) spider.gmap_loader._language = language objects = spider.get_gmap_place_id(place_id) gmap = {} if objects: gmap = objects[0].get_document() gmap.update(language=spider.gmap_loader._language) return gmap
def getTranslate(address): Key = Keygen() translate = {} languages = ["uk","ru","ca", "lv", "en" ,"pl" ,"de" "fr" , "it", "es", "ro", "nl", "el" "cs", "pt", "hu" , "sv", "bg", "sr", "da", "fi", "sk", "sl", "hr", "lt"] for lang in languages: Key.get_key_geocode() cnf = {'geocoding':{'key': Key.get_key_geocode()}} config.set(cnf) spider.gmap_loader._language = lang objects = spider.get_gmap_address(address) gmap = {} if objects: gmap = objects[0].get_document() translate[lang] = {'name': gmap.get('name'), 'short_name': gmap.get('short_name'), 'admin_hierarchy_translate': gmap.get('admin_hierarchy')} return translate
def getPlace_id(lat, lon, gmap_name, gmap_type): Key = Keygen() url = 'https://maps.googleapis.com/maps/api/geocode/json?latlng=' + str( lat) + ',' + str( lon) + '&result_type=' + gmap_type + '&language=es&key=' + str( Key.get_key_geocode()) + '' response = requests.get(url) data = response.json() results = data['results'] i = 0 for item in results: if item['address_components'][0]['long_name'] == gmap_name: print(item['address_components'][0]['long_name'], gmap_name) return results[i]['place_id'] i += 1 return None
def getByPlace(adress, mytype, Name_w_Article): Key = Keygen() keyAPI = Key.get_key_place() if not keyAPI: sys.exit() url = 'https://maps.googleapis.com/maps/api/place/autocomplete/json?input='+str(adress)+'&types=geocode&language=es&key='+str(keyAPI)+'' types = {"Municipio": ["administrative_area_level_4"], "Entidad colectiva" : ["administrative_area_level_5", "neighborhood"], "Otras entidades": ["locality", "neighborhood"], "Capital de municipio":["locality"], "Entidad singular": ["locality"]} print(url) response = requests.get(url) data = response.json() print (types[mytype]) for objects in data['predictions']: if objects['types'][0] in types[mytype] : if Name_w_Article.lower().lstrip().strip() == objects['structured_formatting']['main_text'].lower().lstrip().strip(): print (objects['place_id'], objects['structured_formatting']['main_text']) return objects['place_id'] return False
def getDistance(adress): Key = Keygen() headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0 Mozilla/5.0 (Macintosh; Intel Mac OS X x.y; rv:42.0) Gecko/20100101 Firefox/42.0.', 'From': '*****@*****.**' # This is another valid field } url = 'https://nominatim.openstreetmap.org/search?q='+adress+'&format=json&polygon=1&addressdetails=1' print(url) proxy = {'https':'https://66.82.123.234:8080'} response = requests.get(url, headers=headers, proxies=proxy, timeout=10) print (response) data = response.json() return data
import hashlib from lib.logger.File import File as FileLog from argparse import ArgumentParser import sys import json import requests import pymongo from bson.json_util import dumps from lib.keygen.gmap_keygen import Keygen # from lib.parser.wiki.Spain import Spain as ParserSpain country = 'Spain' config = Config('./config/config.yml') mongo_config = config.get('mongodb') conn = pymongo.MongoClient(mongo_config['host'], mongo_config['port']) Key = Keygen() cnf = {'geocoding': {'key': Key.get_key_geocode()}} config.set(cnf) db = conn.location coll = db.sinoplik_romania # print(config.get('googlemaps').get('geocoding').get('key')) doc_factory = DocFactory(config.get('mongodb')) # try: # skiprows = sys.argv[1] # except Exception as e: # skiprows = 0 # df = pd.read_csv('./data/spain/Spain_notDublicate.csv', skiprows=int(skiprows), low_memory=False) loader = Loader.loader_with_mongodb(config.get('mongodb')) headers = {'User-Agent': 'Mozilla/5.0'} language = 'ro'
"Entidad colectiva": "administrative_area_level_5", "Otras entidades": "locality", "Capital de municipio": "locality", "Entidad singular": "locality" } if doc['12_gmap_type'] == types[doc['25_SNIG_TIPO']]: print("YESSS") else: print("Noooooooo") # types = {"Municipio": "administrative_area_level_4", # "Entidad colectiva" : "administrative_area_level_5", # "Otras entidades": "locality", # "Capital de municipio": "locality", # "Entidad singular": "locality" } # place = get_place_ids_by_address_for_type Key = Keygen() keyAPI = Key.get_key_place() if not keyAPI: sys.exit() cnf = {'googlemaps': {'geocoding': {'key': keyAPI}}} config.set(cnf) print(keyAPI) spider.gmap_loader._language = language print(doc['08_INE_Name_w_Article'], types[doc['25_SNIG_TIPO']]) dta = spider.get_place_ids_by_address_for_type( doc['08_INE_Name_w_Article'] + ', España', types[doc['25_SNIG_TIPO']]) # gmap = {} # if objects: # gmap = objects[0].get_document() if len(dta) > 0: