def getTranslate(place_id):
Key = Keygen()
translate = {}
languages = [
"uk", "ru", "ca", "lv", "en", "pl", "de", "fr", "it", "es", "ro", "nl",
"el", "cs", "pt", "hu", "sv", "bg", "sr", "da", "fi", "sk", "sl", "hr",
"lt"
]
for lang in languages:
keyAPI = Key.get_key_geocode()
if not keyAPI:
sys.exit()
cnf = {'googlemaps': {'geocoding': {'key': keyAPI}}}
config.set(cnf)
spider = Spider(loader_factory=LoaderFactory,
gmap_parser=MapFactory.spain,
wiki_parser=True,
doc_factory=doc_factory,
config=config,
use_cache=True)
spider.gmap_loader._language = lang
objects = spider.get_gmap_place_id(place_id)
gmap = {}
if objects:
gmap = objects[0].get_document()
translate[lang] = {
'name': gmap.get('name'),
}
return translate
def by_place_id(list_places, city_id):
config = Config('./config/config.yml')
Key = Keygen()
keyAPI = Key.get_key_geocode()
if not keyAPI:
sys.exit()
cnf = {'googlemaps': {'geocoding': {'key': keyAPI}}}
config.set(cnf)
language = 'uk'
doc_factory = DocFactory(config.get('mongodb'))
spider = Spider(loader_factory=LoaderFactory,
gmap_parser=MapFactory.spain,
doc_factory=doc_factory,
language=language,
config=config,
use_cache=True)
for loc in list_places:
objects = spider.get_gmap_place_id(loc)
gmap = {}
try:
gmap = objects[0].get_document()
gmap["city_id"] = city_id
exixts = db.ukraine_city_sublocal.find({
"city_id": city_id,
"code": gmap['code']
}).count()
if exixts < 1:
gmap.pop('_id', None)
db.ukraine_city_sublocal.save(gmap)
print(gmap)
except Exception as e:
print(str(e))
def getTranslate(place_id):
Key = Keygen()
translate = {}
languages = [
"uk", "ru", "ca", "lv", "en", "pl", "de", "fr", "it", "es", "ro", "nl",
"el", "cs", "pt", "hu", "sv", "bg", "sr", "da", "fi", "sk", "sl", "hr",
"lt"
]
for lang in languages:
keyAPI = Key.get_key_geocode()
if not keyAPI:
sys.exit()
cnf = {'googlemaps': {'geocoding': {'key': keyAPI}}}
config.set(cnf)
objects = spider.get_gmap_place_id(place_id)
gmap = {}
if objects:
gmap = objects[0].get_document()
translate[lang] = {
'name': gmap.get('name'),
}
return translate
def getTranslate(address):
Key = Keygen()
translate = {}
languages = ["uk","ru","ca", "lv", "en" ,"pl" ,"de" "fr" , "it", "es", "ro", "nl", "el" "cs", "pt", "hu" , "sv", "bg", "sr", "da", "fi", "sk", "sl", "hr", "lt"]
for lang in languages:
Key.get_key_geocode()
cnf = {'geocoding':{'key': Key.get_key_geocode()}}
config.set(cnf)
spider.gmap_loader._language = lang
objects = spider.get_gmap_address(address)
gmap = {}
if objects:
gmap = objects[0].get_document()
translate[lang] = {'name': gmap.get('name'),
'short_name': gmap.get('short_name'),
'admin_hierarchy_translate': gmap.get('admin_hierarchy')}
return translate
def gmap_by_place_id(place_id):
Key = Keygen()
keyAPI = Key.get_key_geocode()
if not keyAPI:
sys.exit()
cnf = {'googlemaps':{'geocoding':{'key': keyAPI}}}
config.set(cnf)
spider.gmap_loader._language = language
objects = spider.get_gmap_place_id(place_id)
gmap = {}
if objects:
gmap = objects[0].get_document()
gmap.update(language=spider.gmap_loader._language)
return gmap
def getPlace_id(lat, lon, gmap_name, gmap_type):
Key = Keygen()
url = 'https://maps.googleapis.com/maps/api/geocode/json?latlng=' + str(
lat) + ',' + str(
lon) + '&result_type=' + gmap_type + '&language=es&key=' + str(
Key.get_key_geocode()) + ''
response = requests.get(url)
data = response.json()
results = data['results']
i = 0
for item in results:
if item['address_components'][0]['long_name'] == gmap_name:
print(item['address_components'][0]['long_name'], gmap_name)
return results[i]['place_id']
i += 1
return None
from lib.logger.File import File as FileLog
from argparse import ArgumentParser
import sys
import json
import requests
import pymongo
from bson.json_util import dumps
from lib.keygen.gmap_keygen import Keygen
# from lib.parser.wiki.Spain import Spain as ParserSpain
country = 'Spain'
config = Config('./config/config.yml')
mongo_config = config.get('mongodb')
conn = pymongo.MongoClient(mongo_config['host'], mongo_config['port'])
Key = Keygen()
cnf = {'geocoding': {'key': Key.get_key_geocode()}}
config.set(cnf)
db = conn.location
coll = db.sinoplik_romania
# print(config.get('googlemaps').get('geocoding').get('key'))
doc_factory = DocFactory(config.get('mongodb'))
# try:
# skiprows = sys.argv[1]
# except Exception as e:
# skiprows = 0
# df = pd.read_csv('./data/spain/Spain_notDublicate.csv', skiprows=int(skiprows), low_memory=False)
loader = Loader.loader_with_mongodb(config.get('mongodb'))
headers = {'User-Agent': 'Mozilla/5.0'}
language = 'ro'
# spider = Spider(