def create_location_model(data):
task_list = data['task_list']
geopy.geocoders.options.default_timeout = 15
location = []
#not_found = 0
for index, task in enumerate(task_list):
t = (0, 0)
try:
task["lat"]
# lat not given, try reversing address
except KeyError:
geolocator = Here("Pe8yMxfGgxyn9yWsRDU9", "OrUmfFZvhbc2KNVLpvffrw")
address = clean_address(task["address"])
glocation = geolocator.geocode(address)
#check if lat lon is available, else move it to the antartic zone :)
try:
glocation.address
except AttributeError:
t = (0, 0)
else:
t = (glocation.latitude, glocation.longitude)
else:
t = (task["lat"], task["lon"])
task["address"] = index
finally:
print(task["address"])
print('{"lat":"', t[0], '","lon":"', t[1], '"},')
location.append(t)
return location
def geocoding():
result = request.get_json(force=True)
data = {}
try:
data['address'] = result['address']
except KeyError:
response = {}
response["status"] = False
response["message"] = "address is required"
response["request"] = result
return json.dumps(response)
t = "0,0"
geolocator = Here(app_id="brUMhDedQJcTEBv3WRm2",
app_code="gEuqige5xqyAJ3Gi5x1Qaw")
#geolocator = Here("Pe8yMxfGgxyn9yWsRDU9","OrUmfFZvhbc2KNVLpvffrw") #old auth
address = clean_address(data["address"])
glocation = geolocator.geocode(address)
#check if lat lon is available, else move it to the antartic zone :)
try:
glocation.address
except KeyError:
t = "0,0"
else:
t = str(glocation.latitude) + "," + str(glocation.longitude)
data["coordinate"] = t
return json.dumps(data)
def create_location_model(data):
task_list = data['task_list']
location = []
not_found = 0
for task in task_list:
t = (0, 0)
try:
task["lat"]
# lat not given, try reversing address
except KeyError:
#georeverse address
geolocator = Here("brUMhDedQJcTEBv3WRm2", "gEuqige5xqyAJ3Gi5x1Qaw")
#geodata = {
#"street": clean_address(task["address"]),
#"country": "IDN"
#}
#print (geodata["street"])
address = clean_address(task["address"])
#print(address)
glocation = geolocator.geocode(address)
#check if lat lon is available, else move it to the antartic zone :)
try:
glocation.address
except AttributeError:
t = (0, 0)
#not_found = not_found +1
#print (not_found)
else:
t = (glocation.latitude, glocation.longitude)
else:
t = (task["lat"], task["lon"])
finally:
print('{"lat":"', t[0], '","lon":"', t[1], '"},')
location.append(t)
return location
def get_geo_coordinates_from_address(self, address):
if not address:
return {'error': True, 'message': 'Address is required'}
geocoder = Here(self.app_id, self.app_code)
response = geocoder.geocode(address)
return response
def get_lat_lon(address):
global APP_ID_HERE, APP_CODE_HERE
geocoder = Here(APP_ID_HERE, APP_CODE_HERE)
result = geocoder.geocode(address)
point = result.point
return point
def coordonnees_adresse(self):
''' Définition des latitude et longitude de l'Adresse.'''
geoloc = Here(apikey=env('HERE_APIKEY'))
try:
loc = geoloc.geocode(self.voirie+', '+self.numero+', '+\
self.localite).raw['Location']['NavigationPosition'][0]
return (loc['Latitude'], loc['Longitude'])
except:
return (0,0)
def geocoding(ctx, query, apikey, forward, raw, display):
"""
HERE's geocoding service.
\f
:param ctx: A context dictionary.
:param query: A string to represent address query for geocoding.
:param apikey: An API key for authentication.
:param forward: A boolean flag for forward/reverse geocoding.
:param raw: A boolean flag to show api response as it is.
:param display: A boolean flag to show result in web browser.
:return: None.
"""
apikey = apikey or os.environ.get("HERE_APIKEY")
if apikey is None:
raise ApiKeyNotFoundError(
"Please pass HERE API KEY as --apikey or set it as environment "
"variable in HERE_APIKEY "
)
ctx.obj["apikey"] = apikey
geolocator = Here(apikey=ctx.obj["apikey"])
if forward:
location = geolocator.geocode(query)
if raw:
click.secho(json.dumps(location.raw, indent=2), fg="green")
elif display:
feature = get_feature_from_lat_lon(location.latitude, location.longitude)
geo_display(feature)
else:
result = {"lat": location.latitude, "lon": location.longitude}
click.secho(json.dumps(result, indent=2), fg="green")
else:
location = geolocator.reverse(query)
if raw:
click.secho(json.dumps(location.raw, indent=2), fg="green")
else:
click.secho(location.address, fg="green")
def get_geolocation(street: str, postal_code: str, city: str) -> dict:
geolocator = Here(settings.HERE_APP_ID, settings.HERE_APP_CODE)
location = geolocator.geocode(" ".join([street, postal_code, city]))
if location:
return {"lat": location.latitude, "lng": location.longitude}
return {"lat": None, "lng": None}
nplayers = []
for ii in data[:,8]:
if ii == '':
nplayers.append(0)
else:
nplayers.append(int(ii))
lookup = []
for f in data:
if(not f[10]):
lookup.append(f[1]+', '+f[2])
lats = []
lons = []
for s in lookup:
location = geolocator.geocode(s)
lats.append(location.latitude)
lons.append(location.longitude)
print(s,location.latitude,location.longitude)
time.sleep(1)
ii = 0
for f in data:
if(not f[10]):
f[9 ] = lats[ii]
f[10] = lons[ii]
ii = ii+1
# save if we had to lookup lat / lon data
if (lookup != []):
np.savetxt(DF,data,fmt="%s",delimiter=",",header=HEADER)
def here(self, address=None, city=None, country=None, key_here=None):
if not key_here:
raise RuntimeError(
"Requires a key! Check https://developer.here.com/ for more information."
)
if not address and not city and not country:
raise RuntimeError(
"Requires an address and/or a city and/or a country!")
addr = "" if address is None else address
#addr = ("" if address is None else ", " + address)
addr += ("" if city is None else ", " + city)
addr += ("" if country is None else ", " + country)
result = self.newResult()
result['service'] = 'here'
result['status'] = 'ZERO_RESULTS'
try:
geolocator_here = Here(apikey=key_here)
location = geolocator_here.geocode(addr,
exactly_one=False,
language="pt-PT")
if location is not None:
answer = location[0].raw
result['status'] = "OK"
result["latitude"] = location[0].latitude
result["longitude"] = location[0].longitude
result["number_of_results"] = len(location)
result["input_string"] = address
result["accuracy"] = answer.get('Relevance')
if answer.get("Location"):
result["formatted_address"] = answer.get("Location").get(
'Address').get('Label')
result["place_id"] = answer.get("Location").get(
"LocationId")
if answer.get("Location"):
if answer.get("Location").get("Address"):
result["postcode"] = answer.get("Location").get(
"Address").get("PostalCode")
# all 4 are not tghrustworthy
result["freguesia"] = answer.get("Location").get(
"Address").get("District")
result["distrito"] = answer.get("Location").get(
"Address").get("County")
result["concelho"] = answer.get("Location").get(
"Address").get("City")
result["localidade"] = answer.get("Location").get(
"Address").get("City")
#if saveraw:
# output["response"] = location[0].raw
except (GeocoderQueryError, GeocoderAuthenticationFailure,
GeocoderInsufficientPrivileges, ConfigurationError):
result['status'] = 'ACCESS_ERROR'
except GeocoderQuotaExceeded:
result['status'] = 'QUOTA_EXCEEDED'
except GeocoderTimedOut:
result['status'] = 'TIME_OUT'
except (GeocoderServiceError, GeocoderUnavailable, GeocoderNotFound):
result['status'] = 'SERVICE_ERROR'
except Exception as e:
result['status'] = 'UNKNOWN_ERROR'
return result
class Geocode():
#SERVICES = []
#IGNORE = []
CURRENT_SERVICE = 0
geolocator_google = None
geolocator_here = None
geolocator_bing = None
geolocator_tomtom = None
geolocator_azure = None
geolocator_nominatum = None
#SHOW_ERRORS = True
def __init__(self, services=None, ignore=None):
self.SERVICES = services
self.IGNORE = ignore
############ SERVICES ############
def initOutput(self):
output = {}
output["formatted_address"] = None
output["latitude"] = None
output["longitude"] = None
output["accuracy"] = None
output["place_id"] = None
output["type"] = None
output["postcode"] = None
output["input_string"] = None
output["number_of_results"] = None
output["status"] = None
output["response"] = None
output["localidade"] = None
output["distrito"] = None
output["concelho"] = None
output["freguesia"] = None
output["service"] = self.SERVICES[self.CURRENT_SERVICE]['service']
return output
def google(self, addr, local, country, saveraw):
output = self.initOutput()
address = "" if addr is None else addr
address = address + ("" if local is None else "," + local)
address = address + ("" if country is None else "," + country)
# init service if not init yet
if not self.geolocator_google:
self.geolocator_google = GoogleV3(
api_key=self.SERVICES[self.CURRENT_SERVICE]['key'])
# geocode address
location = self.geolocator_google.geocode(
address, exactly_one=False) #, components={"country": "PT"})
if location is not None:
answer = location[0].raw
output['status'] = "OK"
output["formatted_address"] = location[0].address
output["latitude"] = location[0].latitude
output["longitude"] = location[0].longitude
output["accuracy"] = answer.get('geometry').get('location_type')
output["place_id"] = answer.get("place_id")
output["type"] = ",".join(answer.get('types'))
output["postcode"] = ",".join([
x['long_name'] for x in answer.get('address_components')
if 'postal_code' in x.get('types')
])
output["input_string"] = address
output["number_of_results"] = len(location)
output["localidade"] = ",".join([
x['long_name'] for x in answer.get('address_components')
if 'locality' in x.get('types')
]).split(',')[0]
output["service"] = self.SERVICES[self.CURRENT_SERVICE]['service']
if saveraw:
output["response"] = location[0].raw
else:
output['status'] = "ZERO_RESULTS"
return output
def tomtom(self, addr, local, country, saveraw):
output = self.initOutput()
# create query
address = "" if addr is None else addr
address = address + ("" if local is None else "," + local)
address = address + ("" if country is None else "," + country)
# init service if not init yet
if not self.geolocator_tomtom:
self.geolocator_tomtom = TomTom(api_key=self.SERVICES[
self.CURRENT_SERVICE]['key']) #,default_scheme = 'https')
# geocode address
location = self.geolocator_tomtom.geocode(
address, exactly_one=False) #, components={"country": "PT"})
if location is not None:
answer = location[0].raw
output['status'] = "OK"
output["latitude"] = location[0].latitude
output["longitude"] = location[0].longitude
output["accuracy"] = answer.get('score')
output["input_string"] = address
output["number_of_results"] = len(
location) #answer.get("numResults")
output["place_id"] = answer.get("id")
if answer.get("address"):
output["distrito"] = answer.get("address").get(
"countrySubdivision")
# maybe?
output["concelho"] = answer.get("address").get("municipality")
output["freguesia"] = answer.get("address").get(
"municipalitySubdivision")
output["formatted_address"] = answer.get('address').get(
'freeformAddress')
CPext = answer.get("address").get('extendedPostalCode')
CP = answer.get("address").get('postalCode')
if CPext:
CPext = CPext.split(',')[0]
CPext = CPext[:4] + '-' + CPext[4:]
output["postcode"] = CPext
elif CP:
output["postcode"] = CP.split(',')[0]
output["type"] = answer.get('type')
#output["query_type"] = answer.get("queryType")
# maybe?
#output["localidade"] = answer.get("address").get("municipality")
output["service"] = self.SERVICES[self.CURRENT_SERVICE]['service']
if saveraw:
output["response"] = location[0].raw
else:
output['status'] = "ZERO_RESULTS"
return output
def nominatim(self, addr, local, country, saveraw):
output = self.initOutput()
# create query
address = "" if addr is None else addr
address = address + ("" if local is None else "," + local)
address = address + ("" if country is None else "," + country)
'''
query = { 'street': data[1],
'city':data[2],
'country': 'Portugal'
}
'''
# init service if not init yet
if not self.geolocator_nominatum:
self.geolocator_nominatum = Nominatim(user_agent="tests_1")
# geocode address
location = self.geolocator_nominatum.geocode(address,
exactly_one=False,
addressdetails=True)
if location is not None:
answer = location[0].raw
output['status'] = "OK"
output["latitude"] = location[0].latitude
output["longitude"] = location[0].longitude
output["number_of_results"] = len(location)
#output["accuracy"] = answer.get('importance')
output["place_id"] = answer.get("osm_id")
output["input_string"] = address
if answer.get("address"):
output["postcode"] = re.sub(
'[^0-9-]+', '',
answer.get("address").get("postcode")) ###???
output["freguesia"] = answer.get("address").get("suburb")
output["localidade"] = answer.get("address").get("city")
if not output["localidade"]:
output["localidade"] = answer.get("address").get("town")
output["formatted_address"] = answer.get('address').get(
'display_name')
output["type"] = answer.get('osm_type')
output["service"] = self.SERVICES[self.CURRENT_SERVICE]['service']
if saveraw:
output["response"] = location[0].raw
else:
output['status'] = "ZERO_RESULTS"
return output
def bing(self, addr, local, country, saveraw):
output = self.initOutput()
# create query
address = "" if addr is None else addr
address = address + ("" if local is None else "," + local)
address = address + ("" if country is None else "," + country)
# init service if not init yet
if not self.geolocator_bing:
self.geolocator_bing = Bing(
api_key=self.SERVICES[self.CURRENT_SERVICE]['key'])
# geocode address
location = self.geolocator_bing.geocode(
address,
exactly_one=False) #culture='PT', include_neighborhood=True,
if location is not None:
answer = location[0].raw
output['status'] = "OK"
output["latitude"] = location[0].latitude
output["longitude"] = location[0].longitude
output["number_of_results"] = len(location)
if answer.get("address"):
output["formatted_address"] = answer.get('address').get(
'formattedAddress')
output["localidade"] = answer.get("address").get("locality")
output["distrito"] = answer.get("address").get("adminDistrict")
output["concelho"] = answer.get("address").get(
"adminDistrict2")
output["freguesia"] = answer.get("address").get("neighborhood")
output["postcode"] = answer.get("address").get("postalCode")
output["accuracy"] = answer.get('confidence')
output["input_string"] = address
output["service"] = self.SERVICES[self.CURRENT_SERVICE]['service']
if saveraw:
output["response"] = location[0].raw
else:
output['status'] = "ZERO_RESULTS"
return output
def here(self, addr, local, country, saveraw):
output = self.initOutput()
# create query
address = "" if addr is None else addr
address = address + ("" if local is None else "," + local)
address = address + ("" if country is None else "," + country)
# init service if not init yet
if not self.geolocator_here:
self.geolocator_here = Here(
app_id=self.SERVICES[self.CURRENT_SERVICE]['app_id'],
app_code=self.SERVICES[self.CURRENT_SERVICE]['app_code'])
# geocode address
location = self.geolocator_here.geocode(address,
exactly_one=False,
language="pt-PT")
if location is not None:
answer = location[0].raw
output['status'] = "OK"
output["latitude"] = location[0].latitude
output["longitude"] = location[0].longitude
output["number_of_results"] = len(location)
output["input_string"] = address
output["accuracy"] = answer.get('Relevance')
if answer.get("Location"):
output["formatted_address"] = answer.get("Location").get(
'Address').get('Label')
output["place_id"] = answer.get("Location").get("LocationId")
if answer.get("Location"):
if answer.get("Location").get("Address"):
output["postcode"] = answer.get("Location").get(
"Address").get("PostalCode")
# all 4 are not tghrustworthy
output["freguesia"] = answer.get("Location").get(
"Address").get("District")
output["distrito"] = answer.get("Location").get(
"Address").get("County")
output["concelho"] = answer.get("Location").get(
"Address").get("City")
output["localidade"] = answer.get("Location").get(
"Address").get("City")
output["service"] = self.SERVICES[self.CURRENT_SERVICE]['service']
if saveraw:
output["response"] = location[0].raw
else:
output['status'] = "ZERO_RESULTS"
return output
###
def azure(self, addr, local, country, saveraw):
output = self.initOutput()
# create query
address = "" if addr is None else addr
address = address + ("" if local is None else "," + local)
address = address + ("" if country is None else "," + country)
# init service if not init yet
if not self.geolocator_azure:
self.geolocator_azure = AzureMaps(
subscription_key=self.SERVICES[self.CURRENT_SERVICE]['key'])
# geocode address
location = self.geolocator_azure.geocode(address,
exactly_one=False,
language="pt-PT")
if location is not None:
answer = location[0].raw
output['status'] = "OK"
output["latitude"] = location[0].latitude
output["longitude"] = location[0].longitude
output["number_of_results"] = len(location)
output["input_string"] = address
output["accuracy"] = answer.get('score')
output["place_id"] = answer.get("id")
if answer.get("address"):
output["formatted_address"] = answer.get('address').get(
'freeformAddress')
output["distrito"] = answer.get("address").get(
"countrySubdivision")
# maybe?
output["concelho"] = answer.get("address").get("municipality")
output["freguesia"] = answer.get("address").get(
"municipalitySubdivision")
CPext = answer.get("address").get('extendedPostalCode')
CP = answer.get("address").get('postalCode')
if CPext:
CPext = CPext.split(',')[0]
CPext = CPext[:4] + '-' + CPext[4:]
output["postcode"] = CPext
elif CP:
output["postcode"] = CP.split(',')[0]
output["type"] = answer.get('type')
output["service"] = self.SERVICES[self.CURRENT_SERVICE]['service']
if saveraw:
output["response"] = location[0].raw
else:
output['status'] = "ZERO_RESULTS"
return output
############ PROCESS FILE ############
def getService(self):
if self.CURRENT_SERVICE >= len(self.SERVICES):
raise UnableToGeocode("Unable to geocode entity.")
if len(self.IGNORE) >= len(self.SERVICES):
raise OutOfServices("No service available.")
for i in self.SERVICES:
if self.SERVICES[self.CURRENT_SERVICE]['service'] in self.IGNORE:
self.CURRENT_SERVICE = self.CURRENT_SERVICE + 1
if self.CURRENT_SERVICE >= len(self.SERVICES):
raise UnableToGeocode("Unable to geocode entity.")
else:
break
if "GOOGLE" in self.SERVICES[self.CURRENT_SERVICE]['service']:
return self.google
elif "TOMTOM" in self.SERVICES[self.CURRENT_SERVICE]['service']:
return self.tomtom
elif "NOMINATUM" in self.SERVICES[self.CURRENT_SERVICE]['service']:
return self.nominatim
elif "BING" in self.SERVICES[self.CURRENT_SERVICE]['service']:
return self.bing
elif "HERE" in self.SERVICES[self.CURRENT_SERVICE]['service']:
return self.here
elif "AZURE" in self.SERVICES[self.CURRENT_SERVICE]['service']:
return self.azure
return None
# service = None => all available
def geocode(self,
addr=None,
local=None,
country="Portugal",
saveraw=True,
service=None):
geocoded = False
self.CURRENT_SERVICE = 0
geocode_result = None
if service:
for s in self.SERVICES:
if s['service'] != service:
self.IGNORE.append(s['service'])
while not geocoded:
try:
serv = self.getService()
geocode_result = serv(addr, local, country, saveraw)
if geocode_result['status'] == "OK":
geocoded = True
break
else:
self.CURRENT_SERVICE = self.CURRENT_SERVICE + 1
'''
else:
if DEBUG:
logger.error ('\n--------------------------------------------------------------------')
logger.error ('ERROR: no addr/name for id_localization [{}].'.format(address.split('|')[0]))
logger.error ('Passing to next address.')
logger.error ('--------------------------------------------------------------------')
CURRENT_SERVICE = 0
geocode_result = initOutput()
geocode_result['id_localization'] = address.split('|')[0]
geocode_result['status'] = "NO_DATA"
break
'''
except UnableToGeocode as e:
if self.SHOW_ERRORS:
pass
#logger.error ('\n--------------------------------------------------------------------')
#logger.error ('ERROR: Unable to geocode addr [{}].'.format(addr))
#logger.error ('Passing to next address.')
#logger.error ('--------------------------------------------------------------------')
self.CURRENT_SERVICE = 0
geocode_result = self.initOutput()
geocode_result['status'] = "UNABLE"
geocode_result['service'] = "ALL"
break
except OutOfServices as e:
#if self.SHOW_ERRORS:
# logger.error ('\n--------------------------------------------------------------------')
# logger.error ('ERROR: you reached the limit on all services. No more services available.')
# logger.error ('Saving the all sucessuful results and exiting the application.')
# logger.error ('--------------------------------------------------------------------')
raise
#return None
except (GeocoderQueryError, GeocoderAuthenticationFailure,
GeocoderInsufficientPrivileges, ConfigurationError):
#if self.SHOW_ERRORS:
# logger.error ('\n--------------------------------------------------------------------')
# logger.error ('ERROR: something wrong with either the service or the query.')
# logger.error ('Check service: [{}]'.format(self.SERVICES[self.CURRENT_SERVICE]['id']))
# logger.error ('Passing to the next service.')
# logger.error ('--------------------------------------------------------------------')
self.IGNORE.append(
self.SERVICES[self.CURRENT_SERVICE]['service'])
except GeocoderQuotaExceeded:
#if self.SHOW_ERRORS:
# logger.error ('\n--------------------------------------------------------------------')
# logger.error ('ERROR: you have reached the end of your quota for service [{}].'.format(self.SERVICES[self.CURRENT_SERVICE]['id']))
# logger.error ('Passing to the next service.')
# logger.error ('--------------------------------------------------------------------')
self.IGNORE.append(
self.SERVICES[self.CURRENT_SERVICE]['service'])
except GeocoderTimedOut:
#if self.SHOW_ERRORS:
# logger.error ('\n--------------------------------------------------------------------')
# logger.error ('TIMEOUT: something went wrong with the geocoding the address: [{}].'.format(addr))
# logger.error ('while using service [{}].'.format(self.SERVICES[self.CURRENT_SERVICE]['id']))
# logger.error ('Passing to the next service.')
# logger.error ('--------------------------------------------------------------------')
self.IGNORE.append(
self.SERVICES[self.CURRENT_SERVICE]['service'])
except (GeocoderServiceError, GeocoderUnavailable):
#if self.SHOW_ERRORS:
# logger.error ('\n--------------------------------------------------------------------')
# logger.error ('ERROR: service unavailable or unknown error for service [{}].'.format(self.SERVICES[self.CURRENT_SERVICE]['id']))
# logger.error ('Passing to the next service.')
# logger.error ('--------------------------------------------------------------------')
self.IGNORE.append(
self.SERVICES[self.CURRENT_SERVICE]['service'])
except GeocoderNotFound:
#if self.SHOW_ERRORS:
# logger.error ('\n--------------------------------------------------------------------')
# logger.error ('ERROR: unknown service > [{}].'.format(self.SERVICES[self.CURRENT_SERVICE]['id']))
# logger.error ('check if this service still exists!')
# logger.error ('Passing to the next service.')
# logger.error ('--------------------------------------------------------------------')
self.IGNORE.append(
self.SERVICES[self.CURRENT_SERVICE]['service'])
except Exception as e:
#logger.error ('\n--------------------------------------------------------------------')
#logger.error("Unknown catastrophic error while processing address: {}".format(addr))
#logger.error('while using service > [{}].'.format(self.SERVICES[self.CURRENT_SERVICE]['id']))
#logger.error("Check the error and correct it before restart the application.")
#logger.error(str(e))
#logger.error('--------------------------------------------------------------------')
raise
#return None
return geocode_result
file = open(r"/Library/WebServer/Documents/JSConferences/data2019.json",
"r",
encoding="utf-8")
data = json.load(file)
features = data["features"]
geojsonFeatures = []
while len(features) > 0:
feat = features.pop(0)
address = feat["city"] + "+" + feat["country"]
print(address)
#geolocator = Nominatim(user_agent="Ralucas-app", ssl_context= ssl.SSLContext())
geolocator = Here(app_id="", app_code="", ssl_context=ssl.SSLContext())
location = geolocator.geocode(address)
if location is not None:
geojsonFeature = {
"type": "Feature",
"properties": feat,
"geometry": {
"type": "Point",
"coordinates": [location.longitude, location.latitude]
}
}
geojsonFeatures.append(geojsonFeature)
print(geojsonFeature)
time.sleep(5)
print("------------------")
geojson = {"type": "FeatureCollection", "features": geojsonFeatures}