def _batch_geocode(self, searches):
bulk_results = {}
pool = Pool(processes=self.PARALLEL_PROCESSES)
for search in searches:
(cartodb_id, street, city, state, country) = search
address = compose_address(street, city, state, country)
if address:
components = self._build_optional_parameters(
city, state, country)
result = pool.apply_async(async_geocoder,
(self.geocoder, address, components))
bulk_results[cartodb_id] = result
pool.close()
pool.join()
try:
results = []
for cartodb_id, bulk_result in bulk_results.items():
try:
lng_lat, metadata = self._process_results(
bulk_result.get())
except Exception as e:
msg = 'Error at Google async_geocoder'
self._logger.error(msg, e)
lng_lat, metadata = geocoder_error_response(msg)
results.append((cartodb_id, lng_lat, metadata))
return results
except Exception as e:
self._logger.error('General error', exception=e)
raise e
def _geocode_searches(self, searches):
try:
location = self._send_batch(searches)
return self._download_results(location)
except Exception as e:
msg = "Error running TomTom batch geocode: {}".format(e)
self._logger.error(msg, e)
return [geocoder_error_response(msg)] * len(searches)
def _serial_geocode(self, searches):
results = []
for search in searches:
(search_id, address, city, state, country) = search
try:
result = self.geocode_meta(searchtext=address, city=city, state=state, country=country)
except Exception as e:
self._logger.error("Error geocoding", e)
result = geocoder_error_response("Error geocoding")
results.append((search_id, result[0], result[1]))
return results
def _parse_response(self, status_code, text):
if status_code == requests.codes.ok:
return self._parse_geocoder_response(text)
elif status_code == requests.codes.bad_request:
return EMPTY_RESPONSE
elif status_code == requests.codes.unprocessable_entity:
return EMPTY_RESPONSE
else:
msg = 'Unknown response {}: {}'.format(str(status_code), text)
self._logger.warning('Error parsing TomTom geocoding response',
data={'msg': msg})
return geocoder_error_response(msg)
def _serial_geocode(self, searches):
results = []
for search in searches:
(cartodb_id, street, city, state, country) = search
try:
lng_lat, metadata = self.geocode_meta(street, city, state,
country)
except Exception as e:
self._logger.error("Error geocoding", e)
lng_lat, metadata = geocoder_error_response("Error geocoding")
results.append((cartodb_id, lng_lat, metadata))
return results
def geocode_free_text_meta(self, free_searches, country=None):
"""
:param free_searches: Free text searches
:param country: Country ISO 3166 code
:return: list of [x, y] on success, [] on error
"""
country = [country] if country else None
try:
free_search = ';'.join([self._escape(fs) for fs in free_searches])
response = self._geocoder.forward(address=free_search.decode('utf-8'),
limit=1,
country=country)
if response.status_code == requests.codes.ok:
return self._parse_geocoder_response(response.text)
elif response.status_code == requests.codes.too_many_requests:
return [TOO_MANY_REQUESTS_ERROR_RESPONSE] * len(free_searches)
elif response.status_code == requests.codes.bad_request:
return EMPTY_BATCH_RESPONSE
elif response.status_code == requests.codes.unprocessable_entity:
return EMPTY_BATCH_RESPONSE
else:
msg = "Unkown status: {}".format(response.status_code)
self._logger.warning(msg, data={"searches": free_searches})
return [geocoder_error_response(msg)] * len(free_searches)
except requests.Timeout as te:
# In case of timeout we want to stop the job because the server
# could be down
msg = 'Timeout connecting to Mapbox geocoding server'
self._logger.error(msg, te)
return [geocoder_error_response(msg)] * len(free_searches)
except requests.ConnectionError as ce:
# Don't raise the exception to continue with the geocoding job
self._logger.error('Error connecting to Mapbox geocoding server',
exception=ce)
return EMPTY_BATCH_RESPONSE
def _geocode_meta(self,
searchtext,
city=None,
state_province=None,
country=None):
if searchtext:
searchtext = searchtext.decode('utf-8')
if city:
city = city.decode('utf-8')
if state_province:
state_province = state_province.decode('utf-8')
if country:
country = country.decode('utf-8')
if not self._validate_input(searchtext, city, state_province, country):
return (EMPTY_RESPONSE, None)
address = []
if searchtext and searchtext.strip():
address.append(normalize(searchtext))
if city:
address.append(normalize(city))
if state_province:
address.append(normalize(state_province))
uri = self._uri(searchtext=', '.join(address), country=country)
try:
response = requests.get(uri)
return (self._parse_response(response.status_code,
response.text), response)
except requests.Timeout as te:
# In case of timeout we want to stop the job because the server
# could be down
msg = 'Timeout connecting to TomTom geocoding server'
self._logger.error(msg, te)
return (geocoder_error_response(msg), None)
except requests.ConnectionError as ce:
# Don't raise the exception to continue with the geocoding job
self._logger.error('Error connecting to TomTom geocoding server',
exception=ce)
return (EMPTY_RESPONSE, None)
