def _batch_geocode(self, searches): bulk_results = {} pool = Pool(processes=self.PARALLEL_PROCESSES) for search in searches: (cartodb_id, street, city, state, country) = search address = compose_address(street, city, state, country) if address: components = self._build_optional_parameters( city, state, country) result = pool.apply_async(async_geocoder, (self.geocoder, address, components)) bulk_results[cartodb_id] = result pool.close() pool.join() try: results = [] for cartodb_id, bulk_result in bulk_results.items(): try: lng_lat, metadata = self._process_results( bulk_result.get()) except Exception as e: msg = 'Error at Google async_geocoder' self._logger.error(msg, e) lng_lat, metadata = geocoder_error_response(msg) results.append((cartodb_id, lng_lat, metadata)) return results except Exception as e: self._logger.error('General error', exception=e) raise e
def _geocode_searches(self, searches): try: location = self._send_batch(searches) return self._download_results(location) except Exception as e: msg = "Error running TomTom batch geocode: {}".format(e) self._logger.error(msg, e) return [geocoder_error_response(msg)] * len(searches)
def _serial_geocode(self, searches): results = [] for search in searches: (search_id, address, city, state, country) = search try: result = self.geocode_meta(searchtext=address, city=city, state=state, country=country) except Exception as e: self._logger.error("Error geocoding", e) result = geocoder_error_response("Error geocoding") results.append((search_id, result[0], result[1])) return results
def _parse_response(self, status_code, text): if status_code == requests.codes.ok: return self._parse_geocoder_response(text) elif status_code == requests.codes.bad_request: return EMPTY_RESPONSE elif status_code == requests.codes.unprocessable_entity: return EMPTY_RESPONSE else: msg = 'Unknown response {}: {}'.format(str(status_code), text) self._logger.warning('Error parsing TomTom geocoding response', data={'msg': msg}) return geocoder_error_response(msg)
def _serial_geocode(self, searches): results = [] for search in searches: (cartodb_id, street, city, state, country) = search try: lng_lat, metadata = self.geocode_meta(street, city, state, country) except Exception as e: self._logger.error("Error geocoding", e) lng_lat, metadata = geocoder_error_response("Error geocoding") results.append((cartodb_id, lng_lat, metadata)) return results
def geocode_free_text_meta(self, free_searches, country=None): """ :param free_searches: Free text searches :param country: Country ISO 3166 code :return: list of [x, y] on success, [] on error """ country = [country] if country else None try: free_search = ';'.join([self._escape(fs) for fs in free_searches]) response = self._geocoder.forward(address=free_search.decode('utf-8'), limit=1, country=country) if response.status_code == requests.codes.ok: return self._parse_geocoder_response(response.text) elif response.status_code == requests.codes.too_many_requests: return [TOO_MANY_REQUESTS_ERROR_RESPONSE] * len(free_searches) elif response.status_code == requests.codes.bad_request: return EMPTY_BATCH_RESPONSE elif response.status_code == requests.codes.unprocessable_entity: return EMPTY_BATCH_RESPONSE else: msg = "Unkown status: {}".format(response.status_code) self._logger.warning(msg, data={"searches": free_searches}) return [geocoder_error_response(msg)] * len(free_searches) except requests.Timeout as te: # In case of timeout we want to stop the job because the server # could be down msg = 'Timeout connecting to Mapbox geocoding server' self._logger.error(msg, te) return [geocoder_error_response(msg)] * len(free_searches) except requests.ConnectionError as ce: # Don't raise the exception to continue with the geocoding job self._logger.error('Error connecting to Mapbox geocoding server', exception=ce) return EMPTY_BATCH_RESPONSE
def _geocode_meta(self, searchtext, city=None, state_province=None, country=None): if searchtext: searchtext = searchtext.decode('utf-8') if city: city = city.decode('utf-8') if state_province: state_province = state_province.decode('utf-8') if country: country = country.decode('utf-8') if not self._validate_input(searchtext, city, state_province, country): return (EMPTY_RESPONSE, None) address = [] if searchtext and searchtext.strip(): address.append(normalize(searchtext)) if city: address.append(normalize(city)) if state_province: address.append(normalize(state_province)) uri = self._uri(searchtext=', '.join(address), country=country) try: response = requests.get(uri) return (self._parse_response(response.status_code, response.text), response) except requests.Timeout as te: # In case of timeout we want to stop the job because the server # could be down msg = 'Timeout connecting to TomTom geocoding server' self._logger.error(msg, te) return (geocoder_error_response(msg), None) except requests.ConnectionError as ce: # Don't raise the exception to continue with the geocoding job self._logger.error('Error connecting to TomTom geocoding server', exception=ce) return (EMPTY_RESPONSE, None)