def parse_json(self, page, exactly_one=True):
if not isinstance(page, basestring):
page = util.decode_page(page)
doc = json.loads(page)
results = doc.get('ResultSet', []).get('Results', [])
def parse_result(place):
line1, line2, line3, line4 = place.get('line1'), place.get('line2'), place.get('line3'), place.get('line4')
address = util.join_filter(", ", [line1, line2, line3, line4])
city = place.get('city')
state = place.get('state')
country = place.get('country')
location = util.join_filter(", ", [address, city, country])
lat, lng = place.get('latitude'), place.get('longitude')
#if lat and lng:
# point = Point(floatlat, lng)
#else:
# point = None
#return (place, location, (float(lat), float(lng)))
return (place)
if exactly_one:
if len(results) > 0:
return parse_result(results[0])
else: return []
else:
return [parse_result(result) for result in results]
def parse_json(self, page, exactly_one):
if not isinstance(page, basestring):
page = util.decode_page(page)
json = simplejson.loads(page)
codes = json.get('postalCodes', [])
if exactly_one and len(codes) != 1:
raise ValueError("Didn't find exactly one code! " \
"(Found %d.)" % len(codes))
def parse_code(code):
place = util.join_filter(", ", [code.get('placeName'),
code.get('countryCode')])
location = util.join_filter(" ", [place,
code.get('postalCode')]) or None
latitude = code.get('lat')
longitude = code.get('lng')
latitude = latitude and float(latitude)
longitude = longitude and float(longitude)
return (location, (latitude, longitude))
if exactly_one:
return parse_code(codes[0])
else:
return (parse_code(code) for code in codes)
def parse_xml(self, page, exactly_one=True):
"""Parse a location name, latitude, and longitude from an XML response.
"""
if not isinstance(page, basestring):
page = util.decode_page(page)
try:
doc = xml.dom.minidom.parseString(page)
except ExpatError:
places = []
else:
places = doc.getElementsByTagName('Placemark')
if exactly_one and len(places) != 1:
raise ValueError("Didn't find exactly one placemark! " \
"(Found %d.)" % len(places))
def parse_place(place):
location = util.get_first_text(place, ['address', 'name']) or None
points = place.getElementsByTagName('Point')
point = points and points[0] or None
coords = util.get_first_text(point, 'coordinates') or None
if coords:
longitude, latitude = [float(f) for f in coords.split(',')[:2]]
else:
latitude = longitude = None
_, (latitude, longitude) = self.geocode(location)
return (location, (latitude, longitude))
if exactly_one:
return parse_place(places[0])
else:
return (parse_place(place) for place in places)
def parse_json(self, page):
if not isinstance(page, basestring):
page = util.decode_page(page)
doc = json.loads(page)
return doc.get('timezoneId', None)
def parse_json(self, page, exactly_one=True):
"""Parse display name, latitude, and longitude from an JSON response."""
if not isinstance(page, basestring):
page = decode_page(page)
resources = json.loads(page)
statuscode = resources.get('info').get('statuscode')
if statuscode == 403:
return "Bad API Key"
resources = resources.get('results')[0].get('locations')
if exactly_one and len(resources) != 1:
from warnings import warn
warn("Didn't find exactly one resource!" + \
"(Found %d.), use exactly_one=False\n" % len(resources)
)
def parse_resource(resource):
city = resource['adminArea5']
county = resource['adminArea4']
state = resource['adminArea3']
country = resource['adminArea1']
latLng = resource['latLng']
latitude, longitude = latLng.get('lat'), latLng.get('lng')
location = join_filter(", ", [city, county, state,country])
if latitude and longitude:
latitude = float(latitude)
longitude = float(longitude)
return (location, (latitude, longitude))
if exactly_one:
return parse_resource(resources[0])
else:
return [parse_resource(resource) for resource in resources]
def _read_http_error_body(self, error):
try:
return decode_page(error)
except Exception:
logger.debug('Unable to fetch body for a non-successful HTTP response',
exc_info=True)
return None
def parse_xml(self, page, exactly_one):
if not isinstance(page, basestring):
page = util.decode_page(page)
doc = xml.dom.minidom.parseString(page)
codes = doc.getElementsByTagName('geoname')
if exactly_one and len(codes) != 1:
raise ValueError("Didn't find exactly one geoname! " \
"(Found %d.)" % len(codes))
def parse_code(code):
place_name = util.get_first_text(code, 'name')
country_code = util.get_first_text(code, 'countryCode')
postal_code = util.get_first_text(code, 'postalcode')
place = util.join_filter(", ", [place_name, country_code])
location = util.join_filter(" ", [place, postal_code]) or None
latitude = util.get_first_text(code, 'lat') or None
longitude = util.get_first_text(code, 'lng') or None
latitude = latitude and float(latitude)
longitude = longitude and float(longitude)
return (location, (latitude, longitude))
if exactly_one:
return parse_code(codes[0])
else:
return (parse_code(code) for code in codes)
def parse_json(self, page, exactly_one=True):
if not isinstance(page, basestring):
page = util.decode_page(page)
doc = json.loads(page)
error = doc.get("ResultSet", []).get("Error", 0)
error_message = doc.get("ResultSet", []).get("ErrorMessage", "")
self._check_error(error, error_message)
results = doc.get("ResultSet", []).get("Results", [])
if not results:
raise ValueError("No results found")
elif exactly_one and len(results) != 1:
raise ValueError("Didn't find exactly one placemark! " "(Found %d.)" % len(results))
def parse_result(place):
line1, line2, line3, line4 = place.get("line1"), place.get("line2"), place.get("line3"), place.get("line4")
address = util.join_filter(", ", [line1, line2, line3, line4])
city = place.get("city")
state = place.get("state")
country = place.get("country")
location = util.join_filter(", ", [address, city, country])
lat, lng = place.get("latitude"), place.get("longitude")
# if lat and lng:
# point = Point(floatlat, lng)
# else:
# point = None
return (location, (float(lat), float(lng)))
if exactly_one:
return parse_result(results[0])
else:
return [parse_result(result) for result in results]
def parse_json(self, page, exactly_one=True):
"""Parse display name, latitude, and longitude from an JSON response."""
if not isinstance(page, basestring):
page = decode_page(page)
resources = json.loads(page)
if exactly_one and len(resources) != 1:
from warnings import warn
warn("Didn't find exactly one resource!" + "(Found %d.), use exactly_one=False\n" % len(resources))
def parse_resource(resource):
location = resource["display_name"]
latitude = resource["lat"] or None
longitude = resource["lon"] or None
if latitude and longitude:
latitude = float(latitude)
longitude = float(longitude)
return (location, (latitude, longitude))
if exactly_one:
return parse_resource(resources[0])
else:
return [parse_resource(resource) for resource in resources]
def parse_json(self, page, exactly_one=True):
if not isinstance(page, basestring):
page = util.decode_page(page)
doc = json.loads(page)
places = doc.get('results', [])
if len(places) == 0:
# Got empty result. Parse out the status code and raise an error if necessary.
status_code = doc.get("status", None)
# status_code = status["code"]
self.check_status_code(status_code)
return None
elif exactly_one and len(places) != 1:
raise ValueError("Didn't find exactly one placemark! " \
"(Found %d.)" % len(places))
def parse_place(place):
location = place.get('vicinity')
latitude = place['geometry']['location']['lat']
longitude = place['geometry']['location']['lng']
return (location, (latitude, longitude))
if exactly_one:
return parse_place(places[0])
else:
return [parse_place(place) for place in places]
def parse_json(self, page, exactly_one):
if not isinstance(page, basestring):
page = decode_page(page)
doc = json.loads(page)
places = doc
if not isinstance (places, list):
places = [places]
if not places:
return None
if exactly_one and len(places) != 1:
raise ValueError("Didn't find exactly one code! " \
"(Found %d.)" % len(places))
def parse_code(place):
latitude = place.get('lat', None)
longitude = place.get('lon', None)
if latitude and longitude:
latitude = float(latitude)
longitude = float(longitude)
else:
return None
placename = place.get('display_name')
return (placename, (latitude, longitude))
if exactly_one:
return parse_code(places[0])
else:
return [parse_code(place) for place in places]
def parse_javascript(self, page, exactly_one=True):
if not isinstance(page, basestring):
page = util.decode_page(page)
matches = self.SINGLE_LOCATION.findall(page)
if not matches:
for match in self.AMBIGUOUS_LOCATION.findall(page):
places = self.AMBIGUOUS_SPLIT.split(match)
matches.extend([place for place in places if place])
if exactly_one and len(matches) != 1:
raise ValueError("Didn't find exactly one location! " \
"(Found %d.)" % len(matches))
def parse_match(match):
json = "[%s]" % self.STRING_QUOTE.sub('"', match.strip('()'))
array = simplejson.loads(json)
if len(array) == 8:
location, (latitude, longitude) = array[0], array[5:7]
else:
location, latitude, longitude = array[:3]
return (location, (latitude, longitude))
if exactly_one:
return parse_match(matches[0])
else:
return (parse_match(match) for match in matches)
def parse_xml(self, page, exactly_one=True):
"""Parse a location name, latitude, and longitude from an XML response.
"""
if not isinstance(page, basestring):
page = util.decode_page(page)
try:
doc = xml.dom.minidom.parseString(page)
except ExpatError:
places = []
doc = None
else:
places = doc.getElementsByTagName('Placemark')
if len(places) == 0 and doc is not None:
# Got empty result. Parse out the status code and raise an error if necessary.
status = doc.getElementsByTagName("Status")
status_code = int(util.get_first_text(status[0], 'code'))
self.check_status_code(status_code)
if exactly_one and len(places) != 1:
raise ValueError("Didn't find exactly one result! " \
"(Found %d.)" % len(places))
if exactly_one:
return self._parse_result(places[0])
else:
return [self._parse_result(place) for place in places]
def parse_js(self, page, exactly_one=True):
"""This parses JavaScript returned by queries the actual Google Maps
interface and could thus break easily. However, this is desirable if
the HTTP geocoder doesn't work for addresses in your country (the
UK, for example).
"""
if not isinstance(page, basestring):
page = util.decode_page(page)
LATITUDE = r"[\s,]lat:\s*(?P<latitude>-?\d+\.\d+)"
LONGITUDE = r"[\s,]lng:\s*(?P<longitude>-?\d+\.\d+)"
LOCATION = r"[\s,]laddr:\s*'(?P<location>.*?)(?<!\\)',"
ADDRESS = r"(?P<address>.*?)(?:(?: \(.*?@)|$)"
MARKER = '.*?'.join([LATITUDE, LONGITUDE, LOCATION])
MARKERS = r"{markers: (?P<markers>\[.*?\]),\s*polylines:"
def parse_marker(marker):
latitude, longitude, location = marker
location = re.match(ADDRESS, location).group('address')
latitude, longitude = float(latitude), float(longitude)
return (location, (latitude, longitude))
match = re.search(MARKERS, page)
markers = match and match.group('markers') or ''
markers = re.findall(MARKER, markers)
if exactly_one:
if len(markers) != 1:
raise ValueError("Didn't find exactly one marker! " \
"(Found %d.)" % len(markers))
marker = markers[0]
return parse_marker(marker)
else:
return (parse_marker(marker) for marker in markers)
def parse_xml(self, page):
if not isinstance(page, basestring):
page = util.decode_page(page)
doc = xml.dom.minidom.parseString(page)
results = doc.getElementsByTagName('Result')
precision = results[0].getAttribute('precision')
def parse_result(result):
strip = ", \n"
address = util.get_first_text(result, 'Address', strip)
city = util.get_first_text(result, 'City', strip)
state = util.get_first_text(result, 'State', strip)
zip = util.get_first_text(result, 'Zip', strip)
country = util.get_first_text(result, 'Country', strip)
city_state = util.join_filter(", ", [city, state])
place = util.join_filter(" ", [city_state, zip])
location = util.join_filter(", ", [address, place, country])
latitude = util.get_first_text(result, 'Latitude') or None
longitude = util.get_first_text(result, 'Longitude') or None
if latitude and longitude:
point = Point(latitude, longitude)
else:
point = None
return Location(location, point, {
'Address': address,
'City': city,
'State': state,
'Zip': zip,
'Country': country,
'precision': precision
})
return [parse_result(result) for result in results]
def _call_geocoder(self, url, timeout=None, raw=False):
"""
For a generated query URL, get the results.
"""
try:
page = self.urlopen(url, timeout=timeout or self.timeout)
except Exception as error: # pylint: disable=W0703
message = str(error) if not py3k else (str(error.args[0]) if len(error.args) else str(error))
if hasattr(self, "_geocoder_exception_handler"):
self._geocoder_exception_handler(error, message) # pylint: disable=E1101
if isinstance(error, HTTPError):
code = error.getcode()
try:
raise ERROR_CODE_MAP[code](message)
except KeyError:
raise GeocoderServiceError(message)
elif isinstance(error, URLError):
if "timed out" in message:
raise GeocoderTimedOut("Service timed out")
elif isinstance(error, SocketTimeout):
raise GeocoderTimedOut("Service timed out")
elif isinstance(error, SSLError):
if "timed out" in message:
raise GeocoderTimedOut("Service timed out")
raise GeocoderServiceError(message)
if raw:
return page
return json.loads(decode_page(page))
def _call_geocoder(self, url, timeout=None, raw=False):
"""
For a generated query URL, get the results.
"""
try:
page = self.urlopen(url, timeout=timeout or self.timeout)
except Exception as error: # pylint: disable=W0703
if hasattr(self, '_geocoder_exception_handler'):
self._geocoder_exception_handler(error) # pylint: disable=E1101
if isinstance(error, HTTPError):
if error.msg.lower().contains("unauthorized"):
raise GeocoderAuthenticationFailure("Unauthorized")
raise GeocoderServiceError(error.getcode(), error.msg)
elif isinstance(error, URLError):
if "timed out" in error.reason:
raise GeocoderTimedOut('Service timed out')
raise
elif isinstance(error, SocketTimeout):
raise GeocoderTimedOut('Service timed out')
elif isinstance(error, SSLError):
if error.message == 'The read operation timed out':
raise GeocoderTimedOut('Service timed out')
raise
else:
raise
if raw:
return page
return json.loads(decode_page(page))
def parse_json(self, page, exactly_one=True):
if not isinstance(page, basestring):
page = util.decode_page(page)
doc = json.loads(page)
results = doc.get('ResultSet', []).get('Results', [])
if not results:
raise ValueError("No results found")
elif exactly_one and len(results) != 1:
raise ValueError("Didn't find exactly one placemark! " \
"(Found %d.)" % len(results))
def parse_result(place):
line1, line2, line3, line4 = place.get('line1'), place.get('line2'), place.get('line3'), place.get('line4')
address = util.join_filter(", ", [line1, line2, line3, line4])
city = place.get('city')
state = place.get('state')
country = place.get('country')
location = util.join_filter(", ", [address, city, country])
lat, lng = place.get('latitude'), place.get('longitude')
#if lat and lng:
# point = Point(floatlat, lng)
#else:
# point = None
return (place, location, (float(lat), float(lng)))
if exactly_one:
return parse_result(results[0])
else:
return [parse_result(result) for result in results]
def parse_json(self, page, exactly_one=True, reverse=False):
if not isinstance(page, basestring):
page = util.decode_page(page)
json = simplejson.loads(page)
status = json.get('Status',{}).get('code')
if status != 200:
raise GeoStatusError(status)
places = json.get('Placemark', [])
print places
if (exactly_one and len(places) != 1) and (not reverse):
raise ValueError("Didn't find exactly one placemark! " \
"(Found %d.)" % len(places))
def parse_place(place):
location = place.get('address')
longitude, latitude = place['Point']['coordinates'][:2]
# Add support for pulling out the canonical name
locality = place.get('AddressDetails',{}).get('Country',{}).get('AdministrativeArea',{}).get('Locality',{}).get('LocalityName')
administrative = place.get('AddressDetails',{}).get('Country',{}).get('AdministrativeArea',{}).get('AdministrativeAreaName')
subadministrative = place.get('AddressDetails',{}).get('Country',{}).get('AdministrativeArea',{}).get('SubAdministrativeArea',{}).get('SubAdministrativeAreaName')
accuracy = place.get('AddressDetails',{}).get('Accuracy')
return util.RichResult((location, (latitude, longitude)), locality=locality, administrative=administrative, subadministrative=subadministrative, accuracy=accuracy)
if exactly_one:
return parse_place(places[0])
else:
return (parse_place(place) for place in places)
def parse_json(self, page, exactly_one=True, reverse=True):
if not isinstance(page, basestring):
page = util.decode_page(page)
doc = json.loads(page)
places = doc.get('Placemark', [])
if len(places) == 0:
# Got empty result. Parse out the status code and raise an error if necessary.
status = doc.get("Status", [])
status_code = status["code"]
self.check_status_code(status_code)
return None
elif exactly_one and len(places) != 1 and not reverse:
raise ValueError("Didn't find exactly one placemark! " \
"(Found %d.)" % len(places))
def parse_place(place):
location = place.get('address')
longitude, latitude = place['Point']['coordinates'][:2]
return (location, (latitude, longitude))
if exactly_one:
return parse_place(places[0])
else:
return [parse_place(place) for place in places]
def parse_json(self, page, exactly_one=True):
'''Returns location, (latitude, longitude) from json feed.'''
if not isinstance(page, basestring):
page = util.decode_page(page)
self.doc = json.loads(page)
places = self.doc.get('results', [])
if not places:
check_status(self.doc.get('status'))
return None
elif exactly_one and len(places) != 1:
raise ValueError(
"Didn't find exactly one placemark! (Found %d)" % len(places))
def parse_place(place):
'''Get the location, lat, lng from a single json place.'''
location = place.get('formatted_address')
latitude = place['geometry']['location']['lat']
longitude = place['geometry']['location']['lng']
return (location, (latitude, longitude))
if exactly_one or len(places) >= 1:
return parse_place(places[0])
else:
return [parse_place(place) for place in places]
def _get_result(url):
try:
result = json.loads(util.decode_page(urlopen_with_retries(url)))
return result
except URLError, e:
logger.error("GeoNames error, could not open URL: %s, error: %s" %
(url, e))
return {}
def _get_result(url):
try:
result = json.loads(util.decode_page(urlopen_with_retries(url)))
return result
except URLError, e:
logger.error("GeoNames error, could not open URL: %s, error: %s" %
(url, e))
return {}
def _read_http_error_body(self, error):
try:
return decode_page(error)
except Exception:
logger.debug(
'Unable to fetch body for a non-successful HTTP response',
exc_info=True)
return None
def _fetch_results(self, q):
params = {'q': q.encode("utf8"), 'key': self.api_key}
url = self.url % urlencode(params)
page = urlopen(url)
if (not isinstance(page, basestring)):
page = util.decode_page(page)
doc = json.loads(page)
return doc["resourceSets"][0]["resources"]
def _call_geocoder(self, url, raw=False):
"""
For a generated query URL, get the results.
"""
try:
page = self.urlopen(url)
except HTTPError as error:
raise GeocoderServiceError(error.getcode(), getattr(error, 'msg', None))
if raw:
return page
return json.loads(decode_page(page))
def parse_json(self, page, exactly_one=True, get_locality=False):
"""Parse a location name, latitude, and longitude from an JSON response."""
if not isinstance(page, basestring):
page = decode_page(page)
doc = json.loads(page)
resources = doc['resourceSets'][0]['resources']
if exactly_one and len(resources) != 1:
raise ValueError("Didn't find exactly one resource! " \
"(Found %d.)" % len(resources))
def parse_locality(resource):
stripchars = ", \n"
a = resource['address']
latitude = resource['point']['coordinates'][0] or None
longitude = resource['point']['coordinates'][1] or None
if latitude and longitude:
latitude = float(latitude)
longitude = float(longitude)
if a.get('locality'):
return (a.get('locality', '').strip(stripchars), (latitude, longitude))
else:
return parse_resource(resource)
def parse_resource(resource):
stripchars = ", \n"
a = resource['address']
address = a.get('addressLine', '').strip(stripchars)
city = a.get('locality', '').strip(stripchars)
state = a.get('adminDistrict', '').strip(stripchars)
zipcode = a.get('postalCode', '').strip(stripchars)
country = a.get('countryRegion', '').strip(stripchars)
city_state = join_filter(", ", [city, state])
place = join_filter(" ", [city_state, zipcode])
location = join_filter(", ", [address, place, country])
latitude = resource['point']['coordinates'][0] or None
longitude = resource['point']['coordinates'][1] or None
if latitude and longitude:
latitude = float(latitude)
longitude = float(longitude)
return (location, (latitude, longitude))
if exactly_one:
if get_locality:
return parse_locality(resources[0])
return parse_resource(resources[0])
else:
if get_locality:
return [parse_locality(resource) for resource in resources]
return [parse_resource(resource) for resource in resources]
def _fetch_results(self, q):
params = {'q': q.encode("utf8"),
'key': self.api_key }
url = self.url % urlencode(params)
page = urlopen(url)
if (not isinstance(page, basestring)):
page = util.decode_page(page);
doc = json.loads(page);
return doc["resourceSets"][0]["resources"]
def parse_json(self, page):
'''Returns timezone (as Olsen format) from json feed.'''
if not isinstance(page, basestring):
page = util.decode_page(page)
self.doc = json.loads(page)
timezone = self.doc.get('rawOffset')
if not timezone:
check_status(self.doc.get('status'))
return None
return timezone
def _call_geocoder(self, url, raw=False):
"""
For a generated query URL, get the results.
"""
try:
page = self.urlopen(url)
except HTTPError as error:
raise GeocoderServiceError(error.getcode(),
getattr(error, 'msg', None))
if raw:
return page
return json.loads(decode_page(page))
def _call_geocoder(self, url, timeout=None, raw=False):
"""
For a generated query URL, get the results.
"""
try:
page = self.urlopen(url, timeout=timeout or self.timeout)
except HTTPError as error:
if hasattr(self, '_geocoder_exception_handler'):
self._geocoder_exception_handler(error) # pylint: disable=E1101
raise GeocoderServiceError(error.getcode(), getattr(error, 'msg', None))
if raw:
return page
return json.loads(decode_page(page))
def _call_geocoder(self, url, timeout=None, raw=False):
"""
For a generated query URL, get the results.
"""
try:
page = self.urlopen(url, timeout=timeout or self.timeout)
except HTTPError as error:
if hasattr(self, '_geocoder_exception_handler'):
self._geocoder_exception_handler(error) # pylint: disable=E1101
raise GeocoderServiceError(error.getcode(),
getattr(error, 'msg', None))
if raw:
return page
return json.loads(decode_page(page))
def parse_raw(self, page, exactly_one=True):
if not isinstance(page, basestring):
page = decode_page(page)
doc = json.loads(page)
resources = doc['resourceSets'][0]['resources']
if exactly_one and len(resources) != 1:
raise ValueError("Didn't find exactly one resource! " \
"(Found %d.)" % len(resources))
if exactly_one:
return resources[0]
else:
return resources
def parse_json(self, page, exactly_one=True):
"""Parse display name, latitude, and longitude from an JSON response."""
if not isinstance(page, basestring):
page = decode_page(page)
resources = json.loads(page)
if exactly_one and len(resources) != 1:
raise ValueError("Didn't find exactly one result! " \
"(Found %d.)" % len(resources))
if exactly_one and len(resources) == 1:
return self._parse_result(resources[0])
else:
return [self._parse_result(resource) for resource in resources]
def parse_json(self, page, exactly_one=True, output='latlon'):
'''Parse returned json feed of geocoded results
Returns various forms of the geocoded 'location' from the JSON feed.
Type of locations include latitude and longitude, address, bounding
box (viewport), geometry information, or the entire JSON feed.
'''
if not isinstance(page, basestring):
page = util.decode_page(page)
self.doc = json.loads(page)
places = self.doc.get('results', [])
if not places:
check_status(self.doc.get('status'))
return None
elif exactly_one and len(places) != 1:
raise ValueError(
"Didn't find exactly one placemark! (Found %d)" % len(places))
outputs = ('latlon', 'alatlon', 'bbox', 'geometry', 'all')
if not output in outputs:
raise ValueError(
"Invalid `output` parameter, must be one of ('%s')" % "', '".join(outputs))
def parse_place(place):
'''Get the location, lat, lng from a single json place.'''
location = place.get('formatted_address')
latitude = place['geometry']['location']['lat']
longitude = place['geometry']['location']['lng']
if output == 'alatlon':
return (location, (latitude, longitude))
elif output == 'bbox':
northeast = place['geometry']['viewport']['northeast']
southwest = place['geometry']['viewport']['southwest']
xmin, xmax = southwest['lng'], northeast['lng']
ymin, ymax = southwest['lat'], northeast['lat']
return (ymin, ymax, xmin, xmax)
elif output == 'geometry':
return place['geometry']
elif output == 'all':
return place
else: # latlon
return (latitude, longitude)
if exactly_one:
return parse_place(places[0])
else:
return [parse_place(place) for place in places]
def parse_json(self, page, exactly_one=True):
if not isinstance(page, basestring):
page = util.decode_page(page)
json = simplejson.loads(page)
places = json.get('Placemark', [])
if exactly_one and len(places) != 1:
raise ValueError("Didn't find exactly one placemark! " \
"(Found %d.)" % len(places))
def parse_place(place):
location = place.get('address')
longitude, latitude = place['Point']['coordinates'][:2]
return (location, (latitude, longitude))
if exactly_one:
return parse_place(places[0])
else:
return (parse_place(place) for place in places)
def reverse_geocode(self, lat, lng):
params = {
"lat": lat,
"lng": lng,
"username": module_config().get("geonames_username"),
"token": module_config().get("geonames_token")
}
url = "http://ws.geonames.net/findNearbyJSON?%s" % urlencode(params)
if (url not in DplaGeonamesGeocoder.resultCache):
result = json.loads(util.decode_page(urlopen(url)))
if ("geonames" in result \
and len(result["geonames"]) > 0):
DplaGeonamesGeocoder.resultCache[url] = result["geonames"][0]
else:
logger.error("geocode: Could not reverse geocode (%s, %s)" % (
lat,
lng,
))
return None
return DplaGeonamesGeocoder.resultCache[url]
def reverse_geocode_hierarchy(self, lat, lng, fcodes=None):
hierarchy = []
geonames_item = self.reverse_geocode(lat, lng)
if (geonames_item):
params = {
"geonameId": geonames_item["geonameId"],
"username": module_config().get("geonames_username"),
"token": module_config().get("geonames_token")
}
url = "http://ws.geonames.net/hierarchyJSON?%s" % urlencode(params)
if (url not in DplaGeonamesGeocoder.resultCache):
result = json.loads(util.decode_page(urlopen(url)))
DplaGeonamesGeocoder.resultCache[url] = result["geonames"]
# Return only the requested fcodes
for place in DplaGeonamesGeocoder.resultCache[url]:
if (("fcode" in place and place["fcode"] in fcodes) \
or fcodes is None):
hierarchy.append(place)
return hierarchy
def parse_xml(self, page, exactly_one=True):
if not isinstance(page, basestring):
page = util.decode_page(page)
doc = xml.dom.minidom.parseString(page)
results = doc.getElementsByTagName('Result')
def parse_result(result):
strip = ", \n"
address = util.get_first_text(result, 'Address', strip)
city = util.get_first_text(result, 'City', strip)
state = util.get_first_text(result, 'State', strip)
zip = util.get_first_text(result, 'Zip', strip)
country = util.get_first_text(result, 'Country', strip)
city_state = util.join_filter(", ", [city, state])
place = util.join_filter(" ", [city_state, zip])
location = util.join_filter(", ", [address, place, country])
latitude = float(util.get_first_text(result, 'Latitude')) or None
longitude = float(util.get_first_text(result, 'Longitude')) or None
# TODO use Point/Location object API in 0.95
#if latitude and longitude:
# point = Point(latitude, longitude)
#else:
# point = Non
#return Location(location, point, {
# 'Address': address,
# 'City': city,
# 'State': state,
# 'Zip': zip,
# 'Country': country
#})
return address, (latitude, longitude)
if exactly_one:
return parse_result(results[0])
else:
return [parse_result(result) for result in results]
def parse_xml(self, page, exactly_one=True, reverse=False):
"""Parse a location name, latitude, and longitude from an XML response.
"""
if not isinstance(page, basestring):
page = util.decode_page(page)
try:
doc = xml.dom.minidom.parseString(page)
except ExpatError:
places = []
doc = None
else:
places = doc.getElementsByTagName('Placemark')
if len(places) == 0 and doc is not None:
# Got empty result. Parse out the status code and raise an error if necessary.
status = doc.getElementsByTagName("Status")
status_code = int(util.get_first_text(status[0], 'code'))
self.check_status_code(status_code)
if (exactly_one and len(places) != 1) and (not reverse):
raise ValueError("Didn't find exactly one placemark! " \
"(Found %d.)" % len(places))
def parse_place(place):
location = util.get_first_text(place, ['address', 'name']) or None
points = place.getElementsByTagName('Point')
point = points and points[0] or None
coords = util.get_first_text(point, 'coordinates') or None
if coords:
longitude, latitude = [float(f) for f in coords.split(',')[:2]]
else:
latitude = longitude = None
_, (latitude, longitude) = self.geocode(location)
return (location, (latitude, longitude))
if exactly_one:
return parse_place(places[0])
else:
return [parse_place(place) for place in places]
def _call_geocoder(self, url, timeout=None, raw=False):
"""
For a generated query URL, get the results.
"""
try:
page = self.urlopen(url, timeout=timeout or self.timeout)
except Exception as error: # pylint: disable=W0703
if hasattr(self, '_geocoder_exception_handler'):
self._geocoder_exception_handler(error) # pylint: disable=E1101
if isinstance(error, HTTPError):
raise GeocoderServiceError(error.getcode(), error.msg)
elif isinstance(error, SSLError):
if error.message == 'The read operation timed out':
raise GeocoderTimedOut(
'Service timed out while using SSL connection')
raise
elif isinstance(error, SocketTimeout):
raise GeocoderTimedOut('Service timed out')
else:
raise
if raw:
return page
return json.loads(decode_page(page))
def parse_json(self, page, exactly_one):
if not isinstance(page, basestring):
page = util.decode_page(page)
doc = json.loads(page)
places = doc.get('geonames', [])
if not places:
return None
if exactly_one and len(places) != 1:
raise ValueError("Didn't find exactly one code! " \
"(Found %d.)" % len(places))
def parse_code(place):
latitude = place.get('lat', None)
longitude = place.get('lng', None)
if latitude and longitude:
latitude = float(latitude)
longitude = float(longitude)
else:
return None
placename = place.get('name')
state = place.get('adminCode1', None)
country = place.get('countryCode', None)
location = ', '.join(filter(lambda x: bool(x),
[placename, state, country]
))
return (location, (latitude, longitude))
if exactly_one:
return parse_code(places[0])
else:
return [parse_code(place) for place in places]
def parse_json(self, page, exactly_one=True):
"""Parse a location name, latitude, and longitude from an JSON response."""
if not isinstance(page, basestring):
page = decode_page(page)
doc = json.loads(page)
resources = doc['resourceSets'][0]['resources']
if exactly_one and len(resources) != 1:
raise ValueError("Didn't find exactly one resource! " \
"(Found %d.)" % len(resources))
def parse_resource(resource):
stripchars = ", \n"
a = resource['address']
address = a.get('addressLine', '').strip(stripchars)
city = a.get('locality', '').strip(stripchars)
state = a.get('adminDistrict', '').strip(stripchars)
zipcode = a.get('postalCode', '').strip(stripchars)
country = a.get('countryRegion', '').strip(stripchars)
city_state = join_filter(", ", [city, state])
place = join_filter(" ", [city_state, zipcode])
location = join_filter(", ", [address, place, country])
latitude = resource['point']['coordinates'][0] or None
longitude = resource['point']['coordinates'][1] or None
if latitude and longitude:
latitude = float(latitude)
longitude = float(longitude)
return (location, (latitude, longitude))
if exactly_one:
return parse_resource(resources[0])
else:
return [parse_resource(resource) for resource in resources]
class DplaBingGeocoder(geocoders.Bing):
countryBBoxCache = {}
resultCache = {}
def __init__(self, **kwargs):
super(DplaBingGeocoder, self).__init__(**kwargs)
def enrich_place(self, place):
"""
Accepts a place and returns a new one with coordinates
added as identified by Bing.
In a better implementation of this geocoder
this function would try to return a fully updated
version of the place.
As it stands, it simply provides wrapper around the
geocode_spatial function to provide a common interface
between geocoders and places.
"""
coords = self.geocode_spatial(place.to_map_json())
if coords:
place.coordinates = floats_to_coordinates(coords)
return place
def geocode_spatial(self, spatial):
'''
Accepts a dictionary and attempts to return a set
of coordinates in format [latitude, longitude] that
match the place.
'''
if (not self.api_key):
logger.warn("No API key set for Bing " +
"(use bing_api_key configuration key)")
return None
address = Address(spatial)
for candidate in address.get_candidates():
# See if this address candidate exists in our cache
if (candidate not in DplaBingGeocoder.resultCache):
results = self._fetch_results(candidate)
DplaBingGeocoder.resultCache[candidate] = list(results)
# Require that a single match, or closely grouped matches be
# returned to avoid bad geocoding results
candidates = len(DplaBingGeocoder.resultCache[candidate])
closely_grouped_results = self._are_closely_grouped_results(
DplaBingGeocoder.resultCache[candidate])
if (candidates == 1 or closely_grouped_results):
result = DplaBingGeocoder.resultCache[candidate][0]
coordinates = (result["geocodePoints"][0]["coordinates"][0],
result["geocodePoints"][0]["coordinates"][1])
valid_result = True
# If we have a specified country, perform a sanity check that
# the returned coordinates is within the country's bounding box
if (address.country and "countryRegion" in result["address"]):
bbox_result = self._is_in_country(coordinates,
address.country)
# If we can't get a country's bbox, assume that we have a
# good result
if (bbox_result is not None):
valid_result = bbox_result
if (not valid_result):
msg = "Geocode result [%s] " % result["name"] + \
"not in the correct country " + \
"[%s], ignoring" % address.country
logger.debug(msg)
if (valid_result):
if ("name" in spatial):
logger.debug("Geocode result: %s => %s (%s)" % (
spatial["name"],
result["name"],
result["point"]["coordinates"],
))
else:
logger.debug("Geocode result: %s => %s (%s)" % (
spatial,
result["name"],
result["point"]["coordinates"],
))
return coordinates
return None
def _are_closely_grouped_results(self, results):
"""
Check to see if all results are within 10km of each other.
"""
if (0 == len(results)):
return False
TOLERANCE_KM = 10
gpoints = "geocodePoints"
coords = "coordinates"
coordinates = [(x[gpoints][0][coords][0], x[gpoints][0][coords][1])
for x in results]
for combination in itertools.combinations(coordinates, 2):
if (TOLERANCE_KM < haversine(combination[0], combination[1])):
return False
return True
def _fetch_results(self, q):
params = {'q': q.encode("utf8"), 'key': self.api_key}
# geopy changed the instance variables on the bing geocoder in
# version 0.96 - this handles the differences
if hasattr(self, 'url'):
url = self.url % urlencode(params)
else:
url = "%s?%s" % (self.api, urlencode(params))
try:
page = urlopen_with_retries(url)
except Exception, e:
logger.error("Geocode error, could not open URL: %s, error: %s" %
(url, e))
return []
if (not isinstance(page, basestring)):
page = util.decode_page(page)
doc = json.loads(page)
return doc["resourceSets"][0]["resources"]
def _call_geocoder(self,
url,
timeout=None,
raw=False,
requester=None,
deserializer=json.loads,
**kwargs):
"""
For a generated query URL, get the results.
"""
requester = requester or self.urlopen
req = Request(url=url, headers=self.headers)
try:
page = requester(req, timeout=(timeout or self.timeout), **kwargs)
except Exception as error: # pylint: disable=W0703
message = (str(error) if not py3k else
(str(error.args[0]) if len(error.args) else str(error)))
if hasattr(self, '_geocoder_exception_handler'):
self._geocoder_exception_handler(error, message) # pylint: disable=E1101
if isinstance(error, HTTPError):
code = error.getcode()
try:
raise ERROR_CODE_MAP[code](message)
except KeyError:
raise GeocoderServiceError(message)
elif isinstance(error, URLError):
if "timed out" in message:
raise GeocoderTimedOut('Service timed out')
elif "unreachable" in message:
raise GeocoderUnavailable('Service not available')
elif isinstance(error, SocketTimeout):
raise GeocoderTimedOut('Service timed out')
elif isinstance(error, SSLError):
if "timed out" in message:
raise GeocoderTimedOut('Service timed out')
raise GeocoderServiceError(message)
if hasattr(page, 'getcode'):
status_code = page.getcode()
elif hasattr(page, 'status_code'):
status_code = page.status_code
else:
status_code = None
if status_code in ERROR_CODE_MAP:
raise ERROR_CODE_MAP[page.status_code]("\n%s" % decode_page(page))
if raw:
return page
page = decode_page(page)
if deserializer is not None:
try:
return deserializer(page)
except ValueError:
raise GeocoderParseError(
"Could not deserialize using deserializer:\n%s" % page)
else:
return page
def _call_geocoder(self,
url,
timeout=DEFAULT_SENTINEL,
raw=False,
requester=None,
deserializer=json.loads,
**kwargs):
"""
For a generated query URL, get the results.
"""
if requester:
req = url # Don't construct an urllib's Request for a custom requester.
# `requester` might be anything which can issue an HTTP request.
# Assume that `requester` is a method of the `requests` library.
# Requests, however, doesn't accept SSL context in its HTTP
# request methods. A custom HTTP adapter has to be created for that.
# So the current usage is not directly compatible with `requests`.
requester = functools.partial(requester,
context=self.ssl_context,
proxies=self.proxies,
headers=self.headers)
else:
if isinstance(url, Request):
# copy Request
headers = self.headers.copy()
headers.update(url.header_items())
req = Request(url=url.get_full_url(), headers=headers)
else:
req = Request(url=url, headers=self.headers)
requester = requester or self.urlopen
if timeout is None:
warnings.warn(
('`timeout=None` has been passed to a geocoder call. Using '
'default geocoder timeout. In geopy 2.0 the '
'behavior will be different: None will mean "no timeout" '
'instead of "default geocoder timeout". Pass '
'geopy.geocoders.base.DEFAULT_SENTINEL instead of None '
'to get rid of this warning.'), DeprecationWarning)
timeout = DEFAULT_SENTINEL
timeout = (timeout
if timeout is not DEFAULT_SENTINEL else self.timeout)
try:
page = requester(req, timeout=timeout, **kwargs)
except Exception as error:
message = (str(error) if not py3k else
(str(error.args[0]) if len(error.args) else str(error)))
self._geocoder_exception_handler(error, message)
if isinstance(error, HTTPError):
code = error.getcode()
try:
raise ERROR_CODE_MAP[code](message)
except KeyError:
raise GeocoderServiceError(message)
elif isinstance(error, URLError):
if "timed out" in message:
raise GeocoderTimedOut('Service timed out')
elif "unreachable" in message:
raise GeocoderUnavailable('Service not available')
elif isinstance(error, SocketTimeout):
raise GeocoderTimedOut('Service timed out')
elif isinstance(error, SSLError):
if "timed out" in message:
raise GeocoderTimedOut('Service timed out')
raise GeocoderServiceError(message)
if hasattr(page, 'getcode'):
status_code = page.getcode()
elif hasattr(page, 'status_code'):
status_code = page.status_code
else:
status_code = None
if status_code in ERROR_CODE_MAP:
raise ERROR_CODE_MAP[page.status_code]("\n%s" % decode_page(page))
if raw:
return page
page = decode_page(page)
if deserializer is not None:
try:
return deserializer(page)
except ValueError:
raise GeocoderParseError(
"Could not deserialize using deserializer:\n%s" % page)
else:
return page
