def clean_location(self):
'''
If the text-based location is a new address for the ModelForm's
internal place instance, turn it into a true Location. Otherwise,
return the stored instance's location.
'''
address = self.cleaned_data['location'].strip()
# if there's already a location stored in this form's instance, just return it assuming the field hasn't changed
if self.instance.location:
if self.instance.location.address.strip() == address:
return self.instance.location
# otherwise, we have to create a new location
location = Location(address=address, postcode='15213',
town='Pittsburgh', state='PA')
# if the form is set to geocode new locations
if self.geocode_location:
# TODO: make this stuff happen after response is sent, maybe via a signal?
try:
resolved = resolve_location(location, retry=0)
if resolved:
location = resolved
except APIError:
pass # do nothing, just go with basic Location
return location
def test_location_resolving(self):
# test basic address lookup -- ensure zip and geocoding info is filled in
# https://maps.googleapis.com/maps/api/geocode/json?address=3411+Blvd+of+the+Allies®ion=US&sensor=false
resolved = pl_outsourcing.resolve_location(Location(address='3411 Blvd of the Allies'))
self.assertEquals(resolved.address,'3411 Boulevard of the Allies')
self.assertEquals(resolved.postcode,'15213')
self.assertEquals(resolved.town,'Pittsburgh')
self.assertEquals(resolved.state,'PA')
self.assertEquals(resolved.country,'US')
self.assertAlmostEquals(resolved.latitude,40.435938,3) # assert equals up to 2 places
self.assertAlmostEquals(resolved.longitude,-79.958309,3)
# test zip codes properly bias searches -- if these fail, make sure the geocoding info
# at the following links matches the expected values below:
# https://maps.googleapis.com/maps/api/geocode/json?address=800+penn+ave%2C+15222®ion=US&sensor=false
# https://maps.googleapis.com/maps/api/geocode/json?address=800+penn+ave%2C+15221®ion=US&sensor=false
resolved = pl_outsourcing.resolve_location(Location(address='800 penn ave',postcode='15222'))
self.assertAlmostEquals(resolved.latitude,40.443290,places=4) # assert equals up to 2 places
self.assertAlmostEquals(resolved.longitude,-79.999092,places=2)
resolved = pl_outsourcing.resolve_location(Location(address='800 penn ave',postcode='15221'))
self.assertAlmostEquals(resolved.latitude,40.442470,4) # assert equals up to 2 places
self.assertAlmostEquals(resolved.longitude,-79.881871,2)
# tests that geocoding info properly biases searches
# expected results are based on the following geocoding API calls:
# http://maps.googleapis.com/maps/api/geocode/json?region=US&sensor=false&bounds=40.438000%2C-80.005000%7C40.448000%2C-79.995000&address=800+penn+ave
# http://maps.googleapis.com/maps/api/geocode/json?region=US&sensor=false&bounds=40.437000%2C-79.905000%7C40.447000%2C-79.895000&address=800+penn+ave
resolved = pl_outsourcing.resolve_location(Location(address='800 penn ave',latitude=40.443,longitude=-80))
self.assertEquals(resolved.postcode,'15222')
resolved = pl_outsourcing.resolve_location(Location(address='800 penn ave',latitude=40.442,longitude=-79.9))
self.assertEquals(resolved.postcode,'15221')
# bad address
unresolved = pl_outsourcing.resolve_location(Location(address='fakey fake double false address'))
self.assertIsNone(unresolved)
def save(self, commit=False):
location = super(SimplePlaceForm, self).save(commit=False)
if self.geocode_location:
# TODO: make this stuff happen after response is sent, maybe via a signal?
try:
resolved = resolve_location(location, retry=0)
if resolved:
location = resolved
except APIError:
pass # do nothing, just go with basic Location
if commit:
location.save()
place = self.place_instance if self.place_instance else Place(location=location)
place.name = self.cleaned_data.get('name', '')
if commit:
place.save()
return place
def insert_row(row, idx):
logging.info(u"Importing row %d (%s)" % (int(idx), row["name"]))
try:
lat = decimal.Decimal(row.get("lat"))
except (TypeError, decimal.InvalidOperation):
lat = None
try:
lng = decimal.Decimal(row.get("lng"))
except (TypeError, decimal.InvalidOperation):
lng = None
address = row.get("street", "")
location = Location(address=address, postcode="15213", latitude=lat, longitude=lng)
# want to resolve location if we have an address worht normalizing and/or we don't have geocoded values
if location.address or location.latitude is None or location.longitude is None:
logging.info('Resolving location "%s"' % unicode(location))
resolved = resolve_location(location)
if resolved:
# hack to get around Google Geocoding appending unviersity names onto many addresses
if "university" in resolved.address.lower() and "university" not in row.get("street", "").lower():
resolved.address = ",".join(resolved.address.split(",")[1:])
resolved.address = resolved.address.strip()
try:
# if exact match exists, use it instead of the newly found one
location = Location.objects.get(address=resolved.address, postcode=location.postcode)
except Location.DoesNotExist:
location = resolved
location.save()
else:
logging.warning("Geocoding failed")
has_corporate_fb = row.get("fb_id", "").startswith("corporate")
place, created = Place.objects.get_or_create(
name=row["name"],
location=location,
url=row.get("url", "")[:200],
phone=row.get("phone", "")[:200],
fb_id=row.get("fb_id", "").split(":")[-1],
description=row.get("description", ""),
listed=bool(int(row.get("listed", "0"))),
)
for t in row.get("tags", "").split(";"):
tag, _ = Tag.objects.get_or_create(name=slugify(t.lower()))
place.tags.add(tag)
place.save()
image_path = row.get("image_url", "")
if re.match("https?\:", image_path):
if re.match("https?\:\/\/profile.ak.fbcdn.net", image_path):
logging.info("Skipping Facebook cdn image")
else:
logging.info("Pulling live image from web")
try:
place.image = utils.imagefile_from_url(image_path)
except IOError:
logging.error("Failed pulling image")
pass
elif image_path != "":
logging.info("Using local image file")
f = open(os.path.join(DATA_DIR, "images", image_path))
place.image = File(f)
place.save()
if place.image:
place.image.close()
if place.fb_id:
logging.info("Supplementing info from Facebook...")
try:
places_fb_import.commit_place(place, corporate=has_corporate_fb)
except places_fb_import.FacebookAPIError as e:
logging.error("Facebook API error (fb id %s): %s" % (str(place.fb_id), unicode(e)))
except IOError as e:
logging.error(unicode(e))
logging.info("Finished import: %s" % unicode(place))
logging.info("")
def resolve_place(new_place, candidate=None):
'''
Will attempt resolving a new place object with an existing one in the
DB using various methods. Helper for add_fbevent.
Returns None if no DB-linked place could be found
'''
print ' attempting place resolve for %s. candidate: %s' % (str(new_place), str(candidate))
# 1: check if the place matches the candidate in name
if candidate and candidate.name and candidate.name.lower() == new_place.name.lower():
print ' using candidate! name match.'
return candidate
# all the rest of the attempts deal with the new place's location. if we don't have that, punt
if not new_place.location:
print ' resolve failed.'
return None
# resolve the location to normalize the address for the next steps
resolved = resolve_location(new_place.location, retry=0)
if resolved:
new_place.location = resolved
new_loc = new_place.location
# 2: see if the new address matches the candidate address - use it if so.
if candidate:
if candidate.location and new_loc.address.lower() == candidate.location.address.lower():
# do a sanity check on distances between candidate and new place
if not new_loc.is_geocoded() or not candidate.location.is_geocoded() or \
new_loc.distance_from(candidate.location) < .8:
print ' using candidate! address match'
return candidate
else:
print ' address matched, but geocoding failure'
# 3: candidate is a bust, next go fishing for the place in the db at large with a location query
# grab all locations in the area
query = None
qs = []
if new_loc.is_geocoded():
qs.append(Q(latitude__lt=(new_loc.latitude + 1e-3), latitude__gt=(new_loc.latitude - 1e-3),
longitude__lt=(new_loc.longitude + 1e-3), longitude__gt=(new_loc.longitude - 1e-3)))
if new_loc.address:
if new_loc.town and new_loc.state:
qs.append(Q(address=new_loc.address, town=new_loc.town, state=new_loc.state))
if new_loc.postcode:
qs.append(Q(address=new_loc.address, postcode=new_loc.postcode))
# construct composite q objects
for q in qs:
if query is None:
query = q
else:
query |= q
# if there are some potential locations, see if we can find a matching place
candidate_locations = Location.objects.filter(query) if query else []
if len(candidate_locations) > 0:
candidates = Place.objects.filter(name=new_place.name, location__in=candidate_locations)
if len(candidates) > 0:
# TODO: rank based on distance
print ' using db match!', candidates[0]
return candidates[0]
# all out of options: punt!
print ' resolve failed.'
return None
