def clean_location(self): ''' If the text-based location is a new address for the ModelForm's internal place instance, turn it into a true Location. Otherwise, return the stored instance's location. ''' address = self.cleaned_data['location'].strip() # if there's already a location stored in this form's instance, just return it assuming the field hasn't changed if self.instance.location: if self.instance.location.address.strip() == address: return self.instance.location # otherwise, we have to create a new location location = Location(address=address, postcode='15213', town='Pittsburgh', state='PA') # if the form is set to geocode new locations if self.geocode_location: # TODO: make this stuff happen after response is sent, maybe via a signal? try: resolved = resolve_location(location, retry=0) if resolved: location = resolved except APIError: pass # do nothing, just go with basic Location return location
def test_location_resolving(self): # test basic address lookup -- ensure zip and geocoding info is filled in # https://maps.googleapis.com/maps/api/geocode/json?address=3411+Blvd+of+the+Allies®ion=US&sensor=false resolved = pl_outsourcing.resolve_location(Location(address='3411 Blvd of the Allies')) self.assertEquals(resolved.address,'3411 Boulevard of the Allies') self.assertEquals(resolved.postcode,'15213') self.assertEquals(resolved.town,'Pittsburgh') self.assertEquals(resolved.state,'PA') self.assertEquals(resolved.country,'US') self.assertAlmostEquals(resolved.latitude,40.435938,3) # assert equals up to 2 places self.assertAlmostEquals(resolved.longitude,-79.958309,3) # test zip codes properly bias searches -- if these fail, make sure the geocoding info # at the following links matches the expected values below: # https://maps.googleapis.com/maps/api/geocode/json?address=800+penn+ave%2C+15222®ion=US&sensor=false # https://maps.googleapis.com/maps/api/geocode/json?address=800+penn+ave%2C+15221®ion=US&sensor=false resolved = pl_outsourcing.resolve_location(Location(address='800 penn ave',postcode='15222')) self.assertAlmostEquals(resolved.latitude,40.443290,places=4) # assert equals up to 2 places self.assertAlmostEquals(resolved.longitude,-79.999092,places=2) resolved = pl_outsourcing.resolve_location(Location(address='800 penn ave',postcode='15221')) self.assertAlmostEquals(resolved.latitude,40.442470,4) # assert equals up to 2 places self.assertAlmostEquals(resolved.longitude,-79.881871,2) # tests that geocoding info properly biases searches # expected results are based on the following geocoding API calls: # http://maps.googleapis.com/maps/api/geocode/json?region=US&sensor=false&bounds=40.438000%2C-80.005000%7C40.448000%2C-79.995000&address=800+penn+ave # http://maps.googleapis.com/maps/api/geocode/json?region=US&sensor=false&bounds=40.437000%2C-79.905000%7C40.447000%2C-79.895000&address=800+penn+ave resolved = pl_outsourcing.resolve_location(Location(address='800 penn ave',latitude=40.443,longitude=-80)) self.assertEquals(resolved.postcode,'15222') resolved = pl_outsourcing.resolve_location(Location(address='800 penn ave',latitude=40.442,longitude=-79.9)) self.assertEquals(resolved.postcode,'15221') # bad address unresolved = pl_outsourcing.resolve_location(Location(address='fakey fake double false address')) self.assertIsNone(unresolved)
def save(self, commit=False): location = super(SimplePlaceForm, self).save(commit=False) if self.geocode_location: # TODO: make this stuff happen after response is sent, maybe via a signal? try: resolved = resolve_location(location, retry=0) if resolved: location = resolved except APIError: pass # do nothing, just go with basic Location if commit: location.save() place = self.place_instance if self.place_instance else Place(location=location) place.name = self.cleaned_data.get('name', '') if commit: place.save() return place
def insert_row(row, idx): logging.info(u"Importing row %d (%s)" % (int(idx), row["name"])) try: lat = decimal.Decimal(row.get("lat")) except (TypeError, decimal.InvalidOperation): lat = None try: lng = decimal.Decimal(row.get("lng")) except (TypeError, decimal.InvalidOperation): lng = None address = row.get("street", "") location = Location(address=address, postcode="15213", latitude=lat, longitude=lng) # want to resolve location if we have an address worht normalizing and/or we don't have geocoded values if location.address or location.latitude is None or location.longitude is None: logging.info('Resolving location "%s"' % unicode(location)) resolved = resolve_location(location) if resolved: # hack to get around Google Geocoding appending unviersity names onto many addresses if "university" in resolved.address.lower() and "university" not in row.get("street", "").lower(): resolved.address = ",".join(resolved.address.split(",")[1:]) resolved.address = resolved.address.strip() try: # if exact match exists, use it instead of the newly found one location = Location.objects.get(address=resolved.address, postcode=location.postcode) except Location.DoesNotExist: location = resolved location.save() else: logging.warning("Geocoding failed") has_corporate_fb = row.get("fb_id", "").startswith("corporate") place, created = Place.objects.get_or_create( name=row["name"], location=location, url=row.get("url", "")[:200], phone=row.get("phone", "")[:200], fb_id=row.get("fb_id", "").split(":")[-1], description=row.get("description", ""), listed=bool(int(row.get("listed", "0"))), ) for t in row.get("tags", "").split(";"): tag, _ = Tag.objects.get_or_create(name=slugify(t.lower())) place.tags.add(tag) place.save() image_path = row.get("image_url", "") if re.match("https?\:", image_path): if re.match("https?\:\/\/profile.ak.fbcdn.net", image_path): logging.info("Skipping Facebook cdn image") else: logging.info("Pulling live image from web") try: place.image = utils.imagefile_from_url(image_path) except IOError: logging.error("Failed pulling image") pass elif image_path != "": logging.info("Using local image file") f = open(os.path.join(DATA_DIR, "images", image_path)) place.image = File(f) place.save() if place.image: place.image.close() if place.fb_id: logging.info("Supplementing info from Facebook...") try: places_fb_import.commit_place(place, corporate=has_corporate_fb) except places_fb_import.FacebookAPIError as e: logging.error("Facebook API error (fb id %s): %s" % (str(place.fb_id), unicode(e))) except IOError as e: logging.error(unicode(e)) logging.info("Finished import: %s" % unicode(place)) logging.info("")
def resolve_place(new_place, candidate=None): ''' Will attempt resolving a new place object with an existing one in the DB using various methods. Helper for add_fbevent. Returns None if no DB-linked place could be found ''' print ' attempting place resolve for %s. candidate: %s' % (str(new_place), str(candidate)) # 1: check if the place matches the candidate in name if candidate and candidate.name and candidate.name.lower() == new_place.name.lower(): print ' using candidate! name match.' return candidate # all the rest of the attempts deal with the new place's location. if we don't have that, punt if not new_place.location: print ' resolve failed.' return None # resolve the location to normalize the address for the next steps resolved = resolve_location(new_place.location, retry=0) if resolved: new_place.location = resolved new_loc = new_place.location # 2: see if the new address matches the candidate address - use it if so. if candidate: if candidate.location and new_loc.address.lower() == candidate.location.address.lower(): # do a sanity check on distances between candidate and new place if not new_loc.is_geocoded() or not candidate.location.is_geocoded() or \ new_loc.distance_from(candidate.location) < .8: print ' using candidate! address match' return candidate else: print ' address matched, but geocoding failure' # 3: candidate is a bust, next go fishing for the place in the db at large with a location query # grab all locations in the area query = None qs = [] if new_loc.is_geocoded(): qs.append(Q(latitude__lt=(new_loc.latitude + 1e-3), latitude__gt=(new_loc.latitude - 1e-3), longitude__lt=(new_loc.longitude + 1e-3), longitude__gt=(new_loc.longitude - 1e-3))) if new_loc.address: if new_loc.town and new_loc.state: qs.append(Q(address=new_loc.address, town=new_loc.town, state=new_loc.state)) if new_loc.postcode: qs.append(Q(address=new_loc.address, postcode=new_loc.postcode)) # construct composite q objects for q in qs: if query is None: query = q else: query |= q # if there are some potential locations, see if we can find a matching place candidate_locations = Location.objects.filter(query) if query else [] if len(candidate_locations) > 0: candidates = Place.objects.filter(name=new_place.name, location__in=candidate_locations) if len(candidates) > 0: # TODO: rank based on distance print ' using db match!', candidates[0] return candidates[0] # all out of options: punt! print ' resolve failed.' return None