def get_locations_from_posting(posting): bounds = None # if posting.location_bounds: # bounds = json.loads(posting.location_bounds) return reverseGeocoder.calc_locations( float(posting.location_latitude), float(posting.location_longitude), bounds=bounds, accuracy=posting.location_accuracy )
def process_postings(parsed_postings): """ Process a batch of parsed postings. 'parsed_postings' should be a list of postings that have been successfully checked by the posting parser. Each entry in this list will be a dictionary with the following entries: 'posting' The posting itself, as a dictionary. The fields in this dictionary match the attributes with the same name in the Posting object. 'annotations' A list of annotation values to associate with this posting. Each string will be of the form "key:value" 'images' A list of images to associate with this posting. Each image will be a dictionary with 'full_url' and 'thumbnail_url' entries, as appropriate. We process the postings, adding them to the system as appropriate. Note that this involves the following steps: 1. Calling the Geolocator API if the postings need to be geolocated. 2. Storing the postings into the database. 3. Sending the postings out via the Notification API. """ eventRecorder.record("POSTING_API", "POSTINGS_DEQUEUED", len(parsed_postings)) start_time = time.time() # If necessary, geolocate the postings. for src in parsed_postings: posting = src['posting'] if "location" in posting: raw_loc = posting['location'] has_lat_long = False # initially. if "latitude" in raw_loc and "longitude" in raw_loc: has_lat_long = True has_loc_codes = False # initially. for field in ["country", "state", "metro", "region", "county", "city", "locality", "zipcode"]: if field in raw_loc: has_loc_codes = True break if has_lat_long and not has_loc_codes: # This posting has a lat/long value but no location codes -> # reverse geocode the posting to see which locations it belongs # to. locs = reverseGeocoder.calc_locations(raw_loc['latitude'], raw_loc['longitude'], raw_loc.get("bounds"), raw_loc.get("accuracy")) for level,loc in locs.items(): raw_loc[level] = loc # If we were supplied a bounds array, convert it to a string for # storage. if "bounds" in raw_loc: raw_loc['bounds'] = repr(bounds['raw_loc']) # Get the Annotation objects used by these postings. Since these # objects hold unique annotation values, they can be shared across # postings -- we use the existing Annotation object if it exists, and # create new ones where necessary. annotations = {} # Maps annotation string to Annotation object. for src in parsed_postings: for annotation_value in src['annotations']: if annotation_value not in annotations: # The following attempts to work around a database deadlock # issue. We attempt to get_or_create the given Annotation # record, and if this results in a database deadlock, we wait a # moment before trying again. while True: try: annotation,created = Annotation.objects.get_or_create( annotation=annotation_value) except DatabaseError,e: if "deadlock" in str(e): logger.debug("DEADLOCK DETECTED!!! TRYING AGAIN") time.sleep(0.1) continue else: raise else: break annotations[annotation_value] = annotation