def get_locations_from_posting(posting):
bounds = None
# if posting.location_bounds:
# bounds = json.loads(posting.location_bounds)
return reverseGeocoder.calc_locations(
float(posting.location_latitude),
float(posting.location_longitude),
bounds=bounds,
accuracy=posting.location_accuracy
)
def process_postings(parsed_postings):
""" Process a batch of parsed postings.
'parsed_postings' should be a list of postings that have been
successfully checked by the posting parser. Each entry in this list
will be a dictionary with the following entries:
'posting'
The posting itself, as a dictionary. The fields in this
dictionary match the attributes with the same name in the
Posting object.
'annotations'
A list of annotation values to associate with this posting.
Each string will be of the form "key:value"
'images'
A list of images to associate with this posting. Each image
will be a dictionary with 'full_url' and 'thumbnail_url'
entries, as appropriate.
We process the postings, adding them to the system as appropriate.
Note that this involves the following steps:
1. Calling the Geolocator API if the postings need to be
geolocated.
2. Storing the postings into the database.
3. Sending the postings out via the Notification API.
"""
eventRecorder.record("POSTING_API", "POSTINGS_DEQUEUED",
len(parsed_postings))
start_time = time.time()
# If necessary, geolocate the postings.
for src in parsed_postings:
posting = src['posting']
if "location" in posting:
raw_loc = posting['location']
has_lat_long = False # initially.
if "latitude" in raw_loc and "longitude" in raw_loc:
has_lat_long = True
has_loc_codes = False # initially.
for field in ["country", "state", "metro", "region", "county",
"city", "locality", "zipcode"]:
if field in raw_loc:
has_loc_codes = True
break
if has_lat_long and not has_loc_codes:
# This posting has a lat/long value but no location codes ->
# reverse geocode the posting to see which locations it belongs
# to.
locs = reverseGeocoder.calc_locations(raw_loc['latitude'],
raw_loc['longitude'],
raw_loc.get("bounds"),
raw_loc.get("accuracy"))
for level,loc in locs.items():
raw_loc[level] = loc
# If we were supplied a bounds array, convert it to a string for
# storage.
if "bounds" in raw_loc:
raw_loc['bounds'] = repr(bounds['raw_loc'])
# Get the Annotation objects used by these postings. Since these
# objects hold unique annotation values, they can be shared across
# postings -- we use the existing Annotation object if it exists, and
# create new ones where necessary.
annotations = {} # Maps annotation string to Annotation object.
for src in parsed_postings:
for annotation_value in src['annotations']:
if annotation_value not in annotations:
# The following attempts to work around a database deadlock
# issue. We attempt to get_or_create the given Annotation
# record, and if this results in a database deadlock, we wait a
# moment before trying again.
while True:
try:
annotation,created = Annotation.objects.get_or_create(
annotation=annotation_value)
except DatabaseError,e:
if "deadlock" in str(e):
logger.debug("DEADLOCK DETECTED!!! TRYING AGAIN")
time.sleep(0.1)
continue
else:
raise
else:
break
annotations[annotation_value] = annotation
