def datetime_to_seconds(timestamp, timezone_offset=None):
""" Convert a datetime.datetime object to a unix time value in seconds.
The parameters are as follows:
'timestamp'
A datetime.datetime object representing a date/time value, in
UTC.
'timezone_offset'
The caller's timezone offset, if any.
If a timezone offset was supplied, we convert the date/time value from
UTC into the caller's timezone.
Upon completion, we return the number of seconds that have elapsed
between the 1st of January 1970 and the given timestamp. This is the
"unix time" equivalent for the given date/time object.
"""
seconds = dateHelpers.datetime_to_seconds(timestamp)
if timezone_offset != None:
seconds = seconds - (timezone_offset * 60)
return seconds
def search(request):
""" Respond to the "/api/latest/search" URL.
We search for postings based on the given search criteria.
"""
start_time = time.time()
if request.method != "GET":
return HttpResponseNotAllowed(["GET"])
# Check the caller's authentication.
# ...eventually.
# Extract the search criteria.
criteria = {}
for param in ["category_group", "category", "country", "state", "metro",
"region", "county", "city", "locality", "zipcode", "source",
"heading", "body", "text", "timestamp", "id", "price",
"currency", "annotations", "external_id", "status",
"has_image", "include_deleted", "only_deleted"]:
if param in request.GET:
criteria[param] = request.GET[param]
# If the caller didn't supply a timestamp, add a default timestamp to the
# search query.
if "timestamp" not in criteria:
criteria['timestamp'] = str(int((time.time() - 24*3600))) + '..' + \
str(int(time.time()))
# Extract our other parameters.
if "rpp" in request.GET:
try:
rpp = int(request.GET['rpp'])
except ValueError:
return HttpResponse(json.dumps(
{'success' : False,
'error' : "Invalid 'rpp' value"}),
mimetype="application/json")
if rpp < 1 or rpp > 100:
return HttpResponse(json.dumps(
{'success' : False,
'error' : "'rpp' value out of range"}),
mimetype="application/json")
else:
rpp = 10
if "retvals" in request.GET:
retvals = set()
for field in request.GET['retvals'].split(","):
if field in ["id", "account_id", "source", "category",
"category_group", "location", "external_id",
"external_url", "heading", "body", "html",
"timestamp", "expires", "language", "price",
"currency", "images", "annotations", "status",
"immortal"]:
retvals.add(field)
else:
return HttpResponse(json.dumps(
{'success' : False,
'error' : "invalid 'retvals' " +
"value: " + repr(field)}),
mimetype="application/json")
else:
retvals = set(["id", "source", "category", "location", "external_id",
"external_url", "heading", "timestamp"])
if "anchor" in request.GET:
anchor = request.GET['anchor']
else:
anchor = None
if "page" in request.GET:
try:
page = int(request.GET['page'])
except ValueError:
return HttpResponse(json.dumps(
{'success' : False,
'error' : "Invalid 'page' value"}),
mimetype="application/json")
else:
page = 0
# Construct a search query based on the supplied parameters.
success,result = searchHelpers.build_search_query(criteria)
if not success:
return HttpResponse(json.dumps({'success' : False,
'error' : result}),
mimetype="application/json")
else:
query = result
if anchor != None:
query = query.filter(id__lte=anchor)
num_matches = query.count()
query = query.order_by("-timestamp")
query = query[page*rpp:page*rpp+rpp]
sql = str(query.query)
# Testing: If the caller provided a "return_sql" parameter, return the raw
# SQL statement rather than running it.
if (request.GET.get("return_sql") == "1"):
return HttpResponse(sql)
# Before running the query, set a timeout so we don't hang if the query
# takes too long.
cursor = connection.cursor()
cursor.execute("SET STATEMENT_TIMEOUT=%s" % settings.QUERY_TIMEOUT)
# Process the search query, and assemble our search results.
found_postings = []
new_anchor = None
try:
for posting in query:
if anchor == None and new_anchor == None:
# Remember the ID of the first (ie, most recent) found posting.
# This will be our anchor for subsequent requests.
new_anchor = str(posting.id)
found_posting = {}
if "id" in retvals:
found_posting['id'] = posting.id
if "account_id" in retvals:
found_posting['account_id'] = posting.account_id
if "source" in retvals:
found_posting['source'] = posting.source.code
if "category" in retvals:
found_posting['category'] = posting.category.code
if "category_group" in retvals:
found_posting['category_group'] = posting.category_group.code
if "location" in retvals:
loc = {}
if posting.location_latitude != None:
loc['latitude'] = posting.location_latitude
if posting.location_longitude != None:
loc['longitude'] = posting.location_longitude
if posting.location_accuracy != None:
loc['accuracy'] = posting.location_accuracy
if posting.location_country != None:
loc['country'] = posting.location_country.code
if posting.location_state != None:
loc['state'] = posting.location_state.code
if posting.location_metro != None:
loc['metro'] = posting.location_metro.code
if posting.location_region != None:
loc['region'] = posting.location_region.code
if posting.location_county != None:
loc['county'] = posting.location_county.code
if posting.location_city != None:
loc['city'] = posting.location_city.code
if posting.location_locality != None:
loc['locality'] = posting.location_locality.code
if posting.location_zipcode != None:
loc['zipcode'] = posting.location_zipcode.code
found_posting['location'] = loc
if "external_id" in retvals:
found_posting['external_id'] = posting.external_id
if "external_url" in retvals:
found_posting['external_url'] = posting.external_url
if "heading" in retvals:
found_posting['heading'] = posting.heading
if "body" in retvals:
found_posting['body'] = posting.body
if "html" in retvals:
found_posting['html'] = posting.html
if "timestamp" in retvals:
found_posting['timestamp'] = datetime_to_seconds(
posting.timestamp)
if "expires" in retvals:
found_posting['expires'] = datetime_to_seconds(posting.expires)
if "language" in retvals:
found_posting['language'] = posting.language
if "price" in retvals:
found_posting['price'] = posting.price
if "currency" in retvals:
found_posting['currency'] = posting.currency
if "images" in retvals:
images = []
for image in posting.imagereference_set.all():
dst_image = {}
if image.full_url != None:
dst_image['full_url'] = image.full_url
if image.full_width != None:
dst_image['full_width'] = image.full_width
if image.full_height != None:
dst_image['full_height'] = image.full_height
if image.thumbnail_url != None:
dst_image['thumbnail_url'] = image.thumbnail_url
if image.thumbnail_width != None:
dst_image['thumbnail_width'] = image.thumbnail_width
if image.thumbnail_height != None:
dst_image['thumbnail_height'] = image.thumbnail_height
images.append(dst_image)
found_posting['images'] = images
if "annotations" in retvals:
annotations = {}
for posting_annotation in posting.postingannotation_set.all():
s = posting_annotation.annotation.annotation
key,value = s.split(":", 1)
annotations[key] = value
found_posting['annotations'] = annotations
if "status" in retvals:
status = {}
status['offered'] = posting.status_offered
status['lost'] = posting.status_lost
status['stolen'] = posting.status_stolen
status['found'] = posting.status_found
status['deleted'] = posting.status_deleted
found_posting['status'] = status
if "immortal" in retvals:
found_posting['immortal'] = posting.immortal
found_postings.append(found_posting)
except DatabaseError, e:
if "statement timeout" in str(e):
# The query timed out. Tell the user the bad news.
sql = str(query.query)
logger.debug("DATABASE TIMEOUT, query=" + sql)
eventRecorder.record("SEARCH_API", "QUERY_TIMED_OUT", text=sql)
return HttpResponse(json.dumps({'success' : False,
'error' : "Database timeout"}),
mimetype="application/json")
else:
logger.exception(e)
sql = str(query.query)
return HttpResponse(json.dumps({'success' : False,
'error' : "Database error"}),
mimetype="application/json")
