def ExecuteApiQueryTask(api_query):
"""Executes a refresh of an API Query from the task queue.
Attempts to fetch and update an API Query and will also log any errors.
Schedules the API Query for next execution.
Args:
api_query: The API Query to refresh.
Returns:
A boolean. True if the API refresh was a success and False if the API
Query is not valid or an error was logged.
"""
if api_query:
query_id = str(api_query.key())
api_query.in_queue = False
api_response_content = FetchApiQueryResponse(api_query)
if not api_response_content or api_response_content.get('error'):
InsertApiQueryError(api_query, api_response_content)
if api_query.is_error_limit_reached:
api_query.is_scheduled = False
SaveApiQuery(api_query)
# Since it failed, execute the query again unless the refresh interval of
# query is less than the random countdown, then schedule it normally.
if api_query.refresh_interval < co.MAX_RANDOM_COUNTDOWN:
schedule_helper.ScheduleApiQuery(api_query) # Run at normal interval.
else:
schedule_helper.ScheduleApiQuery(api_query, randomize=True, countdown=0)
return False
else:
SaveApiQueryResponse(api_query, api_response_content)
# Check that public endpoint wasn't disabled after task added to queue.
if api_query.is_active:
memcache.set_multi({'api_query': api_query,
co.DEFAULT_FORMAT: api_response_content},
key_prefix=query_id,
time=api_query.refresh_interval)
# Delete the transformed content in memcache since it will be updated
# at the next request.
delete_keys = set(co.SUPPORTED_FORMATS) - set([co.DEFAULT_FORMAT])
memcache.delete_multi(list(delete_keys), key_prefix=query_id)
SaveApiQuery(api_query)
schedule_helper.ScheduleApiQuery(api_query)
return True
# Save the query state just in case the user disabled it
# while it was in the task queue.
SaveApiQuery(api_query)
return False
def post(self):
"""Delete API query error responses."""
query_id = self.request.get('query_id')
redirect = self.request.get('redirect', co.LINKS['owner_index'])
api_query = query_helper.GetApiQuery(query_id)
query_helper.DeleteApiQueryErrors(api_query)
schedule_helper.ScheduleApiQuery(api_query, randomize=True, countdown=0)
self.redirect(redirect)
def post(self):
"""Starts/Stops API Query Scheduling."""
query_id = self.request.get('query_id')
api_query = query_helper.GetApiQuery(query_id)
if api_query:
schedule_helper.SetApiQueryScheduleStatus(api_query)
schedule_helper.ScheduleApiQuery(api_query, randomize=True, countdown=0)
api_query_links = template_helper.GetLinksForTemplate(
api_query, self.request.host_url)
self.redirect(api_query_links.get('manage_link', '/'))
return
self.redirect(co.LINKS['owner_index'])
def ScheduleAndSaveApiQuery(api_query, **kwargs):
"""Schedules and saves an API Query.
Args:
api_query: The API Query to save and schedule.
**kwargs: Additional properties to set for the API Query before saving.
Returns:
If successful the API Query that was saved or None if the save was
unsuccessful.
"""
if api_query:
api_query.is_active = True
api_query.is_scheduled = True
saved = SaveApiQuery(api_query, **kwargs)
if saved:
schedule_helper.ScheduleApiQuery(api_query, randomize=True, countdown=0)
return api_query
return None
def GetPublicEndpointResponse(query_id=None,
requested_format=None,
transform=None):
"""Returns the public response for an external user request.
This handles all the steps required to get the latest successful API
response for an API Query.
1) Check Memcache, if found skip to #4.
2) If not in memcache, check if the stored response is abandoned and needs
to be refreshed.
3) Retrieve response from datastore.
4) Perform any transforms and return the formatted response to the user.
Args:
query_id: The query id to retrieve the response for.
requested_format: The format type requested for the response.
transform: The transform instance to use to transform the content to the
requested format, if required.
Returns:
A tuple contatining the response content, and status code to
render. e.g. (CONTENT, 200)
"""
transformed_response_content = None
schedule_query = False
must_store_memcache = False
if not requested_format or requested_format not in co.SUPPORTED_FORMATS:
requested_format = co.DEFAULT_FORMAT
response = GetApiQueryResponseFromMemcache(query_id, requested_format)
# 1. Check Memcache
if response and response.get('api_query') and response.get('content'):
api_query = response.get('api_query')
response_content = response.get('content')
transformed_response_content = response.get('transformed_content')
response_status = 200
else:
api_query = GetApiQuery(query_id)
# 2. Check if this is an abandoned query
# if (api_query is not None and api_query.is_active
# and not api_query.is_error_limit_reached
# and api_query.is_abandoned):
# RefreshApiQueryResponse(api_query)
# 3. Retrieve response from datastore
response = GetApiQueryResponseFromDb(api_query)
response_content = response.get('content')
response_status = response.get('status')
# Flag to schedule query later on if there is a successful response.
if api_query:
schedule_query = not api_query.in_queue
must_store_memcache = True
# 4. Return the formatted response.
if response_status == 200:
# UpdateApiQueryCounter(query_id)
# UpdateApiQueryTimestamp(query_id)
if co.ANONYMIZE_RESPONSES:
response_content = transformers.RemoveKeys(response_content)
if not transformed_response_content:
try:
transformed_response_content = transform.Transform(
response_content)
except (KeyError, TypeError, AttributeError):
# If the transformation fails then return the original content.
transformed_response_content = response_content
if must_store_memcache:
memcache_keys = {
'api_query': api_query,
co.DEFAULT_FORMAT: response_content,
requested_format: transformed_response_content
}
memcache.add_multi(memcache_keys,
key_prefix=query_id,
time=api_query.refresh_interval)
# Attempt to schedule query if required.
if schedule_query:
schedule_helper.ScheduleApiQuery(api_query)
response_content = transformed_response_content
else:
raise errors.GaSuperProxyHttpError(response_content, response_status)
return (response_content, response_status)