def test_bulkadd(self):
app_id = 'test_app'
bulk_add_request = taskqueue_service_pb.TaskQueueBulkAddRequest()
item = bulk_add_request.add_add_request()
item.set_app_id(app_id)
item.set_queue_name('default')
item.set_task_name('babaganoose')
item.set_eta_usec(0)
item.set_method(taskqueue_service_pb.TaskQueueAddRequest.GET)
item.set_mode(taskqueue_service_pb.TaskQueueMode.PUSH)
retry = item.mutable_retry_parameters()
retry.set_age_limit_sec(10)
host = socket.gethostbyname(socket.gethostname())
item.set_url('http://' + host + ':17446/doesnotexist_expires')
host = socket.gethostbyname(socket.gethostname())
req = urllib2.Request('http://' + host + ':17446')
api_request = remote_api_pb.Request()
api_request.set_method("BulkAdd")
api_request.set_service_name("taskqueue")
api_request.set_request(bulk_add_request.Encode())
remote_request = api_request.Encode()
req.add_header('Content-Length', str(len(remote_request)))
req.add_header('protocolbuffertype', 'Request')
req.add_header('appdata', app_id)
response = urllib2.urlopen(req, remote_request)
api_response = response.read()
api_response = remote_api_pb.Response(api_response)
bulk_add_response = taskqueue_service_pb.TaskQueueBulkAddResponse(
api_response.response())
print bulk_add_response
self.assertEquals(response.getcode(), 200)
def _DeliverMatches(self, subscriptions, match_request):
"""Deliver list of subscriptions as batches using taskqueue.
Args:
subscriptions: list of subscription ids
match_request: MatchRequest
"""
parameters = {'topic': match_request.topic()}
if match_request.has_result_python_document_class():
python_document_class = match_request.result_python_document_class()
parameters['python_document_class'] = python_document_class
parameters['document'] = base64.urlsafe_b64encode(
match_request.document().Encode())
if match_request.has_result_key():
parameters['key'] = match_request.result_key()
taskqueue_request = taskqueue_service_pb.TaskQueueBulkAddRequest()
batch_size = match_request.result_batch_size()
for i in range(0, len(subscriptions), batch_size):
add_request = taskqueue_request.add_add_request()
add_request.set_queue_name(match_request.result_task_queue())
add_request.set_task_name('')
add_request.set_eta_usec(0)
add_request.set_url(match_request.result_relative_url())
add_request.set_description('prospective_search::matches')
header = add_request.add_header()
header.set_key('content-type')
header.set_value('application/x-www-form-urlencoded; charset=utf-8')
parameters['results_count'] = len(subscriptions)
parameters['results_offset'] = i
parameters['id'] = subscriptions[i:i+batch_size]
add_request.set_body(six.moves.urllib.parse.urlencode(parameters, doseq=True))
taskqueue_response = taskqueue_service_pb.TaskQueueBulkAddResponse()
self.taskqueue_stub._Dynamic_BulkAdd(taskqueue_request, taskqueue_response)
def add(self, app_id, http_data):
""" Adds a single task to the task queue.
Args:
app_id: The application ID.
http_data: The payload containing the protocol buffer request.
Returns:
A tuple of a encoded response, error code, and error detail.
"""
# Just call bulk add with one task.
request = taskqueue_service_pb.TaskQueueAddRequest(http_data)
request.set_app_id(app_id)
response = taskqueue_service_pb.TaskQueueAddResponse()
bulk_request = taskqueue_service_pb.TaskQueueBulkAddRequest()
bulk_response = taskqueue_service_pb.TaskQueueBulkAddResponse()
bulk_request.add_add_request().CopyFrom(request)
self.__bulk_add(bulk_request, bulk_response)
if bulk_response.taskresult_size() == 1:
result = bulk_response.taskresult(0).result()
else:
err_code = taskqueue_service_pb.TaskQueueServiceError.INTERNAL_ERROR
return (response.Encode(), err_code,
"Task did not receive a task response.")
if result != taskqueue_service_pb.TaskQueueServiceError.OK:
return (response.Encode(), result,
"Task did not get an OK status.")
elif bulk_response.taskresult(0).has_chosen_task_name():
response.set_chosen_task_name(
bulk_response.taskresult(0).chosen_task_name())
return (response.Encode(), 0, "")
def add_actions_request(self, app_id, http_request_data):
""" High level function for adding transactional tasks.
Args:
app_id: Name of the application.
http_request_data: Stores the protocol buffer request from the AppServer.
Returns:
An encoded AddActions response.
"""
global datastore_access
req_pb = taskqueue_service_pb.TaskQueueBulkAddRequest(
http_request_data)
resp_pb = taskqueue_service_pb.TaskQueueBulkAddResponse()
if READ_ONLY:
logger.warning(
'Unable to add transactional tasks in read-only mode')
return (resp_pb.Encode(), datastore_pb.Error.CAPABILITY_DISABLED,
'Datastore is in read-only mode.')
try:
datastore_access.dynamic_add_actions(app_id, req_pb)
return resp_pb.Encode(), 0, ""
except dbconstants.ExcessiveTasks as error:
return (resp_pb.Encode(), datastore_pb.Error.BAD_REQUEST,
str(error))
except dbconstants.AppScaleDBConnectionError:
logger.exception('DB connection error')
return (
resp_pb.Encode(), datastore_pb.Error.INTERNAL_ERROR,
'Datastore connection error when adding transaction tasks.')
def __AddTasks(self, tasks, transactional, fill_request):
"""Internal implementation of adding tasks where tasks must be a list."""
if len(tasks) > MAX_TASKS_PER_ADD:
raise TooManyTasksError(
'No more than %d tasks can be added in a single call' %
MAX_TASKS_PER_ADD)
request = taskqueue_service_pb.TaskQueueBulkAddRequest()
response = taskqueue_service_pb.TaskQueueBulkAddResponse()
task_names = set()
for task in tasks:
if task.name:
if task.name in task_names:
raise DuplicateTaskNameError(
'The task name %r is used more than once in the request' %
task.name)
task_names.add(task.name)
if task.was_enqueued:
raise BadTaskStateError('Task has already been enqueued.')
fill_request(task, request.add_add_request(), transactional)
if transactional and (request.ByteSize() >
MAX_TRANSACTIONAL_REQUEST_SIZE_BYTES):
raise TransactionalRequestTooLargeError(
'Transactional request size must be less than %d; found %d' %
(MAX_TRANSACTIONAL_REQUEST_SIZE_BYTES, request.ByteSize()))
try:
apiproxy_stub_map.MakeSyncCall('taskqueue', 'BulkAdd', request, response)
except apiproxy_errors.ApplicationError, e:
raise self.__TranslateError(e.application_error, e.error_detail)
def __AddTasks(self, tasks, transactional):
"""Internal implementation of .add() where tasks must be a list."""
if len(tasks) > MAX_TASKS_PER_ADD:
raise TooManyTasksError(
'No more than %d tasks can be added in a single add call' %
MAX_TASKS_PER_ADD)
request = taskqueue_service_pb.TaskQueueBulkAddRequest()
response = taskqueue_service_pb.TaskQueueBulkAddResponse()
task_names = set()
for task in tasks:
if task.name:
if task.name in task_names:
raise DuplicateTaskNameError(
'The task name %r is used more than once in the request' %
task.name)
task_names.add(task.name)
self.__FillAddRequest(task, request.add_add_request(), transactional)
try:
apiproxy_stub_map.MakeSyncCall('taskqueue', 'BulkAdd', request, response)
except apiproxy_errors.ApplicationError, e:
raise self.__TranslateError(e.application_error, e.error_detail)
def bulk_add(self, app_id, http_data):
""" Adds multiple tasks to the task queue.
Args:
app_id: The application ID.
http_data: The payload containing the protocol buffer request.
Returns:
A tuple of a encoded response, error code, and error detail.
"""
request = taskqueue_service_pb.TaskQueueBulkAddRequest(http_data)
response = taskqueue_service_pb.TaskQueueBulkAddResponse()
self.__bulk_add(request, response)
return (response.Encode(), 0, "")
def bulk_add(self, source_info, http_data):
""" Adds multiple tasks to the task queue.
Args:
source_info: A dictionary containing the application, module, and version
ID that is sending this request.
http_data: The payload containing the protocol buffer request.
Returns:
A tuple of a encoded response, error code, and error detail.
"""
request = taskqueue_service_pb.TaskQueueBulkAddRequest(http_data)
response = taskqueue_service_pb.TaskQueueBulkAddResponse()
self.__bulk_add(source_info, request, response)
return (response.Encode(), 0, "")
def add_actions_request(self, app_id, http_request_data, service_id,
version_id):
""" High level function for adding transactional tasks.
Args:
app_id: Name of the application.
http_request_data: Stores the protocol buffer request from the AppServer.
service_id: A string specifying the client's service ID.
version_id: A string specifying the client's version ID.
Returns:
An encoded AddActions response.
"""
global datastore_access
req_pb = taskqueue_service_pb.TaskQueueBulkAddRequest(http_request_data)
resp_pb = taskqueue_service_pb.TaskQueueBulkAddResponse()
if service_id is None:
raise gen.Return(
('', datastore_pb.Error.BAD_REQUEST,
'Module header must be defined'))
if version_id is None:
raise gen.Return(
('', datastore_pb.Error.BAD_REQUEST,
'Version header must be defined'))
if READ_ONLY:
logger.warning('Unable to add transactional tasks in read-only mode')
raise gen.Return(
('', datastore_pb.Error.CAPABILITY_DISABLED,
'Datastore is in read-only mode.'))
try:
yield datastore_access.dynamic_add_actions(
app_id, req_pb, service_id, version_id)
raise gen.Return((resp_pb.Encode(), 0, ''))
except dbconstants.ExcessiveTasks as error:
raise gen.Return(('', datastore_pb.Error.BAD_REQUEST, str(error)))
except dbconstants.AppScaleDBConnectionError:
logger.exception('DB connection error')
raise gen.Return(
('', datastore_pb.Error.INTERNAL_ERROR,
'Datastore connection error when adding transaction tasks.'))
def bulk_add(self, source_info, http_data):
""" Adds multiple tasks to the task queue.
Args:
source_info: A dictionary containing the application, module, and version
ID that is sending this request.
http_data: The payload containing the protocol buffer request.
Returns:
A tuple of a encoded response, error code, and error detail.
"""
request = taskqueue_service_pb.TaskQueueBulkAddRequest(http_data)
response = taskqueue_service_pb.TaskQueueBulkAddResponse()
try:
self.__bulk_add(source_info, request, response)
except QueueNotFound as error:
return '', TaskQueueServiceError.UNKNOWN_QUEUE, str(error)
except TransientError as error:
return '', TaskQueueServiceError.TRANSIENT_ERROR, str(error)
return (response.Encode(), 0, "")
def add(self, source_info, http_data):
""" Adds a single task to the task queue.
Args:
source_info: A dictionary containing the application, module, and version
ID that is sending this request.
http_data: The payload containing the protocol buffer request.
Returns:
A tuple of a encoded response, error code, and error detail.
"""
# Just call bulk add with one task.
request = taskqueue_service_pb.TaskQueueAddRequest(http_data)
request.set_app_id(source_info['app_id'])
response = taskqueue_service_pb.TaskQueueAddResponse()
bulk_request = taskqueue_service_pb.TaskQueueBulkAddRequest()
bulk_response = taskqueue_service_pb.TaskQueueBulkAddResponse()
bulk_request.add_add_request().CopyFrom(request)
try:
self.__bulk_add(source_info, bulk_request, bulk_response)
except TransientError as error:
return '', TaskQueueServiceError.TRANSIENT_ERROR, str(error)
except QueueNotFound as error:
return '', TaskQueueServiceError.UNKNOWN_QUEUE, str(error)
if bulk_response.taskresult_size() == 1:
result = bulk_response.taskresult(0).result()
else:
return (response.Encode(), TaskQueueServiceError.INTERNAL_ERROR,
"Task did not receive a task response.")
if result != TaskQueueServiceError.OK:
return (response.Encode(), result,
"Task did not get an OK status.")
elif bulk_response.taskresult(0).has_chosen_task_name():
response.set_chosen_task_name(
bulk_response.taskresult(0).chosen_task_name())
return (response.Encode(), 0, "")
def add_tasks(self, project_id, service_id, version_id, add_requests):
""" Makes a call to the TaskQueue service to enqueue tasks.
Args:
project_id: A string specifying the project ID.
service_id: A string specifying the service ID.
version_id: A string specifying the version ID.
add_requests: A list of TaskQueueAddRequest messages.
Raises:
EnqueueError if unable to enqueue tasks.
"""
request = taskqueue_service_pb.TaskQueueBulkAddRequest()
for add_request in add_requests:
request.add_add_request().MergeFrom(add_request)
api_request = remote_api_pb.Request()
api_request.set_method('BulkAdd')
api_request.set_service_name('taskqueue')
api_request.set_request(request.Encode())
encoded_api_request = api_request.Encode()
headers = {
'ProtocolBufferType': 'Request',
'AppData': project_id,
'Module': service_id,
'Version': version_id
}
api_response = None
for location in self._locations:
url = 'http://{}'.format(location)
try:
response = yield self._client.fetch(url,
method='POST',
body=encoded_api_request,
headers=headers)
except socket_error:
# Try a different location if the load balancer is not available.
continue
except HTTPError as error:
raise EnqueueError(str(error))
api_response = remote_api_pb.Response(response.body)
break
if api_response is None:
raise EnqueueError('Unable to connect to any load balancers')
if api_response.has_application_error():
error_pb = api_response.application_error()
raise EnqueueError(error_pb.detail())
if api_response.has_exception():
raise EnqueueError(api_response.exception())
bulk_response = taskqueue_service_pb.TaskQueueBulkAddResponse(
api_response.response())
if bulk_response.taskresult_size() != len(add_requests):
raise EnqueueError('Unexpected number of task results')
for task_result in bulk_response.taskresult_list():
if task_result.result(
) != taskqueue_service_pb.TaskQueueServiceError.OK:
raise EnqueueError(
'Unable to enqueue task: {}'.format(task_result))