def remote_request(self, app_id, http_request_data): """ Receives a remote request to which it should give the correct response. The http_request_data holds an encoded protocol buffer of a certain type. Each type has a particular response type. Args: app_id: The application ID that is sending this request. http_request_data: Encoded protocol buffer. """ global task_queue apirequest = remote_api_pb.Request() apirequest.ParseFromString(http_request_data) apiresponse = remote_api_pb.Response() response = None errcode = 0 errdetail = "" method = "" http_request_data = "" if not apirequest.has_method(): errcode = taskqueue_service_pb.TaskQueueServiceError.INVALID_REQUEST errdetail = "Method was not set in request" apirequest.set_method("NOT_FOUND") else: method = apirequest.method() if not apirequest.has_request(): errcode = taskqueue_service_pb.TaskQueueServiceError.INVALID_REQUEST errdetail = "Request missing in call" apirequest.set_method("NOT_FOUND") apirequest.clear_request() else: http_request_data = apirequest.request() start_time = time.time() request_log = method if apirequest.has_request_id(): request_log += ': {}'.format(apirequest.request_id()) logger.debug(request_log) if method == "FetchQueueStats": response, errcode, errdetail = task_queue.fetch_queue_stats( app_id, http_request_data) elif method == "PurgeQueue": response, errcode, errdetail = task_queue.purge_queue( app_id, http_request_data) elif method == "Delete": response, errcode, errdetail = task_queue.delete( app_id, http_request_data) elif method == "QueryAndOwnTasks": response, errcode, errdetail = task_queue.query_and_own_tasks( app_id, http_request_data) elif method == "Add": response, errcode, errdetail = task_queue.add( app_id, http_request_data) elif method == "BulkAdd": response, errcode, errdetail = task_queue.bulk_add( app_id, http_request_data) elif method == "ModifyTaskLease": response, errcode, errdetail = task_queue.modify_task_lease( app_id, http_request_data) elif method == "UpdateQueue": response = taskqueue_service_pb.TaskQueueUpdateQueueResponse() response, errcode, errdetail = response.Encode(), 0, "" elif method == "FetchQueues": response, errcode, errdetail = task_queue.fetch_queue( app_id, http_request_data) elif method == "QueryTasks": response, errcode, errdetail = task_queue.query_tasks( app_id, http_request_data) elif method == "FetchTask": response, errcode, errdetail = task_queue.fetch_task( app_id, http_request_data) elif method == "ForceRun": response, errcode, errdetail = task_queue.force_run( app_id, http_request_data) elif method == "DeleteQueue": response = taskqueue_service_pb.TaskQueueDeleteQueueResponse() response, errcode, errdetail = response.Encode(), 0, "" elif method == "PauseQueue": response, errcode, errdetail = task_queue.pause_queue( app_id, http_request_data) elif method == "DeleteGroup": response, errcode, errdetail = task_queue.delete_group( app_id, http_request_data) elif method == "UpdateStorageLimit": response, errcode, errdetail = task_queue.update_storage_limit( app_id, http_request_data) timing_log = 'Elapsed: {}'.format(round(time.time() - start_time, 3)) if apirequest.has_request_id(): timing_log += ' ({})'.format(apirequest.request_id()) logger.debug(timing_log) if response is not None: apiresponse.set_response(response) # If there was an error add it to the response. if errcode != 0: apperror_pb = apiresponse.mutable_application_error() apperror_pb.set_code(errcode) apperror_pb.set_detail(errdetail) self.write(apiresponse.Encode())
def _handle_POST(self, environ, start_response): """Handles a POST request containing a serialized remote_api_pb.Request. Args: environ: An environ dict for the request as defined in PEP-333. start_response: A start_response function with semantics defined in PEP-333. Returns: A single element list containing the string body of the HTTP response. """ start_response('200 OK', [('Content-Type', 'application/octet-stream')]) start_time = time.time() response = remote_api_pb.Response() try: request = remote_api_pb.Request() # NOTE: Exceptions encountered when parsing the PB or handling the request # will be propagated back to the caller the same way as exceptions raised # by the actual API call. if environ.get('HTTP_TRANSFER_ENCODING') == 'chunked': # CherryPy concatenates all chunks when 'wsgi.input' is read but v3.2.2 # will not return even when all of the data in all chunks has been # read. See: https://bitbucket.org/cherrypy/cherrypy/issue/1131. wsgi_input = environ['wsgi.input'].read(2**32) else: wsgi_input = environ['wsgi.input'].read( int(environ['CONTENT_LENGTH'])) request.ParseFromString(wsgi_input) service = request.service_name() if service == 'datastore_v3' and self._datastore_emulator_stub: response = grpc_proxy_util.make_grpc_call_from_remote_api( self._datastore_emulator_stub, request) else: if request.has_request_id(): request_id = request.request_id() service_stub = apiproxy_stub_map.apiproxy.GetStub(service) environ['HTTP_HOST'] = self._balanced_address op = getattr(service_stub.request_data, 'register_request_id', None) if callable(op): op(environ, request_id) api_response = _execute_request(request).Encode() response.set_response(api_response) except Exception, e: if isinstance(e, apiproxy_errors.ApplicationError): level = logging.DEBUG application_error = response.mutable_application_error() application_error.set_code(e.application_error) application_error.set_detail(e.error_detail) else: # If the runtime instance is not Python, it won't be able to unpickle # the exception so use level that won't be ignored by default. level = logging.ERROR # Even if the runtime is Python, the exception may be unpicklable if # it requires importing a class blocked by the sandbox so just send # back the exception representation. # But due to our use of the remote API, at least some apiproxy errors # are generated in the Dev App Server main instance and not in the # language runtime and wrapping them causes different behavior from # prod so don't wrap them. if not isinstance(e, apiproxy_errors.Error): e = RuntimeError(repr(e)) # While not strictly necessary for ApplicationError, do this to limit # differences with remote_api:handler.py. response.set_exception(pickle.dumps(e)) logging.log(level, 'Exception while handling %s\n%s', request, traceback.format_exc())
def remote_request(self, app_id, http_request_data): """ Receives a remote request to which it should give the correct response. The http_request_data holds an encoded protocol buffer of a certain type. Each type has a particular response type. Args: app_id: The application ID that is sending this request. http_request_data: Encoded protocol buffer. """ apirequest = remote_api_pb.Request() apirequest.ParseFromString(http_request_data) apiresponse = remote_api_pb.Response() response = None errcode = 0 errdetail = "" apperror_pb = None if not apirequest.has_method(): errcode = datastore_pb.Error.BAD_REQUEST errdetail = "Method was not set in request" apirequest.set_method("NOT_FOUND") if not apirequest.has_request(): errcode = datastore_pb.Error.BAD_REQUEST errdetail = "Request missing in call" apirequest.set_method("NOT_FOUND") apirequest.clear_request() method = apirequest.method() http_request_data = apirequest.request() start = time.time() logger.debug('Request type: {}'.format(method)) if method == "Put": response, errcode, errdetail = self.put_request(app_id, http_request_data) elif method == "Get": response, errcode, errdetail = self.get_request(app_id, http_request_data) elif method == "Delete": response, errcode, errdetail = self.delete_request(app_id, http_request_data) elif method == "RunQuery": response, errcode, errdetail = self.run_query(http_request_data) elif method == "BeginTransaction": response, errcode, errdetail = self.begin_transaction_request( app_id, http_request_data) elif method == "Commit": response, errcode, errdetail = self.commit_transaction_request( app_id, http_request_data) elif method == "Rollback": response, errcode, errdetail = self.rollback_transaction_request( app_id, http_request_data) elif method == "AllocateIds": response, errcode, errdetail = self.allocate_ids_request( app_id, http_request_data) elif method == "CreateIndex": response, errcode, errdetail = self.create_index_request(app_id, http_request_data) elif method == "GetIndices": response, errcode, errdetail = self.get_indices_request(app_id) elif method == "UpdateIndex": response, errcode, errdetail = self.update_index_request(app_id, http_request_data) elif method == "DeleteIndex": response, errcode, errdetail = self.delete_index_request(app_id, http_request_data) elif method == 'AddActions': response, errcode, errdetail = self.add_actions_request( app_id, http_request_data) else: errcode = datastore_pb.Error.BAD_REQUEST errdetail = "Unknown datastore message" time_taken = time.time() - start if method in STATS: if errcode in STATS[method]: prev_req, pre_time = STATS[method][errcode] STATS[method][errcode] = prev_req + 1, pre_time + time_taken else: STATS[method][errcode] = (1, time_taken) else: STATS[method] = {} STATS[method][errcode] = (1, time_taken) apiresponse.set_response(response) if errcode != 0: apperror_pb = apiresponse.mutable_application_error() apperror_pb.set_code(errcode) apperror_pb.set_detail(errdetail) self.write(apiresponse.Encode())
def _RemoteSend(self, request, response, method, request_id=None): """Sends a request remotely to the datstore server. """ tag = self.project_id self._maybeSetDefaultAuthDomain() user = users.GetCurrentUser() if user != None: tag += ":" + user.email() tag += ":" + user.nickname() tag += ":" + user.auth_domain() api_request = remote_api_pb.Request() api_request.set_method(method) api_request.set_service_name("datastore_v3") api_request.set_request(request.Encode()) if request_id is not None: api_request.set_request_id(request_id) api_response = remote_api_pb.Response() retry_count = 0 max_retries = 5 location = self.__datastore_location while True: try: api_response = api_request.sendCommand(location, tag, api_response, 1, self.__is_encrypted, KEY_LOCATION, CERT_LOCATION) break except socket.error as socket_error: if socket_error.errno in (errno.ECONNREFUSED, errno.EHOSTUNREACH): retry_count += 1 if retry_count > max_retries: raise location = get_random_lb() continue if socket_error.errno == errno.ETIMEDOUT: raise apiproxy_errors.ApplicationError( datastore_pb.Error.TIMEOUT, 'Connection timed out when making datastore request') raise # AppScale: Interpret ProtocolBuffer.ProtocolBufferReturnError as # datastore_errors.InternalError except ProtocolBuffer.ProtocolBufferReturnError as e: raise datastore_errors.InternalError(e) if not api_response or not api_response.has_response(): raise datastore_errors.InternalError( 'No response from db server on %s requests.' % method) if api_response.has_application_error(): error_pb = api_response.application_error() logging.error(error_pb.detail()) raise apiproxy_errors.ApplicationError(error_pb.code(), error_pb.detail()) if api_response.has_exception(): raise api_response.exception() response.ParseFromString(api_response.response())
def _WaitImpl(self): try: already_finishing = False with self.lock: if self._state == apiproxy_rpc.RPC.FINISHING: already_finishing = True else: self._state = apiproxy_rpc.RPC.FINISHING if already_finishing: self.event.wait() return True try: response = self._result_future.get() if response.status_code != requests.codes.ok: raise apiproxy_errors.RPCFailedError( 'Proxy returned HTTP status %s %s' % (response.status_code, response.reason)) except requests.exceptions.Timeout: raise self._ErrorException(*_DEADLINE_EXCEEDED_EXCEPTION) except requests.exceptions.RequestException: raise self._ErrorException(*_DEFAULT_EXCEPTION) parsed_response = remote_api_pb.Response(response.content) if (parsed_response.has_application_error() or parsed_response.has_rpc_error()): raise self._TranslateToError(parsed_response) self.response.ParseFromString(parsed_response.response()) except Exception: _, exc, tb = sys.exc_info() self._exception = exc self._traceback = tb try: self._Callback() return True finally: self.event.set()
def _RemoteSend(self, request, response, method, request_id=None, service_id=None, version_id=None): """Sends a request remotely to the taskqueue server. Args: request: A protocol buffer request. response: A protocol buffer response. method: The function which is calling the remote server. request_id: A string specifying a request ID. service_id: A string specifying the client service ID. version_id: A string specifying the client version ID. Raises: taskqueue_service_pb.InternalError: """ tag = self.__app_id api_request = remote_api_pb.Request() api_request.set_method(method) api_request.set_service_name("taskqueue") api_request.set_request(request.Encode()) if request_id is not None: api_request.set_request_id(request_id) api_response = remote_api_pb.Response() retry_count = 0 max_retries = 3 location = random.choice(self.__tq_locations) while True: try: api_request.sendCommand(location, tag, api_response, 1, False, KEY_LOCATION, CERT_LOCATION) break except socket.error as socket_error: if socket_error.errno in (errno.ECONNREFUSED, errno.EHOSTUNREACH): backoff_ms = 500 * 3**retry_count # 0.5s, 1.5s, 4.5s retry_count += 1 if retry_count > max_retries: raise logging.warning( 'Failed to call {} method of TaskQueue ({}). Retry #{} in {}ms.' .format(method, socket_error, retry_count, backoff_ms)) time.sleep(float(backoff_ms) / 1000) location = random.choice(self.__tq_locations) api_response = remote_api_pb.Response() continue if socket_error.errno == errno.ETIMEDOUT: raise apiproxy_errors.ApplicationError( taskqueue_service_pb.TaskQueueServiceError. INTERNAL_ERROR, 'Connection timed out when making taskqueue request') raise # AppScale: Interpret ProtocolBuffer.ProtocolBufferReturnError as # datastore_errors.InternalError except ProtocolBuffer.ProtocolBufferReturnError as e: raise apiproxy_errors.ApplicationError( taskqueue_service_pb.TaskQueueServiceError.INTERNAL_ERROR, str(e)) if not api_response or not api_response.has_response(): raise apiproxy_errors.ApplicationError( taskqueue_service_pb.TaskQueueServiceError.INTERNAL_ERROR) if api_response.has_application_error(): error_pb = api_response.application_error() logging.error(error_pb.detail()) raise apiproxy_errors.ApplicationError(error_pb.code(), error_pb.detail()) if api_response.has_exception(): raise api_response.exception() response.ParseFromString(api_response.response())
req = remote_api_pb.Request() req.ParseFromString(self.data[:self.n]) self.data, self.n = self.data[self.n:], -1 rapi_result = None rapi_error = 'unknown error' try: rapi_result = RAPI_HANDLER.ExecuteRequest(req) except apiproxy_errors.CallNotFoundError, e: service_name = req.service_name() method = req.method() rapi_error = 'call not found for %s/%s' % (service_name, method) except Exception, e: rapi_error = str(e) res = remote_api_pb.Response() if rapi_result: res.set_response(rapi_result.Encode()) else: ae = res.mutable_application_error() ae.set_code(1) ae.set_detail(rapi_error) res1 = res.Encode() self.send('%d\n' % len(res1)) self.send(res1) def find_app_files(basedir): if not basedir.endswith(os.path.sep): basedir = basedir + os.path.sep
def RemoteApiResponse(self, response): """Return a filled in remote_api response proto.""" remote_response = remote_api_pb.Response() remote_response.mutable_response().set_contents(response.Encode()) return remote_response
def add_tasks(self, project_id, service_id, version_id, add_requests): """ Makes a call to the TaskQueue service to enqueue tasks. Args: project_id: A string specifying the project ID. service_id: A string specifying the service ID. version_id: A string specifying the version ID. add_requests: A list of TaskQueueAddRequest messages. Raises: EnqueueError if unable to enqueue tasks. """ request = taskqueue_service_pb.TaskQueueBulkAddRequest() for add_request in add_requests: request.add_add_request().MergeFrom(add_request) api_request = remote_api_pb.Request() api_request.set_method('BulkAdd') api_request.set_service_name('taskqueue') api_request.set_request(request.Encode()) encoded_api_request = api_request.Encode() headers = { 'ProtocolBufferType': 'Request', 'AppData': project_id, 'Module': service_id, 'Version': version_id } api_response = None for location in self._locations: url = 'http://{}'.format(location) try: response = yield self._client.fetch(url, method='POST', body=encoded_api_request, headers=headers) except socket_error: # Try a different location if the load balancer is not available. continue except HTTPError as error: raise EnqueueError(str(error)) api_response = remote_api_pb.Response(response.body) break if api_response is None: raise EnqueueError('Unable to connect to any load balancers') if api_response.has_application_error(): error_pb = api_response.application_error() raise EnqueueError(error_pb.detail()) if api_response.has_exception(): raise EnqueueError(api_response.exception()) bulk_response = taskqueue_service_pb.TaskQueueBulkAddResponse( api_response.response()) if bulk_response.taskresult_size() != len(add_requests): raise EnqueueError('Unexpected number of task results') for task_result in bulk_response.taskresult_list(): if task_result.result( ) != taskqueue_service_pb.TaskQueueServiceError.OK: raise EnqueueError( 'Unable to enqueue task: {}'.format(task_result))