def main(): """Parses command line options and launches the API server.""" shutdown.install_signal_handlers() options = PARSER.parse_args() logging.getLogger().setLevel(constants.LOG_LEVEL_TO_PYTHON_CONSTANT[ options.dev_appserver_log_level]) # Parse the application configuration if config_paths are provided, else # provide sensible defaults. if options.config_paths: app_config = application_configuration.ApplicationConfiguration( options.config_paths, options.app_id) app_id = app_config.app_id app_root = app_config.modules[0].application_root else: app_id = (options.app_id_prefix + options.app_id if options.app_id else DEFAULT_API_SERVER_APP_ID) app_root = tempfile.mkdtemp() util.setup_environ(app_id) # pylint: disable=protected-access if options.java_app_base_url: # If the user specified a java_app_base_url, then the actual app is running # via the classic Java SDK, so we use the appropriate dispatcher that will # send requests to the Java app rather than forward them internally to a # devappserver2 module. dispatcher = _LocalJavaAppDispatcher( java_app_base_url=options.java_app_base_url) else: # TODO: Rename LocalFakeDispatcher or re-implement for # api_server.py. dispatcher = request_info_lib._LocalFakeDispatcher() request_info = wsgi_request_info.WSGIRequestInfo(dispatcher) # pylint: enable=protected-access metrics_logger = metrics.GetMetricsLogger() metrics_logger.Start( options.google_analytics_client_id, user_agent=options.google_analytics_user_agent, support_datastore_emulator=options.support_datastore_emulator, category=metrics.API_SERVER_CATEGORY) # When Cloud Datastore Emulator is invoked from api_server, it should be in # test mode, which stores in memory. options.datastore_emulator_is_test_mode = True server = create_api_server(request_info=request_info, storage_path=get_storage_path( options.storage_path, app_id), options=options, app_id=app_id, app_root=app_root) try: server.start() shutdown.wait_until_shutdown() finally: metrics.GetMetricsLogger().Stop() server.quit()
def stop(self): """Stops all running devappserver2 modules and report metrics.""" while self._running_modules: self._running_modules.pop().quit() if self._dispatcher: self._dispatcher.quit() if self._options.google_analytics_client_id: kwargs = {} watcher_results = (self._dispatcher.get_watcher_results() if self._dispatcher else None) # get_watcher_results() only returns results for modules that have at # least one record of file change. Hence avoiding divide by zero error # when computing avg_time. if watcher_results: zipped = zip(*watcher_results) total_time = sum(zipped[0]) total_changes = sum(zipped[1]) # Google Analytics Event value cannot be float numbers, so we round the # value into integers, and measure in microseconds to ensure accuracy. avg_time = int(1000000 * total_time / total_changes) # watcher_class is same on all modules. watcher_class = zipped[2][0] kwargs = { metrics.GOOGLE_ANALYTICS_DIMENSIONS['FileWatcherType']: watcher_class, metrics.GOOGLE_ANALYTICS_METRICS['FileChangeDetectionAverageTime']: avg_time, metrics.GOOGLE_ANALYTICS_METRICS['FileChangeEventCount']: total_changes } metrics.GetMetricsLogger().Stop(**kwargs)
def stop(self): """Stops all running devappserver2 modules.""" while self._running_modules: self._running_modules.pop().quit() if self._dispatcher: self._dispatcher.quit() if self._options.google_analytics_client_id: metrics.GetMetricsLogger().Stop()
def new_instance(self, instance_id, expect_ready_request=False): """Create and return a new Instance. Args: instance_id: A string or integer representing the unique (per module) id of the instance. expect_ready_request: If True then the instance will be sent a special request (i.e. /_ah/warmup or /_ah/start) before it can handle external requests. Returns: The newly created instance.Instance. """ def instance_config_getter(): runtime_config = self._runtime_config_getter() runtime_config.instance_id = str(instance_id) return runtime_config with self._application_lock: try: if self._go_application.maybe_build( self._modified_since_last_build): if self._last_build_error: logging.info('Go application successfully built.') self._last_build_error = None except go_errors.BuildError as e: logging.error('Failed to build Go application: %s', e) # Deploy a failure proxy now and each time a new instance is requested. self._last_build_error = e metrics.GetMetricsLogger().LogOnceOnStop( metrics.DEVAPPSERVER_CATEGORY, metrics.ERROR_ACTION, label=repr(e)) self._modified_since_last_build = False if self._last_build_error: logging.debug('Deploying new instance of failure proxy.') proxy = _GoBuildFailureRuntimeProxy(self._last_build_error) else: environ = self._go_application.get_environment() # Add in the environment settings from app_yaml "env_variables:" runtime_config = self._runtime_config_getter() for kv in runtime_config.environ: environ[kv.key] = kv.value proxy = http_runtime.HttpRuntimeProxy( [self._go_application.go_executable], instance_config_getter, self._module_configuration, environ, start_process_flavor=self._start_process_flavor) return instance.Instance(self.request_data, instance_id, proxy, self.max_concurrent_requests, self.max_background_threads, expect_ready_request)
def main(): shutdown.install_signal_handlers() # The timezone must be set in the devappserver2 process rather than just in # the runtime so printed log timestamps are consistent and the taskqueue stub # expects the timezone to be UTC. The runtime inherits the environment. os.environ['TZ'] = 'UTC' if hasattr(time, 'tzset'): # time.tzet() should be called on Unix, but doesn't exist on Windows. time.tzset() options = PARSER.parse_args() dev_server = DevelopmentServer() try: dev_server.start(options) shutdown.wait_until_shutdown() except: # pylint: disable=bare-except metrics.GetMetricsLogger().LogOnceOnStop( metrics.DEVAPPSERVER_CATEGORY, metrics.ERROR_ACTION, label=metrics.GetErrorDetails()) raise finally: dev_server.stop()
def _execute_request(request, use_proto3=False): """Executes an API method call and returns the response object. Args: request: A remote_api_pb.Request object representing the API call e.g. a call to memcache.Get. use_proto3: A boolean representing is request is in proto3. Returns: A ProtocolBuffer.ProtocolMessage representing the API response e.g. a memcache_service_pb.MemcacheGetResponse. Raises: apiproxy_errors.CallNotFoundError: if the requested method doesn't exist. apiproxy_errors.ApplicationError: if the API method calls fails. """ logging.debug('API server executing remote_api_pb.Request: \n%s', request) if use_proto3: service = request.service_name method = request.method if request.request_id: request_id = request.request_id else: # This logging could be time consuming. Hence set as debug level. logging.debug('Received a request without request_id: %s', request) request_id = None else: service = request.service_name() method = request.method() if request.has_request_id(): request_id = request.request_id() else: logging.error('Received a request without request_id: %s', request) request_id = None service_methods = ( _DATASTORE_V4_METHODS if service == 'datastore_v4' else remote_api_services.STUB_SERVICE_PB_MAP.get(service, {})) # We do this rather than making a new map that is a superset of # remote_api_services.SERVICE_PB_MAP because that map is not initialized # all in one place, so we would have to be careful about where we made # our new map. request_class, response_class = service_methods.get(method, (None, None)) if not request_class: raise apiproxy_errors.CallNotFoundError('%s.%s does not exist' % (service, method)) # TODO: Remove after the Files API is really gone. if not _FILESAPI_ENABLED and service == 'file': raise apiproxy_errors.CallNotFoundError( 'Files API method %s.%s is disabled. Further information: ' 'https://cloud.google.com/appengine/docs/deprecations/files_api' % (service, method)) request_data = request_class() if use_proto3: request_data.ParseFromString(request.request) else: request_data.ParseFromString(request.request()) response_data = response_class() service_stub = apiproxy_stub_map.apiproxy.GetStub(service) def make_request(): # TODO: Remove after the Files API is really gone. if (_FILESAPI_USE_TRACKER is not None and service == 'file' and request_id is not None): _FILESAPI_USE_TRACKER.set_filesapi_used(request_id) service_stub.MakeSyncCall(service, method, request_data, response_data, request_id) # If the service has not declared itself as threadsafe acquire # GLOBAL_API_LOCK. if service_stub.THREADSAFE: make_request() else: with GLOBAL_API_LOCK: make_request() metrics.GetMetricsLogger().LogOnceOnStop( metrics.API_STUB_USAGE_CATEGORY, metrics.API_STUB_USAGE_ACTION_TEMPLATE % service) logging.debug('API server responding with remote_api_pb.Response: \n%s', response_data) return response_data
def start(self, options): """Start devappserver2 servers based on the provided command line arguments. Args: options: An argparse.Namespace containing the command line arguments. """ self._options = options logging.getLogger().setLevel(constants.LOG_LEVEL_TO_PYTHON_CONSTANT[ options.dev_appserver_log_level]) parsed_env_variables = dict(options.env_variables or []) configuration = application_configuration.ApplicationConfiguration( config_paths=options.config_paths, app_id=options.app_id, runtime=options.runtime, env_variables=parsed_env_variables) if options.google_analytics_client_id: metrics_logger = metrics.GetMetricsLogger() metrics_logger.Start( options.google_analytics_client_id, options.google_analytics_user_agent, {module.runtime for module in configuration.modules}, {module.env or 'standard' for module in configuration.modules}) if options.skip_sdk_update_check: logging.info('Skipping SDK update check.') else: update_checker.check_for_updates(configuration) # There is no good way to set the default encoding from application code # (it needs to be done during interpreter initialization in site.py or # sitecustomize.py) so just warn developers if they have a different # encoding than production. if sys.getdefaultencoding() != constants.PROD_DEFAULT_ENCODING: logging.warning( 'The default encoding of your local Python interpreter is set to %r ' 'while App Engine\'s production environment uses %r; as a result ' 'your code may behave differently when deployed.', sys.getdefaultencoding(), constants.PROD_DEFAULT_ENCODING) if options.port == 0: logging.warn( 'DEFAULT_VERSION_HOSTNAME will not be set correctly with ' '--port=0') _setup_environ(configuration.app_id) self._dispatcher = dispatcher.Dispatcher( configuration, options.host, options.port, options.auth_domain, constants.LOG_LEVEL_TO_RUNTIME_CONSTANT[options.log_level], self._create_php_config(options), self._create_python_config(options), self._create_java_config(options), self._create_go_config(options), self._create_custom_config(options), self._create_cloud_sql_config(options), self._create_vm_config(options), self._create_module_to_setting(options.max_module_instances, configuration, '--max_module_instances'), options.use_mtime_file_watcher, options.watcher_ignore_re, options.automatic_restart, options.allow_skipped_files, self._create_module_to_setting(options.threadsafe_override, configuration, '--threadsafe_override'), options.external_port) wsgi_request_info_ = wsgi_request_info.WSGIRequestInfo( self._dispatcher) storage_path = api_server.get_storage_path(options.storage_path, configuration.app_id) apiserver = api_server.create_api_server( wsgi_request_info_, storage_path, options, configuration.app_id, configuration.modules[0].application_root) apiserver.start() self._running_modules.append(apiserver) self._dispatcher.start(options.api_host, apiserver.port, wsgi_request_info_) xsrf_path = os.path.join(storage_path, 'xsrf') admin = admin_server.AdminServer(options.admin_host, options.admin_port, self._dispatcher, configuration, xsrf_path) admin.start() self._running_modules.append(admin) try: default = self._dispatcher.get_module_by_name('default') apiserver.set_balanced_address(default.balanced_address) except request_info.ModuleDoesNotExistError: logging.warning('No default module found. Ignoring.')
def _handle_POST(self, environ, start_response): """Handles a POST request containing a serialized remote_api_pb.Request. Args: environ: An environ dict for the request as defined in PEP-333. start_response: A start_response function with semantics defined in PEP-333. Returns: A single element list containing the string body of the HTTP response. """ start_response('200 OK', [('Content-Type', 'application/octet-stream')]) start_time = time.time() response = remote_api_pb.Response() try: request = remote_api_pb.Request() # NOTE: Exceptions encountered when parsing the PB or handling the request # will be propagated back to the caller the same way as exceptions raised # by the actual API call. if environ.get('HTTP_TRANSFER_ENCODING') == 'chunked': # CherryPy concatenates all chunks when 'wsgi.input' is read but v3.2.2 # will not return even when all of the data in all chunks has been # read. See: https://bitbucket.org/cherrypy/cherrypy/issue/1131. wsgi_input = environ['wsgi.input'].read(2**32) else: wsgi_input = environ['wsgi.input'].read( int(environ['CONTENT_LENGTH'])) request.ParseFromString(wsgi_input) service = request.service_name() service_stub = apiproxy_stub_map.apiproxy.GetStub(service) if isinstance(service_stub, datastore_grpc_stub.DatastoreGrpcStub): # len(request.request()) is equivalent to calling ByteSize() on # deserialized request.request. if len(request.request()) > apiproxy_stub.MAX_REQUEST_SIZE: raise apiproxy_errors.RequestTooLargeError( apiproxy_stub.REQ_SIZE_EXCEEDS_LIMIT_MSG_TEMPLATE % (service, request.method())) response = service_stub.MakeSyncCallForRemoteApi(request) metrics.GetMetricsLogger().LogOnceOnStop( metrics.API_STUB_USAGE_CATEGORY, metrics.API_STUB_USAGE_ACTION_TEMPLATE % 'datastore_v3_with_cloud_datastore_emulator') else: if request.has_request_id(): request_id = request.request_id() environ['HTTP_HOST'] = self._balanced_address op = getattr(service_stub.request_data, 'register_request_id', None) if callable(op): op(environ, request_id) api_response = _execute_request(request).Encode() response.set_response(api_response) except Exception, e: if isinstance(e, apiproxy_errors.ApplicationError): level = logging.DEBUG application_error = response.mutable_application_error() application_error.set_code(e.application_error) application_error.set_detail(e.error_detail) else: # If the runtime instance is not Python, it won't be able to unpickle # the exception so use level that won't be ignored by default. level = logging.ERROR # Even if the runtime is Python, the exception may be unpicklable if # it requires importing a class blocked by the sandbox so just send # back the exception representation. # But due to our use of the remote API, at least some apiproxy errors # are generated in the Dev App Server main instance and not in the # language runtime and wrapping them causes different behavior from # prod so don't wrap them. if not isinstance(e, apiproxy_errors.Error): e = RuntimeError(repr(e)) # While not strictly necessary for ApplicationError, do this to limit # differences with remote_api:handler.py. response.set_exception(pickle.dumps(e)) logging.log(level, 'Exception while handling %s.%s()\n%s', request.service_name(), request.method(), traceback.format_exc())
def start(self, options): """Start devappserver2 servers based on the provided command line arguments. Args: options: An argparse.Namespace containing the command line arguments. """ self._options = options logging.getLogger().setLevel(constants.LOG_LEVEL_TO_PYTHON_CONSTANT[ options.dev_appserver_log_level]) parsed_env_variables = dict(options.env_variables or []) configuration = application_configuration.ApplicationConfiguration( config_paths=options.config_paths, app_id=options.app_id, runtime=options.runtime, env_variables=parsed_env_variables) if options.google_analytics_client_id: metrics_logger = metrics.GetMetricsLogger() metrics_logger.Start( options.google_analytics_client_id, options.google_analytics_user_agent, {module.runtime for module in configuration.modules}, {module.env or 'standard' for module in configuration.modules}) if options.skip_sdk_update_check: logging.info('Skipping SDK update check.') else: update_checker.check_for_updates(configuration) # There is no good way to set the default encoding from application code # (it needs to be done during interpreter initialization in site.py or # sitecustomize.py) so just warn developers if they have a different # encoding than production. if sys.getdefaultencoding() != constants.PROD_DEFAULT_ENCODING: logging.warning( 'The default encoding of your local Python interpreter is set to %r ' 'while App Engine\'s production environment uses %r; as a result ' 'your code may behave differently when deployed.', sys.getdefaultencoding(), constants.PROD_DEFAULT_ENCODING) if options.port == 0: logging.warn( 'DEFAULT_VERSION_HOSTNAME will not be set correctly with ' '--port=0') _setup_environ(configuration.app_id) # grpc_proxy is only needed for python2 because remote_api_stub.py is # imported in local python runtime sandbox. For more details, see # grpc_proxy_util.py. grpc_proxy_port = portpicker.PickUnusedPort() self._dispatcher = dispatcher.Dispatcher( configuration, options.host, options.port, options.auth_domain, constants.LOG_LEVEL_TO_RUNTIME_CONSTANT[options.log_level], self._create_php_config(options), self._create_python_config(options, grpc_proxy_port), self._create_java_config(options), self._create_go_config(options), self._create_custom_config(options), self._create_cloud_sql_config(options), self._create_vm_config(options), self._create_module_to_setting(options.max_module_instances, configuration, '--max_module_instances'), options.use_mtime_file_watcher, options.watcher_ignore_re, options.automatic_restart, options.allow_skipped_files, self._create_module_to_setting(options.threadsafe_override, configuration, '--threadsafe_override'), options.external_port) wsgi_request_info_ = wsgi_request_info.WSGIRequestInfo( self._dispatcher) storage_path = api_server.get_storage_path(options.storage_path, configuration.app_id) datastore_emulator_host = ( parsed_env_variables['DATASTORE_EMULATOR_HOST'] if 'DATASTORE_EMULATOR_HOST' in parsed_env_variables else None) apiserver = api_server.create_api_server( wsgi_request_info_, storage_path, options, configuration.app_id, configuration.modules[0].application_root, datastore_emulator_host) apiserver.start() self._running_modules.append(apiserver) if options.grpc_apis: grpc_apiserver = api_server.GRPCAPIServer(options.grpc_api_port) grpc_apiserver.start() self._running_modules.append(grpc_apiserver) # We declare grpc_proxy_util as global, otherwise it cannot be accessed # from outside of this function. global grpc_proxy_util # pylint: disable=g-import-not-at-top # We lazy import here because grpc binaries are not always present. from google.appengine.tools.devappserver2 import grpc_proxy_util grpc_proxy = grpc_proxy_util.GrpcProxyServer(grpc_proxy_port) grpc_proxy.start() self._running_modules.append(grpc_proxy) self._dispatcher.start(options.api_host, apiserver.port, wsgi_request_info_, options.grpc_apis) xsrf_path = os.path.join(storage_path, 'xsrf') admin = admin_server.AdminServer(options.admin_host, options.admin_port, self._dispatcher, configuration, xsrf_path) admin.start() self._running_modules.append(admin) try: default = self._dispatcher.get_module_by_name('default') apiserver.set_balanced_address(default.balanced_address) except request_info.ModuleDoesNotExistError: logging.warning('No default module found. Ignoring.')
def start(self, options): """Start devappserver2 servers based on the provided command line arguments. Args: options: An argparse.Namespace containing the command line arguments. Raises: PhpPathError: php executable path is not specified for php72. MissingDatastoreEmulatorError: dev_appserver.py is not invoked from the right directory. """ self._options = options self._options.datastore_emulator_cmd = self._correct_datastore_emulator_cmd( self._options.datastore_emulator_cmd) self._check_datastore_emulator_support() logging.getLogger().setLevel(constants.LOG_LEVEL_TO_PYTHON_CONSTANT[ options.dev_appserver_log_level]) parsed_env_variables = dict(options.env_variables or []) configuration = application_configuration.ApplicationConfiguration( config_paths=options.config_paths, app_id=options.app_id, runtime=options.runtime, env_variables=parsed_env_variables) all_module_runtimes = { module.runtime for module in configuration.modules } self._check_platform_support(all_module_runtimes) storage_path = api_server.get_storage_path(options.storage_path, configuration.app_id) datastore_path = api_server.get_datastore_path(storage_path, options.datastore_path) datastore_data_type = ( datastore_converter.get_stub_type(datastore_path) if os.path.isfile(datastore_path) else None) if options.skip_sdk_update_check: logging.info('Skipping SDK update check.') else: update_checker.check_for_updates(configuration) # There is no good way to set the default encoding from application code # (it needs to be done during interpreter initialization in site.py or # sitecustomize.py) so just warn developers if they have a different # encoding than production. if sys.getdefaultencoding() != constants.PROD_DEFAULT_ENCODING: logging.warning( 'The default encoding of your local Python interpreter is set to %r ' 'while App Engine\'s production environment uses %r; as a result ' 'your code may behave differently when deployed.', sys.getdefaultencoding(), constants.PROD_DEFAULT_ENCODING) if options.port == 0: logging.warn( 'DEFAULT_VERSION_HOSTNAME will not be set correctly with ' '--port=0') util.setup_environ(configuration.app_id) php_version = self._get_php_runtime(configuration) if not options.php_executable_path and php_version == 'php72': raise PhpPathError( 'For php72, --php_executable_path must be specified.') if options.ssl_certificate_path and options.ssl_certificate_key_path: ssl_certificate_paths = self._create_ssl_certificate_paths_if_valid( options.ssl_certificate_path, options.ssl_certificate_key_path) else: if options.ssl_certificate_path or options.ssl_certificate_key_path: logging.warn('Must provide both --ssl_certificate_path and ' '--ssl_certificate_key_path to enable SSL. Since ' 'only one flag was provided, not using SSL.') ssl_certificate_paths = None if options.google_analytics_client_id: metrics_logger = metrics.GetMetricsLogger() metrics_logger.Start( options.google_analytics_client_id, options.google_analytics_user_agent, all_module_runtimes, {module.env or 'standard' for module in configuration.modules}, options.support_datastore_emulator, datastore_data_type, bool(ssl_certificate_paths), options, multi_module=len(configuration.modules) > 1, dispatch_config=configuration.dispatch is not None, ) self._dispatcher = dispatcher.Dispatcher( configuration, options.host, options.port, options.auth_domain, constants.LOG_LEVEL_TO_RUNTIME_CONSTANT[options.log_level], self._create_php_config(options, php_version), self._create_python_config(options), self._create_java_config(options), self._create_go_config(options), self._create_custom_config(options), self._create_cloud_sql_config(options), self._create_vm_config(options), self._create_module_to_setting(options.max_module_instances, configuration, '--max_module_instances'), options.use_mtime_file_watcher, options.watcher_ignore_re, options.automatic_restart, options.allow_skipped_files, self._create_module_to_setting(options.threadsafe_override, configuration, '--threadsafe_override'), options.external_port, options.specified_service_ports, options.enable_host_checking, ssl_certificate_paths) wsgi_request_info_ = wsgi_request_info.WSGIRequestInfo( self._dispatcher) apiserver = api_server.create_api_server( wsgi_request_info_, storage_path, options, configuration.app_id, configuration.modules[0].application_root) apiserver.start() self._running_modules.append(apiserver) self._dispatcher.start(options.api_host, apiserver.port, wsgi_request_info_) xsrf_path = os.path.join(storage_path, 'xsrf') admin = admin_server.AdminServer(options.admin_host, options.admin_port, self._dispatcher, configuration, xsrf_path, options.enable_host_checking, options.enable_console) admin.start() self._running_modules.append(admin) try: default = self._dispatcher.get_module_by_name('default') apiserver.set_balanced_address(default.balanced_address) except request_info.ModuleDoesNotExistError: logging.warning('No default module found. Ignoring.')