def main():
"""Parses command line options and launches the API server."""
shutdown.install_signal_handlers()
options = PARSER.parse_args()
logging.getLogger().setLevel(constants.LOG_LEVEL_TO_PYTHON_CONSTANT[
options.dev_appserver_log_level])
# Parse the application configuration if config_paths are provided, else
# provide sensible defaults.
if options.config_paths:
app_config = application_configuration.ApplicationConfiguration(
options.config_paths, options.app_id)
app_id = app_config.app_id
app_root = app_config.modules[0].application_root
else:
app_id = (options.app_id_prefix + options.app_id
if options.app_id else DEFAULT_API_SERVER_APP_ID)
app_root = tempfile.mkdtemp()
util.setup_environ(app_id)
# pylint: disable=protected-access
if options.java_app_base_url:
# If the user specified a java_app_base_url, then the actual app is running
# via the classic Java SDK, so we use the appropriate dispatcher that will
# send requests to the Java app rather than forward them internally to a
# devappserver2 module.
dispatcher = _LocalJavaAppDispatcher(
java_app_base_url=options.java_app_base_url)
else:
# TODO: Rename LocalFakeDispatcher or re-implement for
# api_server.py.
dispatcher = request_info_lib._LocalFakeDispatcher()
request_info = wsgi_request_info.WSGIRequestInfo(dispatcher)
# pylint: enable=protected-access
metrics_logger = metrics.GetMetricsLogger()
metrics_logger.Start(
options.google_analytics_client_id,
user_agent=options.google_analytics_user_agent,
support_datastore_emulator=options.support_datastore_emulator,
category=metrics.API_SERVER_CATEGORY)
# When Cloud Datastore Emulator is invoked from api_server, it should be in
# test mode, which stores in memory.
options.datastore_emulator_is_test_mode = True
server = create_api_server(request_info=request_info,
storage_path=get_storage_path(
options.storage_path, app_id),
options=options,
app_id=app_id,
app_root=app_root)
try:
server.start()
shutdown.wait_until_shutdown()
finally:
metrics.GetMetricsLogger().Stop()
server.quit()
def stop(self):
"""Stops all running devappserver2 modules and report metrics."""
while self._running_modules:
self._running_modules.pop().quit()
if self._dispatcher:
self._dispatcher.quit()
if self._options.google_analytics_client_id:
kwargs = {}
watcher_results = (self._dispatcher.get_watcher_results()
if self._dispatcher else None)
# get_watcher_results() only returns results for modules that have at
# least one record of file change. Hence avoiding divide by zero error
# when computing avg_time.
if watcher_results:
zipped = zip(*watcher_results)
total_time = sum(zipped[0])
total_changes = sum(zipped[1])
# Google Analytics Event value cannot be float numbers, so we round the
# value into integers, and measure in microseconds to ensure accuracy.
avg_time = int(1000000 * total_time / total_changes)
# watcher_class is same on all modules.
watcher_class = zipped[2][0]
kwargs = {
metrics.GOOGLE_ANALYTICS_DIMENSIONS['FileWatcherType']:
watcher_class,
metrics.GOOGLE_ANALYTICS_METRICS['FileChangeDetectionAverageTime']:
avg_time,
metrics.GOOGLE_ANALYTICS_METRICS['FileChangeEventCount']:
total_changes
}
metrics.GetMetricsLogger().Stop(**kwargs)
def stop(self):
"""Stops all running devappserver2 modules."""
while self._running_modules:
self._running_modules.pop().quit()
if self._dispatcher:
self._dispatcher.quit()
if self._options.google_analytics_client_id:
metrics.GetMetricsLogger().Stop()
def new_instance(self, instance_id, expect_ready_request=False):
"""Create and return a new Instance.
Args:
instance_id: A string or integer representing the unique (per module) id
of the instance.
expect_ready_request: If True then the instance will be sent a special
request (i.e. /_ah/warmup or /_ah/start) before it can handle external
requests.
Returns:
The newly created instance.Instance.
"""
def instance_config_getter():
runtime_config = self._runtime_config_getter()
runtime_config.instance_id = str(instance_id)
return runtime_config
with self._application_lock:
try:
if self._go_application.maybe_build(
self._modified_since_last_build):
if self._last_build_error:
logging.info('Go application successfully built.')
self._last_build_error = None
except go_errors.BuildError as e:
logging.error('Failed to build Go application: %s', e)
# Deploy a failure proxy now and each time a new instance is requested.
self._last_build_error = e
metrics.GetMetricsLogger().LogOnceOnStop(
metrics.DEVAPPSERVER_CATEGORY,
metrics.ERROR_ACTION,
label=repr(e))
self._modified_since_last_build = False
if self._last_build_error:
logging.debug('Deploying new instance of failure proxy.')
proxy = _GoBuildFailureRuntimeProxy(self._last_build_error)
else:
environ = self._go_application.get_environment()
# Add in the environment settings from app_yaml "env_variables:"
runtime_config = self._runtime_config_getter()
for kv in runtime_config.environ:
environ[kv.key] = kv.value
proxy = http_runtime.HttpRuntimeProxy(
[self._go_application.go_executable],
instance_config_getter,
self._module_configuration,
environ,
start_process_flavor=self._start_process_flavor)
return instance.Instance(self.request_data, instance_id, proxy,
self.max_concurrent_requests,
self.max_background_threads,
expect_ready_request)
def main():
shutdown.install_signal_handlers()
# The timezone must be set in the devappserver2 process rather than just in
# the runtime so printed log timestamps are consistent and the taskqueue stub
# expects the timezone to be UTC. The runtime inherits the environment.
os.environ['TZ'] = 'UTC'
if hasattr(time, 'tzset'):
# time.tzet() should be called on Unix, but doesn't exist on Windows.
time.tzset()
options = PARSER.parse_args()
dev_server = DevelopmentServer()
try:
dev_server.start(options)
shutdown.wait_until_shutdown()
except: # pylint: disable=bare-except
metrics.GetMetricsLogger().LogOnceOnStop(
metrics.DEVAPPSERVER_CATEGORY, metrics.ERROR_ACTION,
label=metrics.GetErrorDetails())
raise
finally:
dev_server.stop()
def _execute_request(request, use_proto3=False):
"""Executes an API method call and returns the response object.
Args:
request: A remote_api_pb.Request object representing the API call e.g. a
call to memcache.Get.
use_proto3: A boolean representing is request is in proto3.
Returns:
A ProtocolBuffer.ProtocolMessage representing the API response e.g. a
memcache_service_pb.MemcacheGetResponse.
Raises:
apiproxy_errors.CallNotFoundError: if the requested method doesn't exist.
apiproxy_errors.ApplicationError: if the API method calls fails.
"""
logging.debug('API server executing remote_api_pb.Request: \n%s', request)
if use_proto3:
service = request.service_name
method = request.method
if request.request_id:
request_id = request.request_id
else:
# This logging could be time consuming. Hence set as debug level.
logging.debug('Received a request without request_id: %s', request)
request_id = None
else:
service = request.service_name()
method = request.method()
if request.has_request_id():
request_id = request.request_id()
else:
logging.error('Received a request without request_id: %s', request)
request_id = None
service_methods = (
_DATASTORE_V4_METHODS if service == 'datastore_v4'
else remote_api_services.STUB_SERVICE_PB_MAP.get(service, {}))
# We do this rather than making a new map that is a superset of
# remote_api_services.SERVICE_PB_MAP because that map is not initialized
# all in one place, so we would have to be careful about where we made
# our new map.
request_class, response_class = service_methods.get(method, (None, None))
if not request_class:
raise apiproxy_errors.CallNotFoundError('%s.%s does not exist' % (service,
method))
# TODO: Remove after the Files API is really gone.
if not _FILESAPI_ENABLED and service == 'file':
raise apiproxy_errors.CallNotFoundError(
'Files API method %s.%s is disabled. Further information: '
'https://cloud.google.com/appengine/docs/deprecations/files_api'
% (service, method))
request_data = request_class()
if use_proto3:
request_data.ParseFromString(request.request)
else:
request_data.ParseFromString(request.request())
response_data = response_class()
service_stub = apiproxy_stub_map.apiproxy.GetStub(service)
def make_request():
# TODO: Remove after the Files API is really gone.
if (_FILESAPI_USE_TRACKER is not None
and service == 'file' and request_id is not None):
_FILESAPI_USE_TRACKER.set_filesapi_used(request_id)
service_stub.MakeSyncCall(service,
method,
request_data,
response_data,
request_id)
# If the service has not declared itself as threadsafe acquire
# GLOBAL_API_LOCK.
if service_stub.THREADSAFE:
make_request()
else:
with GLOBAL_API_LOCK:
make_request()
metrics.GetMetricsLogger().LogOnceOnStop(
metrics.API_STUB_USAGE_CATEGORY,
metrics.API_STUB_USAGE_ACTION_TEMPLATE % service)
logging.debug('API server responding with remote_api_pb.Response: \n%s',
response_data)
return response_data
def start(self, options):
"""Start devappserver2 servers based on the provided command line arguments.
Args:
options: An argparse.Namespace containing the command line arguments.
"""
self._options = options
logging.getLogger().setLevel(constants.LOG_LEVEL_TO_PYTHON_CONSTANT[
options.dev_appserver_log_level])
parsed_env_variables = dict(options.env_variables or [])
configuration = application_configuration.ApplicationConfiguration(
config_paths=options.config_paths,
app_id=options.app_id,
runtime=options.runtime,
env_variables=parsed_env_variables)
if options.google_analytics_client_id:
metrics_logger = metrics.GetMetricsLogger()
metrics_logger.Start(
options.google_analytics_client_id,
options.google_analytics_user_agent,
{module.runtime
for module in configuration.modules},
{module.env or 'standard'
for module in configuration.modules})
if options.skip_sdk_update_check:
logging.info('Skipping SDK update check.')
else:
update_checker.check_for_updates(configuration)
# There is no good way to set the default encoding from application code
# (it needs to be done during interpreter initialization in site.py or
# sitecustomize.py) so just warn developers if they have a different
# encoding than production.
if sys.getdefaultencoding() != constants.PROD_DEFAULT_ENCODING:
logging.warning(
'The default encoding of your local Python interpreter is set to %r '
'while App Engine\'s production environment uses %r; as a result '
'your code may behave differently when deployed.',
sys.getdefaultencoding(), constants.PROD_DEFAULT_ENCODING)
if options.port == 0:
logging.warn(
'DEFAULT_VERSION_HOSTNAME will not be set correctly with '
'--port=0')
_setup_environ(configuration.app_id)
self._dispatcher = dispatcher.Dispatcher(
configuration, options.host, options.port, options.auth_domain,
constants.LOG_LEVEL_TO_RUNTIME_CONSTANT[options.log_level],
self._create_php_config(options),
self._create_python_config(options),
self._create_java_config(options), self._create_go_config(options),
self._create_custom_config(options),
self._create_cloud_sql_config(options),
self._create_vm_config(options),
self._create_module_to_setting(options.max_module_instances,
configuration,
'--max_module_instances'),
options.use_mtime_file_watcher, options.watcher_ignore_re,
options.automatic_restart, options.allow_skipped_files,
self._create_module_to_setting(options.threadsafe_override,
configuration,
'--threadsafe_override'),
options.external_port)
wsgi_request_info_ = wsgi_request_info.WSGIRequestInfo(
self._dispatcher)
storage_path = api_server.get_storage_path(options.storage_path,
configuration.app_id)
apiserver = api_server.create_api_server(
wsgi_request_info_, storage_path, options, configuration.app_id,
configuration.modules[0].application_root)
apiserver.start()
self._running_modules.append(apiserver)
self._dispatcher.start(options.api_host, apiserver.port,
wsgi_request_info_)
xsrf_path = os.path.join(storage_path, 'xsrf')
admin = admin_server.AdminServer(options.admin_host,
options.admin_port, self._dispatcher,
configuration, xsrf_path)
admin.start()
self._running_modules.append(admin)
try:
default = self._dispatcher.get_module_by_name('default')
apiserver.set_balanced_address(default.balanced_address)
except request_info.ModuleDoesNotExistError:
logging.warning('No default module found. Ignoring.')
def _handle_POST(self, environ, start_response):
"""Handles a POST request containing a serialized remote_api_pb.Request.
Args:
environ: An environ dict for the request as defined in PEP-333.
start_response: A start_response function with semantics defined in
PEP-333.
Returns:
A single element list containing the string body of the HTTP response.
"""
start_response('200 OK',
[('Content-Type', 'application/octet-stream')])
start_time = time.time()
response = remote_api_pb.Response()
try:
request = remote_api_pb.Request()
# NOTE: Exceptions encountered when parsing the PB or handling the request
# will be propagated back to the caller the same way as exceptions raised
# by the actual API call.
if environ.get('HTTP_TRANSFER_ENCODING') == 'chunked':
# CherryPy concatenates all chunks when 'wsgi.input' is read but v3.2.2
# will not return even when all of the data in all chunks has been
# read. See: https://bitbucket.org/cherrypy/cherrypy/issue/1131.
wsgi_input = environ['wsgi.input'].read(2**32)
else:
wsgi_input = environ['wsgi.input'].read(
int(environ['CONTENT_LENGTH']))
request.ParseFromString(wsgi_input)
service = request.service_name()
service_stub = apiproxy_stub_map.apiproxy.GetStub(service)
if isinstance(service_stub, datastore_grpc_stub.DatastoreGrpcStub):
# len(request.request()) is equivalent to calling ByteSize() on
# deserialized request.request.
if len(request.request()) > apiproxy_stub.MAX_REQUEST_SIZE:
raise apiproxy_errors.RequestTooLargeError(
apiproxy_stub.REQ_SIZE_EXCEEDS_LIMIT_MSG_TEMPLATE %
(service, request.method()))
response = service_stub.MakeSyncCallForRemoteApi(request)
metrics.GetMetricsLogger().LogOnceOnStop(
metrics.API_STUB_USAGE_CATEGORY,
metrics.API_STUB_USAGE_ACTION_TEMPLATE %
'datastore_v3_with_cloud_datastore_emulator')
else:
if request.has_request_id():
request_id = request.request_id()
environ['HTTP_HOST'] = self._balanced_address
op = getattr(service_stub.request_data,
'register_request_id', None)
if callable(op):
op(environ, request_id)
api_response = _execute_request(request).Encode()
response.set_response(api_response)
except Exception, e:
if isinstance(e, apiproxy_errors.ApplicationError):
level = logging.DEBUG
application_error = response.mutable_application_error()
application_error.set_code(e.application_error)
application_error.set_detail(e.error_detail)
else:
# If the runtime instance is not Python, it won't be able to unpickle
# the exception so use level that won't be ignored by default.
level = logging.ERROR
# Even if the runtime is Python, the exception may be unpicklable if
# it requires importing a class blocked by the sandbox so just send
# back the exception representation.
# But due to our use of the remote API, at least some apiproxy errors
# are generated in the Dev App Server main instance and not in the
# language runtime and wrapping them causes different behavior from
# prod so don't wrap them.
if not isinstance(e, apiproxy_errors.Error):
e = RuntimeError(repr(e))
# While not strictly necessary for ApplicationError, do this to limit
# differences with remote_api:handler.py.
response.set_exception(pickle.dumps(e))
logging.log(level, 'Exception while handling %s.%s()\n%s',
request.service_name(), request.method(),
traceback.format_exc())
def start(self, options):
"""Start devappserver2 servers based on the provided command line arguments.
Args:
options: An argparse.Namespace containing the command line arguments.
"""
self._options = options
logging.getLogger().setLevel(constants.LOG_LEVEL_TO_PYTHON_CONSTANT[
options.dev_appserver_log_level])
parsed_env_variables = dict(options.env_variables or [])
configuration = application_configuration.ApplicationConfiguration(
config_paths=options.config_paths,
app_id=options.app_id,
runtime=options.runtime,
env_variables=parsed_env_variables)
if options.google_analytics_client_id:
metrics_logger = metrics.GetMetricsLogger()
metrics_logger.Start(
options.google_analytics_client_id,
options.google_analytics_user_agent,
{module.runtime
for module in configuration.modules},
{module.env or 'standard'
for module in configuration.modules})
if options.skip_sdk_update_check:
logging.info('Skipping SDK update check.')
else:
update_checker.check_for_updates(configuration)
# There is no good way to set the default encoding from application code
# (it needs to be done during interpreter initialization in site.py or
# sitecustomize.py) so just warn developers if they have a different
# encoding than production.
if sys.getdefaultencoding() != constants.PROD_DEFAULT_ENCODING:
logging.warning(
'The default encoding of your local Python interpreter is set to %r '
'while App Engine\'s production environment uses %r; as a result '
'your code may behave differently when deployed.',
sys.getdefaultencoding(), constants.PROD_DEFAULT_ENCODING)
if options.port == 0:
logging.warn(
'DEFAULT_VERSION_HOSTNAME will not be set correctly with '
'--port=0')
_setup_environ(configuration.app_id)
# grpc_proxy is only needed for python2 because remote_api_stub.py is
# imported in local python runtime sandbox. For more details, see
# grpc_proxy_util.py.
grpc_proxy_port = portpicker.PickUnusedPort()
self._dispatcher = dispatcher.Dispatcher(
configuration, options.host, options.port, options.auth_domain,
constants.LOG_LEVEL_TO_RUNTIME_CONSTANT[options.log_level],
self._create_php_config(options),
self._create_python_config(options, grpc_proxy_port),
self._create_java_config(options), self._create_go_config(options),
self._create_custom_config(options),
self._create_cloud_sql_config(options),
self._create_vm_config(options),
self._create_module_to_setting(options.max_module_instances,
configuration,
'--max_module_instances'),
options.use_mtime_file_watcher, options.watcher_ignore_re,
options.automatic_restart, options.allow_skipped_files,
self._create_module_to_setting(options.threadsafe_override,
configuration,
'--threadsafe_override'),
options.external_port)
wsgi_request_info_ = wsgi_request_info.WSGIRequestInfo(
self._dispatcher)
storage_path = api_server.get_storage_path(options.storage_path,
configuration.app_id)
datastore_emulator_host = (
parsed_env_variables['DATASTORE_EMULATOR_HOST']
if 'DATASTORE_EMULATOR_HOST' in parsed_env_variables else None)
apiserver = api_server.create_api_server(
wsgi_request_info_, storage_path, options, configuration.app_id,
configuration.modules[0].application_root, datastore_emulator_host)
apiserver.start()
self._running_modules.append(apiserver)
if options.grpc_apis:
grpc_apiserver = api_server.GRPCAPIServer(options.grpc_api_port)
grpc_apiserver.start()
self._running_modules.append(grpc_apiserver)
# We declare grpc_proxy_util as global, otherwise it cannot be accessed
# from outside of this function.
global grpc_proxy_util
# pylint: disable=g-import-not-at-top
# We lazy import here because grpc binaries are not always present.
from google.appengine.tools.devappserver2 import grpc_proxy_util
grpc_proxy = grpc_proxy_util.GrpcProxyServer(grpc_proxy_port)
grpc_proxy.start()
self._running_modules.append(grpc_proxy)
self._dispatcher.start(options.api_host, apiserver.port,
wsgi_request_info_, options.grpc_apis)
xsrf_path = os.path.join(storage_path, 'xsrf')
admin = admin_server.AdminServer(options.admin_host,
options.admin_port, self._dispatcher,
configuration, xsrf_path)
admin.start()
self._running_modules.append(admin)
try:
default = self._dispatcher.get_module_by_name('default')
apiserver.set_balanced_address(default.balanced_address)
except request_info.ModuleDoesNotExistError:
logging.warning('No default module found. Ignoring.')
def start(self, options):
"""Start devappserver2 servers based on the provided command line arguments.
Args:
options: An argparse.Namespace containing the command line arguments.
Raises:
PhpPathError: php executable path is not specified for php72.
MissingDatastoreEmulatorError: dev_appserver.py is not invoked from the right
directory.
"""
self._options = options
self._options.datastore_emulator_cmd = self._correct_datastore_emulator_cmd(
self._options.datastore_emulator_cmd)
self._check_datastore_emulator_support()
logging.getLogger().setLevel(constants.LOG_LEVEL_TO_PYTHON_CONSTANT[
options.dev_appserver_log_level])
parsed_env_variables = dict(options.env_variables or [])
configuration = application_configuration.ApplicationConfiguration(
config_paths=options.config_paths,
app_id=options.app_id,
runtime=options.runtime,
env_variables=parsed_env_variables)
all_module_runtimes = {
module.runtime
for module in configuration.modules
}
self._check_platform_support(all_module_runtimes)
storage_path = api_server.get_storage_path(options.storage_path,
configuration.app_id)
datastore_path = api_server.get_datastore_path(storage_path,
options.datastore_path)
datastore_data_type = (
datastore_converter.get_stub_type(datastore_path)
if os.path.isfile(datastore_path) else None)
if options.skip_sdk_update_check:
logging.info('Skipping SDK update check.')
else:
update_checker.check_for_updates(configuration)
# There is no good way to set the default encoding from application code
# (it needs to be done during interpreter initialization in site.py or
# sitecustomize.py) so just warn developers if they have a different
# encoding than production.
if sys.getdefaultencoding() != constants.PROD_DEFAULT_ENCODING:
logging.warning(
'The default encoding of your local Python interpreter is set to %r '
'while App Engine\'s production environment uses %r; as a result '
'your code may behave differently when deployed.',
sys.getdefaultencoding(), constants.PROD_DEFAULT_ENCODING)
if options.port == 0:
logging.warn(
'DEFAULT_VERSION_HOSTNAME will not be set correctly with '
'--port=0')
util.setup_environ(configuration.app_id)
php_version = self._get_php_runtime(configuration)
if not options.php_executable_path and php_version == 'php72':
raise PhpPathError(
'For php72, --php_executable_path must be specified.')
if options.ssl_certificate_path and options.ssl_certificate_key_path:
ssl_certificate_paths = self._create_ssl_certificate_paths_if_valid(
options.ssl_certificate_path, options.ssl_certificate_key_path)
else:
if options.ssl_certificate_path or options.ssl_certificate_key_path:
logging.warn('Must provide both --ssl_certificate_path and '
'--ssl_certificate_key_path to enable SSL. Since '
'only one flag was provided, not using SSL.')
ssl_certificate_paths = None
if options.google_analytics_client_id:
metrics_logger = metrics.GetMetricsLogger()
metrics_logger.Start(
options.google_analytics_client_id,
options.google_analytics_user_agent,
all_module_runtimes,
{module.env or 'standard'
for module in configuration.modules},
options.support_datastore_emulator,
datastore_data_type,
bool(ssl_certificate_paths),
options,
multi_module=len(configuration.modules) > 1,
dispatch_config=configuration.dispatch is not None,
)
self._dispatcher = dispatcher.Dispatcher(
configuration, options.host, options.port, options.auth_domain,
constants.LOG_LEVEL_TO_RUNTIME_CONSTANT[options.log_level],
self._create_php_config(options, php_version),
self._create_python_config(options),
self._create_java_config(options), self._create_go_config(options),
self._create_custom_config(options),
self._create_cloud_sql_config(options),
self._create_vm_config(options),
self._create_module_to_setting(options.max_module_instances,
configuration,
'--max_module_instances'),
options.use_mtime_file_watcher, options.watcher_ignore_re,
options.automatic_restart, options.allow_skipped_files,
self._create_module_to_setting(options.threadsafe_override,
configuration,
'--threadsafe_override'),
options.external_port, options.specified_service_ports,
options.enable_host_checking, ssl_certificate_paths)
wsgi_request_info_ = wsgi_request_info.WSGIRequestInfo(
self._dispatcher)
apiserver = api_server.create_api_server(
wsgi_request_info_, storage_path, options, configuration.app_id,
configuration.modules[0].application_root)
apiserver.start()
self._running_modules.append(apiserver)
self._dispatcher.start(options.api_host, apiserver.port,
wsgi_request_info_)
xsrf_path = os.path.join(storage_path, 'xsrf')
admin = admin_server.AdminServer(options.admin_host,
options.admin_port, self._dispatcher,
configuration, xsrf_path,
options.enable_host_checking,
options.enable_console)
admin.start()
self._running_modules.append(admin)
try:
default = self._dispatcher.get_module_by_name('default')
apiserver.set_balanced_address(default.balanced_address)
except request_info.ModuleDoesNotExistError:
logging.warning('No default module found. Ignoring.')
