def create_api_server(request_info, storage_path, options, app_id, app_root):
"""Creates an API server.
Args:
request_info: An apiproxy_stub.RequestInfo instance used by the stubs to
lookup information about the request associated with an API call.
storage_path: A string directory for storing API stub data.
options: An instance of argparse.Namespace containing command line flags.
app_id: String representing an application ID, used for configuring paths
and string constants in API stubs.
app_root: The path to the directory containing the user's
application e.g. "/home/joe/myapp", used for locating application yaml
files, eg index.yaml for the datastore stub.
Returns:
An instance of APIServer.
Raises:
DatastoreFileError: Cloud Datastore emulator is used while stub_type is
datastore_converter.StubTypes.PYTHON_FILE_STUB.
"""
datastore_path = get_datastore_path(storage_path, options.datastore_path)
logs_path = options.logs_path or os.path.join(storage_path, 'logs.db')
search_index_path = options.search_indexes_path or os.path.join(
storage_path, 'search_indexes')
blobstore_path = options.blobstore_path or os.path.join(
storage_path, 'blobs')
if options.clear_datastore:
_clear_datastore_storage(datastore_path)
if options.clear_search_indexes:
_clear_search_indexes_storage(search_index_path)
if options.auto_id_policy == datastore_stub_util.SEQUENTIAL:
logging.warn(
"--auto_id_policy='sequential' is deprecated. This option "
"will be removed in a future release.")
application_address = '%s' % options.host
if options.port and options.port != 80:
application_address += ':' + str(options.port)
user_login_url = '/%s?%s=%%s' % (login.LOGIN_URL_RELATIVE,
login.CONTINUE_PARAM)
user_logout_url = '/%s?%s=%%s' % (login.LOGOUT_URL_RELATIVE,
login.CONTINUE_PARAM)
if options.datastore_consistency_policy == 'time':
consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy()
elif options.datastore_consistency_policy == 'random':
consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy()
elif options.datastore_consistency_policy == 'consistent':
consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0)
else:
assert 0, ('unknown consistency policy: %r' %
options.datastore_consistency_policy)
stub_type = (datastore_converter.get_stub_type(datastore_path)
if os.path.exists(datastore_path) else None)
gcd_emulator_launching_thread = None
if options.support_datastore_emulator:
if stub_type == datastore_converter.StubTypes.PYTHON_FILE_STUB:
raise DatastoreFileError(
'The datastore file %s cannot be recognized by dev_appserver. Please '
'restart dev_appserver with --clear_datastore=1' %
datastore_path)
env_emulator_host = os.environ.get('DATASTORE_EMULATOR_HOST')
if env_emulator_host: # emulator already running, reuse it.
logging.warning(
'Detected environment variable DATASTORE_EMULATOR_HOST=%s, '
'dev_appserver will speak to the Cloud Datastore emulator running on '
'this address. The datastore_path %s will be neglected.\nIf you '
'want datastore to store on %s, remove DATASTORE_EMULATOR_HOST '
'from environment variables and restart dev_appserver',
env_emulator_host, datastore_path, datastore_path)
else:
gcd_emulator_launching_thread = _launch_gcd_emulator(
app_id=app_id,
emulator_port=(options.datastore_emulator_port
if options.datastore_emulator_port else
portpicker.PickUnusedPort()),
silent=options.dev_appserver_log_level != 'debug',
index_file=os.path.join(app_root, 'index.yaml'),
require_indexes=options.require_indexes,
datastore_path=datastore_path,
stub_type=stub_type,
cmd=options.datastore_emulator_cmd,
is_test=options.datastore_emulator_is_test_mode)
else:
# Use SQLite stub.
# For historic reason we are still supporting conversion from file stub to
# SQLite stub data. But this conversion will go away.
if stub_type == datastore_converter.StubTypes.PYTHON_FILE_STUB:
datastore_converter.convert_datastore_file_stub_data_to_sqlite(
app_id, datastore_path)
stub_util.setup_stubs(
request_data=request_info,
app_id=app_id,
application_root=app_root,
# The "trusted" flag is only relevant for Google administrative
# applications.
trusted=getattr(options, 'trusted', False),
appidentity_email_address=options.appidentity_email_address,
appidentity_private_key_path=os.path.abspath(
options.appidentity_private_key_path)
if options.appidentity_private_key_path else None,
blobstore_path=blobstore_path,
datastore_path=datastore_path,
datastore_consistency=consistency,
datastore_require_indexes=options.require_indexes,
datastore_auto_id_policy=options.auto_id_policy,
images_host_prefix='http://%s' % application_address,
logs_path=logs_path,
mail_smtp_host=options.smtp_host,
mail_smtp_port=options.smtp_port,
mail_smtp_user=options.smtp_user,
mail_smtp_password=options.smtp_password,
mail_enable_sendmail=options.enable_sendmail,
mail_show_mail_body=options.show_mail_body,
mail_allow_tls=options.smtp_allow_tls,
search_index_path=search_index_path,
taskqueue_auto_run_tasks=options.enable_task_running,
taskqueue_default_http_server=application_address,
user_login_url=user_login_url,
user_logout_url=user_logout_url,
default_gcs_bucket_name=options.default_gcs_bucket_name,
appidentity_oauth_url=options.appidentity_oauth_url,
datastore_grpc_stub_class=(datastore_grpc_stub.DatastoreGrpcStub
if options.support_datastore_emulator else
None))
return APIServer(
options.api_host, options.api_port, app_id,
options.api_server_supports_grpc or options.support_datastore_emulator,
options.grpc_api_port, options.enable_host_checking,
gcd_emulator_launching_thread)
def start(self, options):
"""Start devappserver2 servers based on the provided command line arguments.
Args:
options: An argparse.Namespace containing the command line arguments.
Raises:
PhpPathError: php executable path is not specified for php72.
MissingDatastoreEmulatorError: dev_appserver.py is not invoked from the right
directory.
"""
self._options = options
self._options.datastore_emulator_cmd = self._correct_datastore_emulator_cmd(
self._options.datastore_emulator_cmd)
self._check_datastore_emulator_support()
logging.getLogger().setLevel(constants.LOG_LEVEL_TO_PYTHON_CONSTANT[
options.dev_appserver_log_level])
parsed_env_variables = dict(options.env_variables or [])
configuration = application_configuration.ApplicationConfiguration(
config_paths=options.config_paths,
app_id=options.app_id,
runtime=options.runtime,
env_variables=parsed_env_variables)
all_module_runtimes = {
module.runtime
for module in configuration.modules
}
self._check_platform_support(all_module_runtimes)
storage_path = api_server.get_storage_path(options.storage_path,
configuration.app_id)
datastore_path = api_server.get_datastore_path(storage_path,
options.datastore_path)
datastore_data_type = (
datastore_converter.get_stub_type(datastore_path)
if os.path.isfile(datastore_path) else None)
if options.skip_sdk_update_check:
logging.info('Skipping SDK update check.')
else:
update_checker.check_for_updates(configuration)
# There is no good way to set the default encoding from application code
# (it needs to be done during interpreter initialization in site.py or
# sitecustomize.py) so just warn developers if they have a different
# encoding than production.
if sys.getdefaultencoding() != constants.PROD_DEFAULT_ENCODING:
logging.warning(
'The default encoding of your local Python interpreter is set to %r '
'while App Engine\'s production environment uses %r; as a result '
'your code may behave differently when deployed.',
sys.getdefaultencoding(), constants.PROD_DEFAULT_ENCODING)
if options.port == 0:
logging.warn(
'DEFAULT_VERSION_HOSTNAME will not be set correctly with '
'--port=0')
util.setup_environ(configuration.app_id)
php_version = self._get_php_runtime(configuration)
if not options.php_executable_path and php_version == 'php72':
raise PhpPathError(
'For php72, --php_executable_path must be specified.')
if options.ssl_certificate_path and options.ssl_certificate_key_path:
ssl_certificate_paths = self._create_ssl_certificate_paths_if_valid(
options.ssl_certificate_path, options.ssl_certificate_key_path)
else:
if options.ssl_certificate_path or options.ssl_certificate_key_path:
logging.warn('Must provide both --ssl_certificate_path and '
'--ssl_certificate_key_path to enable SSL. Since '
'only one flag was provided, not using SSL.')
ssl_certificate_paths = None
if options.google_analytics_client_id:
metrics_logger = metrics.GetMetricsLogger()
metrics_logger.Start(
options.google_analytics_client_id,
options.google_analytics_user_agent,
all_module_runtimes,
{module.env or 'standard'
for module in configuration.modules},
options.support_datastore_emulator,
datastore_data_type,
bool(ssl_certificate_paths),
options,
multi_module=len(configuration.modules) > 1,
dispatch_config=configuration.dispatch is not None,
)
self._dispatcher = dispatcher.Dispatcher(
configuration, options.host, options.port, options.auth_domain,
constants.LOG_LEVEL_TO_RUNTIME_CONSTANT[options.log_level],
self._create_php_config(options, php_version),
self._create_python_config(options),
self._create_java_config(options), self._create_go_config(options),
self._create_custom_config(options),
self._create_cloud_sql_config(options),
self._create_vm_config(options),
self._create_module_to_setting(options.max_module_instances,
configuration,
'--max_module_instances'),
options.use_mtime_file_watcher, options.watcher_ignore_re,
options.automatic_restart, options.allow_skipped_files,
self._create_module_to_setting(options.threadsafe_override,
configuration,
'--threadsafe_override'),
options.external_port, options.specified_service_ports,
options.enable_host_checking, ssl_certificate_paths)
wsgi_request_info_ = wsgi_request_info.WSGIRequestInfo(
self._dispatcher)
apiserver = api_server.create_api_server(
wsgi_request_info_, storage_path, options, configuration.app_id,
configuration.modules[0].application_root)
apiserver.start()
self._running_modules.append(apiserver)
self._dispatcher.start(options.api_host, apiserver.port,
wsgi_request_info_)
xsrf_path = os.path.join(storage_path, 'xsrf')
admin = admin_server.AdminServer(options.admin_host,
options.admin_port, self._dispatcher,
configuration, xsrf_path,
options.enable_host_checking,
options.enable_console)
admin.start()
self._running_modules.append(admin)
try:
default = self._dispatcher.get_module_by_name('default')
apiserver.set_balanced_address(default.balanced_address)
except request_info.ModuleDoesNotExistError:
logging.warning('No default module found. Ignoring.')
def create_api_server(
request_info, storage_path, options, app_id, app_root):
"""Creates an API server.
Args:
request_info: An apiproxy_stub.RequestInfo instance used by the stubs to
lookup information about the request associated with an API call.
storage_path: A string directory for storing API stub data.
options: An instance of argparse.Namespace containing command line flags.
app_id: String representing an application ID, used for configuring paths
and string constants in API stubs.
app_root: The path to the directory containing the user's
application e.g. "/home/joe/myapp", used for locating application yaml
files, eg index.yaml for the datastore stub.
Returns:
An instance of APIServer.
Raises:
DatastoreFileError: Cloud Datastore emulator is used while stub_type is
datastore_converter.StubTypes.PYTHON_FILE_STUB.
"""
datastore_path = get_datastore_path(storage_path, options.datastore_path)
logs_path = options.logs_path or os.path.join(storage_path, 'logs.db')
search_index_path = options.search_indexes_path or os.path.join(
storage_path, 'search_indexes')
blobstore_path = options.blobstore_path or os.path.join(
storage_path, 'blobs')
if options.clear_datastore:
_clear_datastore_storage(datastore_path)
if options.clear_search_indexes:
_clear_search_indexes_storage(search_index_path)
if options.auto_id_policy == datastore_stub_util.SEQUENTIAL:
logging.warn("--auto_id_policy='sequential' is deprecated. This option "
"will be removed in a future release.")
application_address = '%s' % options.host
if options.port and options.port != 80:
application_address += ':' + str(options.port)
user_login_url = '/%s?%s=%%s' % (
login.LOGIN_URL_RELATIVE, login.CONTINUE_PARAM)
user_logout_url = '/%s?%s=%%s' % (
login.LOGOUT_URL_RELATIVE, login.CONTINUE_PARAM)
if options.datastore_consistency_policy == 'time':
consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy()
elif options.datastore_consistency_policy == 'random':
consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy()
elif options.datastore_consistency_policy == 'consistent':
consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0)
else:
assert 0, ('unknown consistency policy: %r' %
options.datastore_consistency_policy)
stub_type = (datastore_converter.get_stub_type(datastore_path)
if os.path.exists(datastore_path) else None)
gcd_emulator_launching_thread = None
if options.support_datastore_emulator:
if stub_type == datastore_converter.StubTypes.PYTHON_FILE_STUB:
raise DatastoreFileError(
'The datastore file %s cannot be recognized by dev_appserver. Please '
'restart dev_appserver with --clear_datastore=1' % datastore_path)
# The flag should override environment variable regarding emulator host.
if options.running_datastore_emulator_host:
os.environ['DATASTORE_EMULATOR_HOST'] = (
options.running_datastore_emulator_host)
env_emulator_host = os.environ.get('DATASTORE_EMULATOR_HOST')
if env_emulator_host: # emulator already running, reuse it.
logging.warning(
'Detected environment variable DATASTORE_EMULATOR_HOST=%s, '
'dev_appserver will speak to the Cloud Datastore emulator running on '
'this address. The datastore_path %s will be neglected.\nIf you '
'want datastore to store on %s, remove DATASTORE_EMULATOR_HOST '
'from environment variables and restart dev_appserver',
env_emulator_host, datastore_path, datastore_path)
else:
gcd_emulator_launching_thread = _launch_gcd_emulator(
app_id=app_id,
emulator_port=(options.datastore_emulator_port
if options.datastore_emulator_port else
portpicker.pick_unused_port()),
silent=options.dev_appserver_log_level != 'debug',
index_file=os.path.join(app_root, 'index.yaml'),
require_indexes=options.require_indexes,
datastore_path=datastore_path,
stub_type=stub_type,
cmd=options.datastore_emulator_cmd,
is_test=options.datastore_emulator_is_test_mode,
auto_id_policy=options.auto_id_policy)
else:
# Use SQLite stub.
# For historic reason we are still supporting conversion from file stub to
# SQLite stub data. But this conversion will go away.
if stub_type == datastore_converter.StubTypes.PYTHON_FILE_STUB:
datastore_converter.convert_datastore_file_stub_data_to_sqlite(
app_id, datastore_path)
stub_util.setup_stubs(
request_data=request_info,
app_id=app_id,
application_root=app_root,
# The "trusted" flag is only relevant for Google administrative
# applications.
trusted=getattr(options, 'trusted', False),
appidentity_email_address=options.appidentity_email_address,
appidentity_private_key_path=os.path.abspath(
options.appidentity_private_key_path)
if options.appidentity_private_key_path else None,
blobstore_path=blobstore_path,
datastore_path=datastore_path,
datastore_consistency=consistency,
datastore_require_indexes=options.require_indexes,
datastore_auto_id_policy=options.auto_id_policy,
images_host_prefix='http://%s' % application_address,
logs_path=logs_path,
mail_smtp_host=options.smtp_host,
mail_smtp_port=options.smtp_port,
mail_smtp_user=options.smtp_user,
mail_smtp_password=options.smtp_password,
mail_enable_sendmail=options.enable_sendmail,
mail_show_mail_body=options.show_mail_body,
mail_allow_tls=options.smtp_allow_tls,
search_index_path=search_index_path,
taskqueue_auto_run_tasks=options.enable_task_running,
taskqueue_default_http_server=application_address,
user_login_url=user_login_url,
user_logout_url=user_logout_url,
default_gcs_bucket_name=options.default_gcs_bucket_name,
appidentity_oauth_url=options.appidentity_oauth_url,
datastore_grpc_stub_class=(
datastore_grpc_stub.DatastoreGrpcStub
if options.support_datastore_emulator else None)
)
return APIServer(
options.api_host, options.api_port, app_id,
options.api_server_supports_grpc or options.support_datastore_emulator,
options.grpc_api_port, options.enable_host_checking,
gcd_emulator_launching_thread)
def start(self, options):
"""Start devappserver2 servers based on the provided command line arguments.
Args:
options: An argparse.Namespace containing the command line arguments.
Raises:
PhpPathError: php executable path is not specified for php72.
MissingDatastoreEmulatorError: dev_appserver.py is not invoked from the right
directory.
"""
self._options = options
self._correct_datastore_emulator_cmd()
self._fail_for_using_datastore_emulator_from_legacy_sdk()
self._decide_use_datastore_emulator()
logging.getLogger().setLevel(
constants.LOG_LEVEL_TO_PYTHON_CONSTANT[options.dev_appserver_log_level])
parsed_env_variables = dict(options.env_variables or [])
if options.dev_appserver_log_setup_script:
try:
execfile(options.dev_appserver_log_setup_script, {}, {})
except Exception as e:
logging.exception("Error executing log setup script at %r.",
options.dev_appserver_log_setup_script)
configuration = application_configuration.ApplicationConfiguration(
config_paths=options.config_paths,
app_id=options.app_id,
runtime=options.runtime,
env_variables=parsed_env_variables)
all_module_runtimes = {module.runtime for module in configuration.modules}
self._check_platform_support(all_module_runtimes)
storage_path = api_server.get_storage_path(
options.storage_path, configuration.app_id)
datastore_path = api_server.get_datastore_path(
storage_path, options.datastore_path)
datastore_data_type = (datastore_converter.get_stub_type(datastore_path)
if os.path.isfile(datastore_path) else None)
if options.skip_sdk_update_check:
logging.info('Skipping SDK update check.')
else:
update_checker.check_for_updates(configuration)
# There is no good way to set the default encoding from application code
# (it needs to be done during interpreter initialization in site.py or
# sitecustomize.py) so just warn developers if they have a different
# encoding than production.
if sys.getdefaultencoding() != constants.PROD_DEFAULT_ENCODING:
logging.warning(
'The default encoding of your local Python interpreter is set to %r '
'while App Engine\'s production environment uses %r; as a result '
'your code may behave differently when deployed.',
sys.getdefaultencoding(), constants.PROD_DEFAULT_ENCODING)
if options.port == 0:
logging.warn('DEFAULT_VERSION_HOSTNAME will not be set correctly with '
'--port=0')
util.setup_environ(configuration.app_id)
php_version = self._get_php_runtime(configuration)
if not options.php_executable_path and php_version == 'php72':
raise PhpPathError('For php72, --php_executable_path must be specified.')
if options.ssl_certificate_path and options.ssl_certificate_key_path:
ssl_certificate_paths = self._create_ssl_certificate_paths_if_valid(
options.ssl_certificate_path, options.ssl_certificate_key_path)
else:
if options.ssl_certificate_path or options.ssl_certificate_key_path:
logging.warn('Must provide both --ssl_certificate_path and '
'--ssl_certificate_key_path to enable SSL. Since '
'only one flag was provided, not using SSL.')
ssl_certificate_paths = None
if options.google_analytics_client_id:
metrics_logger = metrics.GetMetricsLogger()
metrics_logger.Start(
options.google_analytics_client_id,
options.google_analytics_user_agent,
all_module_runtimes,
{module.env or 'standard' for module in configuration.modules},
options.support_datastore_emulator, datastore_data_type,
bool(ssl_certificate_paths), options,
multi_module=len(configuration.modules) > 1,
dispatch_config=configuration.dispatch is not None,
grpc_import_report=self.grpc_import_report,
java_major_version=self.java_major_version
)
self._dispatcher = dispatcher.Dispatcher(
configuration, options.host, options.port, options.auth_domain,
constants.LOG_LEVEL_TO_RUNTIME_CONSTANT[options.log_level],
self._create_php_config(options, php_version),
self._create_python_config(options),
self._create_java_config(options),
self._create_go_config(options),
self._create_custom_config(options),
self._create_cloud_sql_config(options),
self._create_vm_config(options),
self._create_module_to_setting(options.max_module_instances,
configuration, '--max_module_instances'),
options.use_mtime_file_watcher, options.watcher_ignore_re,
options.automatic_restart, options.allow_skipped_files,
self._create_module_to_setting(
options.threadsafe_override,
configuration,
'--threadsafe_override'),
options.external_port,
options.specified_service_ports,
options.enable_host_checking,
ssl_certificate_paths)
wsgi_request_info_ = wsgi_request_info.WSGIRequestInfo(self._dispatcher)
apiserver = api_server.create_api_server(
wsgi_request_info_, storage_path, options, configuration.app_id,
configuration.modules[0].application_root)
apiserver.start()
self._running_modules.append(apiserver)
self._dispatcher.start(
options.api_host, apiserver.port, wsgi_request_info_)
xsrf_path = os.path.join(storage_path, 'xsrf')
admin = admin_server.AdminServer(options.admin_host, options.admin_port,
self._dispatcher, configuration, xsrf_path,
options.enable_host_checking,
options.enable_console)
admin.start()
self._running_modules.append(admin)
if options.enable_datastore_translator:
# We do a late import just in case of any problems with importing
# protobuf (although in theory python/wrapper_util.py's path-munging has
# handled them all).
from google.appengine.tools.devappserver2.datastore_translator import (
datastore_translator_server)
translator = datastore_translator_server.DatastoreTranslatorServer(
options.datastore_translator_host, options.datastore_translator_port,
options.enable_host_checking)
translator.start()
self._running_modules.append(translator)
try:
default = self._dispatcher.get_module_by_name('default')
apiserver.set_balanced_address(default.balanced_address)
except request_info.ModuleDoesNotExistError:
logging.warning('No default module found. Ignoring.')