def create_api_server(request_info, storage_path, options, app_id, app_root): """Creates an API server. Args: request_info: An apiproxy_stub.RequestInfo instance used by the stubs to lookup information about the request associated with an API call. storage_path: A string directory for storing API stub data. options: An instance of argparse.Namespace containing command line flags. app_id: String representing an application ID, used for configuring paths and string constants in API stubs. app_root: The path to the directory containing the user's application e.g. "/home/joe/myapp", used for locating application yaml files, eg index.yaml for the datastore stub. Returns: An instance of APIServer. Raises: DatastoreFileError: Cloud Datastore emulator is used while stub_type is datastore_converter.StubTypes.PYTHON_FILE_STUB. """ datastore_path = get_datastore_path(storage_path, options.datastore_path) logs_path = options.logs_path or os.path.join(storage_path, 'logs.db') search_index_path = options.search_indexes_path or os.path.join( storage_path, 'search_indexes') blobstore_path = options.blobstore_path or os.path.join( storage_path, 'blobs') if options.clear_datastore: _clear_datastore_storage(datastore_path) if options.clear_search_indexes: _clear_search_indexes_storage(search_index_path) if options.auto_id_policy == datastore_stub_util.SEQUENTIAL: logging.warn( "--auto_id_policy='sequential' is deprecated. This option " "will be removed in a future release.") application_address = '%s' % options.host if options.port and options.port != 80: application_address += ':' + str(options.port) user_login_url = '/%s?%s=%%s' % (login.LOGIN_URL_RELATIVE, login.CONTINUE_PARAM) user_logout_url = '/%s?%s=%%s' % (login.LOGOUT_URL_RELATIVE, login.CONTINUE_PARAM) if options.datastore_consistency_policy == 'time': consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy() elif options.datastore_consistency_policy == 'random': consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy() elif options.datastore_consistency_policy == 'consistent': consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0) else: assert 0, ('unknown consistency policy: %r' % options.datastore_consistency_policy) stub_type = (datastore_converter.get_stub_type(datastore_path) if os.path.exists(datastore_path) else None) gcd_emulator_launching_thread = None if options.support_datastore_emulator: if stub_type == datastore_converter.StubTypes.PYTHON_FILE_STUB: raise DatastoreFileError( 'The datastore file %s cannot be recognized by dev_appserver. Please ' 'restart dev_appserver with --clear_datastore=1' % datastore_path) env_emulator_host = os.environ.get('DATASTORE_EMULATOR_HOST') if env_emulator_host: # emulator already running, reuse it. logging.warning( 'Detected environment variable DATASTORE_EMULATOR_HOST=%s, ' 'dev_appserver will speak to the Cloud Datastore emulator running on ' 'this address. The datastore_path %s will be neglected.\nIf you ' 'want datastore to store on %s, remove DATASTORE_EMULATOR_HOST ' 'from environment variables and restart dev_appserver', env_emulator_host, datastore_path, datastore_path) else: gcd_emulator_launching_thread = _launch_gcd_emulator( app_id=app_id, emulator_port=(options.datastore_emulator_port if options.datastore_emulator_port else portpicker.PickUnusedPort()), silent=options.dev_appserver_log_level != 'debug', index_file=os.path.join(app_root, 'index.yaml'), require_indexes=options.require_indexes, datastore_path=datastore_path, stub_type=stub_type, cmd=options.datastore_emulator_cmd, is_test=options.datastore_emulator_is_test_mode) else: # Use SQLite stub. # For historic reason we are still supporting conversion from file stub to # SQLite stub data. But this conversion will go away. if stub_type == datastore_converter.StubTypes.PYTHON_FILE_STUB: datastore_converter.convert_datastore_file_stub_data_to_sqlite( app_id, datastore_path) stub_util.setup_stubs( request_data=request_info, app_id=app_id, application_root=app_root, # The "trusted" flag is only relevant for Google administrative # applications. trusted=getattr(options, 'trusted', False), appidentity_email_address=options.appidentity_email_address, appidentity_private_key_path=os.path.abspath( options.appidentity_private_key_path) if options.appidentity_private_key_path else None, blobstore_path=blobstore_path, datastore_path=datastore_path, datastore_consistency=consistency, datastore_require_indexes=options.require_indexes, datastore_auto_id_policy=options.auto_id_policy, images_host_prefix='http://%s' % application_address, logs_path=logs_path, mail_smtp_host=options.smtp_host, mail_smtp_port=options.smtp_port, mail_smtp_user=options.smtp_user, mail_smtp_password=options.smtp_password, mail_enable_sendmail=options.enable_sendmail, mail_show_mail_body=options.show_mail_body, mail_allow_tls=options.smtp_allow_tls, search_index_path=search_index_path, taskqueue_auto_run_tasks=options.enable_task_running, taskqueue_default_http_server=application_address, user_login_url=user_login_url, user_logout_url=user_logout_url, default_gcs_bucket_name=options.default_gcs_bucket_name, appidentity_oauth_url=options.appidentity_oauth_url, datastore_grpc_stub_class=(datastore_grpc_stub.DatastoreGrpcStub if options.support_datastore_emulator else None)) return APIServer( options.api_host, options.api_port, app_id, options.api_server_supports_grpc or options.support_datastore_emulator, options.grpc_api_port, options.enable_host_checking, gcd_emulator_launching_thread)
def start(self, options): """Start devappserver2 servers based on the provided command line arguments. Args: options: An argparse.Namespace containing the command line arguments. Raises: PhpPathError: php executable path is not specified for php72. MissingDatastoreEmulatorError: dev_appserver.py is not invoked from the right directory. """ self._options = options self._options.datastore_emulator_cmd = self._correct_datastore_emulator_cmd( self._options.datastore_emulator_cmd) self._check_datastore_emulator_support() logging.getLogger().setLevel(constants.LOG_LEVEL_TO_PYTHON_CONSTANT[ options.dev_appserver_log_level]) parsed_env_variables = dict(options.env_variables or []) configuration = application_configuration.ApplicationConfiguration( config_paths=options.config_paths, app_id=options.app_id, runtime=options.runtime, env_variables=parsed_env_variables) all_module_runtimes = { module.runtime for module in configuration.modules } self._check_platform_support(all_module_runtimes) storage_path = api_server.get_storage_path(options.storage_path, configuration.app_id) datastore_path = api_server.get_datastore_path(storage_path, options.datastore_path) datastore_data_type = ( datastore_converter.get_stub_type(datastore_path) if os.path.isfile(datastore_path) else None) if options.skip_sdk_update_check: logging.info('Skipping SDK update check.') else: update_checker.check_for_updates(configuration) # There is no good way to set the default encoding from application code # (it needs to be done during interpreter initialization in site.py or # sitecustomize.py) so just warn developers if they have a different # encoding than production. if sys.getdefaultencoding() != constants.PROD_DEFAULT_ENCODING: logging.warning( 'The default encoding of your local Python interpreter is set to %r ' 'while App Engine\'s production environment uses %r; as a result ' 'your code may behave differently when deployed.', sys.getdefaultencoding(), constants.PROD_DEFAULT_ENCODING) if options.port == 0: logging.warn( 'DEFAULT_VERSION_HOSTNAME will not be set correctly with ' '--port=0') util.setup_environ(configuration.app_id) php_version = self._get_php_runtime(configuration) if not options.php_executable_path and php_version == 'php72': raise PhpPathError( 'For php72, --php_executable_path must be specified.') if options.ssl_certificate_path and options.ssl_certificate_key_path: ssl_certificate_paths = self._create_ssl_certificate_paths_if_valid( options.ssl_certificate_path, options.ssl_certificate_key_path) else: if options.ssl_certificate_path or options.ssl_certificate_key_path: logging.warn('Must provide both --ssl_certificate_path and ' '--ssl_certificate_key_path to enable SSL. Since ' 'only one flag was provided, not using SSL.') ssl_certificate_paths = None if options.google_analytics_client_id: metrics_logger = metrics.GetMetricsLogger() metrics_logger.Start( options.google_analytics_client_id, options.google_analytics_user_agent, all_module_runtimes, {module.env or 'standard' for module in configuration.modules}, options.support_datastore_emulator, datastore_data_type, bool(ssl_certificate_paths), options, multi_module=len(configuration.modules) > 1, dispatch_config=configuration.dispatch is not None, ) self._dispatcher = dispatcher.Dispatcher( configuration, options.host, options.port, options.auth_domain, constants.LOG_LEVEL_TO_RUNTIME_CONSTANT[options.log_level], self._create_php_config(options, php_version), self._create_python_config(options), self._create_java_config(options), self._create_go_config(options), self._create_custom_config(options), self._create_cloud_sql_config(options), self._create_vm_config(options), self._create_module_to_setting(options.max_module_instances, configuration, '--max_module_instances'), options.use_mtime_file_watcher, options.watcher_ignore_re, options.automatic_restart, options.allow_skipped_files, self._create_module_to_setting(options.threadsafe_override, configuration, '--threadsafe_override'), options.external_port, options.specified_service_ports, options.enable_host_checking, ssl_certificate_paths) wsgi_request_info_ = wsgi_request_info.WSGIRequestInfo( self._dispatcher) apiserver = api_server.create_api_server( wsgi_request_info_, storage_path, options, configuration.app_id, configuration.modules[0].application_root) apiserver.start() self._running_modules.append(apiserver) self._dispatcher.start(options.api_host, apiserver.port, wsgi_request_info_) xsrf_path = os.path.join(storage_path, 'xsrf') admin = admin_server.AdminServer(options.admin_host, options.admin_port, self._dispatcher, configuration, xsrf_path, options.enable_host_checking, options.enable_console) admin.start() self._running_modules.append(admin) try: default = self._dispatcher.get_module_by_name('default') apiserver.set_balanced_address(default.balanced_address) except request_info.ModuleDoesNotExistError: logging.warning('No default module found. Ignoring.')
def create_api_server( request_info, storage_path, options, app_id, app_root): """Creates an API server. Args: request_info: An apiproxy_stub.RequestInfo instance used by the stubs to lookup information about the request associated with an API call. storage_path: A string directory for storing API stub data. options: An instance of argparse.Namespace containing command line flags. app_id: String representing an application ID, used for configuring paths and string constants in API stubs. app_root: The path to the directory containing the user's application e.g. "/home/joe/myapp", used for locating application yaml files, eg index.yaml for the datastore stub. Returns: An instance of APIServer. Raises: DatastoreFileError: Cloud Datastore emulator is used while stub_type is datastore_converter.StubTypes.PYTHON_FILE_STUB. """ datastore_path = get_datastore_path(storage_path, options.datastore_path) logs_path = options.logs_path or os.path.join(storage_path, 'logs.db') search_index_path = options.search_indexes_path or os.path.join( storage_path, 'search_indexes') blobstore_path = options.blobstore_path or os.path.join( storage_path, 'blobs') if options.clear_datastore: _clear_datastore_storage(datastore_path) if options.clear_search_indexes: _clear_search_indexes_storage(search_index_path) if options.auto_id_policy == datastore_stub_util.SEQUENTIAL: logging.warn("--auto_id_policy='sequential' is deprecated. This option " "will be removed in a future release.") application_address = '%s' % options.host if options.port and options.port != 80: application_address += ':' + str(options.port) user_login_url = '/%s?%s=%%s' % ( login.LOGIN_URL_RELATIVE, login.CONTINUE_PARAM) user_logout_url = '/%s?%s=%%s' % ( login.LOGOUT_URL_RELATIVE, login.CONTINUE_PARAM) if options.datastore_consistency_policy == 'time': consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy() elif options.datastore_consistency_policy == 'random': consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy() elif options.datastore_consistency_policy == 'consistent': consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0) else: assert 0, ('unknown consistency policy: %r' % options.datastore_consistency_policy) stub_type = (datastore_converter.get_stub_type(datastore_path) if os.path.exists(datastore_path) else None) gcd_emulator_launching_thread = None if options.support_datastore_emulator: if stub_type == datastore_converter.StubTypes.PYTHON_FILE_STUB: raise DatastoreFileError( 'The datastore file %s cannot be recognized by dev_appserver. Please ' 'restart dev_appserver with --clear_datastore=1' % datastore_path) # The flag should override environment variable regarding emulator host. if options.running_datastore_emulator_host: os.environ['DATASTORE_EMULATOR_HOST'] = ( options.running_datastore_emulator_host) env_emulator_host = os.environ.get('DATASTORE_EMULATOR_HOST') if env_emulator_host: # emulator already running, reuse it. logging.warning( 'Detected environment variable DATASTORE_EMULATOR_HOST=%s, ' 'dev_appserver will speak to the Cloud Datastore emulator running on ' 'this address. The datastore_path %s will be neglected.\nIf you ' 'want datastore to store on %s, remove DATASTORE_EMULATOR_HOST ' 'from environment variables and restart dev_appserver', env_emulator_host, datastore_path, datastore_path) else: gcd_emulator_launching_thread = _launch_gcd_emulator( app_id=app_id, emulator_port=(options.datastore_emulator_port if options.datastore_emulator_port else portpicker.pick_unused_port()), silent=options.dev_appserver_log_level != 'debug', index_file=os.path.join(app_root, 'index.yaml'), require_indexes=options.require_indexes, datastore_path=datastore_path, stub_type=stub_type, cmd=options.datastore_emulator_cmd, is_test=options.datastore_emulator_is_test_mode, auto_id_policy=options.auto_id_policy) else: # Use SQLite stub. # For historic reason we are still supporting conversion from file stub to # SQLite stub data. But this conversion will go away. if stub_type == datastore_converter.StubTypes.PYTHON_FILE_STUB: datastore_converter.convert_datastore_file_stub_data_to_sqlite( app_id, datastore_path) stub_util.setup_stubs( request_data=request_info, app_id=app_id, application_root=app_root, # The "trusted" flag is only relevant for Google administrative # applications. trusted=getattr(options, 'trusted', False), appidentity_email_address=options.appidentity_email_address, appidentity_private_key_path=os.path.abspath( options.appidentity_private_key_path) if options.appidentity_private_key_path else None, blobstore_path=blobstore_path, datastore_path=datastore_path, datastore_consistency=consistency, datastore_require_indexes=options.require_indexes, datastore_auto_id_policy=options.auto_id_policy, images_host_prefix='http://%s' % application_address, logs_path=logs_path, mail_smtp_host=options.smtp_host, mail_smtp_port=options.smtp_port, mail_smtp_user=options.smtp_user, mail_smtp_password=options.smtp_password, mail_enable_sendmail=options.enable_sendmail, mail_show_mail_body=options.show_mail_body, mail_allow_tls=options.smtp_allow_tls, search_index_path=search_index_path, taskqueue_auto_run_tasks=options.enable_task_running, taskqueue_default_http_server=application_address, user_login_url=user_login_url, user_logout_url=user_logout_url, default_gcs_bucket_name=options.default_gcs_bucket_name, appidentity_oauth_url=options.appidentity_oauth_url, datastore_grpc_stub_class=( datastore_grpc_stub.DatastoreGrpcStub if options.support_datastore_emulator else None) ) return APIServer( options.api_host, options.api_port, app_id, options.api_server_supports_grpc or options.support_datastore_emulator, options.grpc_api_port, options.enable_host_checking, gcd_emulator_launching_thread)
def start(self, options): """Start devappserver2 servers based on the provided command line arguments. Args: options: An argparse.Namespace containing the command line arguments. Raises: PhpPathError: php executable path is not specified for php72. MissingDatastoreEmulatorError: dev_appserver.py is not invoked from the right directory. """ self._options = options self._correct_datastore_emulator_cmd() self._fail_for_using_datastore_emulator_from_legacy_sdk() self._decide_use_datastore_emulator() logging.getLogger().setLevel( constants.LOG_LEVEL_TO_PYTHON_CONSTANT[options.dev_appserver_log_level]) parsed_env_variables = dict(options.env_variables or []) if options.dev_appserver_log_setup_script: try: execfile(options.dev_appserver_log_setup_script, {}, {}) except Exception as e: logging.exception("Error executing log setup script at %r.", options.dev_appserver_log_setup_script) configuration = application_configuration.ApplicationConfiguration( config_paths=options.config_paths, app_id=options.app_id, runtime=options.runtime, env_variables=parsed_env_variables) all_module_runtimes = {module.runtime for module in configuration.modules} self._check_platform_support(all_module_runtimes) storage_path = api_server.get_storage_path( options.storage_path, configuration.app_id) datastore_path = api_server.get_datastore_path( storage_path, options.datastore_path) datastore_data_type = (datastore_converter.get_stub_type(datastore_path) if os.path.isfile(datastore_path) else None) if options.skip_sdk_update_check: logging.info('Skipping SDK update check.') else: update_checker.check_for_updates(configuration) # There is no good way to set the default encoding from application code # (it needs to be done during interpreter initialization in site.py or # sitecustomize.py) so just warn developers if they have a different # encoding than production. if sys.getdefaultencoding() != constants.PROD_DEFAULT_ENCODING: logging.warning( 'The default encoding of your local Python interpreter is set to %r ' 'while App Engine\'s production environment uses %r; as a result ' 'your code may behave differently when deployed.', sys.getdefaultencoding(), constants.PROD_DEFAULT_ENCODING) if options.port == 0: logging.warn('DEFAULT_VERSION_HOSTNAME will not be set correctly with ' '--port=0') util.setup_environ(configuration.app_id) php_version = self._get_php_runtime(configuration) if not options.php_executable_path and php_version == 'php72': raise PhpPathError('For php72, --php_executable_path must be specified.') if options.ssl_certificate_path and options.ssl_certificate_key_path: ssl_certificate_paths = self._create_ssl_certificate_paths_if_valid( options.ssl_certificate_path, options.ssl_certificate_key_path) else: if options.ssl_certificate_path or options.ssl_certificate_key_path: logging.warn('Must provide both --ssl_certificate_path and ' '--ssl_certificate_key_path to enable SSL. Since ' 'only one flag was provided, not using SSL.') ssl_certificate_paths = None if options.google_analytics_client_id: metrics_logger = metrics.GetMetricsLogger() metrics_logger.Start( options.google_analytics_client_id, options.google_analytics_user_agent, all_module_runtimes, {module.env or 'standard' for module in configuration.modules}, options.support_datastore_emulator, datastore_data_type, bool(ssl_certificate_paths), options, multi_module=len(configuration.modules) > 1, dispatch_config=configuration.dispatch is not None, grpc_import_report=self.grpc_import_report, java_major_version=self.java_major_version ) self._dispatcher = dispatcher.Dispatcher( configuration, options.host, options.port, options.auth_domain, constants.LOG_LEVEL_TO_RUNTIME_CONSTANT[options.log_level], self._create_php_config(options, php_version), self._create_python_config(options), self._create_java_config(options), self._create_go_config(options), self._create_custom_config(options), self._create_cloud_sql_config(options), self._create_vm_config(options), self._create_module_to_setting(options.max_module_instances, configuration, '--max_module_instances'), options.use_mtime_file_watcher, options.watcher_ignore_re, options.automatic_restart, options.allow_skipped_files, self._create_module_to_setting( options.threadsafe_override, configuration, '--threadsafe_override'), options.external_port, options.specified_service_ports, options.enable_host_checking, ssl_certificate_paths) wsgi_request_info_ = wsgi_request_info.WSGIRequestInfo(self._dispatcher) apiserver = api_server.create_api_server( wsgi_request_info_, storage_path, options, configuration.app_id, configuration.modules[0].application_root) apiserver.start() self._running_modules.append(apiserver) self._dispatcher.start( options.api_host, apiserver.port, wsgi_request_info_) xsrf_path = os.path.join(storage_path, 'xsrf') admin = admin_server.AdminServer(options.admin_host, options.admin_port, self._dispatcher, configuration, xsrf_path, options.enable_host_checking, options.enable_console) admin.start() self._running_modules.append(admin) if options.enable_datastore_translator: # We do a late import just in case of any problems with importing # protobuf (although in theory python/wrapper_util.py's path-munging has # handled them all). from google.appengine.tools.devappserver2.datastore_translator import ( datastore_translator_server) translator = datastore_translator_server.DatastoreTranslatorServer( options.datastore_translator_host, options.datastore_translator_port, options.enable_host_checking) translator.start() self._running_modules.append(translator) try: default = self._dispatcher.get_module_by_name('default') apiserver.set_balanced_address(default.balanced_address) except request_info.ModuleDoesNotExistError: logging.warning('No default module found. Ignoring.')