def setup_stub(high_replication=False): apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap() stub = datastore_file_stub.DatastoreFileStub('test','/dev/null', '/dev/null', trusted=True) if high_replication: stub.SetConsistencyPolicy( datastore_stub_util.TimeBasedHRConsistencyPolicy()) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', stub) apiproxy_stub_map.apiproxy.RegisterStub( 'user', user_service_stub.UserServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'memcache', memcache_stub.MemcacheServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'urlfetch', urlfetch_stub.URLFetchServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'taskqueue', taskqueue_stub.TaskQueueServiceStub()) try: apiproxy_stub_map.apiproxy.RegisterStub( 'images', images_stub.ImagesServiceStub()) except NameError: pass
def setup_stubs(): """Setup the API stubs. This can only be done once.""" api_server.setup_test_stubs( request_data, app_id=APP_ID, application_root=APPLICATION_ROOT, trusted=TRUSTED, blobstore_path=BLOBSTORE_PATH, datastore_consistency=datastore_stub_util.TimeBasedHRConsistencyPolicy( ), datastore_path=DATASTORE_PATH, datastore_require_indexes=DATASTORE_REQUIRE_INDEXES, images_host_prefix=IMAGES_HOST_PREFIX, logs_path=':memory:', mail_smtp_host=MAIL_SMTP_HOST, mail_smtp_port=MAIL_SMTP_PORT, mail_smtp_user=MAIL_SMTP_USER, mail_smtp_password=MAIL_SMTP_PASSWORD, mail_enable_sendmail=MAIL_ENABLE_SENDMAIL, mail_show_mail_body=MAIL_SHOW_MAIL_BODY, taskqueue_auto_run_tasks=TASKQUEUE_AUTO_RUN_TASKS, taskqueue_default_http_server=TASKQUEUE_DEFAULT_HTTP_SERVER, user_login_url=USER_LOGIN_URL, user_logout_url=USER_LOGOUT_URL) apiproxy_stub_map.apiproxy.ReplaceStub('urlfetch', FakeURLFetchServiceStub()) apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v4', FakeDatastoreV4ServiceStub())
def InitAppHostingApi(): """Initialize stubs for various app hosting APIs.""" # clear ndb's context cache # see https://developers.google.com/appengine/docs/python/ndb/cache ndb.get_context().clear_cache() # Pretend we're running in the dev_appserver os.environ['SERVER_SOFTWARE'] = 'Development/unittests' # used by app_identity.get_default_version_hostname() os.environ['DEFAULT_VERSION_HOSTNAME'] = 'localhost:8080' appid = os.environ['APPLICATION_ID'] = 'app' apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap() # Need an HRD stub to support XG transactions hrd_policy = datastore_stub_util.TimeBasedHRConsistencyPolicy() apiproxy_stub_map.apiproxy.RegisterStub( 'datastore_v3', datastore_file_stub.DatastoreFileStub(appid, '/dev/null', '/dev/null', trusted=True, require_indexes=True, consistency_policy=hrd_policy)) # memcache stub apiproxy_stub_map.apiproxy.RegisterStub( 'memcache', memcache_stub.MemcacheServiceStub()) # urlfetch stub apiproxy_stub_map.apiproxy.RegisterStub( 'urlfetch', urlfetch_stub.URLFetchServiceStub()) # blobstore stub temp_dir = tempfile.gettempdir() storage_directory = os.path.join(temp_dir, 'blob_storage') if os.access(storage_directory, os.F_OK): shutil.rmtree(storage_directory) blob_storage = file_blob_storage.FileBlobStorage(storage_directory, appid) apiproxy_stub_map.apiproxy.RegisterStub( 'blobstore', blobstore_stub.BlobstoreServiceStub(blob_storage)) # file stub, required by blobstore stub apiproxy_stub_map.apiproxy.RegisterStub( 'file', file_service_stub.FileServiceStub(blob_storage))
def setup_datastore_stubs(): if "test" in sys.argv: return from google.appengine.datastore import datastore_sqlite_stub from google.appengine.api import apiproxy_stub_map from google.appengine.datastore import datastore_stub_util app_id = application_id() datastore = datastore_sqlite_stub.DatastoreSqliteStub( "dev~" + app_id, os.path.join(data_root(), "datastore.db"), require_indexes=False, trusted=False, root_path=find_project_root(), use_atexit=True) datastore.SetConsistencyPolicy( datastore_stub_util.TimeBasedHRConsistencyPolicy()) apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', datastore)
def create_api_server( request_info, storage_path, options, app_id, app_root): """Creates an API server. Args: request_info: An apiproxy_stub.RequestInfo instance used by the stubs to lookup information about the request associated with an API call. storage_path: A string directory for storing API stub data. options: An instance of argparse.Namespace containing command line flags. app_id: String representing an application ID, used for configuring paths and string constants in API stubs. app_root: The path to the directory containing the user's application e.g. "/home/joe/myapp", used for locating application yaml files, eg index.yaml for the datastore stub. Returns: An instance of APIServer. """ datastore_path = options.datastore_path or os.path.join( storage_path, 'datastore.db') logs_path = options.logs_path or os.path.join(storage_path, 'logs.db') search_index_path = options.search_indexes_path or os.path.join( storage_path, 'search_indexes') blobstore_path = options.blobstore_path or os.path.join( storage_path, 'blobs') if options.clear_datastore: _clear_datastore_storage(datastore_path) if options.clear_search_indexes: _clear_search_indexes_storage(search_index_path) if options.auto_id_policy == datastore_stub_util.SEQUENTIAL: logging.warn("--auto_id_policy='sequential' is deprecated. This option " "will be removed in a future release.") application_address = '%s' % options.host if options.port and options.port != 80: application_address += ':' + str(options.port) user_login_url = '/%s?%s=%%s' % ( login.LOGIN_URL_RELATIVE, login.CONTINUE_PARAM) user_logout_url = '/%s?%s=%%s' % ( login.LOGOUT_URL_RELATIVE, login.CONTINUE_PARAM) if options.datastore_consistency_policy == 'time': consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy() elif options.datastore_consistency_policy == 'random': consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy() elif options.datastore_consistency_policy == 'consistent': consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0) else: assert 0, ('unknown consistency policy: %r' % options.datastore_consistency_policy) maybe_convert_datastore_file_stub_data_to_sqlite(app_id, datastore_path) setup_stubs( request_data=request_info, app_id=app_id, application_root=app_root, # The "trusted" flag is only relevant for Google administrative # applications. trusted=getattr(options, 'trusted', False), appidentity_email_address=options.appidentity_email_address, appidentity_private_key_path=os.path.abspath( options.appidentity_private_key_path) if options.appidentity_private_key_path else None, blobstore_path=blobstore_path, datastore_path=datastore_path, datastore_consistency=consistency, datastore_require_indexes=options.require_indexes, datastore_auto_id_policy=options.auto_id_policy, images_host_prefix='http://%s' % application_address, logs_path=logs_path, mail_smtp_host=options.smtp_host, mail_smtp_port=options.smtp_port, mail_smtp_user=options.smtp_user, mail_smtp_password=options.smtp_password, mail_enable_sendmail=options.enable_sendmail, mail_show_mail_body=options.show_mail_body, mail_allow_tls=options.smtp_allow_tls, search_index_path=search_index_path, taskqueue_auto_run_tasks=options.enable_task_running, taskqueue_default_http_server=application_address, user_login_url=user_login_url, user_logout_url=user_logout_url, default_gcs_bucket_name=options.default_gcs_bucket_name, appidentity_oauth_url=options.appidentity_oauth_url, support_datastore_emulator=options.support_datastore_emulator) return APIServer( options.api_host, options.api_port, app_id, options.api_server_supports_grpc or options.support_datastore_emulator, options.grpc_api_port)
def _SetupStubs( app_id, application_root, trusted, blobstore_path, use_sqlite, auto_id_policy, high_replication, datastore_path, datastore_require_indexes, images_host_prefix, logs_path, mail_smtp_host, mail_smtp_port, mail_smtp_user, mail_smtp_password, mail_enable_sendmail, mail_show_mail_body, matcher_prospective_search_path, taskqueue_auto_run_tasks, taskqueue_task_retry_seconds, taskqueue_default_http_server, user_login_url, user_logout_url): """Configures the APIs hosted by this server. Args: app_id: The str application id e.g. "guestbook". application_root: The path to the directory containing the user's application e.g. "/home/bquinlan/myapp". trusted: A bool indicating if privileged APIs should be made available. blobstore_path: The path to the file that should be used for blobstore storage. use_sqlite: A bool indicating whether DatastoreSqliteStub or DatastoreFileStub should be used. auto_id_policy: One of datastore_stub_util.SEQUENTIAL or .SCATTERED, indicating whether the Datastore stub should assign IDs sequentially or scattered. high_replication: A bool indicating whether to use the high replication consistency model. datastore_path: The path to the file that should be used for datastore storage. datastore_require_indexes: A bool indicating if the same production datastore indexes requirements should be enforced i.e. if True then a google.appengine.ext.db.NeedIndexError will be be raised if a query is executed without the required indexes. images_host_prefix: The URL prefix (protocol://host:port) to preprend to image urls on calls to images.GetUrlBase. logs_path: Path to the file to store the logs data in. mail_smtp_host: The SMTP hostname that should be used when sending e-mails. If None then the mail_enable_sendmail argument is considered. mail_smtp_port: The SMTP port number that should be used when sending e-mails. If this value is None then mail_smtp_host must also be None. mail_smtp_user: The username to use when authenticating with the SMTP server. This value may be None if mail_smtp_host is also None or if the SMTP server does not require authentication. mail_smtp_password: The password to use when authenticating with the SMTP server. This value may be None if mail_smtp_host or mail_smtp_user is also None. mail_enable_sendmail: A bool indicating if sendmail should be used when sending e-mails. This argument is ignored if mail_smtp_host is not None. mail_show_mail_body: A bool indicating whether the body of sent e-mails should be written to the logs. matcher_prospective_search_path: The path to the file that should be used to save prospective search subscriptions. taskqueue_auto_run_tasks: A bool indicating whether taskqueue tasks should be run automatically or it the must be manually triggered. taskqueue_task_retry_seconds: An int representing the number of seconds to wait before a retrying a failed taskqueue task. taskqueue_default_http_server: A str containing the address of the http server that should be used to execute tasks. user_login_url: A str containing the url that should be used for user login. user_logout_url: A str containing the url that should be used for user logout. """ os.environ['APPLICATION_ID'] = app_id apiproxy_stub_map.apiproxy.RegisterStub( 'app_identity_service', app_identity_stub.AppIdentityServiceStub()) blob_storage = file_blob_storage.FileBlobStorage(blobstore_path, app_id) apiproxy_stub_map.apiproxy.RegisterStub( 'blobstore', blobstore_stub.BlobstoreServiceStub(blob_storage)) apiproxy_stub_map.apiproxy.RegisterStub( 'capability_service', capability_stub.CapabilityServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'channel', channel_service_stub.ChannelServiceStub()) if use_sqlite: datastore = datastore_sqlite_stub.DatastoreSqliteStub( app_id, datastore_path, datastore_require_indexes, trusted, root_path=application_root, auto_id_policy=auto_id_policy) else: datastore = datastore_file_stub.DatastoreFileStub( app_id, datastore_path, datastore_require_indexes, trusted, root_path=application_root, auto_id_policy=auto_id_policy) if high_replication: datastore.SetConsistencyPolicy( datastore_stub_util.TimeBasedHRConsistencyPolicy()) apiproxy_stub_map.apiproxy.RegisterStub( 'datastore_v3', datastore) apiproxy_stub_map.apiproxy.RegisterStub( 'file', file_service_stub.FileServiceStub(blob_storage)) try: from google.appengine.api.images import images_stub except ImportError: logging.warning('Could not initialize images API; you are likely missing ' 'the Python "PIL" module.') from google.appengine.api.images import images_not_implemented_stub apiproxy_stub_map.apiproxy.RegisterStub( 'images', images_not_implemented_stub.ImagesNotImplementedServiceStub()) else: apiproxy_stub_map.apiproxy.RegisterStub( 'images', images_stub.ImagesServiceStub(host_prefix=images_host_prefix)) apiproxy_stub_map.apiproxy.RegisterStub( 'logservice', logservice_stub.LogServiceStub(persist=True)) apiproxy_stub_map.apiproxy.RegisterStub( 'mail', mail_stub.MailServiceStub(mail_smtp_host, mail_smtp_port, mail_smtp_user, mail_smtp_password, enable_sendmail=mail_enable_sendmail, show_mail_body=mail_show_mail_body)) apiproxy_stub_map.apiproxy.RegisterStub( 'memcache', memcache_stub.MemcacheServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'search', simple_search_stub.SearchServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub('system', system_stub.SystemServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'taskqueue', taskqueue_stub.TaskQueueServiceStub( root_path=application_root, auto_task_running=taskqueue_auto_run_tasks, task_retry_seconds=taskqueue_task_retry_seconds, default_http_server=taskqueue_default_http_server)) apiproxy_stub_map.apiproxy.GetStub('taskqueue').StartBackgroundExecution() apiproxy_stub_map.apiproxy.RegisterStub( 'urlfetch', urlfetch_stub.URLFetchServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'user', user_service_stub.UserServiceStub(login_url=user_login_url, logout_url=user_logout_url)) apiproxy_stub_map.apiproxy.RegisterStub( 'xmpp', xmpp_service_stub.XmppServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'matcher', prospective_search_stub.ProspectiveSearchStub( matcher_prospective_search_path, apiproxy_stub_map.apiproxy.GetStub('taskqueue')))
def _create_api_server(request_data, storage_path, options, configuration): datastore_path = options.datastore_path or os.path.join( storage_path, 'datastore.db') logs_path = options.logs_path or os.path.join(storage_path, 'logs.db') search_index_path = options.search_indexes_path or os.path.join( storage_path, 'search_indexes') prospective_search_path = options.prospective_search_path or os.path.join( storage_path, 'prospective-search') blobstore_path = options.blobstore_path or os.path.join( storage_path, 'blobs') if options.clear_datastore: _clear_datastore_storage(datastore_path) if options.clear_prospective_search: _clear_prospective_search_storage(prospective_search_path) if options.clear_search_indexes: _clear_search_indexes_storage(search_index_path) if options.auto_id_policy == datastore_stub_util.SEQUENTIAL: logging.warn( "--auto_id_policy='sequential' is deprecated. This option " "will be removed in a future release.") application_address = '%s' % options.host if options.port and options.port != 80: application_address += ':' + str(options.port) user_login_url = '/%s?%s=%%s' % (login.LOGIN_URL_RELATIVE, login.CONTINUE_PARAM) user_logout_url = '%s&%s=%s' % (user_login_url, login.ACTION_PARAM, login.LOGOUT_ACTION) if options.datastore_consistency_policy == 'time': consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy() elif options.datastore_consistency_policy == 'random': consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy() elif options.datastore_consistency_policy == 'consistent': consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy( 1.0) else: assert 0, ('unknown consistency policy: %r' % options.datastore_consistency_policy) api_server.maybe_convert_datastore_file_stub_data_to_sqlite( configuration.app_id, datastore_path) api_server.setup_stubs( request_data=request_data, app_id=configuration.app_id, application_root=configuration.modules[0].application_root, # The "trusted" flag is only relevant for Google administrative # applications. trusted=getattr(options, 'trusted', False), appidentity_email_address=options.appidentity_email_address, appidentity_private_key_path=os.path.abspath( options.appidentity_private_key_path) if options.appidentity_private_key_path else None, blobstore_path=blobstore_path, datastore_path=datastore_path, datastore_consistency=consistency, datastore_require_indexes=options.require_indexes, datastore_auto_id_policy=options.auto_id_policy, images_host_prefix='http://%s' % application_address, logs_path=logs_path, mail_smtp_host=options.smtp_host, mail_smtp_port=options.smtp_port, mail_smtp_user=options.smtp_user, mail_smtp_password=options.smtp_password, mail_enable_sendmail=options.enable_sendmail, mail_show_mail_body=options.show_mail_body, mail_allow_tls=options.smtp_allow_tls, matcher_prospective_search_path=prospective_search_path, search_index_path=search_index_path, taskqueue_auto_run_tasks=options.enable_task_running, taskqueue_default_http_server=application_address, user_login_url=user_login_url, user_logout_url=user_logout_url, default_gcs_bucket_name=options.default_gcs_bucket_name) return api_server.APIServer(options.api_host, options.api_port, configuration.app_id)
def create_api_server(request_info, storage_path, options, app_id, app_root): """Creates an API server. Args: request_info: An apiproxy_stub.RequestInfo instance used by the stubs to lookup information about the request associated with an API call. storage_path: A string directory for storing API stub data. options: An instance of argparse.Namespace containing command line flags. app_id: String representing an application ID, used for configuring paths and string constants in API stubs. app_root: The path to the directory containing the user's application e.g. "/home/joe/myapp", used for locating application yaml files, eg index.yaml for the datastore stub. Returns: An instance of APIServer. Raises: DatastoreFileError: Cloud Datastore emulator is used while stub_type is datastore_converter.StubTypes.PYTHON_FILE_STUB. """ datastore_path = get_datastore_path(storage_path, options.datastore_path) logs_path = options.logs_path or os.path.join(storage_path, 'logs.db') search_index_path = options.search_indexes_path or os.path.join( storage_path, 'search_indexes') blobstore_path = options.blobstore_path or os.path.join( storage_path, 'blobs') if options.clear_datastore: _clear_datastore_storage(datastore_path) if options.clear_search_indexes: _clear_search_indexes_storage(search_index_path) if options.auto_id_policy == datastore_stub_util.SEQUENTIAL: logging.warn( "--auto_id_policy='sequential' is deprecated. This option " "will be removed in a future release.") application_address = '%s' % options.host if options.port and options.port != 80: application_address += ':' + str(options.port) user_login_url = '/%s?%s=%%s' % (login.LOGIN_URL_RELATIVE, login.CONTINUE_PARAM) user_logout_url = '/%s?%s=%%s' % (login.LOGOUT_URL_RELATIVE, login.CONTINUE_PARAM) if options.datastore_consistency_policy == 'time': consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy() elif options.datastore_consistency_policy == 'random': consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy() elif options.datastore_consistency_policy == 'consistent': consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0) else: assert 0, ('unknown consistency policy: %r' % options.datastore_consistency_policy) stub_type = (datastore_converter.get_stub_type(datastore_path) if os.path.exists(datastore_path) else None) gcd_emulator_launching_thread = None if options.support_datastore_emulator: if stub_type == datastore_converter.StubTypes.PYTHON_FILE_STUB: raise DatastoreFileError( 'The datastore file %s cannot be recognized by dev_appserver. Please ' 'restart dev_appserver with --clear_datastore=1' % datastore_path) env_emulator_host = os.environ.get('DATASTORE_EMULATOR_HOST') if env_emulator_host: # emulator already running, reuse it. logging.warning( 'Detected environment variable DATASTORE_EMULATOR_HOST=%s, ' 'dev_appserver will speak to the Cloud Datastore emulator running on ' 'this address. The datastore_path %s will be neglected.\nIf you ' 'want datastore to store on %s, remove DATASTORE_EMULATOR_HOST ' 'from environment variables and restart dev_appserver', env_emulator_host, datastore_path, datastore_path) else: gcd_emulator_launching_thread = _launch_gcd_emulator( app_id=app_id, emulator_port=(options.datastore_emulator_port if options.datastore_emulator_port else portpicker.PickUnusedPort()), silent=options.dev_appserver_log_level != 'debug', index_file=os.path.join(app_root, 'index.yaml'), require_indexes=options.require_indexes, datastore_path=datastore_path, stub_type=stub_type, cmd=options.datastore_emulator_cmd, is_test=options.datastore_emulator_is_test_mode) else: # Use SQLite stub. # For historic reason we are still supporting conversion from file stub to # SQLite stub data. But this conversion will go away. if stub_type == datastore_converter.StubTypes.PYTHON_FILE_STUB: datastore_converter.convert_datastore_file_stub_data_to_sqlite( app_id, datastore_path) stub_util.setup_stubs( request_data=request_info, app_id=app_id, application_root=app_root, # The "trusted" flag is only relevant for Google administrative # applications. trusted=getattr(options, 'trusted', False), appidentity_email_address=options.appidentity_email_address, appidentity_private_key_path=os.path.abspath( options.appidentity_private_key_path) if options.appidentity_private_key_path else None, blobstore_path=blobstore_path, datastore_path=datastore_path, datastore_consistency=consistency, datastore_require_indexes=options.require_indexes, datastore_auto_id_policy=options.auto_id_policy, images_host_prefix='http://%s' % application_address, logs_path=logs_path, mail_smtp_host=options.smtp_host, mail_smtp_port=options.smtp_port, mail_smtp_user=options.smtp_user, mail_smtp_password=options.smtp_password, mail_enable_sendmail=options.enable_sendmail, mail_show_mail_body=options.show_mail_body, mail_allow_tls=options.smtp_allow_tls, search_index_path=search_index_path, taskqueue_auto_run_tasks=options.enable_task_running, taskqueue_default_http_server=application_address, user_login_url=user_login_url, user_logout_url=user_logout_url, default_gcs_bucket_name=options.default_gcs_bucket_name, appidentity_oauth_url=options.appidentity_oauth_url, datastore_grpc_stub_class=(datastore_grpc_stub.DatastoreGrpcStub if options.support_datastore_emulator else None)) return APIServer( options.api_host, options.api_port, app_id, options.api_server_supports_grpc or options.support_datastore_emulator, options.grpc_api_port, options.enable_host_checking, gcd_emulator_launching_thread)
def start(self, options): """Start devappserver2 servers based on the provided command line arguments. Args: options: An argparse.Namespace containing the command line arguments. """ logging.getLogger().setLevel( _LOG_LEVEL_TO_PYTHON_CONSTANT[options.dev_appserver_log_level]) configuration = application_configuration.ApplicationConfiguration( options.yaml_files, options.app_id) if options.skip_sdk_update_check: logging.info('Skipping SDK update check.') else: update_checker.check_for_updates(configuration) if options.port == 0: logging.warn('DEFAULT_VERSION_HOSTNAME will not be set correctly with ' '--port=0') _setup_environ(configuration.app_id, configuration.modules[0].version_id) python_config = runtime_config_pb2.PythonConfig() if options.python_startup_script: python_config.startup_script = os.path.abspath( options.python_startup_script) if options.python_startup_args: python_config.startup_args = options.python_startup_args cloud_sql_config = runtime_config_pb2.CloudSQL() cloud_sql_config.mysql_host = options.mysql_host cloud_sql_config.mysql_port = options.mysql_port cloud_sql_config.mysql_user = options.mysql_user cloud_sql_config.mysql_password = options.mysql_password if options.mysql_socket: cloud_sql_config.mysql_socket = options.mysql_socket if options.max_module_instances is None: module_to_max_instances = {} elif isinstance(options.max_module_instances, int): module_to_max_instances = { module_configuration.module_name: options.max_module_instances for module_configuration in configuration.modules} else: module_to_max_instances = options.max_module_instances self._dispatcher = dispatcher.Dispatcher( configuration, options.host, options.port, options.auth_domain, _LOG_LEVEL_TO_RUNTIME_CONSTANT[options.log_level], options.php_executable_path, options.php_remote_debugging, python_config, cloud_sql_config, module_to_max_instances, options.use_mtime_file_watcher, options.automatic_restart, options.allow_skipped_files, options.external_api_port) request_data = wsgi_request_info.WSGIRequestInfo(self._dispatcher) storage_path = _get_storage_path(options.storage_path, configuration.app_id) datastore_path = options.datastore_path or os.path.join(storage_path, 'datastore.db') logs_path = options.logs_path or os.path.join(storage_path, 'logs.db') xsrf_path = os.path.join(storage_path, 'xsrf') search_index_path = options.search_indexes_path or os.path.join( storage_path, 'search_indexes') prospective_search_path = options.prospective_search_path or os.path.join( storage_path, 'prospective-search') blobstore_path = options.blobstore_path or os.path.join(storage_path, 'blobs') if options.clear_datastore: _clear_datastore_storage(datastore_path) if options.clear_prospective_search: _clear_prospective_search_storage(prospective_search_path) if options.clear_search_indexes: _clear_search_indexes_storage(search_index_path) if options.auto_id_policy==datastore_stub_util.SEQUENTIAL: logging.warn("--auto_id_policy='sequential' is deprecated. This option " "will be removed in a future release.") application_address = '%s' % options.host if options.port and options.port != 80: application_address += ':' + str(options.port) user_login_url = '/%s?%s=%%s' % (login.LOGIN_URL_RELATIVE, login.CONTINUE_PARAM) user_logout_url = '%s&%s=%s' % (user_login_url, login.ACTION_PARAM, login.LOGOUT_ACTION) if options.datastore_consistency_policy == 'time': consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy() elif options.datastore_consistency_policy == 'random': consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy() elif options.datastore_consistency_policy == 'consistent': consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0) else: assert 0, ('unknown consistency policy: %r' % options.datastore_consistency_policy) api_server.maybe_convert_datastore_file_stub_data_to_sqlite( configuration.app_id, datastore_path) api_server.setup_stubs( request_data=request_data, app_id=configuration.app_id, application_root=configuration.modules[0].application_root, # The "trusted" flag is only relevant for Google administrative # applications. trusted=getattr(options, 'trusted', False), blobstore_path=blobstore_path, datastore_path=datastore_path, datastore_consistency=consistency, datastore_require_indexes=options.require_indexes, datastore_auto_id_policy=options.auto_id_policy, images_host_prefix='http://%s' % application_address, logs_path=logs_path, mail_smtp_host=options.smtp_host, mail_smtp_port=options.smtp_port, mail_smtp_user=options.smtp_user, mail_smtp_password=options.smtp_password, mail_enable_sendmail=options.enable_sendmail, mail_show_mail_body=options.show_mail_body, matcher_prospective_search_path=prospective_search_path, search_index_path=search_index_path, taskqueue_auto_run_tasks=options.enable_task_running, taskqueue_default_http_server=application_address, uaserver_path=options.uaserver_path, user_login_url=user_login_url, user_logout_url=user_logout_url, xmpp_path=options.xmpp_path) # The APIServer must bind to localhost because that is what the runtime # instances talk to. apis = api_server.APIServer('localhost', options.api_port, configuration.app_id) apis.start() self._running_modules.append(apis) self._running_modules.append(self._dispatcher) self._dispatcher.start(apis.port, request_data) admin = admin_server.AdminServer(options.admin_host, options.admin_port, self._dispatcher, configuration, xsrf_path) admin.start() self._running_modules.append(admin)
def create_api_server(request_info, storage_path, options, app_id, app_root): """Creates an API server. Args: request_info: An apiproxy_stub.RequestInfo instance used by the stubs to lookup information about the request associated with an API call. storage_path: A string directory for storing API stub data. options: An instance of argparse.Namespace containing command line flags. app_id: String representing an application ID, used for configuring paths and string constants in API stubs. app_root: The path to the directory containing the user's application e.g. "/home/joe/myapp", used for locating application yaml files, eg index.yaml for the datastore stub. Returns: An instance of APIServer. """ emulator_launching_thread = None if options.support_datastore_emulator and not os.environ.get( 'DATASTORE_EMULATOR_HOST'): gcd_emulator_port = portpicker.PickUnusedPort() emulator_launching_thread = threading.Thread( target=GCD_EMULATOR_MANAGER.launch, args=[ gcd_emulator_port, options.dev_appserver_log_level != 'debug', os.path.join(app_root, 'index.yaml'), options.require_indexes ]) emulator_launching_thread.start() os.environ[ 'DATASTORE_EMULATOR_HOST'] = 'localhost:%d' % gcd_emulator_port datastore_path = options.datastore_path or os.path.join( storage_path, 'datastore.db') logs_path = options.logs_path or os.path.join(storage_path, 'logs.db') search_index_path = options.search_indexes_path or os.path.join( storage_path, 'search_indexes') blobstore_path = options.blobstore_path or os.path.join( storage_path, 'blobs') if options.clear_datastore: _clear_datastore_storage(datastore_path) if options.clear_search_indexes: _clear_search_indexes_storage(search_index_path) if options.auto_id_policy == datastore_stub_util.SEQUENTIAL: logging.warn( "--auto_id_policy='sequential' is deprecated. This option " "will be removed in a future release.") application_address = '%s' % options.host if options.port and options.port != 80: application_address += ':' + str(options.port) user_login_url = '/%s?%s=%%s' % (login.LOGIN_URL_RELATIVE, login.CONTINUE_PARAM) user_logout_url = '/%s?%s=%%s' % (login.LOGOUT_URL_RELATIVE, login.CONTINUE_PARAM) if options.datastore_consistency_policy == 'time': consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy() elif options.datastore_consistency_policy == 'random': consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy() elif options.datastore_consistency_policy == 'consistent': consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0) else: assert 0, ('unknown consistency policy: %r' % options.datastore_consistency_policy) # Check if local datastore data should be converted. # Using GCD Emulator this could convert python file stub or sqlite stub data # to Emulator data format; Without GCD Emulator this converts python file stub # to sqlite stub data. if os.path.exists(datastore_path): data_type = datastore_converter.get_data_type(datastore_path) if options.support_datastore_emulator: if data_type in [ datastore_converter.StubTypes.PYTHON_FILE_STUB, datastore_converter.StubTypes.PYTHON_SQLITE_STUB ]: if emulator_launching_thread: emulator_launching_thread.join() gcd_emulator_host = os.environ.get('DATASTORE_EMULATOR_HOST') datastore_converter.convert_python_data_to_emulator( app_id, data_type, datastore_path, gcd_emulator_host) else: if data_type != datastore_converter.StubTypes.PYTHON_SQLITE_STUB: datastore_converter.convert_datastore_file_stub_data_to_sqlite( app_id, datastore_path) stub_util.setup_stubs( request_data=request_info, app_id=app_id, application_root=app_root, # The "trusted" flag is only relevant for Google administrative # applications. trusted=getattr(options, 'trusted', False), appidentity_email_address=options.appidentity_email_address, appidentity_private_key_path=os.path.abspath( options.appidentity_private_key_path) if options.appidentity_private_key_path else None, blobstore_path=blobstore_path, datastore_path=datastore_path, datastore_consistency=consistency, datastore_require_indexes=options.require_indexes, datastore_auto_id_policy=options.auto_id_policy, images_host_prefix='http://%s' % application_address, logs_path=logs_path, mail_smtp_host=options.smtp_host, mail_smtp_port=options.smtp_port, mail_smtp_user=options.smtp_user, mail_smtp_password=options.smtp_password, mail_enable_sendmail=options.enable_sendmail, mail_show_mail_body=options.show_mail_body, mail_allow_tls=options.smtp_allow_tls, search_index_path=search_index_path, taskqueue_auto_run_tasks=options.enable_task_running, taskqueue_default_http_server=application_address, user_login_url=user_login_url, user_logout_url=user_logout_url, default_gcs_bucket_name=options.default_gcs_bucket_name, appidentity_oauth_url=options.appidentity_oauth_url, datastore_grpc_stub_class=(datastore_grpc_stub.DatastoreGrpcStub if options.support_datastore_emulator else None)) if emulator_launching_thread: emulator_launching_thread.join() return APIServer( options.api_host, options.api_port, app_id, options.api_server_supports_grpc or options.support_datastore_emulator, options.grpc_api_port, options.enable_host_checking)