def setup_stub(high_replication=False):
apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
stub = datastore_file_stub.DatastoreFileStub('test','/dev/null',
'/dev/null', trusted=True)
if high_replication:
stub.SetConsistencyPolicy(
datastore_stub_util.TimeBasedHRConsistencyPolicy())
apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', stub)
apiproxy_stub_map.apiproxy.RegisterStub(
'user', user_service_stub.UserServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'memcache', memcache_stub.MemcacheServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'urlfetch', urlfetch_stub.URLFetchServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'taskqueue', taskqueue_stub.TaskQueueServiceStub())
try:
apiproxy_stub_map.apiproxy.RegisterStub(
'images', images_stub.ImagesServiceStub())
except NameError:
pass
def setup_stubs():
"""Setup the API stubs. This can only be done once."""
api_server.setup_test_stubs(
request_data,
app_id=APP_ID,
application_root=APPLICATION_ROOT,
trusted=TRUSTED,
blobstore_path=BLOBSTORE_PATH,
datastore_consistency=datastore_stub_util.TimeBasedHRConsistencyPolicy(
),
datastore_path=DATASTORE_PATH,
datastore_require_indexes=DATASTORE_REQUIRE_INDEXES,
images_host_prefix=IMAGES_HOST_PREFIX,
logs_path=':memory:',
mail_smtp_host=MAIL_SMTP_HOST,
mail_smtp_port=MAIL_SMTP_PORT,
mail_smtp_user=MAIL_SMTP_USER,
mail_smtp_password=MAIL_SMTP_PASSWORD,
mail_enable_sendmail=MAIL_ENABLE_SENDMAIL,
mail_show_mail_body=MAIL_SHOW_MAIL_BODY,
taskqueue_auto_run_tasks=TASKQUEUE_AUTO_RUN_TASKS,
taskqueue_default_http_server=TASKQUEUE_DEFAULT_HTTP_SERVER,
user_login_url=USER_LOGIN_URL,
user_logout_url=USER_LOGOUT_URL)
apiproxy_stub_map.apiproxy.ReplaceStub('urlfetch',
FakeURLFetchServiceStub())
apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v4',
FakeDatastoreV4ServiceStub())
def InitAppHostingApi():
"""Initialize stubs for various app hosting APIs."""
# clear ndb's context cache
# see https://developers.google.com/appengine/docs/python/ndb/cache
ndb.get_context().clear_cache()
# Pretend we're running in the dev_appserver
os.environ['SERVER_SOFTWARE'] = 'Development/unittests'
# used by app_identity.get_default_version_hostname()
os.environ['DEFAULT_VERSION_HOSTNAME'] = 'localhost:8080'
appid = os.environ['APPLICATION_ID'] = 'app'
apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
# Need an HRD stub to support XG transactions
hrd_policy = datastore_stub_util.TimeBasedHRConsistencyPolicy()
apiproxy_stub_map.apiproxy.RegisterStub(
'datastore_v3',
datastore_file_stub.DatastoreFileStub(appid,
'/dev/null',
'/dev/null',
trusted=True,
require_indexes=True,
consistency_policy=hrd_policy))
# memcache stub
apiproxy_stub_map.apiproxy.RegisterStub(
'memcache', memcache_stub.MemcacheServiceStub())
# urlfetch stub
apiproxy_stub_map.apiproxy.RegisterStub(
'urlfetch', urlfetch_stub.URLFetchServiceStub())
# blobstore stub
temp_dir = tempfile.gettempdir()
storage_directory = os.path.join(temp_dir, 'blob_storage')
if os.access(storage_directory, os.F_OK):
shutil.rmtree(storage_directory)
blob_storage = file_blob_storage.FileBlobStorage(storage_directory, appid)
apiproxy_stub_map.apiproxy.RegisterStub(
'blobstore', blobstore_stub.BlobstoreServiceStub(blob_storage))
# file stub, required by blobstore stub
apiproxy_stub_map.apiproxy.RegisterStub(
'file', file_service_stub.FileServiceStub(blob_storage))
def setup_datastore_stubs():
if "test" in sys.argv:
return
from google.appengine.datastore import datastore_sqlite_stub
from google.appengine.api import apiproxy_stub_map
from google.appengine.datastore import datastore_stub_util
app_id = application_id()
datastore = datastore_sqlite_stub.DatastoreSqliteStub(
"dev~" + app_id,
os.path.join(data_root(), "datastore.db"),
require_indexes=False,
trusted=False,
root_path=find_project_root(),
use_atexit=True)
datastore.SetConsistencyPolicy(
datastore_stub_util.TimeBasedHRConsistencyPolicy())
apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', datastore)
def create_api_server(
request_info, storage_path, options, app_id, app_root):
"""Creates an API server.
Args:
request_info: An apiproxy_stub.RequestInfo instance used by the stubs to
lookup information about the request associated with an API call.
storage_path: A string directory for storing API stub data.
options: An instance of argparse.Namespace containing command line flags.
app_id: String representing an application ID, used for configuring paths
and string constants in API stubs.
app_root: The path to the directory containing the user's
application e.g. "/home/joe/myapp", used for locating application yaml
files, eg index.yaml for the datastore stub.
Returns:
An instance of APIServer.
"""
datastore_path = options.datastore_path or os.path.join(
storage_path, 'datastore.db')
logs_path = options.logs_path or os.path.join(storage_path, 'logs.db')
search_index_path = options.search_indexes_path or os.path.join(
storage_path, 'search_indexes')
blobstore_path = options.blobstore_path or os.path.join(
storage_path, 'blobs')
if options.clear_datastore:
_clear_datastore_storage(datastore_path)
if options.clear_search_indexes:
_clear_search_indexes_storage(search_index_path)
if options.auto_id_policy == datastore_stub_util.SEQUENTIAL:
logging.warn("--auto_id_policy='sequential' is deprecated. This option "
"will be removed in a future release.")
application_address = '%s' % options.host
if options.port and options.port != 80:
application_address += ':' + str(options.port)
user_login_url = '/%s?%s=%%s' % (
login.LOGIN_URL_RELATIVE, login.CONTINUE_PARAM)
user_logout_url = '/%s?%s=%%s' % (
login.LOGOUT_URL_RELATIVE, login.CONTINUE_PARAM)
if options.datastore_consistency_policy == 'time':
consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy()
elif options.datastore_consistency_policy == 'random':
consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy()
elif options.datastore_consistency_policy == 'consistent':
consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0)
else:
assert 0, ('unknown consistency policy: %r' %
options.datastore_consistency_policy)
maybe_convert_datastore_file_stub_data_to_sqlite(app_id, datastore_path)
setup_stubs(
request_data=request_info,
app_id=app_id,
application_root=app_root,
# The "trusted" flag is only relevant for Google administrative
# applications.
trusted=getattr(options, 'trusted', False),
appidentity_email_address=options.appidentity_email_address,
appidentity_private_key_path=os.path.abspath(
options.appidentity_private_key_path)
if options.appidentity_private_key_path else None,
blobstore_path=blobstore_path,
datastore_path=datastore_path,
datastore_consistency=consistency,
datastore_require_indexes=options.require_indexes,
datastore_auto_id_policy=options.auto_id_policy,
images_host_prefix='http://%s' % application_address,
logs_path=logs_path,
mail_smtp_host=options.smtp_host,
mail_smtp_port=options.smtp_port,
mail_smtp_user=options.smtp_user,
mail_smtp_password=options.smtp_password,
mail_enable_sendmail=options.enable_sendmail,
mail_show_mail_body=options.show_mail_body,
mail_allow_tls=options.smtp_allow_tls,
search_index_path=search_index_path,
taskqueue_auto_run_tasks=options.enable_task_running,
taskqueue_default_http_server=application_address,
user_login_url=user_login_url,
user_logout_url=user_logout_url,
default_gcs_bucket_name=options.default_gcs_bucket_name,
appidentity_oauth_url=options.appidentity_oauth_url,
support_datastore_emulator=options.support_datastore_emulator)
return APIServer(
options.api_host, options.api_port, app_id,
options.api_server_supports_grpc or options.support_datastore_emulator,
options.grpc_api_port)
def _SetupStubs(
app_id,
application_root,
trusted,
blobstore_path,
use_sqlite,
auto_id_policy,
high_replication,
datastore_path,
datastore_require_indexes,
images_host_prefix,
logs_path,
mail_smtp_host,
mail_smtp_port,
mail_smtp_user,
mail_smtp_password,
mail_enable_sendmail,
mail_show_mail_body,
matcher_prospective_search_path,
taskqueue_auto_run_tasks,
taskqueue_task_retry_seconds,
taskqueue_default_http_server,
user_login_url,
user_logout_url):
"""Configures the APIs hosted by this server.
Args:
app_id: The str application id e.g. "guestbook".
application_root: The path to the directory containing the user's
application e.g. "/home/bquinlan/myapp".
trusted: A bool indicating if privileged APIs should be made available.
blobstore_path: The path to the file that should be used for blobstore
storage.
use_sqlite: A bool indicating whether DatastoreSqliteStub or
DatastoreFileStub should be used.
auto_id_policy: One of datastore_stub_util.SEQUENTIAL or .SCATTERED,
indicating whether the Datastore stub should assign IDs sequentially
or scattered.
high_replication: A bool indicating whether to use the high replication
consistency model.
datastore_path: The path to the file that should be used for datastore
storage.
datastore_require_indexes: A bool indicating if the same production
datastore indexes requirements should be enforced i.e. if True then
a google.appengine.ext.db.NeedIndexError will be be raised if a query
is executed without the required indexes.
images_host_prefix: The URL prefix (protocol://host:port) to preprend to
image urls on calls to images.GetUrlBase.
logs_path: Path to the file to store the logs data in.
mail_smtp_host: The SMTP hostname that should be used when sending e-mails.
If None then the mail_enable_sendmail argument is considered.
mail_smtp_port: The SMTP port number that should be used when sending
e-mails. If this value is None then mail_smtp_host must also be None.
mail_smtp_user: The username to use when authenticating with the
SMTP server. This value may be None if mail_smtp_host is also None or if
the SMTP server does not require authentication.
mail_smtp_password: The password to use when authenticating with the
SMTP server. This value may be None if mail_smtp_host or mail_smtp_user
is also None.
mail_enable_sendmail: A bool indicating if sendmail should be used when
sending e-mails. This argument is ignored if mail_smtp_host is not None.
mail_show_mail_body: A bool indicating whether the body of sent e-mails
should be written to the logs.
matcher_prospective_search_path: The path to the file that should be used to
save prospective search subscriptions.
taskqueue_auto_run_tasks: A bool indicating whether taskqueue tasks should
be run automatically or it the must be manually triggered.
taskqueue_task_retry_seconds: An int representing the number of seconds to
wait before a retrying a failed taskqueue task.
taskqueue_default_http_server: A str containing the address of the http
server that should be used to execute tasks.
user_login_url: A str containing the url that should be used for user login.
user_logout_url: A str containing the url that should be used for user
logout.
"""
os.environ['APPLICATION_ID'] = app_id
apiproxy_stub_map.apiproxy.RegisterStub(
'app_identity_service',
app_identity_stub.AppIdentityServiceStub())
blob_storage = file_blob_storage.FileBlobStorage(blobstore_path, app_id)
apiproxy_stub_map.apiproxy.RegisterStub(
'blobstore',
blobstore_stub.BlobstoreServiceStub(blob_storage))
apiproxy_stub_map.apiproxy.RegisterStub(
'capability_service',
capability_stub.CapabilityServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'channel',
channel_service_stub.ChannelServiceStub())
if use_sqlite:
datastore = datastore_sqlite_stub.DatastoreSqliteStub(
app_id,
datastore_path,
datastore_require_indexes,
trusted,
root_path=application_root,
auto_id_policy=auto_id_policy)
else:
datastore = datastore_file_stub.DatastoreFileStub(
app_id,
datastore_path,
datastore_require_indexes,
trusted,
root_path=application_root,
auto_id_policy=auto_id_policy)
if high_replication:
datastore.SetConsistencyPolicy(
datastore_stub_util.TimeBasedHRConsistencyPolicy())
apiproxy_stub_map.apiproxy.RegisterStub(
'datastore_v3', datastore)
apiproxy_stub_map.apiproxy.RegisterStub(
'file',
file_service_stub.FileServiceStub(blob_storage))
try:
from google.appengine.api.images import images_stub
except ImportError:
logging.warning('Could not initialize images API; you are likely missing '
'the Python "PIL" module.')
from google.appengine.api.images import images_not_implemented_stub
apiproxy_stub_map.apiproxy.RegisterStub(
'images',
images_not_implemented_stub.ImagesNotImplementedServiceStub())
else:
apiproxy_stub_map.apiproxy.RegisterStub(
'images',
images_stub.ImagesServiceStub(host_prefix=images_host_prefix))
apiproxy_stub_map.apiproxy.RegisterStub(
'logservice',
logservice_stub.LogServiceStub(persist=True))
apiproxy_stub_map.apiproxy.RegisterStub(
'mail',
mail_stub.MailServiceStub(mail_smtp_host,
mail_smtp_port,
mail_smtp_user,
mail_smtp_password,
enable_sendmail=mail_enable_sendmail,
show_mail_body=mail_show_mail_body))
apiproxy_stub_map.apiproxy.RegisterStub(
'memcache',
memcache_stub.MemcacheServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'search',
simple_search_stub.SearchServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub('system',
system_stub.SystemServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'taskqueue',
taskqueue_stub.TaskQueueServiceStub(
root_path=application_root,
auto_task_running=taskqueue_auto_run_tasks,
task_retry_seconds=taskqueue_task_retry_seconds,
default_http_server=taskqueue_default_http_server))
apiproxy_stub_map.apiproxy.GetStub('taskqueue').StartBackgroundExecution()
apiproxy_stub_map.apiproxy.RegisterStub(
'urlfetch',
urlfetch_stub.URLFetchServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'user',
user_service_stub.UserServiceStub(login_url=user_login_url,
logout_url=user_logout_url))
apiproxy_stub_map.apiproxy.RegisterStub(
'xmpp',
xmpp_service_stub.XmppServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'matcher',
prospective_search_stub.ProspectiveSearchStub(
matcher_prospective_search_path,
apiproxy_stub_map.apiproxy.GetStub('taskqueue')))
def _create_api_server(request_data, storage_path, options, configuration):
datastore_path = options.datastore_path or os.path.join(
storage_path, 'datastore.db')
logs_path = options.logs_path or os.path.join(storage_path, 'logs.db')
search_index_path = options.search_indexes_path or os.path.join(
storage_path, 'search_indexes')
prospective_search_path = options.prospective_search_path or os.path.join(
storage_path, 'prospective-search')
blobstore_path = options.blobstore_path or os.path.join(
storage_path, 'blobs')
if options.clear_datastore:
_clear_datastore_storage(datastore_path)
if options.clear_prospective_search:
_clear_prospective_search_storage(prospective_search_path)
if options.clear_search_indexes:
_clear_search_indexes_storage(search_index_path)
if options.auto_id_policy == datastore_stub_util.SEQUENTIAL:
logging.warn(
"--auto_id_policy='sequential' is deprecated. This option "
"will be removed in a future release.")
application_address = '%s' % options.host
if options.port and options.port != 80:
application_address += ':' + str(options.port)
user_login_url = '/%s?%s=%%s' % (login.LOGIN_URL_RELATIVE,
login.CONTINUE_PARAM)
user_logout_url = '%s&%s=%s' % (user_login_url, login.ACTION_PARAM,
login.LOGOUT_ACTION)
if options.datastore_consistency_policy == 'time':
consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy()
elif options.datastore_consistency_policy == 'random':
consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy()
elif options.datastore_consistency_policy == 'consistent':
consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(
1.0)
else:
assert 0, ('unknown consistency policy: %r' %
options.datastore_consistency_policy)
api_server.maybe_convert_datastore_file_stub_data_to_sqlite(
configuration.app_id, datastore_path)
api_server.setup_stubs(
request_data=request_data,
app_id=configuration.app_id,
application_root=configuration.modules[0].application_root,
# The "trusted" flag is only relevant for Google administrative
# applications.
trusted=getattr(options, 'trusted', False),
appidentity_email_address=options.appidentity_email_address,
appidentity_private_key_path=os.path.abspath(
options.appidentity_private_key_path)
if options.appidentity_private_key_path else None,
blobstore_path=blobstore_path,
datastore_path=datastore_path,
datastore_consistency=consistency,
datastore_require_indexes=options.require_indexes,
datastore_auto_id_policy=options.auto_id_policy,
images_host_prefix='http://%s' % application_address,
logs_path=logs_path,
mail_smtp_host=options.smtp_host,
mail_smtp_port=options.smtp_port,
mail_smtp_user=options.smtp_user,
mail_smtp_password=options.smtp_password,
mail_enable_sendmail=options.enable_sendmail,
mail_show_mail_body=options.show_mail_body,
mail_allow_tls=options.smtp_allow_tls,
matcher_prospective_search_path=prospective_search_path,
search_index_path=search_index_path,
taskqueue_auto_run_tasks=options.enable_task_running,
taskqueue_default_http_server=application_address,
user_login_url=user_login_url,
user_logout_url=user_logout_url,
default_gcs_bucket_name=options.default_gcs_bucket_name)
return api_server.APIServer(options.api_host, options.api_port,
configuration.app_id)
def create_api_server(request_info, storage_path, options, app_id, app_root):
"""Creates an API server.
Args:
request_info: An apiproxy_stub.RequestInfo instance used by the stubs to
lookup information about the request associated with an API call.
storage_path: A string directory for storing API stub data.
options: An instance of argparse.Namespace containing command line flags.
app_id: String representing an application ID, used for configuring paths
and string constants in API stubs.
app_root: The path to the directory containing the user's
application e.g. "/home/joe/myapp", used for locating application yaml
files, eg index.yaml for the datastore stub.
Returns:
An instance of APIServer.
Raises:
DatastoreFileError: Cloud Datastore emulator is used while stub_type is
datastore_converter.StubTypes.PYTHON_FILE_STUB.
"""
datastore_path = get_datastore_path(storage_path, options.datastore_path)
logs_path = options.logs_path or os.path.join(storage_path, 'logs.db')
search_index_path = options.search_indexes_path or os.path.join(
storage_path, 'search_indexes')
blobstore_path = options.blobstore_path or os.path.join(
storage_path, 'blobs')
if options.clear_datastore:
_clear_datastore_storage(datastore_path)
if options.clear_search_indexes:
_clear_search_indexes_storage(search_index_path)
if options.auto_id_policy == datastore_stub_util.SEQUENTIAL:
logging.warn(
"--auto_id_policy='sequential' is deprecated. This option "
"will be removed in a future release.")
application_address = '%s' % options.host
if options.port and options.port != 80:
application_address += ':' + str(options.port)
user_login_url = '/%s?%s=%%s' % (login.LOGIN_URL_RELATIVE,
login.CONTINUE_PARAM)
user_logout_url = '/%s?%s=%%s' % (login.LOGOUT_URL_RELATIVE,
login.CONTINUE_PARAM)
if options.datastore_consistency_policy == 'time':
consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy()
elif options.datastore_consistency_policy == 'random':
consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy()
elif options.datastore_consistency_policy == 'consistent':
consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0)
else:
assert 0, ('unknown consistency policy: %r' %
options.datastore_consistency_policy)
stub_type = (datastore_converter.get_stub_type(datastore_path)
if os.path.exists(datastore_path) else None)
gcd_emulator_launching_thread = None
if options.support_datastore_emulator:
if stub_type == datastore_converter.StubTypes.PYTHON_FILE_STUB:
raise DatastoreFileError(
'The datastore file %s cannot be recognized by dev_appserver. Please '
'restart dev_appserver with --clear_datastore=1' %
datastore_path)
env_emulator_host = os.environ.get('DATASTORE_EMULATOR_HOST')
if env_emulator_host: # emulator already running, reuse it.
logging.warning(
'Detected environment variable DATASTORE_EMULATOR_HOST=%s, '
'dev_appserver will speak to the Cloud Datastore emulator running on '
'this address. The datastore_path %s will be neglected.\nIf you '
'want datastore to store on %s, remove DATASTORE_EMULATOR_HOST '
'from environment variables and restart dev_appserver',
env_emulator_host, datastore_path, datastore_path)
else:
gcd_emulator_launching_thread = _launch_gcd_emulator(
app_id=app_id,
emulator_port=(options.datastore_emulator_port
if options.datastore_emulator_port else
portpicker.PickUnusedPort()),
silent=options.dev_appserver_log_level != 'debug',
index_file=os.path.join(app_root, 'index.yaml'),
require_indexes=options.require_indexes,
datastore_path=datastore_path,
stub_type=stub_type,
cmd=options.datastore_emulator_cmd,
is_test=options.datastore_emulator_is_test_mode)
else:
# Use SQLite stub.
# For historic reason we are still supporting conversion from file stub to
# SQLite stub data. But this conversion will go away.
if stub_type == datastore_converter.StubTypes.PYTHON_FILE_STUB:
datastore_converter.convert_datastore_file_stub_data_to_sqlite(
app_id, datastore_path)
stub_util.setup_stubs(
request_data=request_info,
app_id=app_id,
application_root=app_root,
# The "trusted" flag is only relevant for Google administrative
# applications.
trusted=getattr(options, 'trusted', False),
appidentity_email_address=options.appidentity_email_address,
appidentity_private_key_path=os.path.abspath(
options.appidentity_private_key_path)
if options.appidentity_private_key_path else None,
blobstore_path=blobstore_path,
datastore_path=datastore_path,
datastore_consistency=consistency,
datastore_require_indexes=options.require_indexes,
datastore_auto_id_policy=options.auto_id_policy,
images_host_prefix='http://%s' % application_address,
logs_path=logs_path,
mail_smtp_host=options.smtp_host,
mail_smtp_port=options.smtp_port,
mail_smtp_user=options.smtp_user,
mail_smtp_password=options.smtp_password,
mail_enable_sendmail=options.enable_sendmail,
mail_show_mail_body=options.show_mail_body,
mail_allow_tls=options.smtp_allow_tls,
search_index_path=search_index_path,
taskqueue_auto_run_tasks=options.enable_task_running,
taskqueue_default_http_server=application_address,
user_login_url=user_login_url,
user_logout_url=user_logout_url,
default_gcs_bucket_name=options.default_gcs_bucket_name,
appidentity_oauth_url=options.appidentity_oauth_url,
datastore_grpc_stub_class=(datastore_grpc_stub.DatastoreGrpcStub
if options.support_datastore_emulator else
None))
return APIServer(
options.api_host, options.api_port, app_id,
options.api_server_supports_grpc or options.support_datastore_emulator,
options.grpc_api_port, options.enable_host_checking,
gcd_emulator_launching_thread)
def start(self, options):
"""Start devappserver2 servers based on the provided command line arguments.
Args:
options: An argparse.Namespace containing the command line arguments.
"""
logging.getLogger().setLevel(
_LOG_LEVEL_TO_PYTHON_CONSTANT[options.dev_appserver_log_level])
configuration = application_configuration.ApplicationConfiguration(
options.yaml_files, options.app_id)
if options.skip_sdk_update_check:
logging.info('Skipping SDK update check.')
else:
update_checker.check_for_updates(configuration)
if options.port == 0:
logging.warn('DEFAULT_VERSION_HOSTNAME will not be set correctly with '
'--port=0')
_setup_environ(configuration.app_id, configuration.modules[0].version_id)
python_config = runtime_config_pb2.PythonConfig()
if options.python_startup_script:
python_config.startup_script = os.path.abspath(
options.python_startup_script)
if options.python_startup_args:
python_config.startup_args = options.python_startup_args
cloud_sql_config = runtime_config_pb2.CloudSQL()
cloud_sql_config.mysql_host = options.mysql_host
cloud_sql_config.mysql_port = options.mysql_port
cloud_sql_config.mysql_user = options.mysql_user
cloud_sql_config.mysql_password = options.mysql_password
if options.mysql_socket:
cloud_sql_config.mysql_socket = options.mysql_socket
if options.max_module_instances is None:
module_to_max_instances = {}
elif isinstance(options.max_module_instances, int):
module_to_max_instances = {
module_configuration.module_name: options.max_module_instances
for module_configuration in configuration.modules}
else:
module_to_max_instances = options.max_module_instances
self._dispatcher = dispatcher.Dispatcher(
configuration,
options.host,
options.port,
options.auth_domain,
_LOG_LEVEL_TO_RUNTIME_CONSTANT[options.log_level],
options.php_executable_path,
options.php_remote_debugging,
python_config,
cloud_sql_config,
module_to_max_instances,
options.use_mtime_file_watcher,
options.automatic_restart,
options.allow_skipped_files,
options.external_api_port)
request_data = wsgi_request_info.WSGIRequestInfo(self._dispatcher)
storage_path = _get_storage_path(options.storage_path, configuration.app_id)
datastore_path = options.datastore_path or os.path.join(storage_path,
'datastore.db')
logs_path = options.logs_path or os.path.join(storage_path, 'logs.db')
xsrf_path = os.path.join(storage_path, 'xsrf')
search_index_path = options.search_indexes_path or os.path.join(
storage_path, 'search_indexes')
prospective_search_path = options.prospective_search_path or os.path.join(
storage_path, 'prospective-search')
blobstore_path = options.blobstore_path or os.path.join(storage_path,
'blobs')
if options.clear_datastore:
_clear_datastore_storage(datastore_path)
if options.clear_prospective_search:
_clear_prospective_search_storage(prospective_search_path)
if options.clear_search_indexes:
_clear_search_indexes_storage(search_index_path)
if options.auto_id_policy==datastore_stub_util.SEQUENTIAL:
logging.warn("--auto_id_policy='sequential' is deprecated. This option "
"will be removed in a future release.")
application_address = '%s' % options.host
if options.port and options.port != 80:
application_address += ':' + str(options.port)
user_login_url = '/%s?%s=%%s' % (login.LOGIN_URL_RELATIVE,
login.CONTINUE_PARAM)
user_logout_url = '%s&%s=%s' % (user_login_url, login.ACTION_PARAM,
login.LOGOUT_ACTION)
if options.datastore_consistency_policy == 'time':
consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy()
elif options.datastore_consistency_policy == 'random':
consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy()
elif options.datastore_consistency_policy == 'consistent':
consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0)
else:
assert 0, ('unknown consistency policy: %r' %
options.datastore_consistency_policy)
api_server.maybe_convert_datastore_file_stub_data_to_sqlite(
configuration.app_id, datastore_path)
api_server.setup_stubs(
request_data=request_data,
app_id=configuration.app_id,
application_root=configuration.modules[0].application_root,
# The "trusted" flag is only relevant for Google administrative
# applications.
trusted=getattr(options, 'trusted', False),
blobstore_path=blobstore_path,
datastore_path=datastore_path,
datastore_consistency=consistency,
datastore_require_indexes=options.require_indexes,
datastore_auto_id_policy=options.auto_id_policy,
images_host_prefix='http://%s' % application_address,
logs_path=logs_path,
mail_smtp_host=options.smtp_host,
mail_smtp_port=options.smtp_port,
mail_smtp_user=options.smtp_user,
mail_smtp_password=options.smtp_password,
mail_enable_sendmail=options.enable_sendmail,
mail_show_mail_body=options.show_mail_body,
matcher_prospective_search_path=prospective_search_path,
search_index_path=search_index_path,
taskqueue_auto_run_tasks=options.enable_task_running,
taskqueue_default_http_server=application_address,
uaserver_path=options.uaserver_path,
user_login_url=user_login_url,
user_logout_url=user_logout_url,
xmpp_path=options.xmpp_path)
# The APIServer must bind to localhost because that is what the runtime
# instances talk to.
apis = api_server.APIServer('localhost', options.api_port,
configuration.app_id)
apis.start()
self._running_modules.append(apis)
self._running_modules.append(self._dispatcher)
self._dispatcher.start(apis.port, request_data)
admin = admin_server.AdminServer(options.admin_host, options.admin_port,
self._dispatcher, configuration, xsrf_path)
admin.start()
self._running_modules.append(admin)
def create_api_server(request_info, storage_path, options, app_id, app_root):
"""Creates an API server.
Args:
request_info: An apiproxy_stub.RequestInfo instance used by the stubs to
lookup information about the request associated with an API call.
storage_path: A string directory for storing API stub data.
options: An instance of argparse.Namespace containing command line flags.
app_id: String representing an application ID, used for configuring paths
and string constants in API stubs.
app_root: The path to the directory containing the user's
application e.g. "/home/joe/myapp", used for locating application yaml
files, eg index.yaml for the datastore stub.
Returns:
An instance of APIServer.
"""
emulator_launching_thread = None
if options.support_datastore_emulator and not os.environ.get(
'DATASTORE_EMULATOR_HOST'):
gcd_emulator_port = portpicker.PickUnusedPort()
emulator_launching_thread = threading.Thread(
target=GCD_EMULATOR_MANAGER.launch,
args=[
gcd_emulator_port, options.dev_appserver_log_level != 'debug',
os.path.join(app_root, 'index.yaml'), options.require_indexes
])
emulator_launching_thread.start()
os.environ[
'DATASTORE_EMULATOR_HOST'] = 'localhost:%d' % gcd_emulator_port
datastore_path = options.datastore_path or os.path.join(
storage_path, 'datastore.db')
logs_path = options.logs_path or os.path.join(storage_path, 'logs.db')
search_index_path = options.search_indexes_path or os.path.join(
storage_path, 'search_indexes')
blobstore_path = options.blobstore_path or os.path.join(
storage_path, 'blobs')
if options.clear_datastore:
_clear_datastore_storage(datastore_path)
if options.clear_search_indexes:
_clear_search_indexes_storage(search_index_path)
if options.auto_id_policy == datastore_stub_util.SEQUENTIAL:
logging.warn(
"--auto_id_policy='sequential' is deprecated. This option "
"will be removed in a future release.")
application_address = '%s' % options.host
if options.port and options.port != 80:
application_address += ':' + str(options.port)
user_login_url = '/%s?%s=%%s' % (login.LOGIN_URL_RELATIVE,
login.CONTINUE_PARAM)
user_logout_url = '/%s?%s=%%s' % (login.LOGOUT_URL_RELATIVE,
login.CONTINUE_PARAM)
if options.datastore_consistency_policy == 'time':
consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy()
elif options.datastore_consistency_policy == 'random':
consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy()
elif options.datastore_consistency_policy == 'consistent':
consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0)
else:
assert 0, ('unknown consistency policy: %r' %
options.datastore_consistency_policy)
# Check if local datastore data should be converted.
# Using GCD Emulator this could convert python file stub or sqlite stub data
# to Emulator data format; Without GCD Emulator this converts python file stub
# to sqlite stub data.
if os.path.exists(datastore_path):
data_type = datastore_converter.get_data_type(datastore_path)
if options.support_datastore_emulator:
if data_type in [
datastore_converter.StubTypes.PYTHON_FILE_STUB,
datastore_converter.StubTypes.PYTHON_SQLITE_STUB
]:
if emulator_launching_thread:
emulator_launching_thread.join()
gcd_emulator_host = os.environ.get('DATASTORE_EMULATOR_HOST')
datastore_converter.convert_python_data_to_emulator(
app_id, data_type, datastore_path, gcd_emulator_host)
else:
if data_type != datastore_converter.StubTypes.PYTHON_SQLITE_STUB:
datastore_converter.convert_datastore_file_stub_data_to_sqlite(
app_id, datastore_path)
stub_util.setup_stubs(
request_data=request_info,
app_id=app_id,
application_root=app_root,
# The "trusted" flag is only relevant for Google administrative
# applications.
trusted=getattr(options, 'trusted', False),
appidentity_email_address=options.appidentity_email_address,
appidentity_private_key_path=os.path.abspath(
options.appidentity_private_key_path)
if options.appidentity_private_key_path else None,
blobstore_path=blobstore_path,
datastore_path=datastore_path,
datastore_consistency=consistency,
datastore_require_indexes=options.require_indexes,
datastore_auto_id_policy=options.auto_id_policy,
images_host_prefix='http://%s' % application_address,
logs_path=logs_path,
mail_smtp_host=options.smtp_host,
mail_smtp_port=options.smtp_port,
mail_smtp_user=options.smtp_user,
mail_smtp_password=options.smtp_password,
mail_enable_sendmail=options.enable_sendmail,
mail_show_mail_body=options.show_mail_body,
mail_allow_tls=options.smtp_allow_tls,
search_index_path=search_index_path,
taskqueue_auto_run_tasks=options.enable_task_running,
taskqueue_default_http_server=application_address,
user_login_url=user_login_url,
user_logout_url=user_logout_url,
default_gcs_bucket_name=options.default_gcs_bucket_name,
appidentity_oauth_url=options.appidentity_oauth_url,
datastore_grpc_stub_class=(datastore_grpc_stub.DatastoreGrpcStub
if options.support_datastore_emulator else
None))
if emulator_launching_thread:
emulator_launching_thread.join()
return APIServer(
options.api_host, options.api_port, app_id,
options.api_server_supports_grpc or options.support_datastore_emulator,
options.grpc_api_port, options.enable_host_checking)
