def _convert_datastore_file_stub_data_to_sqlite(app_id, datastore_path):
logging.info('Converting datastore stub data to sqlite.')
previous_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
try:
apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
datastore_stub = datastore_file_stub.DatastoreFileStub(
app_id, datastore_path, trusted=True, save_changes=False)
apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', datastore_stub)
entities = _fetch_all_datastore_entities()
sqlite_datastore_stub = datastore_sqlite_stub.DatastoreSqliteStub(
app_id, datastore_path + '.sqlite', trusted=True)
apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3',
sqlite_datastore_stub)
datastore.Put(entities)
sqlite_datastore_stub.Close()
finally:
apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', previous_stub)
shutil.copy(datastore_path, datastore_path + '.filestub')
os.remove(datastore_path)
shutil.move(datastore_path + '.sqlite', datastore_path)
logging.info(
'Datastore conversion complete. File stub data has been backed '
'up in %s', datastore_path + '.filestub')
def init_datastore_v3_stub(self,
enable=True,
datastore_file=None,
use_sqlite=False,
auto_id_policy=AUTO_ID_POLICY_SEQUENTIAL,
**stub_kw_args):
"""Enable the datastore stub.
The 'datastore_file' argument can be the path to an existing
datastore file, or None (default) to use an in-memory datastore
that is initially empty. If you use the sqlite stub and have
'datastore_file' defined, changes you apply in a test will be
written to the file. If you use the default datastore stub,
changes are _not_ saved to disk unless you set save_changes=True.
Note that you can only access those entities of the datastore file
which have the same application ID associated with them as the
test run. You can change the application ID for a test with
setup_env().
Args:
enable: True if the fake service should be enabled, False if real
service should be disabled.
datastore_file: Filename of a dev_appserver datastore file.
use_sqlite: True to use the Sqlite stub, False (default) for file stub.
auto_id_policy: How datastore stub assigns auto IDs. Either
AUTO_ID_POLICY_SEQUENTIAL or AUTO_ID_POLICY_SCATTERED.
stub_kw_args: Keyword arguments passed on to the service stub.
"""
if not enable:
self._disable_stub(DATASTORE_SERVICE_NAME)
self._disable_stub('datastore_v4')
return
if use_sqlite:
if datastore_sqlite_stub is None:
raise StubNotSupportedError(
'The sqlite stub is not supported in production.')
stub = datastore_sqlite_stub.DatastoreSqliteStub(
os.environ['APPLICATION_ID'],
datastore_file,
use_atexit=False,
auto_id_policy=auto_id_policy,
**stub_kw_args)
else:
stub_kw_args.setdefault('save_changes', False)
stub = datastore_file_stub.DatastoreFileStub(
os.environ['APPLICATION_ID'],
datastore_file,
use_atexit=False,
auto_id_policy=auto_id_policy,
**stub_kw_args)
self._register_stub(DATASTORE_SERVICE_NAME, stub,
self._deactivate_datastore_v3_stub)
v4_stub = datastore_v4_stub.DatastoreV4Stub(
os.environ['APPLICATION_ID'])
self._register_stub('datastore_v4', v4_stub)
def __init__(self, addr, app, apps_root, var, verbose=False, secure=False):
"""Initialize the WSGI server.
Args:
app: A webapp.WSGIApplication.
addr: Use this address to initialize the HTTP server.
apps_root: Applications root directory.
var: Directory for platform independent data.
verbose: Boolean, default False. If True, enable verbose mode.
secure: Boolean, default False. If True, enable secure connection.
"""
assert isinstance(app, webapp.WSGIApplication)
self.app = app
host, port = addr.split(':')
self.host = host
self.port = int(port)
self.apps_root = apps_root
self.secure = secure
# Setup signals
signal.signal(signal.SIGHUP, self._handleSignal)
signal.signal(signal.SIGQUIT, self._handleSignal)
signal.signal(signal.SIGABRT, self._handleSignal)
signal.signal(signal.SIGTERM, self._handleSignal)
# Setup environment
os.environ['APPLICATION_ID'] = APPLICATION_ID
# Setup Datastore
# We use the SQLite Datastore stub for development. Later, we take
# the production Datastore.
from google.appengine.datastore import datastore_sqlite_stub
apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
datastore_path = os.path.join(var, APPLICATION_ID + '.sqlite')
datastore = datastore_sqlite_stub.DatastoreSqliteStub(
APPLICATION_ID, datastore_path)
apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', datastore)
logging.debug('Using datastore: %s', datastore_path)
# Setup Urlfetch
from google.appengine.api import urlfetch_stub
urlfetch_stub.MAX_REQUEST_SIZE = 5012 << 30
apiproxy_stub_map.apiproxy.RegisterStub(
'urlfetch', urlfetch_stub.URLFetchServiceStub())
def setupDatastore(options, conf, datastore_file, history, require_indexes,
trusted):
"""Sets up datastore."""
name = options.datastore.lower()
if name == 'mongodb':
from typhoonae.mongodb import datastore_mongo_stub
datastore = datastore_mongo_stub.DatastoreMongoStub(
conf.application, require_indexes=require_indexes)
elif name == 'bdbdatastore':
from notdot.bdbdatastore import socket_apiproxy_stub
datastore = socket_apiproxy_stub.RecordingSocketApiProxyStub(
('localhost', 9123))
global end_request_hook
end_request_hook = datastore.closeSession
elif name == 'mysql':
from typhoonae.mysql import datastore_mysql_stub
database_info = {
"host": options.mysql_host,
"user": options.mysql_user,
"passwd": options.mysql_passwd,
"db": options.mysql_db
}
datastore = datastore_mysql_stub.DatastoreMySQLStub(
conf.application, database_info, verbose=options.debug_mode)
elif name == 'sqlite':
from google.appengine.datastore import datastore_sqlite_stub
datastore = datastore_sqlite_stub.DatastoreSqliteStub(
conf.application,
datastore_file,
require_indexes=require_indexes,
trusted=trusted)
else:
raise RuntimeError, "unknown datastore"
apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', datastore)
if name == 'bdbdatastore':
from google.appengine.tools import dev_appserver_index
app_root = os.getcwd()
logging.info("%s" % app_root)
dev_appserver_index.SetupIndexes(conf.application, app_root)
dev_appserver_index.IndexYamlUpdater(app_root)
def shell(datastore_path='',
history_path='',
useful_imports=True,
use_ipython=True,
use_sqlite=False):
""" Start a new interactive python session."""
banner = 'Interactive Kay Shell'
if useful_imports:
namespace = create_useful_locals()
else:
namespace = {}
appid = get_appid()
os.environ['APPLICATION_ID'] = appid
p = get_datastore_paths()
if not datastore_path:
datastore_path = p[0]
if not history_path:
history_path = p[1]
apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
if use_sqlite:
from google.appengine.datastore import datastore_sqlite_stub
stub = datastore_sqlite_stub.DatastoreSqliteStub(
appid, datastore_path, history_path)
else:
stub = datastore_file_stub.DatastoreFileStub(appid, datastore_path,
history_path)
apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', stub)
if use_ipython:
try:
import IPython
except ImportError:
pass
else:
sh = IPython.Shell.IPShellEmbed(argv='', banner=banner)
sh(global_ns={}, local_ns=namespace)
return
sys.ps1 = '%s> ' % appid
if readline is not None:
readline.parse_and_bind('tab: complete')
atexit.register(lambda: readline.write_history_file(HISTORY_PATH))
if os.path.exists(HISTORY_PATH):
readline.read_history_file(HISTORY_PATH)
from code import interact
interact(banner, local=namespace)
def setup_stubs(storage_path, options, configuration):
datastore_path = options.datastore_path or os.path.join(storage_path, 'datastore.db')
search_index_path = options.search_indexes_path or os.path.join(storage_path, 'search_indexes')
blobstore_path = options.blobstore_path or os.path.join(storage_path, 'blobs')
# Init the proxy map and stubs
from google.appengine.api import apiproxy_stub_map
apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
# DB
from google.appengine.datastore import datastore_sqlite_stub
apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', datastore_sqlite_stub.DatastoreSqliteStub(configuration.app_id, datastore_path))
# Search service
from google.appengine.api.search import simple_search_stub
apiproxy_stub_map.apiproxy.RegisterStub('search', simple_search_stub.SearchServiceStub(index_file=search_index_path))
from google.appengine.api.blobstore import file_blob_storage
blob_storage = file_blob_storage.FileBlobStorage(blobstore_path, configuration.app_id)
from google.appengine.api.blobstore import blobstore_stub
apiproxy_stub_map.apiproxy.RegisterStub('blobstore', blobstore_stub.BlobstoreServiceStub(blob_storage))
from google.appengine.api.app_identity import app_identity_stub
apiproxy_stub_map.apiproxy.RegisterStub('app_identity_service', app_identity_stub.AppIdentityServiceStub())
# Capability
from google.appengine.api.capabilities import capability_stub
apiproxy_stub_map.apiproxy.RegisterStub('capability_service', capability_stub.CapabilityServiceStub())
# Memcache
from google.appengine.api.memcache import memcache_stub
apiproxy_stub_map.apiproxy.RegisterStub('memcache', memcache_stub.MemcacheServiceStub())
# Task queues
from google.appengine.api.taskqueue import taskqueue_stub
apiproxy_stub_map.apiproxy.RegisterStub('taskqueue', taskqueue_stub.TaskQueueServiceStub())
# URLfetch service
from google.appengine.api import urlfetch_stub
apiproxy_stub_map.apiproxy.RegisterStub('urlfetch', urlfetch_stub.URLFetchServiceStub())
def convert_sqlite_data_to_emulator(app_id, filename, gcd_emulator_host):
"""Convert datastore sqlite stub data to cloud emulator data.
Args:
app_id: A String representing application ID.
filename: A String representing the absolute path to SQLite data.
gcd_emulator_host: A String in the format of host:port indicate the hostname
and port number of gcd emulator.
Raises:
PersistException: if the call to emulator's /persist endpoint fails.
"""
previous_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
sqlite_stub = datastore_sqlite_stub.DatastoreSqliteStub(app_id,
filename,
trusted=True,
use_atexit=False)
apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', sqlite_stub)
entities = _fetch_all_datastore_entities()
if entities:
logging.info('Fetched %d entities from %s', len(entities), filename)
grpc_stub = datastore_grpc_stub.DatastoreGrpcStub(gcd_emulator_host)
grpc_stub.get_or_set_call_handler_stub()
apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', grpc_stub)
datastore.Put(entities)
# persist entities to disk in emulator's data format.
conn = httplib.HTTPConnection(gcd_emulator_host)
conn.request('POST', '/persist')
response = conn.getresponse()
msg = response.read()
if httplib.OK != response.status:
raise PersistException(msg)
logging.info('Datastore conversion complete')
else:
logging.warning(
'Fetched 0 entity from %s, will not create cloud '
'datastore emulator file', filename)
sqlite_stub.Close()
apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', previous_stub)
def reset_stub(stub_name, stub_options):
"""Reset a stub currently in place with a new one.
You might wish to swap out a currently running stub with another. For
example, in cases where you are running tests that use the devappserver and
you want to reset the stub in between test cases.
Currently the datastore is the only stub that can be reset, but others could
be added here if needed.
Args:
stub_name: string
stub_options: a dictionary of options to be used when resetting the stub.
When stub_name == 'datastore_v3', we expect an object that looks like:
{
'app_id': 'dev~id-of-application',
'db_consistency_probability': 1.0,
'datastore_path': /path/to/sqlite.sql
}
"""
if stub_name != 'datastore_v3':
raise NotImplementedError("Only datastore stubs can be currently reset.")
# TODO(dhruv): Figure out when it is safe to close a datastore stub (it is
# not currently in read-only mode) and do that here before replacing it.
# Create a new stub and replace it in the api stub map
policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy(
probability=stub_options['db_consistency_probability'])
new_stub = (
datastore_sqlite_stub.DatastoreSqliteStub(
stub_options['app_id'],
stub_options['datastore_path'],
consistency_policy=policy
)
)
apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', new_stub)
def setup_datastore_stubs():
if "test" in sys.argv:
return
from google.appengine.datastore import datastore_sqlite_stub
from google.appengine.api import apiproxy_stub_map
from google.appengine.datastore import datastore_stub_util
app_id = application_id()
datastore = datastore_sqlite_stub.DatastoreSqliteStub(
"dev~" + app_id,
os.path.join(data_root(), "datastore.db"),
require_indexes=False,
trusted=False,
root_path=find_project_root(),
use_atexit=True)
datastore.SetConsistencyPolicy(
datastore_stub_util.TimeBasedHRConsistencyPolicy())
apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', datastore)
def convert_datastore_file_stub_data_to_sqlite(app_id, datastore_file):
"""Convert datastore_file_stub data into sqlite data.
Args:
app_id: String indicating application id.
datastore_file: String indicating the file name of datastore_file_stub data.
Raises:
IOError: if datastore_file is not writeable.
"""
if not os.access(datastore_file, os.W_OK):
raise IOError('Does not have write access to %s' % datastore_file)
logging.info('Converting datastore file stub data to sqlite.')
previous_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
sqlite_file_name = datastore_file + '.sqlite'
try:
apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
datastore_stub = datastore_file_stub.DatastoreFileStub(
app_id, datastore_file, trusted=True, save_changes=False)
apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', datastore_stub)
entities = _fetch_all_datastore_entities()
sqlite_datastore_stub = datastore_sqlite_stub.DatastoreSqliteStub(
app_id, sqlite_file_name, trusted=True)
apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3',
sqlite_datastore_stub)
datastore.Put(entities)
sqlite_datastore_stub.Close()
finally:
apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', previous_stub)
back_up_file_name = datastore_file + '.filestub'
shutil.copy(datastore_file, back_up_file_name)
os.remove(datastore_file)
shutil.move(sqlite_file_name, datastore_file)
logging.info(
'Datastore conversion complete. File stub data has been backed '
'up in %s', back_up_file_name)
def get_new_connection(self, conn_params):
from google.appengine.datastore import datastore_stub_util
from google.appengine.datastore import datastore_sqlite_stub
from google.appengine.api import apiproxy_stub_map
datastore_stub = datastore_sqlite_stub.DatastoreSqliteStub(
"dev~" + application_id(),
os.path.join(data_root(), 'datastore.db'),
False,
trusted=False,
root_path=find_project_root(),
auto_id_policy=datastore_stub_util.SCATTERED)
datastore_stub.SetConsistencyPolicy(
datastore_stub_util.PseudoRandomHRConsistencyPolicy())
apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', datastore_stub)
class FakeConnection:
def commit(self):
pass
return FakeConnection()
def convert_python_data_to_emulator(app_id, stub_type, filename,
gcd_emulator_host):
"""Convert datastore_file_stub or datastore_sqlite_stub data to emulator data.
Args:
app_id: A String representing application ID.
stub_type: A String representing the stub type filename belongs to.
filename: A String representing the absolute path to local data.
gcd_emulator_host: A String in the format of host:port indicate the hostname
and port number of gcd emulator.
"""
previous_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
try:
if stub_type == StubTypes.PYTHON_FILE_STUB:
logging.info(
'Converting datastore_file_stub data to cloud datastore emulator '
'data.')
python_stub = datastore_file_stub.DatastoreFileStub(
app_id, filename, trusted=True, save_changes=False)
else: # Sqlite stub
logging.info(
'Converting datastore_sqlite_stub data to cloud datastore emulator '
'data.')
python_stub = datastore_sqlite_stub.DatastoreSqliteStub(
app_id, filename, trusted=True, use_atexit=False)
apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', python_stub)
entities = _fetch_all_datastore_entities()
grpc_stub = datastore_grpc_stub.DatastoreGrpcStub(gcd_emulator_host)
grpc_stub.get_or_set_call_handler_stub()
apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', grpc_stub)
datastore.Put(entities)
logging.info('Conversion complete.')
python_stub.Close()
finally:
apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', previous_stub)
logging.info('Datastore conversion complete')
def setup_stubs(
request_data,
app_id,
application_root,
trusted,
appidentity_email_address,
appidentity_private_key_path,
blobstore_path,
datastore_consistency,
datastore_path,
datastore_require_indexes,
datastore_auto_id_policy,
images_host_prefix,
logs_path,
mail_smtp_host,
mail_smtp_port,
mail_smtp_user,
mail_smtp_password,
mail_enable_sendmail,
mail_show_mail_body,
mail_allow_tls,
search_index_path,
taskqueue_auto_run_tasks,
taskqueue_default_http_server,
user_login_url,
user_logout_url,
default_gcs_bucket_name,
appidentity_oauth_url=None,
support_datastore_emulator=False):
"""Configures the APIs hosted by this server.
Args:
request_data: An apiproxy_stub.RequestInformation instance used by the
stubs to lookup information about the request associated with an API
call.
app_id: The str application id e.g. "guestbook".
application_root: The path to the directory containing the user's
application e.g. "/home/joe/myapp".
trusted: A bool indicating if privileged APIs should be made available.
appidentity_email_address: Email address associated with a service account
that has a downloadable key. May be None for no local application
identity.
appidentity_private_key_path: Path to private key file associated with
service account (.pem format). Must be set if appidentity_email_address
is set.
blobstore_path: The path to the file that should be used for blobstore
storage.
datastore_consistency: The datastore_stub_util.BaseConsistencyPolicy to
use as the datastore consistency policy.
datastore_path: The path to the file that should be used for datastore
storage.
datastore_require_indexes: A bool indicating if the same production
datastore indexes requirements should be enforced i.e. if True then
a google.appengine.ext.db.NeedIndexError will be be raised if a query
is executed without the required indexes.
datastore_auto_id_policy: The type of sequence from which the datastore
stub assigns auto IDs, either datastore_stub_util.SEQUENTIAL or
datastore_stub_util.SCATTERED.
images_host_prefix: The URL prefix (protocol://host:port) to prepend to
image urls on calls to images.GetUrlBase.
logs_path: Path to the file to store the logs data in.
mail_smtp_host: The SMTP hostname that should be used when sending e-mails.
If None then the mail_enable_sendmail argument is considered.
mail_smtp_port: The SMTP port number that should be used when sending
e-mails. If this value is None then mail_smtp_host must also be None.
mail_smtp_user: The username to use when authenticating with the
SMTP server. This value may be None if mail_smtp_host is also None or if
the SMTP server does not require authentication.
mail_smtp_password: The password to use when authenticating with the
SMTP server. This value may be None if mail_smtp_host or mail_smtp_user
is also None.
mail_enable_sendmail: A bool indicating if sendmail should be used when
sending e-mails. This argument is ignored if mail_smtp_host is not None.
mail_show_mail_body: A bool indicating whether the body of sent e-mails
should be written to the logs.
mail_allow_tls: A bool indicating whether TLS should be allowed when
communicating with an SMTP server. This argument is ignored if
mail_smtp_host is None.
search_index_path: The path to the file that should be used for search index
storage.
taskqueue_auto_run_tasks: A bool indicating whether taskqueue tasks should
be run automatically or it the must be manually triggered.
taskqueue_default_http_server: A str containing the address of the http
server that should be used to execute tasks.
user_login_url: A str containing the url that should be used for user login.
user_logout_url: A str containing the url that should be used for user
logout.
default_gcs_bucket_name: A str, overriding the default bucket behavior.
appidentity_oauth_url: A str containing the url to the oauth2 server to use
to authenticate the private key. If set to None, then the standard
google oauth2 server is used.
support_datastore_emulator: A bool indicating if datastore_emulator is
supported.
"""
identity_stub = app_identity_stub.AppIdentityServiceStub.Create(
email_address=appidentity_email_address,
private_key_path=appidentity_private_key_path,
oauth_url=appidentity_oauth_url)
if default_gcs_bucket_name is not None:
identity_stub.SetDefaultGcsBucketName(default_gcs_bucket_name)
apiproxy_stub_map.apiproxy.RegisterStub('app_identity_service', identity_stub)
blob_storage = file_blob_storage.FileBlobStorage(blobstore_path, app_id)
apiproxy_stub_map.apiproxy.RegisterStub(
'blobstore',
blobstore_stub.BlobstoreServiceStub(blob_storage,
request_data=request_data))
apiproxy_stub_map.apiproxy.RegisterStub(
'capability_service',
capability_stub.CapabilityServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'channel',
channel_service_stub.ChannelServiceStub(request_data=request_data))
if support_datastore_emulator:
datastore_emulator_host = os.environ.get('DATASTORE_EMULATOR_HOST', '')
error_msg = 'DATASTORE_EMULATOR_HOST is not found in environment variables'
assert datastore_emulator_host, error_msg
apiproxy_stub_map.apiproxy.ReplaceStub(
'datastore_v3',
datastore_grpc_stub.DatastoreGrpcStub(datastore_emulator_host)
)
else:
apiproxy_stub_map.apiproxy.ReplaceStub(
'datastore_v3',
datastore_sqlite_stub.DatastoreSqliteStub(
app_id,
datastore_path,
datastore_require_indexes,
trusted,
root_path=application_root,
auto_id_policy=datastore_auto_id_policy,
consistency_policy=datastore_consistency))
apiproxy_stub_map.apiproxy.RegisterStub(
'datastore_v4',
datastore_v4_stub.DatastoreV4Stub(app_id))
apiproxy_stub_map.apiproxy.RegisterStub(
'file',
file_service_stub.FileServiceStub(blob_storage))
try:
from google.appengine.api.images import images_stub
except ImportError:
# We register a stub which throws a NotImplementedError for most RPCs.
from google.appengine.api.images import images_not_implemented_stub
apiproxy_stub_map.apiproxy.RegisterStub(
'images',
images_not_implemented_stub.ImagesNotImplementedServiceStub(
host_prefix=images_host_prefix))
else:
apiproxy_stub_map.apiproxy.RegisterStub(
'images',
images_stub.ImagesServiceStub(host_prefix=images_host_prefix))
apiproxy_stub_map.apiproxy.RegisterStub(
'logservice',
logservice_stub.LogServiceStub(logs_path=logs_path))
apiproxy_stub_map.apiproxy.RegisterStub(
'mail',
mail_stub.MailServiceStub(mail_smtp_host,
mail_smtp_port,
mail_smtp_user,
mail_smtp_password,
enable_sendmail=mail_enable_sendmail,
show_mail_body=mail_show_mail_body,
allow_tls=mail_allow_tls))
apiproxy_stub_map.apiproxy.RegisterStub(
'memcache',
memcache_stub.MemcacheServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'modules',
modules_stub.ModulesServiceStub(request_data))
apiproxy_stub_map.apiproxy.RegisterStub(
'remote_socket',
_remote_socket_stub.RemoteSocketServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'search',
simple_search_stub.SearchServiceStub(index_file=search_index_path))
apiproxy_stub_map.apiproxy.RegisterStub(
'system',
system_stub.SystemServiceStub(request_data=request_data))
apiproxy_stub_map.apiproxy.RegisterStub(
'taskqueue',
taskqueue_stub.TaskQueueServiceStub(
root_path=application_root,
auto_task_running=taskqueue_auto_run_tasks,
default_http_server=taskqueue_default_http_server,
request_data=request_data))
apiproxy_stub_map.apiproxy.GetStub('taskqueue').StartBackgroundExecution()
apiproxy_stub_map.apiproxy.RegisterStub(
'urlfetch',
urlfetch_stub.URLFetchServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'user',
user_service_stub.UserServiceStub(login_url=user_login_url,
logout_url=user_logout_url,
request_data=request_data))
apiproxy_stub_map.apiproxy.RegisterStub(
'xmpp',
xmpp_service_stub.XmppServiceStub())
def _SetupStubs(
app_id,
application_root,
trusted,
blobstore_path,
use_sqlite,
auto_id_policy,
high_replication,
datastore_path,
datastore_require_indexes,
images_host_prefix,
logs_path,
mail_smtp_host,
mail_smtp_port,
mail_smtp_user,
mail_smtp_password,
mail_enable_sendmail,
mail_show_mail_body,
matcher_prospective_search_path,
taskqueue_auto_run_tasks,
taskqueue_task_retry_seconds,
taskqueue_default_http_server,
user_login_url,
user_logout_url):
"""Configures the APIs hosted by this server.
Args:
app_id: The str application id e.g. "guestbook".
application_root: The path to the directory containing the user's
application e.g. "/home/bquinlan/myapp".
trusted: A bool indicating if privileged APIs should be made available.
blobstore_path: The path to the file that should be used for blobstore
storage.
use_sqlite: A bool indicating whether DatastoreSqliteStub or
DatastoreFileStub should be used.
auto_id_policy: One of datastore_stub_util.SEQUENTIAL or .SCATTERED,
indicating whether the Datastore stub should assign IDs sequentially
or scattered.
high_replication: A bool indicating whether to use the high replication
consistency model.
datastore_path: The path to the file that should be used for datastore
storage.
datastore_require_indexes: A bool indicating if the same production
datastore indexes requirements should be enforced i.e. if True then
a google.appengine.ext.db.NeedIndexError will be be raised if a query
is executed without the required indexes.
images_host_prefix: The URL prefix (protocol://host:port) to preprend to
image urls on calls to images.GetUrlBase.
logs_path: Path to the file to store the logs data in.
mail_smtp_host: The SMTP hostname that should be used when sending e-mails.
If None then the mail_enable_sendmail argument is considered.
mail_smtp_port: The SMTP port number that should be used when sending
e-mails. If this value is None then mail_smtp_host must also be None.
mail_smtp_user: The username to use when authenticating with the
SMTP server. This value may be None if mail_smtp_host is also None or if
the SMTP server does not require authentication.
mail_smtp_password: The password to use when authenticating with the
SMTP server. This value may be None if mail_smtp_host or mail_smtp_user
is also None.
mail_enable_sendmail: A bool indicating if sendmail should be used when
sending e-mails. This argument is ignored if mail_smtp_host is not None.
mail_show_mail_body: A bool indicating whether the body of sent e-mails
should be written to the logs.
matcher_prospective_search_path: The path to the file that should be used to
save prospective search subscriptions.
taskqueue_auto_run_tasks: A bool indicating whether taskqueue tasks should
be run automatically or it the must be manually triggered.
taskqueue_task_retry_seconds: An int representing the number of seconds to
wait before a retrying a failed taskqueue task.
taskqueue_default_http_server: A str containing the address of the http
server that should be used to execute tasks.
user_login_url: A str containing the url that should be used for user login.
user_logout_url: A str containing the url that should be used for user
logout.
"""
os.environ['APPLICATION_ID'] = app_id
apiproxy_stub_map.apiproxy.RegisterStub(
'app_identity_service',
app_identity_stub.AppIdentityServiceStub())
blob_storage = file_blob_storage.FileBlobStorage(blobstore_path, app_id)
apiproxy_stub_map.apiproxy.RegisterStub(
'blobstore',
blobstore_stub.BlobstoreServiceStub(blob_storage))
apiproxy_stub_map.apiproxy.RegisterStub(
'capability_service',
capability_stub.CapabilityServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'channel',
channel_service_stub.ChannelServiceStub())
if use_sqlite:
datastore = datastore_sqlite_stub.DatastoreSqliteStub(
app_id,
datastore_path,
datastore_require_indexes,
trusted,
root_path=application_root,
auto_id_policy=auto_id_policy)
else:
datastore = datastore_file_stub.DatastoreFileStub(
app_id,
datastore_path,
datastore_require_indexes,
trusted,
root_path=application_root,
auto_id_policy=auto_id_policy)
if high_replication:
datastore.SetConsistencyPolicy(
datastore_stub_util.TimeBasedHRConsistencyPolicy())
apiproxy_stub_map.apiproxy.RegisterStub(
'datastore_v3', datastore)
apiproxy_stub_map.apiproxy.RegisterStub(
'file',
file_service_stub.FileServiceStub(blob_storage))
try:
from google.appengine.api.images import images_stub
except ImportError:
logging.warning('Could not initialize images API; you are likely missing '
'the Python "PIL" module.')
from google.appengine.api.images import images_not_implemented_stub
apiproxy_stub_map.apiproxy.RegisterStub(
'images',
images_not_implemented_stub.ImagesNotImplementedServiceStub())
else:
apiproxy_stub_map.apiproxy.RegisterStub(
'images',
images_stub.ImagesServiceStub(host_prefix=images_host_prefix))
apiproxy_stub_map.apiproxy.RegisterStub(
'logservice',
logservice_stub.LogServiceStub(persist=True))
apiproxy_stub_map.apiproxy.RegisterStub(
'mail',
mail_stub.MailServiceStub(mail_smtp_host,
mail_smtp_port,
mail_smtp_user,
mail_smtp_password,
enable_sendmail=mail_enable_sendmail,
show_mail_body=mail_show_mail_body))
apiproxy_stub_map.apiproxy.RegisterStub(
'memcache',
memcache_stub.MemcacheServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'search',
simple_search_stub.SearchServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub('system',
system_stub.SystemServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'taskqueue',
taskqueue_stub.TaskQueueServiceStub(
root_path=application_root,
auto_task_running=taskqueue_auto_run_tasks,
task_retry_seconds=taskqueue_task_retry_seconds,
default_http_server=taskqueue_default_http_server))
apiproxy_stub_map.apiproxy.GetStub('taskqueue').StartBackgroundExecution()
apiproxy_stub_map.apiproxy.RegisterStub(
'urlfetch',
urlfetch_stub.URLFetchServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'user',
user_service_stub.UserServiceStub(login_url=user_login_url,
logout_url=user_logout_url))
apiproxy_stub_map.apiproxy.RegisterStub(
'xmpp',
xmpp_service_stub.XmppServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'matcher',
prospective_search_stub.ProspectiveSearchStub(
matcher_prospective_search_path,
apiproxy_stub_map.apiproxy.GetStub('taskqueue')))
def setup_stubs(request_data, app_id, application_root, trusted,
blobstore_path, datastore_consistency, datastore_path,
datastore_require_indexes, datastore_auto_id_policy,
images_host_prefix, logs_path, mail_smtp_host, mail_smtp_port,
mail_smtp_user, mail_smtp_password, mail_enable_sendmail,
mail_show_mail_body, matcher_prospective_search_path,
search_index_path, taskqueue_auto_run_tasks,
taskqueue_default_http_server, user_login_url,
user_logout_url):
"""Configures the APIs hosted by this server.
Args:
request_data: An apiproxy_stub.RequestInformation instance used by the
stubs to lookup information about the request associated with an API
call.
app_id: The str application id e.g. "guestbook".
application_root: The path to the directory containing the user's
application e.g. "/home/joe/myapp".
trusted: A bool indicating if privileged APIs should be made available.
blobstore_path: The path to the file that should be used for blobstore
storage.
datastore_consistency: The datastore_stub_util.BaseConsistencyPolicy to
use as the datastore consistency policy.
datastore_path: The path to the file that should be used for datastore
storage.
datastore_require_indexes: A bool indicating if the same production
datastore indexes requirements should be enforced i.e. if True then
a google.appengine.ext.db.NeedIndexError will be be raised if a query
is executed without the required indexes.
datastore_auto_id_policy: The type of sequence from which the datastore
stub assigns auto IDs, either datastore_stub_util.SEQUENTIAL or
datastore_stub_util.SCATTERED.
images_host_prefix: The URL prefix (protocol://host:port) to prepend to
image urls on calls to images.GetUrlBase.
logs_path: Path to the file to store the logs data in.
mail_smtp_host: The SMTP hostname that should be used when sending e-mails.
If None then the mail_enable_sendmail argument is considered.
mail_smtp_port: The SMTP port number that should be used when sending
e-mails. If this value is None then mail_smtp_host must also be None.
mail_smtp_user: The username to use when authenticating with the
SMTP server. This value may be None if mail_smtp_host is also None or if
the SMTP server does not require authentication.
mail_smtp_password: The password to use when authenticating with the
SMTP server. This value may be None if mail_smtp_host or mail_smtp_user
is also None.
mail_enable_sendmail: A bool indicating if sendmail should be used when
sending e-mails. This argument is ignored if mail_smtp_host is not None.
mail_show_mail_body: A bool indicating whether the body of sent e-mails
should be written to the logs.
matcher_prospective_search_path: The path to the file that should be used to
save prospective search subscriptions.
search_index_path: The path to the file that should be used for search index
storage.
taskqueue_auto_run_tasks: A bool indicating whether taskqueue tasks should
be run automatically or it the must be manually triggered.
taskqueue_default_http_server: A str containing the address of the http
server that should be used to execute tasks.
user_login_url: A str containing the url that should be used for user login.
user_logout_url: A str containing the url that should be used for user
logout.
"""
apiproxy_stub_map.apiproxy.RegisterStub(
'app_identity_service', app_identity_stub.AppIdentityServiceStub())
blob_storage = file_blob_storage.FileBlobStorage(blobstore_path, app_id)
apiproxy_stub_map.apiproxy.RegisterStub(
'blobstore',
blobstore_stub.BlobstoreServiceStub(blob_storage,
request_data=request_data))
apiproxy_stub_map.apiproxy.RegisterStub(
'capability_service', capability_stub.CapabilityServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'channel',
channel_service_stub.ChannelServiceStub(request_data=request_data))
datastore_stub = datastore_sqlite_stub.DatastoreSqliteStub(
app_id,
datastore_path,
datastore_require_indexes,
trusted,
root_path=application_root,
auto_id_policy=datastore_auto_id_policy)
datastore_stub.SetConsistencyPolicy(datastore_consistency)
apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', datastore_stub)
apiproxy_stub_map.apiproxy.RegisterStub(
'file', file_service_stub.FileServiceStub(blob_storage))
try:
from google.appengine.api.images import images_stub
except ImportError:
logging.warning(
'Could not initialize images API; you are likely missing '
'the Python "PIL" module.')
# We register a stub which throws a NotImplementedError for most RPCs.
from google.appengine.api.images import images_not_implemented_stub
apiproxy_stub_map.apiproxy.RegisterStub(
'images',
images_not_implemented_stub.ImagesNotImplementedServiceStub(
host_prefix=images_host_prefix))
else:
apiproxy_stub_map.apiproxy.RegisterStub(
'images',
images_stub.ImagesServiceStub(host_prefix=images_host_prefix))
apiproxy_stub_map.apiproxy.RegisterStub(
'logservice', logservice_stub.LogServiceStub(logs_path=logs_path))
apiproxy_stub_map.apiproxy.RegisterStub(
'mail',
mail_stub.MailServiceStub(mail_smtp_host,
mail_smtp_port,
mail_smtp_user,
mail_smtp_password,
enable_sendmail=mail_enable_sendmail,
show_mail_body=mail_show_mail_body))
apiproxy_stub_map.apiproxy.RegisterStub(
'memcache', memcache_stub.MemcacheServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'search',
simple_search_stub.SearchServiceStub(index_file=search_index_path))
apiproxy_stub_map.apiproxy.RegisterStub(
'servers', servers_stub.ServersServiceStub(request_data))
apiproxy_stub_map.apiproxy.RegisterStub(
'system', system_stub.SystemServiceStub(request_data=request_data))
apiproxy_stub_map.apiproxy.RegisterStub(
'taskqueue',
taskqueue_stub.TaskQueueServiceStub(
root_path=application_root,
auto_task_running=taskqueue_auto_run_tasks,
default_http_server=taskqueue_default_http_server,
request_data=request_data))
apiproxy_stub_map.apiproxy.GetStub('taskqueue').StartBackgroundExecution()
urlmatchers_to_fetch_functions = []
urlmatchers_to_fetch_functions.extend(
gcs_dispatcher.URLMATCHERS_TO_FETCH_FUNCTIONS)
apiproxy_stub_map.apiproxy.RegisterStub(
'urlfetch',
urlfetch_stub.URLFetchServiceStub(
urlmatchers_to_fetch_functions=urlmatchers_to_fetch_functions))
apiproxy_stub_map.apiproxy.RegisterStub(
'user',
user_service_stub.UserServiceStub(login_url=user_login_url,
logout_url=user_logout_url,
request_data=request_data))
apiproxy_stub_map.apiproxy.RegisterStub(
'xmpp', xmpp_service_stub.XmppServiceStub())
apiproxy_stub_map.apiproxy.RegisterStub(
'matcher',
prospective_search_stub.ProspectiveSearchStub(
matcher_prospective_search_path,
apiproxy_stub_map.apiproxy.GetStub('taskqueue')))
apiproxy_stub_map.apiproxy.RegisterStub(
'remote_socket', _remote_socket_stub.RemoteSocketServiceStub())
def init_datastore_v3_stub(self, enable=True, datastore_file=None,
use_sqlite=False,
auto_id_policy=AUTO_ID_POLICY_SEQUENTIAL,
**stub_kw_args):
"""Enables the datastore stub.
The `datastore_file` argument can be set to the path of an existing
datastore file, or `None` (default) to use an in-memory datastore that is
initially empty. If you use the sqlite stub and have defined
`datastore_file`, the changes that you apply in a test will be written to
the file. If you use the default datastore stub, changes are *not* saved to
disk unless you set `save_changes=True`.
Note:
You can only access those entities of the datastore file that use the
same application ID as the test run. You can change the application ID
for a test with `setup_env()`.
Args:
enable: `True` if the fake service should be enabled, or `False` if the
real service should be disabled.
datastore_file: File name of a dev_appserver datastore file.
use_sqlite: `True` to use the Sqlite stub, or `False` (default) to use
the file stub.
auto_id_policy: How datastore stub assigns auto IDs. This value can be
either `AUTO_ID_POLICY_SEQUENTIAL` or `AUTO_ID_POLICY_SCATTERED`.
**stub_kw_args: Keyword arguments passed on to the service stub.
Raises:
StubNotSupportedError: If datastore_sqlite_stub is None.
"""
if USE_DATASTORE_EMULATOR:
self._disable_stub(DATASTORE_SERVICE_NAME)
testserver_util.TESTSERVER_UTIL.reset_emulator()
testserver_util.remote_api_stub.ConfigureRemoteApi(
os.environ['APPLICATION_ID'],
'/',
lambda: ('', ''),
'localhost:%d' % testserver_util.TESTSERVER_UTIL.api_port,
services=[DATASTORE_SERVICE_NAME],
apiproxy=self._test_stub_map,
use_remote_datastore=False)
self._enabled_stubs[DATASTORE_SERVICE_NAME] = None
return
if not enable:
self._disable_stub(DATASTORE_SERVICE_NAME)
self._disable_stub(datastore_v4_stub.SERVICE_NAME)
self._disable_stub(cloud_datastore_v1_stub.SERVICE_NAME)
return
if use_sqlite:
if datastore_sqlite_stub is None:
raise StubNotSupportedError(
'The sqlite stub is not supported in production.')
stub = datastore_sqlite_stub.DatastoreSqliteStub(
os.environ['APPLICATION_ID'],
datastore_file,
use_atexit=False,
auto_id_policy=auto_id_policy,
**stub_kw_args)
else:
stub_kw_args.setdefault('save_changes', False)
stub = datastore_file_stub.DatastoreFileStub(
os.environ['APPLICATION_ID'],
datastore_file,
use_atexit=False,
auto_id_policy=auto_id_policy,
**stub_kw_args)
self._register_stub(DATASTORE_SERVICE_NAME, stub,
self._deactivate_datastore_v3_stub)
v4_stub = datastore_v4_stub.DatastoreV4Stub(os.environ['APPLICATION_ID'])
self._register_stub(datastore_v4_stub.SERVICE_NAME, v4_stub)
if datastore_pbs._CLOUD_DATASTORE_ENABLED:
helper = datastore_pbs.googledatastore.helper
credential_env = helper._DATASTORE_USE_STUB_CREDENTIAL_FOR_TEST_ENV
os.environ[credential_env] = 'True'
cloud_stub = cloud_datastore_v1_stub.CloudDatastoreV1Stub(
os.environ['APPLICATION_ID'])
self._register_stub(cloud_datastore_v1_stub.SERVICE_NAME, cloud_stub)
def init_datastore_v3_stub(self,
enable=True,
datastore_file=None,
use_sqlite=False,
auto_id_policy=AUTO_ID_POLICY_SEQUENTIAL,
**stub_kw_args):
"""Enables the datastore stub.
The `datastore_file` argument can be set to the path of an existing
datastore file, or `None` (default) to use an in-memory datastore that is
initially empty. If you use the sqlite stub and have defined
`datastore_file`, the changes that you apply in a test will be written to
the file. If you use the default datastore stub, changes are *not* saved to
disk unless you set `save_changes=True`.
Note:
You can only access those entities of the datastore file that use the
same application ID as the test run. You can change the application ID
for a test with `setup_env()`.
Args:
enable: `True` if the fake service should be enabled, or `False` if the
real service should be disabled.
datastore_file: File name of a dev_appserver datastore file.
use_sqlite: `True` to use the Sqlite stub, or `False` (default) to use
the file stub.
auto_id_policy: How datastore stub assigns auto IDs. This value can be
either `AUTO_ID_POLICY_SEQUENTIAL` or `AUTO_ID_POLICY_SCATTERED`.
**stub_kw_args: Keyword arguments passed on to the service stub.
Raises:
StubNotSupportedError: If datastore_sqlite_stub is None.
"""
if self._use_datastore_emulator:
self._disable_stub(DATASTORE_SERVICE_NAME)
delegate_stub = (remote_api_stub.DatastoreStubTestbedDelegate(
self.rpc_server,
'/',
stub_kw_args.get('max_request_size',
apiproxy_stub.MAX_REQUEST_SIZE),
emulator_port=self._emulator_port))
delegate_stub.Clear()
self._test_stub_map.RegisterStub(DATASTORE_SERVICE_NAME,
delegate_stub)
consistency_policy = stub_kw_args.get(
'consistency_policy',
datastore_stub_util.PseudoRandomHRConsistencyPolicy(
probability=1.0))
datastore_stub_util.UpdateEmulatorConfig(self._emulator_port,
auto_id_policy,
consistency_policy)
if isinstance(consistency_policy,
datastore_stub_util.PseudoRandomHRConsistencyPolicy):
consistency_policy.is_using_cloud_datastore_emulator = True
consistency_policy.emulator_port = self._emulator_port
self._enabled_stubs[DATASTORE_SERVICE_NAME] = None
return
if not enable:
self._disable_stub(DATASTORE_SERVICE_NAME)
self._disable_stub(datastore_v4_stub.SERVICE_NAME)
self._disable_stub(cloud_datastore_v1_stub.SERVICE_NAME)
return
if use_sqlite:
if datastore_sqlite_stub is None:
raise StubNotSupportedError(
'The sqlite stub is not supported in production.')
stub = datastore_sqlite_stub.DatastoreSqliteStub(
os.environ['APPLICATION_ID'],
datastore_file,
use_atexit=False,
auto_id_policy=auto_id_policy,
**stub_kw_args)
else:
stub_kw_args.setdefault('save_changes', False)
stub = datastore_file_stub.DatastoreFileStub(
os.environ['APPLICATION_ID'],
datastore_file,
use_atexit=False,
auto_id_policy=auto_id_policy,
**stub_kw_args)
self._register_stub(DATASTORE_SERVICE_NAME, stub,
self._deactivate_datastore_v3_stub)
v4_stub = datastore_v4_stub.DatastoreV4Stub(
os.environ['APPLICATION_ID'])
self._register_stub(datastore_v4_stub.SERVICE_NAME, v4_stub)
if datastore_pbs._CLOUD_DATASTORE_ENABLED:
helper = datastore_pbs.googledatastore.helper
credential_env = helper._DATASTORE_USE_STUB_CREDENTIAL_FOR_TEST_ENV
os.environ[credential_env] = 'True'
cloud_stub = cloud_datastore_v1_stub.CloudDatastoreV1Stub(
os.environ['APPLICATION_ID'])
self._register_stub(cloud_datastore_v1_stub.SERVICE_NAME,
cloud_stub)
