def _convert_datastore_file_stub_data_to_sqlite(app_id, datastore_path): logging.info('Converting datastore stub data to sqlite.') previous_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') try: apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap() datastore_stub = datastore_file_stub.DatastoreFileStub( app_id, datastore_path, trusted=True, save_changes=False) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', datastore_stub) entities = _fetch_all_datastore_entities() sqlite_datastore_stub = datastore_sqlite_stub.DatastoreSqliteStub( app_id, datastore_path + '.sqlite', trusted=True) apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', sqlite_datastore_stub) datastore.Put(entities) sqlite_datastore_stub.Close() finally: apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', previous_stub) shutil.copy(datastore_path, datastore_path + '.filestub') os.remove(datastore_path) shutil.move(datastore_path + '.sqlite', datastore_path) logging.info( 'Datastore conversion complete. File stub data has been backed ' 'up in %s', datastore_path + '.filestub')
def init_datastore_v3_stub(self, enable=True, datastore_file=None, use_sqlite=False, auto_id_policy=AUTO_ID_POLICY_SEQUENTIAL, **stub_kw_args): """Enable the datastore stub. The 'datastore_file' argument can be the path to an existing datastore file, or None (default) to use an in-memory datastore that is initially empty. If you use the sqlite stub and have 'datastore_file' defined, changes you apply in a test will be written to the file. If you use the default datastore stub, changes are _not_ saved to disk unless you set save_changes=True. Note that you can only access those entities of the datastore file which have the same application ID associated with them as the test run. You can change the application ID for a test with setup_env(). Args: enable: True if the fake service should be enabled, False if real service should be disabled. datastore_file: Filename of a dev_appserver datastore file. use_sqlite: True to use the Sqlite stub, False (default) for file stub. auto_id_policy: How datastore stub assigns auto IDs. Either AUTO_ID_POLICY_SEQUENTIAL or AUTO_ID_POLICY_SCATTERED. stub_kw_args: Keyword arguments passed on to the service stub. """ if not enable: self._disable_stub(DATASTORE_SERVICE_NAME) self._disable_stub('datastore_v4') return if use_sqlite: if datastore_sqlite_stub is None: raise StubNotSupportedError( 'The sqlite stub is not supported in production.') stub = datastore_sqlite_stub.DatastoreSqliteStub( os.environ['APPLICATION_ID'], datastore_file, use_atexit=False, auto_id_policy=auto_id_policy, **stub_kw_args) else: stub_kw_args.setdefault('save_changes', False) stub = datastore_file_stub.DatastoreFileStub( os.environ['APPLICATION_ID'], datastore_file, use_atexit=False, auto_id_policy=auto_id_policy, **stub_kw_args) self._register_stub(DATASTORE_SERVICE_NAME, stub, self._deactivate_datastore_v3_stub) v4_stub = datastore_v4_stub.DatastoreV4Stub( os.environ['APPLICATION_ID']) self._register_stub('datastore_v4', v4_stub)
def __init__(self, addr, app, apps_root, var, verbose=False, secure=False): """Initialize the WSGI server. Args: app: A webapp.WSGIApplication. addr: Use this address to initialize the HTTP server. apps_root: Applications root directory. var: Directory for platform independent data. verbose: Boolean, default False. If True, enable verbose mode. secure: Boolean, default False. If True, enable secure connection. """ assert isinstance(app, webapp.WSGIApplication) self.app = app host, port = addr.split(':') self.host = host self.port = int(port) self.apps_root = apps_root self.secure = secure # Setup signals signal.signal(signal.SIGHUP, self._handleSignal) signal.signal(signal.SIGQUIT, self._handleSignal) signal.signal(signal.SIGABRT, self._handleSignal) signal.signal(signal.SIGTERM, self._handleSignal) # Setup environment os.environ['APPLICATION_ID'] = APPLICATION_ID # Setup Datastore # We use the SQLite Datastore stub for development. Later, we take # the production Datastore. from google.appengine.datastore import datastore_sqlite_stub apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap() datastore_path = os.path.join(var, APPLICATION_ID + '.sqlite') datastore = datastore_sqlite_stub.DatastoreSqliteStub( APPLICATION_ID, datastore_path) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', datastore) logging.debug('Using datastore: %s', datastore_path) # Setup Urlfetch from google.appengine.api import urlfetch_stub urlfetch_stub.MAX_REQUEST_SIZE = 5012 << 30 apiproxy_stub_map.apiproxy.RegisterStub( 'urlfetch', urlfetch_stub.URLFetchServiceStub())
def setupDatastore(options, conf, datastore_file, history, require_indexes, trusted): """Sets up datastore.""" name = options.datastore.lower() if name == 'mongodb': from typhoonae.mongodb import datastore_mongo_stub datastore = datastore_mongo_stub.DatastoreMongoStub( conf.application, require_indexes=require_indexes) elif name == 'bdbdatastore': from notdot.bdbdatastore import socket_apiproxy_stub datastore = socket_apiproxy_stub.RecordingSocketApiProxyStub( ('localhost', 9123)) global end_request_hook end_request_hook = datastore.closeSession elif name == 'mysql': from typhoonae.mysql import datastore_mysql_stub database_info = { "host": options.mysql_host, "user": options.mysql_user, "passwd": options.mysql_passwd, "db": options.mysql_db } datastore = datastore_mysql_stub.DatastoreMySQLStub( conf.application, database_info, verbose=options.debug_mode) elif name == 'sqlite': from google.appengine.datastore import datastore_sqlite_stub datastore = datastore_sqlite_stub.DatastoreSqliteStub( conf.application, datastore_file, require_indexes=require_indexes, trusted=trusted) else: raise RuntimeError, "unknown datastore" apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', datastore) if name == 'bdbdatastore': from google.appengine.tools import dev_appserver_index app_root = os.getcwd() logging.info("%s" % app_root) dev_appserver_index.SetupIndexes(conf.application, app_root) dev_appserver_index.IndexYamlUpdater(app_root)
def shell(datastore_path='', history_path='', useful_imports=True, use_ipython=True, use_sqlite=False): """ Start a new interactive python session.""" banner = 'Interactive Kay Shell' if useful_imports: namespace = create_useful_locals() else: namespace = {} appid = get_appid() os.environ['APPLICATION_ID'] = appid p = get_datastore_paths() if not datastore_path: datastore_path = p[0] if not history_path: history_path = p[1] apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap() if use_sqlite: from google.appengine.datastore import datastore_sqlite_stub stub = datastore_sqlite_stub.DatastoreSqliteStub( appid, datastore_path, history_path) else: stub = datastore_file_stub.DatastoreFileStub(appid, datastore_path, history_path) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', stub) if use_ipython: try: import IPython except ImportError: pass else: sh = IPython.Shell.IPShellEmbed(argv='', banner=banner) sh(global_ns={}, local_ns=namespace) return sys.ps1 = '%s> ' % appid if readline is not None: readline.parse_and_bind('tab: complete') atexit.register(lambda: readline.write_history_file(HISTORY_PATH)) if os.path.exists(HISTORY_PATH): readline.read_history_file(HISTORY_PATH) from code import interact interact(banner, local=namespace)
def setup_stubs(storage_path, options, configuration): datastore_path = options.datastore_path or os.path.join(storage_path, 'datastore.db') search_index_path = options.search_indexes_path or os.path.join(storage_path, 'search_indexes') blobstore_path = options.blobstore_path or os.path.join(storage_path, 'blobs') # Init the proxy map and stubs from google.appengine.api import apiproxy_stub_map apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap() # DB from google.appengine.datastore import datastore_sqlite_stub apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', datastore_sqlite_stub.DatastoreSqliteStub(configuration.app_id, datastore_path)) # Search service from google.appengine.api.search import simple_search_stub apiproxy_stub_map.apiproxy.RegisterStub('search', simple_search_stub.SearchServiceStub(index_file=search_index_path)) from google.appengine.api.blobstore import file_blob_storage blob_storage = file_blob_storage.FileBlobStorage(blobstore_path, configuration.app_id) from google.appengine.api.blobstore import blobstore_stub apiproxy_stub_map.apiproxy.RegisterStub('blobstore', blobstore_stub.BlobstoreServiceStub(blob_storage)) from google.appengine.api.app_identity import app_identity_stub apiproxy_stub_map.apiproxy.RegisterStub('app_identity_service', app_identity_stub.AppIdentityServiceStub()) # Capability from google.appengine.api.capabilities import capability_stub apiproxy_stub_map.apiproxy.RegisterStub('capability_service', capability_stub.CapabilityServiceStub()) # Memcache from google.appengine.api.memcache import memcache_stub apiproxy_stub_map.apiproxy.RegisterStub('memcache', memcache_stub.MemcacheServiceStub()) # Task queues from google.appengine.api.taskqueue import taskqueue_stub apiproxy_stub_map.apiproxy.RegisterStub('taskqueue', taskqueue_stub.TaskQueueServiceStub()) # URLfetch service from google.appengine.api import urlfetch_stub apiproxy_stub_map.apiproxy.RegisterStub('urlfetch', urlfetch_stub.URLFetchServiceStub())
def convert_sqlite_data_to_emulator(app_id, filename, gcd_emulator_host): """Convert datastore sqlite stub data to cloud emulator data. Args: app_id: A String representing application ID. filename: A String representing the absolute path to SQLite data. gcd_emulator_host: A String in the format of host:port indicate the hostname and port number of gcd emulator. Raises: PersistException: if the call to emulator's /persist endpoint fails. """ previous_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') sqlite_stub = datastore_sqlite_stub.DatastoreSqliteStub(app_id, filename, trusted=True, use_atexit=False) apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', sqlite_stub) entities = _fetch_all_datastore_entities() if entities: logging.info('Fetched %d entities from %s', len(entities), filename) grpc_stub = datastore_grpc_stub.DatastoreGrpcStub(gcd_emulator_host) grpc_stub.get_or_set_call_handler_stub() apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', grpc_stub) datastore.Put(entities) # persist entities to disk in emulator's data format. conn = httplib.HTTPConnection(gcd_emulator_host) conn.request('POST', '/persist') response = conn.getresponse() msg = response.read() if httplib.OK != response.status: raise PersistException(msg) logging.info('Datastore conversion complete') else: logging.warning( 'Fetched 0 entity from %s, will not create cloud ' 'datastore emulator file', filename) sqlite_stub.Close() apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', previous_stub)
def reset_stub(stub_name, stub_options): """Reset a stub currently in place with a new one. You might wish to swap out a currently running stub with another. For example, in cases where you are running tests that use the devappserver and you want to reset the stub in between test cases. Currently the datastore is the only stub that can be reset, but others could be added here if needed. Args: stub_name: string stub_options: a dictionary of options to be used when resetting the stub. When stub_name == 'datastore_v3', we expect an object that looks like: { 'app_id': 'dev~id-of-application', 'db_consistency_probability': 1.0, 'datastore_path': /path/to/sqlite.sql } """ if stub_name != 'datastore_v3': raise NotImplementedError("Only datastore stubs can be currently reset.") # TODO(dhruv): Figure out when it is safe to close a datastore stub (it is # not currently in read-only mode) and do that here before replacing it. # Create a new stub and replace it in the api stub map policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy( probability=stub_options['db_consistency_probability']) new_stub = ( datastore_sqlite_stub.DatastoreSqliteStub( stub_options['app_id'], stub_options['datastore_path'], consistency_policy=policy ) ) apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', new_stub)
def setup_datastore_stubs(): if "test" in sys.argv: return from google.appengine.datastore import datastore_sqlite_stub from google.appengine.api import apiproxy_stub_map from google.appengine.datastore import datastore_stub_util app_id = application_id() datastore = datastore_sqlite_stub.DatastoreSqliteStub( "dev~" + app_id, os.path.join(data_root(), "datastore.db"), require_indexes=False, trusted=False, root_path=find_project_root(), use_atexit=True) datastore.SetConsistencyPolicy( datastore_stub_util.TimeBasedHRConsistencyPolicy()) apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', datastore)
def convert_datastore_file_stub_data_to_sqlite(app_id, datastore_file): """Convert datastore_file_stub data into sqlite data. Args: app_id: String indicating application id. datastore_file: String indicating the file name of datastore_file_stub data. Raises: IOError: if datastore_file is not writeable. """ if not os.access(datastore_file, os.W_OK): raise IOError('Does not have write access to %s' % datastore_file) logging.info('Converting datastore file stub data to sqlite.') previous_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') sqlite_file_name = datastore_file + '.sqlite' try: apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap() datastore_stub = datastore_file_stub.DatastoreFileStub( app_id, datastore_file, trusted=True, save_changes=False) apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', datastore_stub) entities = _fetch_all_datastore_entities() sqlite_datastore_stub = datastore_sqlite_stub.DatastoreSqliteStub( app_id, sqlite_file_name, trusted=True) apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', sqlite_datastore_stub) datastore.Put(entities) sqlite_datastore_stub.Close() finally: apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', previous_stub) back_up_file_name = datastore_file + '.filestub' shutil.copy(datastore_file, back_up_file_name) os.remove(datastore_file) shutil.move(sqlite_file_name, datastore_file) logging.info( 'Datastore conversion complete. File stub data has been backed ' 'up in %s', back_up_file_name)
def get_new_connection(self, conn_params): from google.appengine.datastore import datastore_stub_util from google.appengine.datastore import datastore_sqlite_stub from google.appengine.api import apiproxy_stub_map datastore_stub = datastore_sqlite_stub.DatastoreSqliteStub( "dev~" + application_id(), os.path.join(data_root(), 'datastore.db'), False, trusted=False, root_path=find_project_root(), auto_id_policy=datastore_stub_util.SCATTERED) datastore_stub.SetConsistencyPolicy( datastore_stub_util.PseudoRandomHRConsistencyPolicy()) apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', datastore_stub) class FakeConnection: def commit(self): pass return FakeConnection()
def convert_python_data_to_emulator(app_id, stub_type, filename, gcd_emulator_host): """Convert datastore_file_stub or datastore_sqlite_stub data to emulator data. Args: app_id: A String representing application ID. stub_type: A String representing the stub type filename belongs to. filename: A String representing the absolute path to local data. gcd_emulator_host: A String in the format of host:port indicate the hostname and port number of gcd emulator. """ previous_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') try: if stub_type == StubTypes.PYTHON_FILE_STUB: logging.info( 'Converting datastore_file_stub data to cloud datastore emulator ' 'data.') python_stub = datastore_file_stub.DatastoreFileStub( app_id, filename, trusted=True, save_changes=False) else: # Sqlite stub logging.info( 'Converting datastore_sqlite_stub data to cloud datastore emulator ' 'data.') python_stub = datastore_sqlite_stub.DatastoreSqliteStub( app_id, filename, trusted=True, use_atexit=False) apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', python_stub) entities = _fetch_all_datastore_entities() grpc_stub = datastore_grpc_stub.DatastoreGrpcStub(gcd_emulator_host) grpc_stub.get_or_set_call_handler_stub() apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', grpc_stub) datastore.Put(entities) logging.info('Conversion complete.') python_stub.Close() finally: apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', previous_stub) logging.info('Datastore conversion complete')
def setup_stubs( request_data, app_id, application_root, trusted, appidentity_email_address, appidentity_private_key_path, blobstore_path, datastore_consistency, datastore_path, datastore_require_indexes, datastore_auto_id_policy, images_host_prefix, logs_path, mail_smtp_host, mail_smtp_port, mail_smtp_user, mail_smtp_password, mail_enable_sendmail, mail_show_mail_body, mail_allow_tls, search_index_path, taskqueue_auto_run_tasks, taskqueue_default_http_server, user_login_url, user_logout_url, default_gcs_bucket_name, appidentity_oauth_url=None, support_datastore_emulator=False): """Configures the APIs hosted by this server. Args: request_data: An apiproxy_stub.RequestInformation instance used by the stubs to lookup information about the request associated with an API call. app_id: The str application id e.g. "guestbook". application_root: The path to the directory containing the user's application e.g. "/home/joe/myapp". trusted: A bool indicating if privileged APIs should be made available. appidentity_email_address: Email address associated with a service account that has a downloadable key. May be None for no local application identity. appidentity_private_key_path: Path to private key file associated with service account (.pem format). Must be set if appidentity_email_address is set. blobstore_path: The path to the file that should be used for blobstore storage. datastore_consistency: The datastore_stub_util.BaseConsistencyPolicy to use as the datastore consistency policy. datastore_path: The path to the file that should be used for datastore storage. datastore_require_indexes: A bool indicating if the same production datastore indexes requirements should be enforced i.e. if True then a google.appengine.ext.db.NeedIndexError will be be raised if a query is executed without the required indexes. datastore_auto_id_policy: The type of sequence from which the datastore stub assigns auto IDs, either datastore_stub_util.SEQUENTIAL or datastore_stub_util.SCATTERED. images_host_prefix: The URL prefix (protocol://host:port) to prepend to image urls on calls to images.GetUrlBase. logs_path: Path to the file to store the logs data in. mail_smtp_host: The SMTP hostname that should be used when sending e-mails. If None then the mail_enable_sendmail argument is considered. mail_smtp_port: The SMTP port number that should be used when sending e-mails. If this value is None then mail_smtp_host must also be None. mail_smtp_user: The username to use when authenticating with the SMTP server. This value may be None if mail_smtp_host is also None or if the SMTP server does not require authentication. mail_smtp_password: The password to use when authenticating with the SMTP server. This value may be None if mail_smtp_host or mail_smtp_user is also None. mail_enable_sendmail: A bool indicating if sendmail should be used when sending e-mails. This argument is ignored if mail_smtp_host is not None. mail_show_mail_body: A bool indicating whether the body of sent e-mails should be written to the logs. mail_allow_tls: A bool indicating whether TLS should be allowed when communicating with an SMTP server. This argument is ignored if mail_smtp_host is None. search_index_path: The path to the file that should be used for search index storage. taskqueue_auto_run_tasks: A bool indicating whether taskqueue tasks should be run automatically or it the must be manually triggered. taskqueue_default_http_server: A str containing the address of the http server that should be used to execute tasks. user_login_url: A str containing the url that should be used for user login. user_logout_url: A str containing the url that should be used for user logout. default_gcs_bucket_name: A str, overriding the default bucket behavior. appidentity_oauth_url: A str containing the url to the oauth2 server to use to authenticate the private key. If set to None, then the standard google oauth2 server is used. support_datastore_emulator: A bool indicating if datastore_emulator is supported. """ identity_stub = app_identity_stub.AppIdentityServiceStub.Create( email_address=appidentity_email_address, private_key_path=appidentity_private_key_path, oauth_url=appidentity_oauth_url) if default_gcs_bucket_name is not None: identity_stub.SetDefaultGcsBucketName(default_gcs_bucket_name) apiproxy_stub_map.apiproxy.RegisterStub('app_identity_service', identity_stub) blob_storage = file_blob_storage.FileBlobStorage(blobstore_path, app_id) apiproxy_stub_map.apiproxy.RegisterStub( 'blobstore', blobstore_stub.BlobstoreServiceStub(blob_storage, request_data=request_data)) apiproxy_stub_map.apiproxy.RegisterStub( 'capability_service', capability_stub.CapabilityServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'channel', channel_service_stub.ChannelServiceStub(request_data=request_data)) if support_datastore_emulator: datastore_emulator_host = os.environ.get('DATASTORE_EMULATOR_HOST', '') error_msg = 'DATASTORE_EMULATOR_HOST is not found in environment variables' assert datastore_emulator_host, error_msg apiproxy_stub_map.apiproxy.ReplaceStub( 'datastore_v3', datastore_grpc_stub.DatastoreGrpcStub(datastore_emulator_host) ) else: apiproxy_stub_map.apiproxy.ReplaceStub( 'datastore_v3', datastore_sqlite_stub.DatastoreSqliteStub( app_id, datastore_path, datastore_require_indexes, trusted, root_path=application_root, auto_id_policy=datastore_auto_id_policy, consistency_policy=datastore_consistency)) apiproxy_stub_map.apiproxy.RegisterStub( 'datastore_v4', datastore_v4_stub.DatastoreV4Stub(app_id)) apiproxy_stub_map.apiproxy.RegisterStub( 'file', file_service_stub.FileServiceStub(blob_storage)) try: from google.appengine.api.images import images_stub except ImportError: # We register a stub which throws a NotImplementedError for most RPCs. from google.appengine.api.images import images_not_implemented_stub apiproxy_stub_map.apiproxy.RegisterStub( 'images', images_not_implemented_stub.ImagesNotImplementedServiceStub( host_prefix=images_host_prefix)) else: apiproxy_stub_map.apiproxy.RegisterStub( 'images', images_stub.ImagesServiceStub(host_prefix=images_host_prefix)) apiproxy_stub_map.apiproxy.RegisterStub( 'logservice', logservice_stub.LogServiceStub(logs_path=logs_path)) apiproxy_stub_map.apiproxy.RegisterStub( 'mail', mail_stub.MailServiceStub(mail_smtp_host, mail_smtp_port, mail_smtp_user, mail_smtp_password, enable_sendmail=mail_enable_sendmail, show_mail_body=mail_show_mail_body, allow_tls=mail_allow_tls)) apiproxy_stub_map.apiproxy.RegisterStub( 'memcache', memcache_stub.MemcacheServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'modules', modules_stub.ModulesServiceStub(request_data)) apiproxy_stub_map.apiproxy.RegisterStub( 'remote_socket', _remote_socket_stub.RemoteSocketServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'search', simple_search_stub.SearchServiceStub(index_file=search_index_path)) apiproxy_stub_map.apiproxy.RegisterStub( 'system', system_stub.SystemServiceStub(request_data=request_data)) apiproxy_stub_map.apiproxy.RegisterStub( 'taskqueue', taskqueue_stub.TaskQueueServiceStub( root_path=application_root, auto_task_running=taskqueue_auto_run_tasks, default_http_server=taskqueue_default_http_server, request_data=request_data)) apiproxy_stub_map.apiproxy.GetStub('taskqueue').StartBackgroundExecution() apiproxy_stub_map.apiproxy.RegisterStub( 'urlfetch', urlfetch_stub.URLFetchServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'user', user_service_stub.UserServiceStub(login_url=user_login_url, logout_url=user_logout_url, request_data=request_data)) apiproxy_stub_map.apiproxy.RegisterStub( 'xmpp', xmpp_service_stub.XmppServiceStub())
def _SetupStubs( app_id, application_root, trusted, blobstore_path, use_sqlite, auto_id_policy, high_replication, datastore_path, datastore_require_indexes, images_host_prefix, logs_path, mail_smtp_host, mail_smtp_port, mail_smtp_user, mail_smtp_password, mail_enable_sendmail, mail_show_mail_body, matcher_prospective_search_path, taskqueue_auto_run_tasks, taskqueue_task_retry_seconds, taskqueue_default_http_server, user_login_url, user_logout_url): """Configures the APIs hosted by this server. Args: app_id: The str application id e.g. "guestbook". application_root: The path to the directory containing the user's application e.g. "/home/bquinlan/myapp". trusted: A bool indicating if privileged APIs should be made available. blobstore_path: The path to the file that should be used for blobstore storage. use_sqlite: A bool indicating whether DatastoreSqliteStub or DatastoreFileStub should be used. auto_id_policy: One of datastore_stub_util.SEQUENTIAL or .SCATTERED, indicating whether the Datastore stub should assign IDs sequentially or scattered. high_replication: A bool indicating whether to use the high replication consistency model. datastore_path: The path to the file that should be used for datastore storage. datastore_require_indexes: A bool indicating if the same production datastore indexes requirements should be enforced i.e. if True then a google.appengine.ext.db.NeedIndexError will be be raised if a query is executed without the required indexes. images_host_prefix: The URL prefix (protocol://host:port) to preprend to image urls on calls to images.GetUrlBase. logs_path: Path to the file to store the logs data in. mail_smtp_host: The SMTP hostname that should be used when sending e-mails. If None then the mail_enable_sendmail argument is considered. mail_smtp_port: The SMTP port number that should be used when sending e-mails. If this value is None then mail_smtp_host must also be None. mail_smtp_user: The username to use when authenticating with the SMTP server. This value may be None if mail_smtp_host is also None or if the SMTP server does not require authentication. mail_smtp_password: The password to use when authenticating with the SMTP server. This value may be None if mail_smtp_host or mail_smtp_user is also None. mail_enable_sendmail: A bool indicating if sendmail should be used when sending e-mails. This argument is ignored if mail_smtp_host is not None. mail_show_mail_body: A bool indicating whether the body of sent e-mails should be written to the logs. matcher_prospective_search_path: The path to the file that should be used to save prospective search subscriptions. taskqueue_auto_run_tasks: A bool indicating whether taskqueue tasks should be run automatically or it the must be manually triggered. taskqueue_task_retry_seconds: An int representing the number of seconds to wait before a retrying a failed taskqueue task. taskqueue_default_http_server: A str containing the address of the http server that should be used to execute tasks. user_login_url: A str containing the url that should be used for user login. user_logout_url: A str containing the url that should be used for user logout. """ os.environ['APPLICATION_ID'] = app_id apiproxy_stub_map.apiproxy.RegisterStub( 'app_identity_service', app_identity_stub.AppIdentityServiceStub()) blob_storage = file_blob_storage.FileBlobStorage(blobstore_path, app_id) apiproxy_stub_map.apiproxy.RegisterStub( 'blobstore', blobstore_stub.BlobstoreServiceStub(blob_storage)) apiproxy_stub_map.apiproxy.RegisterStub( 'capability_service', capability_stub.CapabilityServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'channel', channel_service_stub.ChannelServiceStub()) if use_sqlite: datastore = datastore_sqlite_stub.DatastoreSqliteStub( app_id, datastore_path, datastore_require_indexes, trusted, root_path=application_root, auto_id_policy=auto_id_policy) else: datastore = datastore_file_stub.DatastoreFileStub( app_id, datastore_path, datastore_require_indexes, trusted, root_path=application_root, auto_id_policy=auto_id_policy) if high_replication: datastore.SetConsistencyPolicy( datastore_stub_util.TimeBasedHRConsistencyPolicy()) apiproxy_stub_map.apiproxy.RegisterStub( 'datastore_v3', datastore) apiproxy_stub_map.apiproxy.RegisterStub( 'file', file_service_stub.FileServiceStub(blob_storage)) try: from google.appengine.api.images import images_stub except ImportError: logging.warning('Could not initialize images API; you are likely missing ' 'the Python "PIL" module.') from google.appengine.api.images import images_not_implemented_stub apiproxy_stub_map.apiproxy.RegisterStub( 'images', images_not_implemented_stub.ImagesNotImplementedServiceStub()) else: apiproxy_stub_map.apiproxy.RegisterStub( 'images', images_stub.ImagesServiceStub(host_prefix=images_host_prefix)) apiproxy_stub_map.apiproxy.RegisterStub( 'logservice', logservice_stub.LogServiceStub(persist=True)) apiproxy_stub_map.apiproxy.RegisterStub( 'mail', mail_stub.MailServiceStub(mail_smtp_host, mail_smtp_port, mail_smtp_user, mail_smtp_password, enable_sendmail=mail_enable_sendmail, show_mail_body=mail_show_mail_body)) apiproxy_stub_map.apiproxy.RegisterStub( 'memcache', memcache_stub.MemcacheServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'search', simple_search_stub.SearchServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub('system', system_stub.SystemServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'taskqueue', taskqueue_stub.TaskQueueServiceStub( root_path=application_root, auto_task_running=taskqueue_auto_run_tasks, task_retry_seconds=taskqueue_task_retry_seconds, default_http_server=taskqueue_default_http_server)) apiproxy_stub_map.apiproxy.GetStub('taskqueue').StartBackgroundExecution() apiproxy_stub_map.apiproxy.RegisterStub( 'urlfetch', urlfetch_stub.URLFetchServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'user', user_service_stub.UserServiceStub(login_url=user_login_url, logout_url=user_logout_url)) apiproxy_stub_map.apiproxy.RegisterStub( 'xmpp', xmpp_service_stub.XmppServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'matcher', prospective_search_stub.ProspectiveSearchStub( matcher_prospective_search_path, apiproxy_stub_map.apiproxy.GetStub('taskqueue')))
def setup_stubs(request_data, app_id, application_root, trusted, blobstore_path, datastore_consistency, datastore_path, datastore_require_indexes, datastore_auto_id_policy, images_host_prefix, logs_path, mail_smtp_host, mail_smtp_port, mail_smtp_user, mail_smtp_password, mail_enable_sendmail, mail_show_mail_body, matcher_prospective_search_path, search_index_path, taskqueue_auto_run_tasks, taskqueue_default_http_server, user_login_url, user_logout_url): """Configures the APIs hosted by this server. Args: request_data: An apiproxy_stub.RequestInformation instance used by the stubs to lookup information about the request associated with an API call. app_id: The str application id e.g. "guestbook". application_root: The path to the directory containing the user's application e.g. "/home/joe/myapp". trusted: A bool indicating if privileged APIs should be made available. blobstore_path: The path to the file that should be used for blobstore storage. datastore_consistency: The datastore_stub_util.BaseConsistencyPolicy to use as the datastore consistency policy. datastore_path: The path to the file that should be used for datastore storage. datastore_require_indexes: A bool indicating if the same production datastore indexes requirements should be enforced i.e. if True then a google.appengine.ext.db.NeedIndexError will be be raised if a query is executed without the required indexes. datastore_auto_id_policy: The type of sequence from which the datastore stub assigns auto IDs, either datastore_stub_util.SEQUENTIAL or datastore_stub_util.SCATTERED. images_host_prefix: The URL prefix (protocol://host:port) to prepend to image urls on calls to images.GetUrlBase. logs_path: Path to the file to store the logs data in. mail_smtp_host: The SMTP hostname that should be used when sending e-mails. If None then the mail_enable_sendmail argument is considered. mail_smtp_port: The SMTP port number that should be used when sending e-mails. If this value is None then mail_smtp_host must also be None. mail_smtp_user: The username to use when authenticating with the SMTP server. This value may be None if mail_smtp_host is also None or if the SMTP server does not require authentication. mail_smtp_password: The password to use when authenticating with the SMTP server. This value may be None if mail_smtp_host or mail_smtp_user is also None. mail_enable_sendmail: A bool indicating if sendmail should be used when sending e-mails. This argument is ignored if mail_smtp_host is not None. mail_show_mail_body: A bool indicating whether the body of sent e-mails should be written to the logs. matcher_prospective_search_path: The path to the file that should be used to save prospective search subscriptions. search_index_path: The path to the file that should be used for search index storage. taskqueue_auto_run_tasks: A bool indicating whether taskqueue tasks should be run automatically or it the must be manually triggered. taskqueue_default_http_server: A str containing the address of the http server that should be used to execute tasks. user_login_url: A str containing the url that should be used for user login. user_logout_url: A str containing the url that should be used for user logout. """ apiproxy_stub_map.apiproxy.RegisterStub( 'app_identity_service', app_identity_stub.AppIdentityServiceStub()) blob_storage = file_blob_storage.FileBlobStorage(blobstore_path, app_id) apiproxy_stub_map.apiproxy.RegisterStub( 'blobstore', blobstore_stub.BlobstoreServiceStub(blob_storage, request_data=request_data)) apiproxy_stub_map.apiproxy.RegisterStub( 'capability_service', capability_stub.CapabilityServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'channel', channel_service_stub.ChannelServiceStub(request_data=request_data)) datastore_stub = datastore_sqlite_stub.DatastoreSqliteStub( app_id, datastore_path, datastore_require_indexes, trusted, root_path=application_root, auto_id_policy=datastore_auto_id_policy) datastore_stub.SetConsistencyPolicy(datastore_consistency) apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', datastore_stub) apiproxy_stub_map.apiproxy.RegisterStub( 'file', file_service_stub.FileServiceStub(blob_storage)) try: from google.appengine.api.images import images_stub except ImportError: logging.warning( 'Could not initialize images API; you are likely missing ' 'the Python "PIL" module.') # We register a stub which throws a NotImplementedError for most RPCs. from google.appengine.api.images import images_not_implemented_stub apiproxy_stub_map.apiproxy.RegisterStub( 'images', images_not_implemented_stub.ImagesNotImplementedServiceStub( host_prefix=images_host_prefix)) else: apiproxy_stub_map.apiproxy.RegisterStub( 'images', images_stub.ImagesServiceStub(host_prefix=images_host_prefix)) apiproxy_stub_map.apiproxy.RegisterStub( 'logservice', logservice_stub.LogServiceStub(logs_path=logs_path)) apiproxy_stub_map.apiproxy.RegisterStub( 'mail', mail_stub.MailServiceStub(mail_smtp_host, mail_smtp_port, mail_smtp_user, mail_smtp_password, enable_sendmail=mail_enable_sendmail, show_mail_body=mail_show_mail_body)) apiproxy_stub_map.apiproxy.RegisterStub( 'memcache', memcache_stub.MemcacheServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'search', simple_search_stub.SearchServiceStub(index_file=search_index_path)) apiproxy_stub_map.apiproxy.RegisterStub( 'servers', servers_stub.ServersServiceStub(request_data)) apiproxy_stub_map.apiproxy.RegisterStub( 'system', system_stub.SystemServiceStub(request_data=request_data)) apiproxy_stub_map.apiproxy.RegisterStub( 'taskqueue', taskqueue_stub.TaskQueueServiceStub( root_path=application_root, auto_task_running=taskqueue_auto_run_tasks, default_http_server=taskqueue_default_http_server, request_data=request_data)) apiproxy_stub_map.apiproxy.GetStub('taskqueue').StartBackgroundExecution() urlmatchers_to_fetch_functions = [] urlmatchers_to_fetch_functions.extend( gcs_dispatcher.URLMATCHERS_TO_FETCH_FUNCTIONS) apiproxy_stub_map.apiproxy.RegisterStub( 'urlfetch', urlfetch_stub.URLFetchServiceStub( urlmatchers_to_fetch_functions=urlmatchers_to_fetch_functions)) apiproxy_stub_map.apiproxy.RegisterStub( 'user', user_service_stub.UserServiceStub(login_url=user_login_url, logout_url=user_logout_url, request_data=request_data)) apiproxy_stub_map.apiproxy.RegisterStub( 'xmpp', xmpp_service_stub.XmppServiceStub()) apiproxy_stub_map.apiproxy.RegisterStub( 'matcher', prospective_search_stub.ProspectiveSearchStub( matcher_prospective_search_path, apiproxy_stub_map.apiproxy.GetStub('taskqueue'))) apiproxy_stub_map.apiproxy.RegisterStub( 'remote_socket', _remote_socket_stub.RemoteSocketServiceStub())
def init_datastore_v3_stub(self, enable=True, datastore_file=None, use_sqlite=False, auto_id_policy=AUTO_ID_POLICY_SEQUENTIAL, **stub_kw_args): """Enables the datastore stub. The `datastore_file` argument can be set to the path of an existing datastore file, or `None` (default) to use an in-memory datastore that is initially empty. If you use the sqlite stub and have defined `datastore_file`, the changes that you apply in a test will be written to the file. If you use the default datastore stub, changes are *not* saved to disk unless you set `save_changes=True`. Note: You can only access those entities of the datastore file that use the same application ID as the test run. You can change the application ID for a test with `setup_env()`. Args: enable: `True` if the fake service should be enabled, or `False` if the real service should be disabled. datastore_file: File name of a dev_appserver datastore file. use_sqlite: `True` to use the Sqlite stub, or `False` (default) to use the file stub. auto_id_policy: How datastore stub assigns auto IDs. This value can be either `AUTO_ID_POLICY_SEQUENTIAL` or `AUTO_ID_POLICY_SCATTERED`. **stub_kw_args: Keyword arguments passed on to the service stub. Raises: StubNotSupportedError: If datastore_sqlite_stub is None. """ if USE_DATASTORE_EMULATOR: self._disable_stub(DATASTORE_SERVICE_NAME) testserver_util.TESTSERVER_UTIL.reset_emulator() testserver_util.remote_api_stub.ConfigureRemoteApi( os.environ['APPLICATION_ID'], '/', lambda: ('', ''), 'localhost:%d' % testserver_util.TESTSERVER_UTIL.api_port, services=[DATASTORE_SERVICE_NAME], apiproxy=self._test_stub_map, use_remote_datastore=False) self._enabled_stubs[DATASTORE_SERVICE_NAME] = None return if not enable: self._disable_stub(DATASTORE_SERVICE_NAME) self._disable_stub(datastore_v4_stub.SERVICE_NAME) self._disable_stub(cloud_datastore_v1_stub.SERVICE_NAME) return if use_sqlite: if datastore_sqlite_stub is None: raise StubNotSupportedError( 'The sqlite stub is not supported in production.') stub = datastore_sqlite_stub.DatastoreSqliteStub( os.environ['APPLICATION_ID'], datastore_file, use_atexit=False, auto_id_policy=auto_id_policy, **stub_kw_args) else: stub_kw_args.setdefault('save_changes', False) stub = datastore_file_stub.DatastoreFileStub( os.environ['APPLICATION_ID'], datastore_file, use_atexit=False, auto_id_policy=auto_id_policy, **stub_kw_args) self._register_stub(DATASTORE_SERVICE_NAME, stub, self._deactivate_datastore_v3_stub) v4_stub = datastore_v4_stub.DatastoreV4Stub(os.environ['APPLICATION_ID']) self._register_stub(datastore_v4_stub.SERVICE_NAME, v4_stub) if datastore_pbs._CLOUD_DATASTORE_ENABLED: helper = datastore_pbs.googledatastore.helper credential_env = helper._DATASTORE_USE_STUB_CREDENTIAL_FOR_TEST_ENV os.environ[credential_env] = 'True' cloud_stub = cloud_datastore_v1_stub.CloudDatastoreV1Stub( os.environ['APPLICATION_ID']) self._register_stub(cloud_datastore_v1_stub.SERVICE_NAME, cloud_stub)
def init_datastore_v3_stub(self, enable=True, datastore_file=None, use_sqlite=False, auto_id_policy=AUTO_ID_POLICY_SEQUENTIAL, **stub_kw_args): """Enables the datastore stub. The `datastore_file` argument can be set to the path of an existing datastore file, or `None` (default) to use an in-memory datastore that is initially empty. If you use the sqlite stub and have defined `datastore_file`, the changes that you apply in a test will be written to the file. If you use the default datastore stub, changes are *not* saved to disk unless you set `save_changes=True`. Note: You can only access those entities of the datastore file that use the same application ID as the test run. You can change the application ID for a test with `setup_env()`. Args: enable: `True` if the fake service should be enabled, or `False` if the real service should be disabled. datastore_file: File name of a dev_appserver datastore file. use_sqlite: `True` to use the Sqlite stub, or `False` (default) to use the file stub. auto_id_policy: How datastore stub assigns auto IDs. This value can be either `AUTO_ID_POLICY_SEQUENTIAL` or `AUTO_ID_POLICY_SCATTERED`. **stub_kw_args: Keyword arguments passed on to the service stub. Raises: StubNotSupportedError: If datastore_sqlite_stub is None. """ if self._use_datastore_emulator: self._disable_stub(DATASTORE_SERVICE_NAME) delegate_stub = (remote_api_stub.DatastoreStubTestbedDelegate( self.rpc_server, '/', stub_kw_args.get('max_request_size', apiproxy_stub.MAX_REQUEST_SIZE), emulator_port=self._emulator_port)) delegate_stub.Clear() self._test_stub_map.RegisterStub(DATASTORE_SERVICE_NAME, delegate_stub) consistency_policy = stub_kw_args.get( 'consistency_policy', datastore_stub_util.PseudoRandomHRConsistencyPolicy( probability=1.0)) datastore_stub_util.UpdateEmulatorConfig(self._emulator_port, auto_id_policy, consistency_policy) if isinstance(consistency_policy, datastore_stub_util.PseudoRandomHRConsistencyPolicy): consistency_policy.is_using_cloud_datastore_emulator = True consistency_policy.emulator_port = self._emulator_port self._enabled_stubs[DATASTORE_SERVICE_NAME] = None return if not enable: self._disable_stub(DATASTORE_SERVICE_NAME) self._disable_stub(datastore_v4_stub.SERVICE_NAME) self._disable_stub(cloud_datastore_v1_stub.SERVICE_NAME) return if use_sqlite: if datastore_sqlite_stub is None: raise StubNotSupportedError( 'The sqlite stub is not supported in production.') stub = datastore_sqlite_stub.DatastoreSqliteStub( os.environ['APPLICATION_ID'], datastore_file, use_atexit=False, auto_id_policy=auto_id_policy, **stub_kw_args) else: stub_kw_args.setdefault('save_changes', False) stub = datastore_file_stub.DatastoreFileStub( os.environ['APPLICATION_ID'], datastore_file, use_atexit=False, auto_id_policy=auto_id_policy, **stub_kw_args) self._register_stub(DATASTORE_SERVICE_NAME, stub, self._deactivate_datastore_v3_stub) v4_stub = datastore_v4_stub.DatastoreV4Stub( os.environ['APPLICATION_ID']) self._register_stub(datastore_v4_stub.SERVICE_NAME, v4_stub) if datastore_pbs._CLOUD_DATASTORE_ENABLED: helper = datastore_pbs.googledatastore.helper credential_env = helper._DATASTORE_USE_STUB_CREDENTIAL_FOR_TEST_ENV os.environ[credential_env] = 'True' cloud_stub = cloud_datastore_v1_stub.CloudDatastoreV1Stub( os.environ['APPLICATION_ID']) self._register_stub(cloud_datastore_v1_stub.SERVICE_NAME, cloud_stub)