def enable_sandbox(config):
"""Enable the sandbox based on the configuration.
This includes installing import hooks to restrict access to C modules and
stub out functions that are not implemented in production, replacing the file
builtins with read-only versions and add enabled libraries to the path.
Args:
config: The runtime_config_pb2.Config to use to configure the sandbox.
"""
devnull = open(os.path.devnull)
modules = [os, traceback, google]
c_module = _find_shared_object_c_module()
if c_module:
modules.append(c_module)
module_paths = [module.__file__ for module in modules]
module_paths.extend(
[os.path.realpath(module.__file__) for module in modules])
python_lib_paths = [config.application_root]
for path in sys.path:
if any(module_path.startswith(path) for module_path in module_paths):
python_lib_paths.append(path)
python_lib_paths.extend(_enable_libraries(config.libraries))
# Note that the above code (see _find_shared_object_c_module) imports modules
# that must be pruned so please use care if you move the call to
# _prune_sys_modules.
_prune_sys_modules()
path_override_hook = PathOverrideImportHook(
set(
_THIRD_PARTY_LIBRARY_NAME_OVERRIDES.get(lib.name, lib.name)
for lib in config.libraries).intersection(_C_MODULES))
python_lib_paths.extend(path_override_hook.extra_sys_paths)
if not config.vm:
_install_fake_file(config, python_lib_paths, path_override_hook)
_install_open_hooks()
sys.platform = 'linux3'
_install_import_hooks(config, path_override_hook)
sys.path_importer_cache = {}
sys.path = python_lib_paths[:]
thread = __import__('thread')
__import__('%s.threading' % dist27.__name__)
threading = sys.modules['%s.threading' % dist27.__name__]
thread.start_new_thread = _make_request_id_aware_start_new_thread(
thread.start_new_thread)
# This import needs to be after enabling the sandbox so it imports the
# sandboxed version of the logging module.
from google.appengine.runtime import runtime
runtime.PatchStartNewThread(thread)
threading._start_new_thread = thread.start_new_thread
os.chdir(config.application_root)
sandboxed_os = __import__('os')
request_environment.PatchOsEnviron(sandboxed_os)
os.__dict__.update(sandboxed_os.__dict__)
_init_logging(config.stderr_log_level)
pdb_sandbox.install(config)
sys.stdin = devnull
sys.stdout = sys.stderr
def InitRequestEnvironMiddleware(app, copy_gae_application=False):
"""Patch os.environ to be thread local, and stamp it with default values.
When this function is called, we remember the values of os.environ. When the
wrapped inner function (i.e. the WSGI middleware) is called, we patch
os.environ to be thread local, and we fill in the remembered values, and merge
in WSGI env vars.
Args:
app: The WSGI app to wrap.
copy_gae_application: GAE_APPLICATION is copied to APPLICATION_ID if set.
Returns:
The wrapped app, also a WSGI app.
"""
if copy_gae_application:
os.environ['APPLICATION_ID'] = os.environ['GAE_APPLICATION']
original_environ = dict(os.environ)
request_environment.PatchOsEnviron()
os.environ.update(original_environ)
def PatchEnv(wsgi_env, start_response):
"""The middleware WSGI app."""
request_environment.current_request.Init(
errors=None, environ=original_environ.copy())
return app(wsgi_env, start_response)
return PatchEnv
def MakeInitLegacyRequestOsEnvironMiddleware():
"""Patch os.environ to be thread local, and stamp it with default values.
When this function is called, we remember the values of os.environ. When the
wrapped inner function (i.e. the WSGI middleware) is called, we patch
os.environ to be thread local, and we fill in the remembered values, and merge
in WSGI env vars.
Returns:
The InitLegacyRequestOsEnviron Middleware.
"""
original_environ = dict(os.environ)
request_environment.PatchOsEnviron()
os.environ.update(original_environ)
@middleware
def InitLegacyRequestOsEnvironMiddleware(app, wsgi_env, start_response):
"""The middleware WSGI app."""
request_environment.current_request.Init(
errors=None, environ=original_environ.copy())
return app(wsgi_env, start_response)
return InitLegacyRequestOsEnvironMiddleware
def setUp(self):
"""Initializes the App Engine stubs."""
# Evil os-environ patching which mirrors dev_appserver and production.
# This patch turns os.environ into a thread-local object, which also happens
# to support storing more than just strings. This patch must come first.
self._old_os_environ = os.environ.copy()
request_environment.current_request.Clear()
request_environment.PatchOsEnviron()
os.environ.update(self._old_os_environ)
# Setup and activate the testbed.
self.InitTestbed()
# Register the search stub (until included in init_all_stubs).
if (simple_search_stub
and apiproxy_stub_map.apiproxy.GetStub('search') is None):
self.search_stub = simple_search_stub.SearchServiceStub()
apiproxy_stub_map.apiproxy.RegisterStub('search', self.search_stub)
# Fake an always strongly-consistent HR datastore.
policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy(
probability=1)
self.testbed.init_datastore_v3_stub(consistency_policy=policy)
self.datastore_stub = self.testbed.get_stub(
testbed.DATASTORE_SERVICE_NAME)
# Save the taskqueue_stub for use in RunDeferredTasks.
self.testbed.init_taskqueue_stub(_all_queues_valid=True)
self.taskqueue_stub = self.testbed.get_stub(
testbed.TASKQUEUE_SERVICE_NAME)
# Save other stubs for use in helper methods and tests.
self.users_stub = self.testbed.get_stub(testbed.USER_SERVICE_NAME)
self.channel_stub = self.testbed.get_stub(testbed.CHANNEL_SERVICE_NAME)
# Each time setUp is called, treat it like a different request to a
# different app instance.
request_id_hash = ''.join(
random.sample(string.letters + string.digits, 26))
instance_id = ''.join(random.sample(string.letters + string.digits,
26))
# More like the production environment: "testbed-version.123123123", rather
# than the default "testbed-version".
current_version_id = 'testbed-version.%s' % random.randint(
1, 1000000000000)
self.testbed.setup_env(request_id_hash=request_id_hash,
instance_id=instance_id,
current_version_id=current_version_id,
overwrite=True)
self.Logout()
super(AppEngineTestCase, self).setUp()
def PatchEnv(wsgi_env, start_response):
"""The middleware WSGI app."""
request_environment.PatchOsEnviron()
os.environ.clear()
os.environ.update(original_environ)
for key, val in wsgi_env.iteritems():
if isinstance(val, basestring):
os.environ[key] = val
os.environ['SERVER_SOFTWARE'] = appengine_config.server_software
os.environ['APPENGINE_RUNTIME'] = 'python27'
os.environ['APPLICATION_ID'] = '%s~%s' % (appengine_config.partition,
appengine_config.appid)
os.environ['INSTANCE_ID'] = appengine_config.instance
os.environ['BACKEND_ID'] = appengine_config.major_version
os.environ['CURRENT_MODULE_ID'] = appengine_config.module
os.environ['CURRENT_VERSION_ID'] = '%s.%s' % (
appengine_config.major_version, appengine_config.minor_version)
os.environ['DEFAULT_TICKET'] = appengine_config.default_ticket
return app(wsgi_env, start_response)
def fix_subprocess_module(self):
"""
Making the subprocess module work on the dev_appserver is hard work
so I've isloated it all here
"""
if on_production():
return
import sys
from google.appengine import dist27
from google.appengine.tools.devappserver2.python import sandbox
if 'fcntl' not in sandbox._WHITE_LIST_C_MODULES:
sandbox._WHITE_LIST_C_MODULES.extend([
'fcntl'
])
if "subprocess" in dist27.MODULE_OVERRIDES:
dist27.MODULE_OVERRIDES.remove("subprocess")
if "subprocess" in sys.modules:
del sys.modules["subprocess"]
for finder in sys.meta_path:
if isinstance(finder, sandbox.ModuleOverrideImportHook):
del finder.policies['os']
if "os" in sys.modules:
del sys.modules["os"]
request_environment.PatchOsEnviron(__import__("os"))
if "select" in dist27.MODULE_OVERRIDES:
dist27.MODULE_OVERRIDES.remove("select")
if "select" not in sandbox._WHITE_LIST_C_MODULES:
sandbox._WHITE_LIST_C_MODULES.extend(["select"])
if "select" in sys.modules:
del sys.modules["select"]
def enable_sandbox(config):
"""Enable the sandbox based on the configuration.
This includes installing import hooks to restrict access to C modules and
stub out functions that are not implemented in production, replacing the file
builtins with read-only versions and add enabled libraries to the path.
Args:
config: The runtime_config_pb2.Config to use to configure the sandbox.
"""
devnull = open(os.path.devnull)
sys.platform = 'linux3'
sys.meta_path = [
PyCryptoRandomImportHook,
] + sys.meta_path
app_root = config.application_root.decode()
sys.path = [app_root] + sys.path
thread = __import__('_thread')
__import__('%s.threading' % dist27.__name__)
threading = sys.modules['%s.threading' % dist27.__name__]
thread.start_new_thread = _make_request_id_aware_start_new_thread(
thread.start_new_thread)
# This import needs to be after enabling the sandbox so it imports the
# sandboxed version of the logging module.
from google.appengine.runtime import runtime
runtime.PatchStartNewThread(thread)
threading._start_new_thread = thread.start_new_thread
os.chdir(app_root)
sandboxed_os = __import__('os')
request_environment.PatchOsEnviron(sandboxed_os)
os.__dict__.update(sandboxed_os.__dict__)
_init_logging(config.stderr_log_level)
pdb_sandbox.install(config)
sys.stdin = devnull
sys.stdout = sys.stderr
def enable_sandbox(config):
"""Enable the sandbox based on the configuration.
This includes installing import hooks to restrict access to C modules and
stub out functions that are not implemented in production, replacing the file
builtins with read-only versions and add enabled libraries to the path.
Args:
config: The runtime_config_pb2.Config to use to configure the sandbox.
"""
devnull = open(os.path.devnull)
modules = [os, traceback, google]
c_module = _find_shared_object_c_module()
if c_module:
modules.append(c_module)
module_paths = [module.__file__ for module in modules]
module_paths.extend(
[os.path.realpath(module.__file__) for module in modules])
python_lib_paths = [config.application_root]
for path in sys.path:
if any(module_path.startswith(path) for module_path in module_paths):
python_lib_paths.append(path)
python_lib_paths.extend(_enable_libraries(config.libraries))
# Note that the above code (see _find_shared_object_c_module) imports modules
# that must be pruned so please use care if you move the call to
# _prune_sys_modules.
_prune_sys_modules()
path_override_hook = PathOverrideImportHook(
THIRD_PARTY_C_MODULES.get_importable_module_names(config))
python_lib_paths.extend(path_override_hook.extra_sys_paths)
if not config.vm:
_install_fake_file(config, python_lib_paths, path_override_hook)
_install_open_hooks()
# NOTE(user): The sys.platform was a hack needed to solve
# b/7482060. After python version 2.7.4 this is no longer needed.
def was_created_before(ver1, ver2):
"""Returns true if the integer tuple ver1 is less than the tuple ver2."""
if ver1[0] != ver2[0]:
return ver1[0] < ver2[0]
elif ver1[1] != ver2[1]:
return ver1[1] < ver2[1]
else:
return ver1[2] < ver2[2]
if was_created_before(sys.version_info, (2, 7, 4)):
sys.platform = 'linux3'
_install_import_hooks(config, path_override_hook)
sys.path_importer_cache = {}
if not config.vm:
sys.path = python_lib_paths[:]
else:
# Use anything present on the sys.path if the runtime is on a vm.
# This lets users use deps installed with pip.
sys.path.extend(python_lib_paths)
thread = __import__('thread')
__import__('%s.threading' % dist27.__name__)
threading = sys.modules['%s.threading' % dist27.__name__]
thread.start_new_thread = _make_request_id_aware_start_new_thread(
thread.start_new_thread)
# This import needs to be after enabling the sandbox so it imports the
# sandboxed version of the logging module.
from google.appengine.runtime import runtime
runtime.PatchStartNewThread(thread)
threading._start_new_thread = thread.start_new_thread
os.chdir(config.application_root)
sandboxed_os = __import__('os')
request_environment.PatchOsEnviron(sandboxed_os)
os.__dict__.update(sandboxed_os.__dict__)
_init_logging(config.stderr_log_level)
pdb_sandbox.install(config)
sys.stdin = devnull
sys.stdout = sys.stderr
def enable_sandbox(config):
"""Enable the sandbox based on the configuration.
This includes installing import hooks to restrict access to C modules and
stub out functions that are not implemented in production, replacing the file
builtins with read-only versions and add enabled libraries to the path.
Args:
config: The runtime_config_pb2.Config to use to configure the sandbox.
"""
modules = [os, traceback, google, protorpc]
c_module = _find_shared_object_c_module()
if c_module:
modules.append(c_module)
module_paths = [module.__file__ for module in modules]
module_paths.extend(
[os.path.realpath(module.__file__) for module in modules])
python_lib_paths = [config.application_root]
for path in sys.path:
if any(module_path.startswith(path) for module_path in module_paths):
python_lib_paths.append(path)
python_lib_paths.extend(_enable_libraries(config.libraries))
for name in list(sys.modules):
if not _should_keep_module(name):
_removed_modules.append(sys.modules[name])
del sys.modules[name]
path_override_hook = PathOverrideImportHook(
set(
_THIRD_PARTY_LIBRARY_NAME_OVERRIDES.get(lib.name, lib.name)
for lib in config.libraries).intersection(_C_MODULES))
python_lib_paths.extend(path_override_hook.extra_sys_paths)
stubs.FakeFile.set_allowed_paths(
config.application_root,
python_lib_paths[1:] + path_override_hook.extra_accessible_paths)
stubs.FakeFile.set_skip_files(config.skip_files)
stubs.FakeFile.set_static_files(config.static_files)
__builtin__.file = stubs.FakeFile
__builtin__.open = stubs.FakeFile
types.FileType = stubs.FakeFile
sys.platform = 'linux3'
enabled_library_regexes = [
NAME_TO_CMODULE_WHITELIST_REGEX[lib.name] for lib in config.libraries
if lib.name in NAME_TO_CMODULE_WHITELIST_REGEX
]
sys.meta_path = [
StubModuleImportHook(),
ModuleOverrideImportHook(_MODULE_OVERRIDE_POLICIES),
BuiltinImportHook(),
CModuleImportHook(enabled_library_regexes), path_override_hook,
PyCryptoRandomImportHook,
PathRestrictingImportHook(enabled_library_regexes)
]
sys.path_importer_cache = {}
sys.path = python_lib_paths[:]
thread = __import__('thread')
__import__('%s.threading' % dist27.__name__)
threading = sys.modules['%s.threading' % dist27.__name__]
thread.start_new_thread = _make_request_id_aware_start_new_thread(
thread.start_new_thread)
# This import needs to be after enabling the sandbox so it imports the
# sandboxed version of the logging module.
from google.appengine.runtime import runtime
runtime.PatchStartNewThread(thread)
threading._start_new_thread = thread.start_new_thread
os.chdir(config.application_root)
sandboxed_os = __import__('os')
request_environment.PatchOsEnviron(sandboxed_os)
os.__dict__.update(sandboxed_os.__dict__)
_init_logging(config.stderr_log_level)
