def new_instance(self, instance_id, expect_ready_request=False):
"""Create and return a new Instance.
Args:
instance_id: A string or integer representing the unique (per module) id
of the instance.
expect_ready_request: If True then the instance will be sent a special
request (i.e. /_ah/warmup or /_ah/start) before it can handle external
requests.
Returns:
The newly created instance.Instance.
"""
def instance_config_getter():
runtime_config = self._runtime_config_getter()
runtime_config.instance_id = str(instance_id)
return runtime_config
# This is also checked in a more user-friendly fashion earlier in execution.
assert self._runtime_config_getter().custom_config.custom_entrypoint
proxy = http_runtime.HttpRuntimeProxy(
# Split the input from the command line into a Popen-compatible list.
shlex.split(
self._runtime_config_getter().custom_config.custom_entrypoint),
instance_config_getter,
self._module_configuration,
env=dict(os.environ),
start_process_flavor=http_runtime.START_PROCESS_REVERSE_NO_FILE)
return instance.Instance(self.request_data, instance_id, proxy,
self.max_concurrent_requests,
self.max_background_threads, expect_ready_request)
def new_instance(self, instance_id, expect_ready_request=False):
"""Create and return a new Instance.
Args:
instance_id: A string or integer representing the unique (per module) id
of the instance.
expect_ready_request: If True then the instance will be sent a special
request (i.e. /_ah/warmup or /_ah/start) before it can handle external
requests.
Returns:
The newly created instance.Instance.
"""
def instance_config_getter():
runtime_config = self._runtime_config_getter()
runtime_config.instance_id = str(instance_id)
return runtime_config
proxy = http_runtime.HttpRuntimeProxy(
self._GetRuntimeArgs(),
instance_config_getter,
self._module_configuration,
env=self._GetRuntimeEnvironmentVariables(instance_id),
start_process_flavor=self._get_process_flavor())
return instance.Instance(self.request_data,
instance_id,
proxy,
self.max_concurrent_requests,
self.max_background_threads,
expect_ready_request)
def new_instance(self, instance_id, expect_ready_request=False):
"""Create and return a new Instance.
Args:
instance_id: A string or integer representing the unique (per module) id
of the instance.
expect_ready_request: If True then the instance will be sent a special
request (i.e. /_ah/warmup or /_ah/start) before it can handle external
requests.
Returns:
The newly created instance.Instance.
"""
def instance_config_getter():
runtime_config = self._runtime_config_getter()
runtime_config.instance_id = str(instance_id)
return runtime_config
proxy = http_runtime.HttpRuntimeProxy(
_RUNTIME_ARGS,
instance_config_getter,
self._module_configuration,
env=dict(os.environ, PYTHONHASHSEED='random'),
start_process_flavor=http_runtime.START_PROCESS_FILE)
return instance.Instance(self.request_data, instance_id, proxy,
self.max_concurrent_requests,
self.max_background_threads,
expect_ready_request)
def test_connection_error_process_quit(self):
self.proxy = http_runtime.HttpRuntimeProxy(['/runtime'],
self.runtime_config_getter,
appinfo.AppInfoExternal())
self.proxy._process = self.mox.CreateMockAnything()
self.proxy._port = 123
login.get_user_info(None).AndReturn(('', False, ''))
httplib.HTTPConnection.connect().AndRaise(socket.error())
self.proxy._process.poll().AndReturn(1)
shutdown.async_quit()
httplib.HTTPConnection.close()
self.mox.ReplayAll()
expected_headers = {
'Content-Type': 'text/plain',
'Content-Length': '110',
}
expected_content = (
'the runtime process for the instance running on port '
'123 has unexpectedly quit; exiting the development '
'server')
self.assertResponse('500 Internal Server Error',
expected_headers,
expected_content,
self.proxy.handle, {'PATH_INFO': '/'},
url_map=self.url_map,
match=re.match(self.url_map.url, '/get%20error'),
request_id='request id',
request_type=instance.NORMAL_REQUEST)
self.mox.VerifyAll()
def setUp(self):
self.mox = mox.Mox()
self.tmpdir = tempfile.mkdtemp()
module_configuration = ModuleConfigurationStub(
application_root=self.tmpdir,
error_handlers=[
appinfo.ErrorHandlers(error_code='over_quota',
file='foo.html'),
appinfo.ErrorHandlers(error_code='default', file='error.html'),
])
self.runtime_config = runtime_config_pb2.Config()
self.runtime_config.app_id = 'app'
self.runtime_config.version_id = 'version'
self.runtime_config.api_port = 12345
self.runtime_config.application_root = self.tmpdir
self.runtime_config.datacenter = 'us1'
self.runtime_config.instance_id = 'abc3dzac4'
self.runtime_config.auth_domain = 'gmail.com'
self.runtime_config_getter = lambda: self.runtime_config
self.proxy = http_runtime.HttpRuntimeProxy(['/runtime'],
self.runtime_config_getter,
module_configuration,
env={'foo': 'bar'})
self.process = self.mox.CreateMock(subprocess.Popen)
self.process.stdin = self.mox.CreateMockAnything()
self.process.stdout = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(safe_subprocess, 'start_process')
self.mox.StubOutWithMock(httplib.HTTPConnection, 'connect')
self.mox.StubOutWithMock(httplib.HTTPConnection, 'request')
self.mox.StubOutWithMock(httplib.HTTPConnection, 'getresponse')
self.mox.StubOutWithMock(httplib.HTTPConnection, 'close')
self.mox.StubOutWithMock(login, 'get_user_info')
self.mox.StubOutWithMock(shutdown, 'async_quit')
self.url_map = appinfo.URLMap(url=r'/(get|post).*', script=r'\1.py')
def new_instance(self, instance_id, expect_ready_request=False):
"""Create and return a new Instance.
Args:
instance_id: A string or integer representing the unique (per module) id
of the instance.
expect_ready_request: If True then the instance will be sent a special
request (i.e. /_ah/warmup or /_ah/start) before it can handle external
requests.
Returns:
The newly created instance.Instance.
"""
def instance_config_getter():
runtime_config = self._GenerateConfigForRuntime()
runtime_config.instance_id = str(instance_id)
return runtime_config
php_executable_path = (
self._GenerateConfigForRuntime().php_config.php_executable_path)
gae_extension_path = (
self._GenerateConfigForRuntime().php_config.gae_extension_path)
self._check_binaries(php_executable_path, gae_extension_path)
proxy = http_runtime.HttpRuntimeProxy(_RUNTIME_ARGS,
instance_config_getter,
self._module_configuration)
return instance.Instance(self.request_data, instance_id, proxy,
self.max_concurrent_requests,
self.max_background_threads,
expect_ready_request)
def new_instance(self, instance_id, expect_ready_request=False):
"""Create and return a new Instance.
Args:
instance_id: A string or integer representing the unique (per module) id
of the instance.
expect_ready_request: If True then the instance will be sent a special
request (i.e. /_ah/warmup or /_ah/start) before it can handle external
requests.
Returns:
The newly created instance.Instance.
"""
def instance_config_getter():
runtime_config = self._runtime_config_getter()
runtime_config.instance_id = str(instance_id)
return runtime_config
env = self._java_application.get_environment()
with self._application_lock:
proxy = http_runtime.HttpRuntimeProxy(
self._java_command,
instance_config_getter,
self._module_configuration,
env=env,
start_process_flavor=http_runtime.START_PROCESS_FILE)
return instance.Instance(self.request_data, instance_id, proxy,
self.max_concurrent_requests,
self.max_background_threads,
expect_ready_request)
def setUp(self):
self.mox = mox.Mox()
self.tmpdir = tempfile.mkdtemp()
module_configuration = ModuleConfigurationStub(
application_root=self.tmpdir)
self.runtime_config = runtime_config_pb2.Config()
self.runtime_config.app_id = 'app'
self.runtime_config.version_id = 'version'
self.runtime_config.api_port = 12345
self.runtime_config.application_root = self.tmpdir
self.runtime_config.datacenter = 'us1'
self.runtime_config.instance_id = 'abc3dzac4'
self.runtime_config.auth_domain = 'gmail.com'
self.runtime_config_getter = lambda: self.runtime_config
self.proxy = http_runtime.HttpRuntimeProxy(
['/runtime'],
self.runtime_config_getter,
module_configuration,
env={'foo': 'bar'},
start_process_flavor=http_runtime.START_PROCESS_REVERSE)
self.mox.StubOutWithMock(self.proxy, '_process_lock')
self.process = self.mox.CreateMock(subprocess.Popen)
self.process.stdin = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(safe_subprocess, 'start_process_file')
self.mox.StubOutWithMock(os, 'remove')
self.mox.StubOutWithMock(time, 'sleep')
self.url_map = appinfo.URLMap(url=r'/(get|post).*', script=r'\1.py')
self.mox.StubOutWithMock(http_proxy.HttpProxy, 'wait_for_connection')
self.mox.StubOutWithMock(portpicker, 'PickUnusedPort')
http_proxy.HttpProxy.wait_for_connection(self.process)
def new_instance(self, instance_id, expect_ready_request=False):
"""Create and return a new Instance.
Args:
instance_id: A string or integer representing the unique (per module) id
of the instance.
expect_ready_request: If True then the instance will be sent a special
request (i.e. /_ah/warmup or /_ah/start) before it can handle external
requests.
Returns:
The newly created instance.Instance.
"""
def instance_config_getter():
runtime_config = self._runtime_config_getter()
runtime_config.instance_id = str(instance_id)
return runtime_config
with self._application_lock:
try:
if self._go_application.maybe_build(
self._modified_since_last_build):
if self._last_build_error:
logging.info('Go application successfully built.')
self._last_build_error = None
except go_errors.BuildError as e:
logging.error('Failed to build Go application: %s', e)
# Deploy a failure proxy now and each time a new instance is requested.
self._last_build_error = e
metrics.GetMetricsLogger().LogOnceOnStop(
metrics.DEVAPPSERVER_CATEGORY,
metrics.ERROR_ACTION,
label=repr(e))
self._modified_since_last_build = False
if self._last_build_error:
logging.debug('Deploying new instance of failure proxy.')
proxy = _GoBuildFailureRuntimeProxy(self._last_build_error)
else:
environ = self._go_application.get_environment()
# Add in the environment settings from app_yaml "env_variables:"
runtime_config = self._runtime_config_getter()
for kv in runtime_config.environ:
environ[kv.key] = kv.value
proxy = http_runtime.HttpRuntimeProxy(
[self._go_application.go_executable],
instance_config_getter,
self._module_configuration,
environ,
start_process_flavor=self._start_process_flavor)
return instance.Instance(self.request_data, instance_id, proxy,
self.max_concurrent_requests,
self.max_background_threads,
expect_ready_request)
def new_instance(self, instance_id, expect_ready_request=False):
"""Create and return a new Instance.
Args:
instance_id: A string or integer representing the unique (per module) id
of the instance.
expect_ready_request: If True then the instance will be sent a special
request (i.e. /_ah/warmup or /_ah/start) before it can handle external
requests.
Returns:
The newly created instance.Instance.
"""
def instance_config_getter():
runtime_config = self._runtime_config_getter()
runtime_config.instance_id = str(instance_id)
return runtime_config
with self._application_lock:
try:
if self._go_application.maybe_build(self._modified_since_last_build):
if self._last_build_error:
logging.info('Go application successfully built.')
self._last_build_error = None
except go_application.BuildError as e:
logging.error('Failed to build Go application: %s', e)
# Deploy a failure proxy now and each time a new instance is requested.
self._last_build_error = e
self._modified_since_last_build = False
if self._last_build_error:
logging.debug('Deploying new instance of failure proxy.')
proxy = _GoBuildFailureRuntimeProxy(self._last_build_error)
else:
proxy = http_runtime.HttpRuntimeProxy(
self._go_application.go_executable,
instance_config_getter,
self._module_configuration,
self._go_application.get_environment())
return instance.Instance(self.request_data,
instance_id,
proxy,
self.max_concurrent_requests,
self.max_background_threads,
expect_ready_request)
def new_instance(self, instance_id, expect_ready_request=False):
"""Create and return a new Instance.
Args:
instance_id: A string or integer representing the unique (per module) id
of the instance.
expect_ready_request: If True then the instance will be sent a special
request (i.e. /_ah/warmup or /_ah/start) before it can handle external
requests.
Returns:
The newly created instance.Instance.
"""
def instance_config_getter():
runtime_config = self._runtime_config_getter()
runtime_config.instance_id = str(instance_id)
return runtime_config
def extra_args_getter(port):
return 'jetty.port=%s' % port
env = self._java_application.get_environment()
runtime_config = instance_config_getter()
for env_entry in runtime_config.environ:
env[env_entry.key] = env_entry.value
if self._for_jetty9:
start_process_flavor = http_runtime.START_PROCESS_REVERSE_NO_FILE
env['APP_ENGINE_LOG_CONFIG_PATTERN'] = (os.path.join(
tempfile.mkdtemp(suffix='gae'), 'log.%g'))
else:
start_process_flavor = http_runtime.START_PROCESS_FILE
with self._application_lock:
proxy = http_runtime.HttpRuntimeProxy(
self._java_command,
instance_config_getter,
self._module_configuration,
env=env,
start_process_flavor=start_process_flavor,
extra_args_getter=extra_args_getter)
return instance.Instance(self.request_data, instance_id, proxy,
self.max_concurrent_requests,
self.max_background_threads,
expect_ready_request)
def new_instance(self, instance_id, expect_ready_request=False):
"""Create and return a new Instance.
Args:
instance_id: A string or integer representing the unique (per module) id
of the instance.
expect_ready_request: If True then the instance will be sent a special
request (i.e. /_ah/warmup or /_ah/start) before it can handle external
requests.
Returns:
The newly created instance.Instance.
"""
def instance_config_getter():
runtime_config = self._GenerateConfigForRuntime()
runtime_config.instance_id = str(instance_id)
return runtime_config
php_executable_path = (
self._GenerateConfigForRuntime().php_config.php_executable_path)
gae_extension_path = (
self._GenerateConfigForRuntime().php_config.gae_extension_path)
if php_executable_path not in self._php_binary_to_error_proxy:
try:
self._check_binaries(php_executable_path, gae_extension_path)
except Exception as e:
self._php_binary_to_error_proxy[php_executable_path] = (
_BadPHPEnvironmentRuntimeProxy(php_executable_path, e))
logging.exception('The PHP runtime is not available')
else:
self._php_binary_to_error_proxy[php_executable_path] = None
proxy = self._php_binary_to_error_proxy[php_executable_path]
if proxy is None:
proxy = http_runtime.HttpRuntimeProxy(_RUNTIME_ARGS,
instance_config_getter,
self._module_configuration)
return instance.Instance(self.request_data,
instance_id,
proxy,
self.max_concurrent_requests,
self.max_background_threads,
expect_ready_request)
def test_handle_with_error_no_error_handler(self):
self.proxy = http_runtime.HttpRuntimeProxy(['/runtime'],
self.runtime_config_getter,
appinfo.AppInfoExternal())
self.proxy._port = 23456
response = FakeHttpResponse(
500, 'Internal Server Error',
[(http_runtime_constants.ERROR_CODE_HEADER, '1')], '')
login.get_user_info(None).AndReturn(('', False, ''))
httplib.HTTPConnection.connect()
httplib.HTTPConnection.request(
'GET', '/get%20error', '', {
'HEADER': 'value',
http_runtime_constants.REQUEST_ID_HEADER: 'request id',
'X-AppEngine-Country': 'ZZ',
'X-Appengine-Internal-User-Email': '',
'X-Appengine-Internal-User-Id': '',
'X-Appengine-Internal-User-Is-Admin': '0',
'X-Appengine-Internal-User-Nickname': '',
'X-Appengine-Internal-User-Organization': '',
'X-APPENGINE-INTERNAL-SCRIPT': 'get.py',
'X-APPENGINE-INTERNAL-SERVER-NAME': 'localhost',
'X-APPENGINE-INTERNAL-SERVER-PORT': '8080',
'X-APPENGINE-INTERNAL-SERVER-PROTOCOL': 'HTTP/1.1',
})
httplib.HTTPConnection.getresponse().AndReturn(response)
httplib.HTTPConnection.close()
environ = {
'HTTP_HEADER': 'value',
'PATH_INFO': '/get error',
'QUERY_STRING': '',
'HTTP_X_APPENGINE_INTERNAL_USER_ID': '123',
'SERVER_NAME': 'localhost',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.1',
}
self.mox.ReplayAll()
self.assertResponse('500 Internal Server Error', {},
'',
self.proxy.handle,
environ,
url_map=self.url_map,
match=re.match(self.url_map.url, '/get%20error'),
request_id='request id',
request_type=instance.NORMAL_REQUEST)
self.mox.VerifyAll()
def new_instance(self, instance_id, expect_ready_request=False):
"""Create and return a new Instance.
Args:
instance_id: A string or integer representing the unique (per server) id
of the instance.
expect_ready_request: If True then the instance will be sent a special
request (i.e. /_ah/warmup or /_ah/start) before it can handle external
requests.
Returns:
The newly created instance.Instance.
"""
def instance_config_getter():
runtime_config = self._runtime_config_getter()
runtime_config.instance_id = str(instance_id)
return runtime_config
php_executable_path = (
self._runtime_config_getter().php_config.php_executable_path)
if self._php_binary_to_bad_environment_proxy.get(
php_executable_path) is None:
try:
self._check_environment(php_executable_path)
except _PHPEnvironmentError as e:
self._php_binary_to_bad_environment_proxy[php_executable_path] = (
_BadPHPEnvironmentRuntimeProxy(php_executable_path, str(e)))
logging.error('The PHP runtime is not available because: %r', str(e))
else:
self._php_binary_to_bad_environment_proxy[php_executable_path] = None
proxy = self._php_binary_to_bad_environment_proxy[php_executable_path]
if proxy is None:
proxy = http_runtime.HttpRuntimeProxy(_RUNTIME_ARGS,
instance_config_getter,
self._server_configuration)
return instance.Instance(self.request_data,
instance_id,
proxy,
self.max_concurrent_requests,
self.max_background_threads,
expect_ready_request)
def test_connection_error(self):
self.proxy = http_runtime.HttpRuntimeProxy(
['/runtime'], self.runtime_config_getter, appinfo.AppInfoExternal())
self.proxy._process = self.mox.CreateMockAnything()
login.get_user_info(None).AndReturn(('', False, ''))
httplib.HTTPConnection.connect().AndRaise(socket.error())
self.proxy._process.poll().AndReturn(None)
httplib.HTTPConnection.close()
self.mox.ReplayAll()
self.assertRaises(socket.error,
self.proxy.handle(
{'PATH_INFO': '/'},
start_response=None, # Not used.
url_map=self.url_map,
match=re.match(self.url_map.url, '/get%20error'),
request_id='request id',
request_type=instance.NORMAL_REQUEST).next)
self.mox.VerifyAll()
def new_instance(self, instance_id, expect_ready_request=False):
"""Create and return a new Instance.
Args:
instance_id: A string or integer representing the unique (per module) id
of the instance.
expect_ready_request: If True then the instance will be sent a special
request (i.e. /_ah/warmup or /_ah/start) before it can handle external
requests.
Returns:
The newly created instance.Instance.
"""
def instance_config_getter():
runtime_config = self._runtime_config_getter()
runtime_config.instance_id = str(instance_id)
return runtime_config
env = self._java_application.get_environment()
instance_jar = os.path.abspath(os.path.join(
os.path.dirname(google.__file__),
'appengine/tools/devappserver2/java/lib/StandaloneInstance_deploy.jar'))
assert os.path.exists(instance_jar), instance_jar
# TODO: replace this with something smaller and releasable
with self._application_lock:
proxy = http_runtime.HttpRuntimeProxy(
['java', '-jar', instance_jar],
instance_config_getter,
self._module_configuration,
env=env,
start_process_flavor=http_runtime.START_PROCESS_FILE)
return instance.Instance(self.request_data,
instance_id,
proxy,
self.max_concurrent_requests,
self.max_background_threads,
expect_ready_request)
def new_instance(self, instance_id, expect_ready_request=False):
"""Create and return a new Instance.
Args:
instance_id: A string or integer representing the unique (per server) id
of the instance.
expect_ready_request: If True then the instance will be sent a special
request (i.e. /_ah/warmup or /_ah/start) before it can handle external
requests.
Returns:
The newly created instance.Instance.
"""
def instance_config_getter():
runtime_config = self._runtime_config_getter()
runtime_config.instance_id = str(instance_id)
return runtime_config
with self._application_lock:
try:
self._go_application.maybe_build(
self._modified_since_last_build)
except go_application.BuildError as e:
logging.error('Failed to build Go application: %s', e)
proxy = _GoBuildFailureRuntimeProxy(e)
else:
proxy = http_runtime.HttpRuntimeProxy(
self._go_application.go_executable, instance_config_getter,
self._server_configuration,
self._go_application.get_environment())
self._modified_since_last_build = False
return instance.Instance(self.request_data, instance_id, proxy,
self.max_concurrent_requests,
self.max_background_threads,
expect_ready_request)
def new_instance(self, instance_id, expect_ready_request=False):
"""Create and return a new Instance.
Args:
instance_id: A string or integer representing the unique (per module) id
of the instance.
expect_ready_request: If True then the instance will be sent a special
request (i.e. /_ah/warmup or /_ah/start) before it can handle external
requests.
Returns:
The newly created instance.Instance.
"""
def instance_config_getter():
runtime_config = self._runtime_config_getter()
runtime_config.instance_id = str(instance_id)
return runtime_config
# Note, the PORT will be added to the environment when the HttpRuntimeProxy
# is started.
prefix_to_strip = 'dev~'
app_id = self._module_configuration.application
if app_id.startswith('dev~'):
app_id = app_id[len(prefix_to_strip):]
instance_start_time = datetime.datetime.now().strftime('%Y%m%dt%H%M%S')
node_environ = {
'GAE_ENV':
'localdev',
'GAE_INSTANCE':
instance_id,
'GAE_MEMORY_MB':
str(self._module_configuration.memory_limit),
'GAE_RUNTIME':
self._module_configuration.runtime,
'GAE_SERVICE':
self._module_configuration.module_name,
'GAE_VERSION': (self._module_configuration.major_version
or instance_start_time),
# TODO: Determine how to pull the gcloud project from the
# Cloud SDK.
'GOOGLE_CLOUD_PROJECT':
app_id,
}
# Set the runtime config environment variables to pass into the subprocess.
for env_var in self._runtime_config_getter().environ:
if env_var.key not in node_environ:
# We don't allow users to override the standard runtime environment
# variables. In production, no error is raised, and the standard runtime
# environment variables take precendence over user-defined variables
# of the same name.
node_environ[env_var.key] = env_var.value
proxy = http_runtime.HttpRuntimeProxy(
_RUNTIME_ARGS,
instance_config_getter,
self._module_configuration,
env=node_environ,
start_process_flavor=http_runtime.START_PROCESS_REVERSE)
return instance.Instance(self.request_data, instance_id, proxy,
self.max_concurrent_requests,
self.max_background_threads,
expect_ready_request)
