def test_prior_error(self):
error = 'Oh no! Something is broken again!'
self.proxy = http_proxy.HttpProxy(
host=None,
port=None,
instance_died_unexpectedly=None,
instance_logs_getter=get_instance_logs,
error_handler_file=None,
prior_error=error)
# Expect that wait_for_connection does not hang.
self.proxy.wait_for_connection()
expected_headers = {
'Content-Type': 'text/plain',
'Content-Length': str(len(error)),
}
self.assertResponse('500 Internal Server Error',
expected_headers,
error,
self.proxy.handle, {},
url_map=self.url_map,
match=re.match(self.url_map.url, '/get%20request'),
request_id='request id',
request_type=instance.NORMAL_REQUEST)
self.mox.VerifyAll()
def test_connection_error_process_quit(self):
self.proxy = http_proxy.HttpProxy(
host='localhost',
port=123,
instance_died_unexpectedly=lambda: True,
instance_logs_getter=get_instance_logs,
error_handler_file=None)
login.get_user_info(None).AndReturn(('', False, ''))
httplib.HTTPConnection.connect().AndRaise(socket.error())
httplib.HTTPConnection.close()
self.mox.ReplayAll()
expected_headers = {
'Content-Type': 'text/plain',
'Content-Length': '78',
}
expected_content = (
'the runtime process for the instance running on port '
'123 has unexpectedly quit')
self.assertResponse('500 Internal Server Error',
expected_headers,
expected_content,
self.proxy.handle, {'PATH_INFO': '/'},
url_map=self.url_map,
match=re.match(self.url_map.url, '/get%20error'),
request_id='request id',
request_type=instance.NORMAL_REQUEST)
self.mox.VerifyAll()
def test_http_response_early_failure(self):
header = ('the runtime process gave a bad HTTP response: '
'IncompleteRead(0 bytes read)\n\n')
def dave_message():
return "I'm sorry, Dave. I'm afraid I can't do that.\n"
self.proxy = http_proxy.HttpProxy(
host='localhost',
port=23456,
instance_died_unexpectedly=lambda: False,
instance_logs_getter=dave_message,
error_handler_file=None)
login.get_user_info(None).AndReturn(('', False, ''))
httplib.HTTPConnection.connect()
httplib.HTTPConnection.request(
'GET', '/get%20request?key=value', None, {
'HEADER': 'value',
http_runtime_constants.REQUEST_ID_HEADER: 'request id',
'X-AppEngine-Country': 'ZZ',
'X-Appengine-User-Email': '',
'X-Appengine-User-Id': '',
'X-Appengine-User-Is-Admin': '0',
'X-Appengine-User-Nickname': '',
'X-Appengine-User-Organization': '',
'X-APPENGINE-DEV-SCRIPT': 'get.py',
'X-APPENGINE-SERVER-NAME': 'localhost',
'X-APPENGINE-SERVER-PORT': '8080',
'X-APPENGINE-SERVER-PROTOCOL': 'HTTP/1.1',
})
httplib.HTTPConnection.getresponse().AndRaise(
httplib.IncompleteRead(''))
httplib.HTTPConnection.close()
environ = {
'HTTP_HEADER': 'value',
'PATH_INFO': '/get request',
'QUERY_STRING': 'key=value',
'HTTP_X_APPENGINE_USER_ID': '123',
'SERVER_NAME': 'localhost',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.1',
}
self.mox.ReplayAll()
expected_headers = {
'Content-Type': 'text/plain',
'Content-Length': '%d' % (len(header) + len(dave_message()))
}
self.assertResponse('500 Internal Server Error',
expected_headers,
header + dave_message(),
self.proxy.handle,
environ,
url_map=self.url_map,
match=re.match(self.url_map.url, '/get%20request'),
request_id='request id',
request_type=instance.NORMAL_REQUEST)
self.mox.VerifyAll()
def test_handle_with_error(self):
error_handler_file = os.path.join(self.tmpdir, 'error.html')
with open(error_handler_file, 'w') as f:
f.write('error')
self.proxy = http_proxy.HttpProxy(
host='localhost',
port=23456,
instance_died_unexpectedly=lambda: False,
instance_logs_getter=get_instance_logs,
error_handler_file=error_handler_file)
response = FakeHttpResponse(
500, 'Internal Server Error',
[(http_runtime_constants.ERROR_CODE_HEADER, '1')], '')
login.get_user_info(None).AndReturn(('', False, ''))
httplib.HTTPConnection.connect()
httplib.HTTPConnection.request(
'GET', '/get%20error', None, {
'HEADER': 'value',
http_runtime_constants.REQUEST_ID_HEADER: 'request id',
'X-AppEngine-Country': 'ZZ',
'X-Appengine-User-Email': '',
'X-Appengine-User-Id': '',
'X-Appengine-User-Is-Admin': '0',
'X-Appengine-User-Nickname': '',
'X-Appengine-User-Organization': '',
'X-APPENGINE-DEV-SCRIPT': 'get.py',
'X-APPENGINE-SERVER-NAME': 'localhost',
'X-APPENGINE-SERVER-PORT': '8080',
'X-APPENGINE-SERVER-PROTOCOL': 'HTTP/1.1',
})
httplib.HTTPConnection.getresponse().AndReturn(response)
httplib.HTTPConnection.close()
environ = {
'HTTP_HEADER': 'value',
'PATH_INFO': '/get error',
'QUERY_STRING': '',
'HTTP_X_APPENGINE_USER_ID': '123',
'SERVER_NAME': 'localhost',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.1',
}
self.mox.ReplayAll()
expected_headers = {
'Content-Type': 'text/html',
'Content-Length': '5',
}
self.assertResponse('500 Internal Server Error',
expected_headers,
'error',
self.proxy.handle,
environ,
url_map=self.url_map,
match=re.match(self.url_map.url, '/get%20error'),
request_id='request id',
request_type=instance.NORMAL_REQUEST)
self.mox.VerifyAll()
def start(self):
runtime_config = self._runtime_config_getter()
# api_host set to 'localhost' won't be accessible from a docker container
# because container will have it's own 'localhost'.
# TODO: this works only when /etc/hosts is configured properly.
api_host = socket.gethostbyname(socket.gethostname()) if (
runtime_config.api_host == '0.0.0.0') else runtime_config.api_host
# Must be HTTP_PORT from apphosting/ext/vmruntime/vmservice.py
# TODO: update apphosting/ext/vmruntime/vmservice.py to use
# env var set here.
PORT = 8080
self._container = containers.Container(
self._docker_client,
containers.ContainerOptions(
image_opts=containers.ImageOptions(
dockerfile_dir=self._module_configuration.application_root,
tag='vm.%(RUNTIME)s.%(APP_ID)s.%(MODULE)s.%(VERSION)s' % {
'APP_ID': self._module_configuration.application,
'MODULE': self._module_configuration.module_name,
'RUNTIME':
self._module_configuration.effective_runtime,
'VERSION': self._module_configuration.major_version
},
nocache=False),
port=PORT,
environment={
'API_HOST': api_host,
'API_PORT': runtime_config.api_port,
'GAE_LONG_APP_ID':
self._module_configuration.application_external_name,
'GAE_PARTITION': self._module_configuration.partition,
'GAE_MODULE_NAME': self._module_configuration.module_name,
'GAE_MODULE_VERSION':
self._module_configuration.major_version,
'GAE_MINOR_VERSION':
self._module_configuration.minor_version,
'GAE_MODULE_INSTANCE': runtime_config.instance_id
},
volumes={
'/var/log/app_engine/app': {
'bind': '/var/log/app_engine/app'
}
},
volumes_from=None))
self._container.Start()
self._proxy = http_proxy.HttpProxy(
host=self._container.host,
port=self._container.port,
instance_died_unexpectedly=self._instance_died_unexpectedly,
instance_logs_getter=self._get_instance_logs,
error_handler_file=application_configuration.get_app_error_file(
self._module_configuration))
def start(self):
"""Starts the runtime process and waits until it is ready to serve."""
runtime_config = self._runtime_config_getter()
# TODO: Use a different process group to isolate the child process
# from signals sent to the parent. Only available in subprocess in
# Python 2.7.
assert self._start_process_flavor in self._VALID_START_PROCESS_FLAVORS
if self._start_process_flavor == START_PROCESS:
serialized_config = base64.b64encode(
runtime_config.SerializeToString())
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process(
self._args,
serialized_config,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self._env,
cwd=self._module_configuration.application_root)
line = self._process.stdout.readline()
elif self._start_process_flavor == START_PROCESS_FILE:
serialized_config = runtime_config.SerializeToString()
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process_file(
args=self._args,
input_string=serialized_config,
env=self._env,
cwd=self._module_configuration.application_root,
stderr=subprocess.PIPE)
line = self._read_start_process_file()
_remove_retry_sharing_violation(self._process.child_out.name)
# _stderr_tee may be pre-set by unit tests.
if self._stderr_tee is None:
self._stderr_tee = tee.Tee(self._process.stderr, sys.stderr)
self._stderr_tee.start()
port = None
error = None
try:
port = int(line)
except ValueError:
error = 'bad runtime process port [%r]' % line
logging.error(error)
finally:
self._proxy = http_proxy.HttpProxy(
host='localhost',
port=port,
instance_died_unexpectedly=self._instance_died_unexpectedly,
instance_logs_getter=self._get_instance_logs,
error_handler_file=application_configuration.
get_app_error_file(self._module_configuration),
prior_error=error)
self._proxy.wait_for_connection()
def test_http_response_late_failure(self):
line0 = "I know I've made some very poor decisions recently...\n"
def dave_message():
return "I'm afraid. I'm afraid, Dave.\n"
self.proxy = http_proxy.HttpProxy(
host='localhost',
port=23456,
instance_died_unexpectedly=lambda: False,
instance_logs_getter=dave_message,
error_handler_file=None)
response = FakeHttpResponse(200, 'OK', [], line0)
response.partial_read_error = httplib.IncompleteRead('')
login.get_user_info(None).AndReturn(('', False, ''))
httplib.HTTPConnection.connect()
httplib.HTTPConnection.request(
'GET', '/get%20request?key=value', None, {
'HEADER': 'value',
http_runtime_constants.REQUEST_ID_HEADER: 'request id',
'X-AppEngine-Country': 'ZZ',
'X-Appengine-User-Email': '',
'X-Appengine-User-Id': '',
'X-Appengine-User-Is-Admin': '0',
'X-Appengine-User-Nickname': '',
'X-Appengine-User-Organization': '',
'X-APPENGINE-DEV-SCRIPT': 'get.py',
'X-APPENGINE-SERVER-NAME': 'localhost',
'X-APPENGINE-SERVER-PORT': '8080',
'X-APPENGINE-SERVER-PROTOCOL': 'HTTP/1.1',
})
httplib.HTTPConnection.getresponse().AndReturn(response)
httplib.HTTPConnection.close()
environ = {
'HTTP_HEADER': 'value',
'PATH_INFO': '/get request',
'QUERY_STRING': 'key=value',
'HTTP_X_APPENGINE_USER_ID': '123',
'SERVER_NAME': 'localhost',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.1',
}
self.mox.ReplayAll()
self.assertResponse('200 OK', {},
line0,
self.proxy.handle,
environ,
url_map=self.url_map,
match=re.match(self.url_map.url, '/get%20request'),
request_id='request id',
request_type=instance.NORMAL_REQUEST)
self.mox.VerifyAll()
def setUp(self):
self.mox = mox.Mox()
self.tmpdir = tempfile.mkdtemp()
self.proxy = http_proxy.HttpProxy(
host='localhost',
port=23456,
instance_died_unexpectedly=lambda: False,
instance_logs_getter=get_instance_logs,
error_handler_file=None)
self.mox.StubOutWithMock(httplib.HTTPConnection, 'connect')
self.mox.StubOutWithMock(httplib.HTTPConnection, 'request')
self.mox.StubOutWithMock(httplib.HTTPConnection, 'getresponse')
self.mox.StubOutWithMock(httplib.HTTPConnection, 'close')
self.mox.StubOutWithMock(login, 'get_user_info')
self.url_map = appinfo.URLMap(url=r'/(get|post).*', script=r'\1.py')
def start(self, dockerfile_dir=None):
runtime_config = self._runtime_config_getter()
if not self._module_configuration.major_version:
logging.error('Version needs to be specified in your application '
'configuration file.')
raise VersionError()
if not dockerfile_dir:
dockerfile_dir = self._module_configuration.application_root
# api_host set to 'localhost' won't be accessible from a docker container
# because container will have it's own 'localhost'.
# 10.0.2.2 is a special network setup by virtualbox to connect from the
# guest to the host.
api_host = runtime_config.api_host
if runtime_config.api_host in ('0.0.0.0', 'localhost'):
api_host = '10.0.2.2'
image_name = _DOCKER_IMAGE_NAME_FORMAT.format(
# Escape domain if it is present.
display=self._escape_domain(
self._module_configuration.application_external_name),
module=self._module_configuration.module_name,
version=self._module_configuration.major_version)
port_bindings = self._port_bindings if self._port_bindings else {}
port_bindings.setdefault(self._default_port, None)
debug_port = None
environment = {
'API_HOST':
api_host,
'API_PORT':
runtime_config.api_port,
'GAE_LONG_APP_ID':
self._module_configuration.application_external_name,
'GAE_PARTITION':
self._module_configuration.partition,
'GAE_MODULE_NAME':
self._module_configuration.module_name,
'GAE_MODULE_VERSION':
self._module_configuration.major_version,
'GAE_MINOR_VERSION':
self._module_configuration.minor_version,
'GAE_MODULE_INSTANCE':
runtime_config.instance_id,
'GAE_SERVER_PORT':
runtime_config.server_port,
'MODULE_YAML_PATH':
os.path.basename(self._module_configuration.config_path)
}
if self._additional_environment:
environment.update(self._additional_environment)
# Handle user defined environment variables
if self._module_configuration.env_variables:
ev = (environment.viewkeys()
& self._module_configuration.env_variables.viewkeys())
if ev:
raise InvalidEnvVariableError(
'Environment variables [%s] are reserved for App Engine use'
% ', '.join(ev))
environment.update(self._module_configuration.env_variables)
# Publish debug port if running in Debug mode.
if self._module_configuration.env_variables.get('DBG_ENABLE'):
debug_port = int(
self._module_configuration.env_variables.get(
'DBG_PORT', self.DEFAULT_DEBUG_PORT))
environment['DBG_PORT'] = debug_port
port_bindings[debug_port] = _GetPortToPublish(debug_port)
# Publish forwarded ports
# NOTE: fowarded ports are mapped as host_port => container_port,
# port_bindings are mapped the other way around.
for h, c in self._module_configuration.forwarded_ports.iteritems():
if c in port_bindings:
raise InvalidForwardedPortError(
'Port {port} is already used by debugger or runtime specific '
'VM Service. Please use a different forwarded_port.'.
format(port=c))
port_bindings[c] = h
external_logs_path = os.path.join(
'/var/log/app_engine',
self._escape_domain(
self._module_configuration.application_external_name),
self._module_configuration.module_name,
self._module_configuration.major_version,
runtime_config.instance_id)
internal_logs_path = '/var/log/app_engine'
container_name = _DOCKER_CONTAINER_NAME_FORMAT.format(
image_name=image_name, instance_id=runtime_config.instance_id)
self._container = containers.Container(
self._docker_client,
containers.ContainerOptions(
image_opts=containers.ImageOptions(
dockerfile_dir=dockerfile_dir,
tag=image_name,
nocache=False),
port=self._default_port,
port_bindings=port_bindings,
environment=environment,
volumes={external_logs_path: {
'bind': internal_logs_path
}},
name=container_name))
if self._log_server_container:
self._logs_container = self._build_logs_container(
external_logs_path, internal_logs_path,
runtime_config.instance_id)
self._container.Start()
if self._log_server_container:
try:
self._logs_container.Start()
except containers.ImageError:
logging.warning('Image \'log_server\' is not found.'
'Showing logs in admin console is disabled.')
# Print the debug information before connecting to the container
# as debugging might break the runtime during initialization, and
# connecting the debugger is required to start processing requests.
if debug_port:
logging.info(
'To debug module {module} attach to {host}:{port}'.format(
module=self._module_configuration.module_name,
host=self.ContainerHost(),
port=self.PortBinding(debug_port)))
self._proxy = http_proxy.HttpProxy(
host=self._container.host,
port=self._container.port,
instance_died_unexpectedly=self._instance_died_unexpectedly,
instance_logs_getter=self.get_instance_logs,
error_handler_file=application_configuration.get_app_error_file(
self._module_configuration))
# If forwarded ports are used we do not really have to serve on 8080.
# We'll log /_ah/start request fail, but with health-checks disabled
# we should ignore that and continue working (accepting requests on
# our forwarded ports outside of dev server control).
health_check = self._module_configuration.vm_health_check
health_check_enabled = health_check and health_check.enable_health_check
if health_check_enabled or not self._module_configuration.forwarded_ports:
self._proxy.wait_for_connection()
def start(self):
"""Starts the runtime process and waits until it is ready to serve."""
runtime_config = self._runtime_config_getter()
# TODO: Use a different process group to isolate the child process
# from signals sent to the parent. Only available in subprocess in
# Python 2.7.
assert self._start_process_flavor in self._VALID_START_PROCESS_FLAVORS
host = 'localhost'
if self._start_process_flavor == START_PROCESS:
serialized_config = base64.b64encode(runtime_config.SerializeToString())
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process(
self._args,
serialized_config,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self._env,
cwd=self._module_configuration.application_root)
port = self._process.stdout.readline()
if '\t' in port: # Split out the host if present.
host, port = port.split('\t', 1)
elif self._start_process_flavor == START_PROCESS_FILE:
serialized_config = runtime_config.SerializeToString()
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process_file(
args=self._args,
input_string=serialized_config,
env=self._env,
cwd=self._module_configuration.application_root,
stderr=subprocess.PIPE)
port = self._read_start_process_file()
_remove_retry_sharing_violation(self._process.child_out.name)
elif self._start_process_flavor == START_PROCESS_REVERSE:
serialized_config = runtime_config.SerializeToString()
with self._process_lock:
assert not self._process, 'start() can only be called once'
port = portpicker.PickUnusedPort()
self._env['PORT'] = str(port)
# If any of the strings in args contain {port}, replace that substring
# with the selected port. This allows a user-specified runtime to
# pass the port along to the subprocess as a command-line argument.
args = [arg.replace('{port}', str(port)) for arg in self._args]
self._process = safe_subprocess.start_process_file(
args=args,
input_string=serialized_config,
env=self._env,
cwd=self._module_configuration.application_root,
stderr=subprocess.PIPE)
elif self._start_process_flavor == START_PROCESS_REVERSE_NO_FILE:
serialized_config = runtime_config.SerializeToString()
with self._process_lock:
assert not self._process, 'start() can only be called once'
port = portpicker.PickUnusedPort()
if self._extra_args_getter:
self._args.append(self._extra_args_getter(port))
# If any of the strings in _args contain {port}, {api_host}, {api_port},
# replace that substring with the selected port. This allows
# a user-specified runtime to pass the port along to the subprocess
# as a command-line argument.
args = [arg.replace('{port}', str(port))
.replace('{api_port}', str(runtime_config.api_port))
.replace('{api_host}', runtime_config.api_host)
for arg in self._args]
self._process = safe_subprocess.start_process(
args=args,
input_string=serialized_config,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self._env,
cwd=self._module_configuration.application_root)
# _stderr_tee may be pre-set by unit tests.
if self._stderr_tee is None:
self._stderr_tee = tee.Tee(self._process.stderr, sys.stderr)
self._stderr_tee.start()
error = None
try:
port = int(port)
except ValueError:
error = 'bad runtime process port [%r]' % port
logging.error(error)
finally:
self._proxy = http_proxy.HttpProxy(
host=host, port=port,
instance_died_unexpectedly=self._instance_died_unexpectedly,
instance_logs_getter=self._get_instance_logs,
error_handler_file=application_configuration.get_app_error_file(
self._module_configuration),
prior_error=error)
self._proxy.wait_for_connection()
def start(self, dockerfile_dir=None):
runtime_config = self._runtime_config_getter()
if not self._module_configuration.major_version:
logging.error('Version needs to be specified in your application '
'configuration file.')
raise VersionError()
if not dockerfile_dir:
dockerfile_dir = self._module_configuration.application_root
# api_host set to 'localhost' won't be accessible from a docker container
# because container will have it's own 'localhost'.
# 10.0.2.2 is a special network setup by virtualbox to connect from the
# guest to the host.
api_host = runtime_config.api_host
if runtime_config.api_host in ('0.0.0.0', 'localhost'):
api_host = '10.0.2.2'
image_name = _DOCKER_IMAGE_NAME_FORMAT.format(
# Escape domain if it is present.
display=self._escape_domain(
self._module_configuration.application_external_name),
module=self._module_configuration.module_name,
version=self._module_configuration.major_version)
port_bindings = self._port_bindings if self._port_bindings else {}
port_bindings.setdefault(self._default_port, None)
debug_port = None
environment = {
'API_HOST':
api_host,
'API_PORT':
runtime_config.api_port,
'GAE_LONG_APP_ID':
self._module_configuration.application_external_name,
'GAE_PARTITION':
self._module_configuration.partition,
'GAE_MODULE_NAME':
self._module_configuration.module_name,
'GAE_MODULE_VERSION':
self._module_configuration.major_version,
'GAE_MINOR_VERSION':
self._module_configuration.minor_version,
'GAE_MODULE_INSTANCE':
runtime_config.instance_id,
'GAE_SERVER_PORT':
runtime_config.server_port,
'MODULE_YAML_PATH':
os.path.basename(self._module_configuration.config_path)
}
if self._additional_environment:
environment.update(self._additional_environment)
# Handle user defined environment variables
if self._module_configuration.env_variables:
ev = (environment.viewkeys()
& self._module_configuration.env_variables.viewkeys())
if ev:
raise InvalidEnvVariableError(
'Environment variables [%s] are reserved for App Engine use'
% ', '.join(ev))
environment.update(self._module_configuration.env_variables)
# Publish debug port if running in Debug mode.
if self._module_configuration.env_variables.get('DBG_ENABLE'):
debug_port = int(
self._module_configuration.env_variables.get(
'DBG_PORT', self.DEFAULT_DEBUG_PORT))
environment['DBG_PORT'] = debug_port
port_bindings[debug_port] = _GetPortToPublish(debug_port)
external_logs_path = os.path.join(
'/var/log/app_engine',
self._escape_domain(
self._module_configuration.application_external_name),
self._module_configuration.module_name,
self._module_configuration.major_version,
runtime_config.instance_id)
container_name = _DOCKER_CONTAINER_NAME_FORMAT.format(
image_name=image_name,
minor_version=self._module_configuration.minor_version)
self._container = containers.Container(
self._docker_client,
containers.ContainerOptions(
image_opts=containers.ImageOptions(
dockerfile_dir=dockerfile_dir,
tag=image_name,
nocache=False),
port=self._default_port,
port_bindings=port_bindings,
environment=environment,
volumes={external_logs_path: {
'bind': '/var/log/app_engine'
}},
name=container_name))
self._container.Start()
# Print the debug information before connecting to the container
# as debugging might break the runtime during initialization, and
# connecting the debugger is required to start processing requests.
if debug_port:
logging.info(
'To debug module {module} attach to {host}:{port}'.format(
module=self._module_configuration.module_name,
host=self.ContainerHost(),
port=self.PortBinding(debug_port)))
self._proxy = http_proxy.HttpProxy(
host=self._container.host,
port=self._container.port,
instance_died_unexpectedly=self._instance_died_unexpectedly,
instance_logs_getter=self._get_instance_logs,
error_handler_file=application_configuration.get_app_error_file(
self._module_configuration))
self._proxy.wait_for_connection()
