def test_start_and_not_serving(self):
safe_subprocess.start_process(
['/runtime'],
base64.b64encode(self.runtime_config.SerializeToString()),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={
'foo': 'bar'
},
cwd=self.tmpdir).AndReturn(self.process)
self.process.stdout.readline().AndReturn('30002')
self.proxy._stderr_tee = FakeTee('')
httplib.HTTPConnection.connect().AndRaise(socket.error)
httplib.HTTPConnection.close()
self.mox.ReplayAll()
self.proxy.start()
expected_headers = {
'Content-Type': 'text/plain',
'Content-Length': '39',
}
self.assertResponse('500 Internal Server Error',
expected_headers,
'cannot connect to runtime on port 30002',
self.proxy.handle, {},
url_map=self.url_map,
match=re.match(self.url_map.url, '/get%20request'),
request_id='request id',
request_type=instance.NORMAL_REQUEST)
self.mox.VerifyAll()
def test_start_and_not_serving(self):
safe_subprocess.start_process(
['/runtime'],
base64.b64encode(self.runtime_config.SerializeToString()),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={'foo': 'bar'},
cwd=self.tmpdir).AndReturn(self.process)
self.process.stdout.readline().AndReturn('30002')
self.proxy._stderr_tee = FakeTee('')
httplib.HTTPConnection.connect().AndRaise(socket.error)
httplib.HTTPConnection.close()
self.mox.ReplayAll()
self.proxy.start()
expected_headers = {
'Content-Type': 'text/plain',
'Content-Length': '39',
}
self.assertResponse('500 Internal Server Error', expected_headers,
'cannot connect to runtime on port 30002',
self.proxy.handle, {},
url_map=self.url_map,
match=re.match(self.url_map.url, '/get%20request'),
request_id='request id',
request_type=instance.NORMAL_REQUEST)
self.mox.VerifyAll()
def test_start_and_quit(self):
## Test start()
# start()
safe_subprocess.start_process(
['/runtime'],
base64.b64encode(self.runtime_config.SerializeToString()),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={
'foo': 'bar'
},
cwd=self.tmpdir).AndReturn(self.process)
self.process.stdout.readline().AndReturn('30000')
self.proxy._stderr_tee = FakeTee('')
self.mox.ReplayAll()
self.proxy.start()
self.mox.VerifyAll()
self.mox.ResetAll()
## Test quit()
self.process.kill()
self.mox.ReplayAll()
self.proxy.quit()
self.mox.VerifyAll()
def _test_start_and_quit(self, quit_with_sigterm):
## Test start()
# start()
self._saved_quit_with_sigterm = (
http_runtime.HttpRuntimeProxy.stop_runtimes_with_sigterm(
quit_with_sigterm))
safe_subprocess.start_process(
['/runtime'],
base64.b64encode(self.runtime_config.SerializeToString()),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={
'foo': 'bar'
},
cwd=self.tmpdir).AndReturn(self.process)
self.process.stdout.readline().AndReturn('30000')
self.proxy._stderr_tee = FakeTee('')
self.mox.ReplayAll()
self.proxy.start()
self.assertEquals(30000, self.proxy._proxy._port)
self.assertEquals('localhost', self.proxy._proxy._host)
self.mox.VerifyAll()
self.mox.ResetAll()
## Test quit()
if quit_with_sigterm:
self.process.terminate()
else:
self.process.kill()
self.mox.ReplayAll()
self.proxy.quit()
self.mox.VerifyAll()
def _test_start_and_quit(self, quit_with_sigterm):
## Test start()
# start()
self._saved_quit_with_sigterm = (
http_runtime.HttpRuntimeProxy.stop_runtimes_with_sigterm(
quit_with_sigterm))
safe_subprocess.start_process(
['/runtime'],
base64.b64encode(self.runtime_config.SerializeToString()),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={'foo': 'bar'},
cwd=self.tmpdir).AndReturn(self.process)
self.process.stdout.readline().AndReturn('30000')
self.proxy._stderr_tee = FakeTee('')
self.mox.ReplayAll()
self.proxy.start()
self.assertEquals(30000, self.proxy._proxy._port)
self.assertEquals('localhost', self.proxy._proxy._host)
self.mox.VerifyAll()
self.mox.ResetAll()
## Test quit()
if quit_with_sigterm:
self.process.terminate()
else:
self.process.kill()
self.mox.ReplayAll()
self.proxy.quit()
self.mox.VerifyAll()
def test_start_and_quit(self):
## Test start()
# start()
safe_subprocess.start_process(
['/runtime'],
base64.b64encode(self.runtime_config.SerializeToString()),
stdout=subprocess.PIPE,
env={
'foo': 'bar'
},
cwd=self.tmpdir).AndReturn(self.process)
self.process.stdout.readline().AndReturn('34567')
# _can_connect() via _check_serving().
httplib.HTTPConnection.connect()
httplib.HTTPConnection.close()
self.mox.ReplayAll()
self.proxy.start()
self.mox.VerifyAll()
self.mox.ResetAll()
## Test quit()
self.process.kill()
self.mox.ReplayAll()
self.proxy.quit()
self.mox.VerifyAll()
def test_start_bad_port(self):
safe_subprocess.start_process(
['/runtime'],
base64.b64encode(self.runtime_config.SerializeToString()),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={'foo': 'bar'},
cwd=self.tmpdir).AndReturn(self.process)
self.process.stdout.readline().AndReturn('hello 30001')
header = "bad runtime process port ['hello 30001']\n\n"
stderr0 = "I've just picked up a fault in the AE35 unit.\n"
stderr1 = "It's going to go 100% failure in 72 hours.\n"
self.proxy._stderr_tee = FakeTee(stderr0 + stderr1)
self.mox.ReplayAll()
self.proxy.start()
expected_headers = {
'Content-Type': 'text/plain',
'Content-Length': str(len(header) + len(stderr0) + len(stderr1)),
}
self.assertResponse('500 Internal Server Error', expected_headers,
header + stderr0 + stderr1,
self.proxy.handle, {},
url_map=self.url_map,
match=re.match(self.url_map.url, '/get%20request'),
request_id='request id',
request_type=instance.NORMAL_REQUEST)
self.mox.VerifyAll()
def _check_environment(php_executable_path):
if php_executable_path is None:
raise _PHPBinaryError(
'The development server must be started with the '
'--php_executable_path flag set to the path of the '
'php-cgi binary.')
if not os.path.exists(php_executable_path):
raise _PHPBinaryError(
'The path specified with the --php_executable_path '
'flag (%s) does not exist.' % php_executable_path)
if not os.access(php_executable_path, os.X_OK):
raise _PHPBinaryError(
'The path specified with the --php_executable_path '
'flag (%s) is not executable' % php_executable_path)
env = {}
# On Windows, in order to run a side-by-side assembly the specified env
# must include a valid SystemRoot.
if 'SYSTEMROOT' in os.environ:
env['SYSTEMROOT'] = os.environ['SYSTEMROOT']
if not os.access(php_executable_path, os.X_OK):
raise _PHPBinaryError(
'The path specified with the --php_executable_path '
'flag (%s) is not executable' % php_executable_path)
version_process = safe_subprocess.start_process(
[php_executable_path, '-v'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
version_stdout, version_stderr = version_process.communicate()
if version_process.returncode:
raise _PHPEnvironmentError(
'"%s -v" returned an error [%d]\n%s%s' %
(php_executable_path, version_process.returncode,
version_stderr, version_stdout))
version_match = re.search(r'PHP (\d+).(\d+)', version_stdout)
if version_match is None:
raise _PHPEnvironmentError(
'"%s -v" returned an unexpected version string:\n%s%s' %
(php_executable_path, version_stderr, version_stdout))
version = tuple(int(v) for v in version_match.groups())
if version < (5, 4):
raise _PHPEnvironmentError(
'The PHP interpreter must be version >= 5.4, %d.%d found' %
version)
check_process = safe_subprocess.start_process(
[php_executable_path, '-f', _CHECK_ENVIRONMENT_SCRIPT_PATH],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
check_process_stdout, _ = check_process.communicate()
if check_process.returncode:
raise _PHPEnvironmentError(check_process_stdout)
def test_start_and_quit(self):
## Test start()
# start()
safe_subprocess.start_process(
['/runtime'],
base64.b64encode(self.runtime_config.SerializeToString()),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={'foo': 'bar'},
cwd=self.tmpdir).AndReturn(self.process)
self.process.stdout.readline().AndReturn('30000')
self.proxy._stderr_tee = FakeTee('')
# _can_connect() via start().
httplib.HTTPConnection.connect()
httplib.HTTPConnection.close()
self.mox.ReplayAll()
self.proxy.start()
self.mox.VerifyAll()
self.mox.ResetAll()
## Test quit()
self.process.kill()
self.mox.ReplayAll()
self.proxy.quit()
self.mox.VerifyAll()
def test_start_bad_port(self):
safe_subprocess.start_process(
['/runtime'],
base64.b64encode(self.runtime_config.SerializeToString()),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={
'foo': 'bar'
},
cwd=self.tmpdir).AndReturn(self.process)
self.process.stdout.readline().AndReturn('hello 30001')
header = "bad runtime process port ['hello 30001']\n\n"
stderr0 = "I've just picked up a fault in the AE35 unit.\n"
stderr1 = "It's going to go 100% failure in 72 hours.\n"
self.proxy._stderr_tee = FakeTee(stderr0 + stderr1)
self.mox.ReplayAll()
self.proxy.start()
expected_headers = {
'Content-Type': 'text/plain',
'Content-Length': str(len(header) + len(stderr0) + len(stderr1)),
}
self.assertResponse('500 Internal Server Error',
expected_headers,
header + stderr0 + stderr1,
self.proxy.handle, {},
url_map=self.url_map,
match=re.match(self.url_map.url, '/get%20request'),
request_id='request id',
request_type=instance.NORMAL_REQUEST)
self.mox.VerifyAll()
def _check_environment(php_executable_path):
if php_executable_path is None:
raise _PHPBinaryError('The development server must be started with the '
'--php_executable_path flag set to the path of the '
'php-cgi binary.')
if not os.path.exists(php_executable_path):
raise _PHPBinaryError('The path specified with the --php_executable_path '
'flag (%s) does not exist.' % php_executable_path)
if not os.access(php_executable_path, os.X_OK):
raise _PHPBinaryError('The path specified with the --php_executable_path '
'flag (%s) is not executable' % php_executable_path)
env = {}
# On Windows, in order to run a side-by-side assembly the specified env
# must include a valid SystemRoot.
if 'SYSTEMROOT' in os.environ:
env['SYSTEMROOT'] = os.environ['SYSTEMROOT']
if not os.access(php_executable_path, os.X_OK):
raise _PHPBinaryError('The path specified with the --php_executable_path '
'flag (%s) is not executable' % php_executable_path)
version_process = safe_subprocess.start_process([php_executable_path, '-v'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
version_stdout, version_stderr = version_process.communicate()
if version_process.returncode:
raise _PHPEnvironmentError(
'"%s -v" returned an error [%d]\n%s%s' % (
php_executable_path,
version_process.returncode,
version_stderr,
version_stdout))
version_match = re.search(r'PHP (\d+).(\d+)', version_stdout)
if version_match is None:
raise _PHPEnvironmentError(
'"%s -v" returned an unexpected version string:\n%s%s' % (
php_executable_path,
version_stderr,
version_stdout))
version = tuple(int(v) for v in version_match.groups())
if version < (5, 4):
raise _PHPEnvironmentError(
'The PHP interpreter must be version >= 5.4, %d.%d found' % version)
check_process = safe_subprocess.start_process(
[php_executable_path, '-f', _CHECK_ENVIRONMENT_SCRIPT_PATH],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
check_process_stdout, _ = check_process.communicate()
if check_process.returncode:
raise _PHPEnvironmentError(check_process_stdout)
def test_start_bad_port(self):
safe_subprocess.start_process(
['/runtime'],
base64.b64encode(self.runtime_config.SerializeToString()),
stdout=subprocess.PIPE,
env={'foo': 'bar'},
cwd=self.tmpdir).AndReturn(self.process)
self.process.stdout.readline().AndReturn('hello 34567')
shutdown.async_quit()
self.mox.ReplayAll()
self.proxy.start()
self.mox.VerifyAll()
def _check_php_version(cls, php_executable_path, env):
"""Check if php-cgi has the correct version."""
version_process = safe_subprocess.start_process([php_executable_path, '-v'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
version_stdout, version_stderr = version_process.communicate()
if version_process.returncode:
raise _PHPEnvironmentError(
'"%s -v" returned an error [%d]\n%s%s' % (
php_executable_path,
version_process.returncode,
version_stderr,
version_stdout))
version_match = re.search(r'PHP (\d+).(\d+)', version_stdout)
if version_match is None:
raise _PHPEnvironmentError(
'"%s -v" returned an unexpected version string:\n%s%s' % (
php_executable_path,
version_stderr,
version_stdout))
version = tuple(int(v) for v in version_match.groups())
if version < (5, 4):
raise _PHPEnvironmentError(
'The PHP interpreter must be version >= 5.4, %d.%d found' % version)
def test_start_and_not_serving(self):
safe_subprocess.start_process(
['/runtime'],
base64.b64encode(self.runtime_config.SerializeToString()),
stdout=subprocess.PIPE,
env={'foo': 'bar'},
cwd=self.tmpdir).AndReturn(self.process)
self.process.stdout.readline().AndReturn('34567')
httplib.HTTPConnection.connect().AndRaise(socket.error)
httplib.HTTPConnection.close()
shutdown.async_quit()
self.mox.ReplayAll()
self.proxy.start()
self.mox.VerifyAll()
def _build(self):
assert self._go_file_to_mtime, 'no .go files'
logging.debug('Building Go application')
gab_args = _get_base_gab_args(
self._module_configuration.application_root,
self._module_configuration.nobuild_files, self._arch)
gab_args.extend([
'-binary_name', '_go_app',
'-extra_imports', 'appengine_internal/init',
'-work_dir', self._work_dir,
'-gcflags', _escape_tool_flags('-I', self._pkg_path),
'-ldflags', _escape_tool_flags('-L', self._pkg_path),
])
gab_args.extend(self._go_file_to_mtime)
gab_process = safe_subprocess.start_process(gab_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={})
gab_stdout, gab_stderr = gab_process.communicate()
if gab_process.returncode:
raise BuildError(
'(Executed command: %s)\n\n%s\n%s' % (' '.join(gab_args),
gab_stdout,
gab_stderr))
else:
logging.debug('Build succeeded:\n%s\n%s', gab_stdout, gab_stderr)
self._go_executable = os.path.join(self._work_dir, '_go_app')
def start(self):
"""Starts the runtime process and waits until it is ready to serve."""
runtime_config = self._runtime_config_getter()
serialized_config = base64.b64encode(runtime_config.SerializeToString())
# TODO: Use a different process group to isolate the child process
# from signals sent to the parent. Only available in subprocess in
# Python 2.7.
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process(
self._args,
serialized_config,
stdout=subprocess.PIPE,
env=self._env,
cwd=self._server_configuration.application_root)
line = self._process.stdout.readline()
try:
self._port = int(line)
except ValueError:
# The development server is in a bad state. Log an error message and quit.
logging.error('unexpected port response from runtime [%r]; '
'exiting the development server',
line)
shutdown.async_quit()
else:
self._check_serving()
def _run_gab(application_root, nobuild_files, arch, gab_extra_args, env):
"""Run go-app-builder.
Args:
application_root: string path to the root dir of the application.
nobuild_files: regexp identifying which files to not build.
arch: The one-character architecture designator (5, 6, or 8).
gab_extra_args: additional arguments (i.e. other than the standard base
arguments) for go-app-builder.
env: A dict containing environment variables for the subprocess.
Returns:
A tuple of the (stdout, stderr) from the go-app-builder process.
Raises:
BuildError: if the go application builder fails.
"""
gab_args = _get_base_gab_args(application_root, nobuild_files, arch)
gab_args.extend(gab_extra_args)
gab_process = safe_subprocess.start_process(gab_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
gab_stdout, gab_stderr = gab_process.communicate()
if gab_process.returncode:
raise BuildError(
'(Executed command: %s)\n\n%s' % (' '.join(gab_args),
gab_stderr))
return gab_stdout, gab_stderr
def get_app_extras_for_vm(module_config):
"""Returns an iterable describing extra Go files needed to build VM apps.
The Go files are decided based on the production environment linux/amd64.
Args:
module_config: An application_configuration.ModuleConfiguration
instance storing the configuration data for a module.
Returns:
An iterable of pairs, one per extra Go file. The first pair element
is the relative path at which to import the Go file; the second is its
absolute path.
Raises:
BuildError: if the go application builder fails.
"""
gab_args = _get_base_gab_args(module_config, '6')
gab_args.extend(['-print_extras', '-vm'])
gab_args.extend(list_go_files(module_config))
env = {
'GOOS': 'linux',
'GOARCH': 'amd64',
}
gab_process = safe_subprocess.start_process(gab_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
gab_stdout, gab_stderr = gab_process.communicate()
if gab_process.returncode:
raise BuildError(
'(Executed command: %s)\n\n%s' % (' '.join(gab_args),
gab_stderr))
return [l.split('|') for l in gab_stdout.split('\n') if l]
def _check_php_version(cls, php_executable_path, env):
"""Check if php-cgi has the correct version."""
version_process = safe_subprocess.start_process(
[php_executable_path, '-v'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
version_stdout, version_stderr = version_process.communicate()
if version_process.returncode:
raise _PHPEnvironmentError(
'"%s -v" returned an error [%d]\n%s%s' %
(php_executable_path, version_process.returncode,
version_stderr, version_stdout))
version_match = re.search(r'PHP (\d+).(\d+)', version_stdout)
if version_match is None:
raise _PHPEnvironmentError(
'"%s -v" returned an unexpected version string:\n%s%s' %
(php_executable_path, version_stderr, version_stdout))
version = tuple(int(v) for v in version_match.groups())
if version < (5, 4):
raise _PHPEnvironmentError(
'The PHP interpreter must be version >= 5.4, %d.%d found' %
version)
def _check_gae_extension(cls, php_executable_path, gae_extension_path, env):
"""Check if GAE extension can be loaded."""
if not os.path.exists(gae_extension_path):
raise _PHPBinaryError(
"The path specified with the "
"--php_gae_extension_path flag (%s) does not "
"exist." % gae_extension_path
)
# The GAE extension requires APPLICATION_ROOT to be set.
env["APPLICATION_ROOT"] = os.getcwd()
args = [
php_executable_path,
"-m",
"-d",
'extension="%s"' % os.path.basename(gae_extension_path),
"-d",
'extension_dir="%s"' % os.path.dirname(gae_extension_path),
]
ext_process = safe_subprocess.start_process(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
ext_stdout, ext_stderr = ext_process.communicate()
if ext_process.returncode:
raise _PHPEnvironmentError(
'"%s -m" returned an error [%d]\n%s%s'
% (php_executable_path, ext_process.returncode, ext_stderr, ext_stdout)
)
if GAE_EXTENSION_NAME not in ext_stdout:
raise _PHPEnvironmentError("Unable to load GAE runtime module at %s" % gae_extension_path)
def _check_gae_extension(cls, php_executable_path, gae_extension_path,
env):
"""Check if GAE extension can be loaded."""
if not os.path.exists(gae_extension_path):
raise _PHPBinaryError(
'The path specified with the '
'--php_gae_extension_path flag (%s) does not '
'exist.' % gae_extension_path)
# The GAE extension requires APPLICATION_ROOT to be set.
env['APPLICATION_ROOT'] = os.getcwd()
args = [
php_executable_path, '-m', '-d',
'extension="%s"' % os.path.basename(gae_extension_path), '-d',
'extension_dir="%s"' % os.path.dirname(gae_extension_path)
]
ext_process = safe_subprocess.start_process(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
ext_stdout, ext_stderr = ext_process.communicate()
if ext_process.returncode:
raise _PHPEnvironmentError(
'"%s -m" returned an error [%d]\n%s%s' %
(php_executable_path, ext_process.returncode, ext_stderr,
ext_stdout))
if GAE_EXTENSION_NAME not in ext_stdout:
raise _PHPEnvironmentError(
'Unable to load GAE runtime module at %s' % gae_extension_path)
def _get_extras_hash(self):
"""Returns a hash of the names and mtimes of package dependencies.
Returns:
Returns a string representing a hash.
Raises:
BuildError: if the go application builder fails.
"""
gab_args = _get_base_gab_args(
self._module_configuration.application_root,
self._module_configuration.nobuild_files, self._arch)
gab_args.append('-print_extras_hash')
gab_args.extend(self._go_file_to_mtime)
gab_process = safe_subprocess.start_process(gab_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={})
gab_stdout, gab_stderr = gab_process.communicate()
if gab_process.returncode:
raise BuildError('(Executed command: %s)\n\n%s' %
(' '.join(gab_args), gab_stderr))
else:
return gab_stdout
def _run_gab(application_root, nobuild_files, arch, gab_extra_args, env):
"""Run go-app-builder.
Args:
application_root: string path to the root dir of the application.
nobuild_files: regexp identifying which files to not build.
arch: The one-character architecture designator (5, 6, or 8).
gab_extra_args: additional arguments (i.e. other than the standard base
arguments) for go-app-builder.
env: A dict containing environment variables for the subprocess.
Returns:
A tuple of the (stdout, stderr) from the go-app-builder process.
Raises:
BuildError: if the go application builder fails.
"""
gab_args = _get_base_gab_args(application_root, nobuild_files, arch)
gab_args.extend(gab_extra_args)
gab_process = safe_subprocess.start_process(gab_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
gab_stdout, gab_stderr = gab_process.communicate()
if gab_process.returncode:
raise go_errors.BuildError(
'(Executed command: %s)\n%s\n%s' %
(' '.join(gab_args), gab_stdout, gab_stderr))
return gab_stdout, gab_stderr
def _build(self):
assert self._go_file_to_mtime, 'no .go files'
logging.debug('Building Go application')
gab_args = _get_base_gab_args(
self._module_configuration.application_root,
self._module_configuration.nobuild_files, self._arch)
gab_args.extend([
'-binary_name',
'_go_app',
'-extra_imports',
'appengine_internal/init',
'-work_dir',
self._work_dir,
'-gcflags',
_escape_tool_flags('-I', self._pkg_path),
'-ldflags',
_escape_tool_flags('-L', self._pkg_path),
])
gab_args.extend(self._go_file_to_mtime)
gab_process = safe_subprocess.start_process(gab_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={})
gab_stdout, gab_stderr = gab_process.communicate()
if gab_process.returncode:
raise BuildError('(Executed command: %s)\n\n%s\n%s' %
(' '.join(gab_args), gab_stdout, gab_stderr))
else:
logging.debug('Build succeeded:\n%s\n%s', gab_stdout, gab_stderr)
self._go_executable = os.path.join(self._work_dir, '_go_app')
def get_app_extras_for_vm(application_root, nobuild_files, skip_files):
"""Returns an iterable describing extra Go files needed to build VM apps.
The Go files are decided based on the production environment linux/amd64.
Args:
application_root: string path to the root dir of the application.
nobuild_files: regexp identifying which files to not build.
skip_files: regexp identifying which files to omit from app.
Returns:
An iterable of pairs, one per extra Go file. The first pair element
is the relative path at which to import the Go file; the second is its
absolute path.
Raises:
BuildError: if the go application builder fails.
"""
gab_args = _get_base_gab_args(application_root, nobuild_files, '6')
gab_args.extend(['-print_extras', '-vm'])
gab_args.extend(list_go_files(application_root, nobuild_files, skip_files))
env = {
'GOOS': 'linux',
'GOARCH': 'amd64',
}
gab_process = safe_subprocess.start_process(gab_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
gab_stdout, gab_stderr = gab_process.communicate()
if gab_process.returncode:
raise BuildError('(Executed command: %s)\n\n%s' %
(' '.join(gab_args), gab_stderr))
return [l.split('|') for l in gab_stdout.split('\n') if l]
def start(self):
"""Starts the runtime process and waits until it is ready to serve."""
runtime_config = self._runtime_config_getter()
serialized_config = base64.b64encode(runtime_config.SerializeToString())
# TODO: Use a different process group to isolate the child process
# from signals sent to the parent. Only available in subprocess in
# Python 2.7.
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process(
self._args,
serialized_config,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self._env,
cwd=self._module_configuration.application_root)
line = self._process.stdout.readline()
if self._stderr_tee is None:
self._stderr_tee = tee.Tee(self._process.stderr, sys.stderr)
self._stderr_tee.start()
self._prior_error = None
self._port = None
try:
self._port = int(line)
except ValueError:
self._prior_error = 'bad runtime process port [%r]' % line
logging.error(self._prior_error)
else:
# Check if the runtime can serve requests.
if not self._can_connect():
self._prior_error = 'cannot connect to runtime on port %r' % self._port
logging.error(self._prior_error)
def _get_extras_hash(self):
"""Returns a hash of the names and mtimes of package dependencies.
Returns:
Returns a string representing a hash.
Raises:
BuildError: if the go application builder fails.
"""
gab_args = _get_base_gab_args(
self._module_configuration.application_root,
self._module_configuration.nobuild_files, self._arch)
gab_args.append('-print_extras_hash')
gab_args.extend(self._go_file_to_mtime)
gab_process = safe_subprocess.start_process(gab_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={})
gab_stdout, gab_stderr = gab_process.communicate()
if gab_process.returncode:
raise BuildError(
'(Executed command: %s)\n\n%s' % (' '.join(gab_args),
gab_stderr))
else:
return gab_stdout
def _run_tool(tool, extra_args):
"""Run external executable tool.
Args:
tool: string name of the tool to run.
extra_args: additional arguments for tool.
Returns:
A tuple of the (stdout, stderr) from the process.
Raises:
BuildError: if tool fails.
"""
args = [tool]
if sys.platform.startswith('win'):
args = [tool + '.exe']
args.extend(extra_args)
logging.debug('Calling: %s', ' '.join(args))
try:
process = safe_subprocess.start_process(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
except OSError as e:
msg = '%s not found.' % args[0]
raise go_errors.BuildError('%s\n%s' % (msg, e))
if process.returncode:
raise go_errors.BuildError(
'(Executed command: %s)\n\n%s\n%s' % (' '.join(args), stdout, stderr))
return stdout, stderr
def test_start_with_host_port(self):
# start()
safe_subprocess.start_process(
['/runtime'],
base64.b64encode(self.runtime_config.SerializeToString()),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={'foo': 'bar'},
cwd=self.tmpdir).AndReturn(self.process)
self.process.stdout.readline().AndReturn('::1\t34567')
self.proxy._stderr_tee = FakeTee('')
self.mox.ReplayAll()
self.proxy.start()
self.assertEquals(34567, self.proxy._proxy._port)
self.assertEquals('::1', self.proxy._proxy._host)
self.mox.VerifyAll()
def test_start_with_host_port(self):
# start()
safe_subprocess.start_process(
['/runtime'],
base64.b64encode(self.runtime_config.SerializeToString()),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={'foo': 'bar'},
cwd=self.tmpdir).AndReturn(self.process)
self.process.stdout.readline().AndReturn('::1\t34567')
self.proxy._stderr_tee = FakeTee('')
self.mox.ReplayAll()
self.proxy.start()
self.assertEquals(34567, self.proxy._proxy._port)
self.assertEquals('::1', self.proxy._proxy._host)
self.mox.VerifyAll()
def _run_gab(self, gab_extra_args, env):
"""Run go-app-builder.
Args:
gab_extra_args: additional arguments (i.e. other than the standard base
arguments) for go-app-builder.
env: A dict containing environment variables for the subprocess.
Returns:
A tuple of the (stdout, stderr) from the go-app-builder process.
Raises:
BuildError: if the go application builder fails.
"""
gab_path = os.path.join(self._goroot, 'bin', 'go-app-builder')
if sys.platform.startswith('win'):
gab_path += '.exe'
if not os.path.exists(gab_path):
# TODO: This message should be more useful i.e. point the
# user to an SDK that does have the right components.
raise go_errors.BuildError(
'Required Go components are missing from the SDK.')
# Go's regexp package does not implicitly anchor to the start.
gab_args = [
gab_path,
'-app_base',
self._module_configuration.application_root,
'-api_version',
self._module_configuration.api_version,
'-arch',
self._arch,
'-dynamic',
'-goroot',
self._goroot,
'-gopath',
os.environ.get('GOPATH', GOPATH),
'-nobuild_files',
'^' + str(self._module_configuration.nobuild_files),
'-incremental_rebuild',
'-unsafe',
]
gab_args.extend(gab_extra_args)
logging.debug('Calling go-app-builder: env: %s, args: %s', env,
gab_args)
gab_process = safe_subprocess.start_process(gab_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
gab_stdout, gab_stderr = gab_process.communicate()
if gab_process.returncode:
msg = (u'(Executed command: %s)\n%s\n%s' %
(u' '.join(gab_args), gab_stdout.decode('utf-8'),
gab_stderr.decode('utf-8')))
raise go_errors.BuildError(msg.encode('utf-8'))
return gab_stdout, gab_stderr
def _check_environment(cls, php_executable_path, env):
check_process = safe_subprocess.start_process(
[php_executable_path, '-n', '-f', _CHECK_ENVIRONMENT_SCRIPT_PATH],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
check_process_stdout, _ = check_process.communicate()
if check_process.returncode:
raise _PHPEnvironmentError(check_process_stdout)
def start(self):
"""Starts the runtime process and waits until it is ready to serve."""
runtime_config = self._runtime_config_getter()
# TODO: Use a different process group to isolate the child process
# from signals sent to the parent. Only available in subprocess in
# Python 2.7.
assert self._start_process_flavor in self._VALID_START_PROCESS_FLAVORS
if self._start_process_flavor == START_PROCESS:
serialized_config = base64.b64encode(
runtime_config.SerializeToString())
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process(
self._args,
serialized_config,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self._env,
cwd=self._module_configuration.application_root)
line = self._process.stdout.readline()
elif self._start_process_flavor == START_PROCESS_FILE:
serialized_config = runtime_config.SerializeToString()
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process_file(
args=self._args,
input_string=serialized_config,
env=self._env,
cwd=self._module_configuration.application_root,
stderr=subprocess.PIPE)
line = self._read_start_process_file()
_remove_retry_sharing_violation(self._process.child_out.name)
# _stderr_tee may be pre-set by unit tests.
if self._stderr_tee is None:
self._stderr_tee = tee.Tee(self._process.stderr, sys.stderr)
self._stderr_tee.start()
port = None
error = None
try:
port = int(line)
except ValueError:
error = 'bad runtime process port [%r]' % line
logging.error(error)
finally:
self._proxy = http_proxy.HttpProxy(
host='localhost',
port=port,
instance_died_unexpectedly=self._instance_died_unexpectedly,
instance_logs_getter=self._get_instance_logs,
error_handler_file=application_configuration.
get_app_error_file(self._module_configuration),
prior_error=error)
self._proxy.wait_for_connection()
def _check_environment(cls, php_executable_path, env):
# Clear auto_prepend_file & auto_append_file ini directives as they can
# trigger error and cause non-zero return.
args = [php_executable_path, '-f', _CHECK_ENVIRONMENT_SCRIPT_PATH,
'-d', 'auto_prepend_file=NULL', '-d', 'auto_append_file=NULL']
check_process = safe_subprocess.start_process(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
check_process_stdout, _ = check_process.communicate()
if check_process.returncode:
raise _PHPEnvironmentError(check_process_stdout)
def start(self):
"""Starts the runtime process and waits until it is ready to serve."""
runtime_config = self._runtime_config_getter()
# TODO: Use a different process group to isolate the child process
# from signals sent to the parent. Only available in subprocess in
# Python 2.7.
assert self._start_process_flavor in self._VALID_START_PROCESS_FLAVORS
if self._start_process_flavor == START_PROCESS:
serialized_config = base64.b64encode(runtime_config.SerializeToString())
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process(
self._args,
serialized_config,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self._env,
cwd=self._module_configuration.application_root)
line = self._process.stdout.readline()
elif self._start_process_flavor == START_PROCESS_FILE:
serialized_config = runtime_config.SerializeToString()
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process_file(
args=self._args,
input_string=serialized_config,
env=self._env,
cwd=self._module_configuration.application_root,
stderr=subprocess.PIPE)
line = self._read_start_process_file()
_remove_retry_sharing_violation(self._process.child_out.name)
# _stderr_tee may be pre-set by unit tests.
if self._stderr_tee is None:
self._stderr_tee = tee.Tee(self._process.stderr, sys.stderr)
self._stderr_tee.start()
port = None
error = None
try:
port = int(line)
except ValueError:
error = 'bad runtime process port [%r]' % line
logging.error(error)
finally:
self._proxy = http_proxy.HttpProxy(
host='localhost', port=port,
instance_died_unexpectedly=self._instance_died_unexpectedly,
instance_logs_getter=self._get_instance_logs,
error_handler_file=application_configuration.get_app_error_file(
self._module_configuration),
prior_error=error)
self._proxy.wait_for_connection()
def _check_environment(php_executable_path):
if not os.path.exists(php_executable_path):
raise _PHPEnvironmentError(
'the file "%s" does not exist' % php_executable_path)
version_process = safe_subprocess.start_process([php_executable_path, '-v'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={})
version_stdout, version_stderr = version_process.communicate()
if version_process.returncode:
raise _PHPEnvironmentError(
'"%s -v" returned an error [%d]\n%s%s' % (
php_executable_path,
version_process.returncode,
version_stderr,
version_stdout))
version_match = re.search(r'PHP (\d+).(\d+)', version_stdout)
if version_match is None:
raise _PHPEnvironmentError(
'"%s -v" returned an unexpected version string:\n%s%s' % (
php_executable_path,
version_stderr,
version_stdout))
version = tuple(int(v) for v in version_match.groups())
if version < (5, 4):
raise _PHPEnvironmentError(
'The PHP interpreter must be version >= 5.4, %d.%d found' % version)
check_process = safe_subprocess.start_process(
[php_executable_path, '-f', _CHECK_ENVIRONMENT_SCRIPT_PATH],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={})
check_process_stdout, _ = check_process.communicate()
if check_process.returncode:
raise _PHPEnvironmentError(check_process_stdout)
def _check_environment(php_executable_path):
if not os.path.exists(php_executable_path):
raise _PHPEnvironmentError(
'the file "%s" does not exist' % php_executable_path)
version_process = safe_subprocess.start_process([php_executable_path, '-v'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={})
version_stdout, version_stderr = version_process.communicate()
if version_process.returncode:
raise _PHPEnvironmentError(
'"%s -v" returned an error [%d]\n%s%s' % (
php_executable_path,
version_process.returncode,
version_stderr,
version_stdout))
version_match = re.search(r'PHP (\d+).(\d+)', version_stdout)
if version_match is None:
raise _PHPEnvironmentError(
'"%s -v" returned an unexpected version string:\n%s%s' % (
php_executable_path,
version_stderr,
version_stdout))
version = tuple(int(v) for v in version_match.groups())
if version < (5, 4):
raise _PHPEnvironmentError(
'The PHP interpreter must be version >= 5.4, %d.%d found' % version)
check_process = safe_subprocess.start_process(
[php_executable_path, '-f', _CHECK_ENVIRONMENT_SCRIPT_PATH],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={})
check_process_stdout, _ = check_process.communicate()
if check_process.returncode:
raise _PHPEnvironmentError(check_process_stdout)
def start(self):
"""Starts the runtime process and waits until it is ready to serve."""
runtime_config = self._runtime_config_getter()
# TODO: Use a different process group to isolate the child process
# from signals sent to the parent. Only available in subprocess in
# Python 2.7.
assert self._start_process_flavor in self._VALID_START_PROCESS_FLAVORS
if self._start_process_flavor == START_PROCESS:
serialized_config = base64.b64encode(
runtime_config.SerializeToString())
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process(
self._args,
serialized_config,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self._env,
cwd=self._module_configuration.application_root)
line = self._process.stdout.readline()
elif self._start_process_flavor == START_PROCESS_FILE:
serialized_config = runtime_config.SerializeToString()
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process_file(
args=self._args,
input_string=serialized_config,
env=self._env,
cwd=self._module_configuration.application_root,
stderr=subprocess.PIPE)
line = self._read_start_process_file()
_remove_retry_sharing_violation(self._process.child_out.name)
# _stderr_tee may be pre-set by unit tests.
if self._stderr_tee is None:
self._stderr_tee = tee.Tee(self._process.stderr, sys.stderr)
self._stderr_tee.start()
self._prior_error = None
self._port = None
try:
# Older runtimes output just the port, while newer ones prepend the host.
self._port = int(line.split()[-1])
except ValueError:
self._prior_error = 'bad runtime process port [%r]' % line
logging.error(self._prior_error)
else:
# Check if the runtime can serve requests.
if not self._can_connect():
self._prior_error = 'cannot connect to runtime on port %r' % self._port
logging.error(self._prior_error)
def _build(self):
assert self._go_file_to_mtime, "no .go files"
logging.debug("Building Go application")
gab_args = self._get_gab_args()
gab_args.extend(self._go_file_to_mtime)
gab_process = safe_subprocess.start_process(gab_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env={})
gab_stdout, gab_stderr = gab_process.communicate()
if gab_process.returncode:
raise BuildError("(Executed command: %s)\n\n%s\n%s" % (" ".join(gab_args), gab_stdout, gab_stderr))
else:
logging.debug("Build succeeded:\n%s\n%s", gab_stdout, gab_stderr)
self._go_executable = os.path.join(self._work_dir, "_go_app")
def _check_environment(cls, php_executable_path, env):
# Clear auto_prepend_file & auto_append_file ini directives as they can
# trigger error and cause non-zero return.
args = [
php_executable_path, '-f', _CHECK_ENVIRONMENT_SCRIPT_PATH, '-d',
'auto_prepend_file=NULL', '-d', 'auto_append_file=NULL'
]
check_process = safe_subprocess.start_process(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
check_process_stdout, _ = check_process.communicate()
if check_process.returncode:
raise _PHPEnvironmentError(check_process_stdout)
def start(self):
"""Starts the runtime process and waits until it is ready to serve."""
runtime_config = self._runtime_config_getter()
# TODO: Use a different process group to isolate the child process
# from signals sent to the parent. Only available in subprocess in
# Python 2.7.
assert self._start_process_flavor in self._VALID_START_PROCESS_FLAVORS
if self._start_process_flavor == START_PROCESS:
serialized_config = base64.b64encode(runtime_config.SerializeToString())
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process(
self._args,
serialized_config,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self._env,
cwd=self._module_configuration.application_root)
line = self._process.stdout.readline()
elif self._start_process_flavor == START_PROCESS_FILE:
serialized_config = runtime_config.SerializeToString()
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process_file(
args=self._args,
input_string=serialized_config,
env=self._env,
cwd=self._module_configuration.application_root,
stderr=subprocess.PIPE)
line = self._read_start_process_file()
_remove_retry_sharing_violation(self._process.child_out.name)
# _stderr_tee may be pre-set by unit tests.
if self._stderr_tee is None:
self._stderr_tee = tee.Tee(self._process.stderr, sys.stderr)
self._stderr_tee.start()
self._prior_error = None
self._port = None
try:
# Older runtimes output just the port, while newer ones prepend the host.
self._port = int(line.split()[-1])
except ValueError:
self._prior_error = 'bad runtime process port [%r]' % line
logging.error(self._prior_error)
else:
# Check if the runtime can serve requests.
if not self._can_connect():
self._prior_error = 'cannot connect to runtime on port %r' % self._port
logging.error(self._prior_error)
def _run_gab(self, gab_extra_args, env):
"""Run go-app-builder.
Args:
gab_extra_args: additional arguments (i.e. other than the standard base
arguments) for go-app-builder.
env: A dict containing environment variables for the subprocess.
Returns:
A tuple of the (stdout, stderr) from the go-app-builder process.
Raises:
BuildError: if the go application builder fails.
"""
gab_path = os.path.join(self._goroot, 'bin', 'go-app-builder')
if sys.platform.startswith('win'):
gab_path += '.exe'
if not os.path.exists(gab_path):
# TODO: This message should be more useful i.e. point the
# user to an SDK that does have the right components.
raise go_errors.BuildError(
'Required Go components are missing from the SDK.')
# Go's regexp package does not implicitly anchor to the start.
gab_args = [
gab_path,
'-app_base', self._module_configuration.application_root,
'-arch', self._arch,
'-dynamic',
'-goroot', self._goroot,
'-gopath', os.environ.get('GOPATH', GOPATH),
'-nobuild_files', '^' + str(self._module_configuration.nobuild_files),
'-incremental_rebuild',
'-unsafe',
]
gab_args.extend(gab_extra_args)
gab_process = safe_subprocess.start_process(gab_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
gab_stdout, gab_stderr = gab_process.communicate()
if gab_process.returncode:
raise go_errors.BuildError(
'(Executed command: %s)\n%s\n%s' % (' '.join(gab_args),
gab_stdout, gab_stderr))
return gab_stdout, gab_stderr
def _build(self):
assert self._go_file_to_mtime, 'no .go files'
logging.debug('Building Go application')
gab_args = self._get_gab_args()
gab_args.extend(self._go_file_to_mtime)
gab_process = safe_subprocess.start_process(gab_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={})
gab_stdout, gab_stderr = gab_process.communicate()
if gab_process.returncode:
raise BuildError('%s\n%s\n\n(Executed command: %s)' %
(gab_stdout, gab_stderr, ' '.join(gab_args)))
else:
logging.debug('Build succeeded:\n%s\n%s', gab_stdout, gab_stderr)
self._go_executable = os.path.join(self._work_dir, '_go_app')
def _get_extras_hash(self):
"""Returns a hash of the names and mtimes of package dependencies.
Returns:
Returns a string representing a hash.
Raises:
BuildError: if the go application builder fails.
"""
gab_args = self._get_gab_args()
gab_args.append("-print_extras_hash")
gab_args.extend(self._go_file_to_mtime)
gab_process = safe_subprocess.start_process(gab_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env={})
gab_stdout, gab_stderr = gab_process.communicate()
if gab_process.returncode:
raise BuildError("(Executed command: %s)\n\n%s" % (" ".join(gab_args), gab_stderr))
else:
return gab_stdout
def _build(self):
assert self._go_file_to_mtime, 'no .go files'
logging.debug('Building Go application')
gab_args = self._get_gab_args()
gab_args.extend(self._go_file_to_mtime)
gab_process = safe_subprocess.start_process(gab_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={})
gab_stdout, gab_stderr = gab_process.communicate()
if gab_process.returncode:
raise BuildError(
'%s\n%s\n\n(Executed command: %s)' % (gab_stdout,
gab_stderr,
' '.join(gab_args)))
else:
self._go_executable = os.path.join(self._work_dir, '_go_app')
def main():
# Read the runtime configuration from file.
config = runtime_config_pb2.Config()
config.ParseFromString(open(sys.argv[1], 'rb').read())
# Launch the node process. Note, the port is specified in os.environ.
node_app_process = safe_subprocess.start_process(
args=[config.node_config.node_executable_path,
os.path.join(config.application_root, 'server.js')],
env=os.environ.copy(),
cwd=config.application_root,
stdout=sys.stderr,
)
# Wait for the devappserver to kill the process.
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
pass
finally:
sys.stdout.close()
node_app_process.kill()
def get_app_extras_for_vm(application_root, nobuild_files, skip_files):
"""Returns an iterable describing extra Go files needed to build VM apps.
The Go files are decided based on the production environment linux/amd64.
Args:
application_root: string path to the root dir of the application.
nobuild_files: regexp identifying which files to not build.
skip_files: regexp identifying which files to omit from app.
Returns:
An iterable of pairs, one per extra Go file. The first pair element
is the relative path at which to import the Go file; the second is its
absolute path.
Raises:
BuildError: if the go application builder fails.
"""
gab_args = _get_base_gab_args(application_root, nobuild_files, '6')
gab_args.extend(['-print_extras', '-vm'])
gab_args.extend(list_go_files(application_root, nobuild_files, skip_files))
env = {
'GOOS': 'linux',
'GOARCH': 'amd64',
}
gab_process = safe_subprocess.start_process(gab_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
gab_stdout, gab_stderr = gab_process.communicate()
if gab_process.returncode:
raise BuildError(
'(Executed command: %s)\n\n%s' % (' '.join(gab_args),
gab_stderr))
return [l.split('|') for l in gab_stdout.split('\n') if l]
def __call__(self, environ, start_response):
"""Handles an HTTP request for the runtime using a PHP executable.
Args:
environ: An environ dict for the request as defined in PEP-333.
start_response: A function with semantics defined in PEP-333.
Returns:
An iterable over strings containing the body of the HTTP response.
"""
user_environ = self.make_php_cgi_environ(environ)
if 'CONTENT_LENGTH' in environ:
content = environ['wsgi.input'].read(int(environ['CONTENT_LENGTH']))
else:
content = ''
args = self.make_php_cgi_args()
# Handles interactive request.
request_type = environ.pop(http_runtime_constants.REQUEST_TYPE_HEADER, None)
if request_type == 'interactive':
args.extend(['-d', 'html_errors="0"'])
user_environ[http_runtime_constants.REQUEST_TYPE_HEADER] = request_type
try:
p = safe_subprocess.start_process(args,
input_string=content,
env=user_environ,
cwd=self.config.application_root,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
except Exception as e:
logging.exception('Failure to start PHP with: %s', args)
start_response('500 Internal Server Error',
[(http_runtime_constants.ERROR_CODE_HEADER, '1')])
return ['Failure to start the PHP subprocess with %r:\n%s' % (args, e)]
if p.returncode:
if request_type == 'interactive':
start_response('200 OK', [('Content-Type', 'text/plain')])
message = httplib.HTTPMessage(cStringIO.StringIO(stdout))
return [message.fp.read()]
else:
logging.error('php failure (%r) with:\nstdout:\n%sstderr:\n%s',
p.returncode, stdout, stderr)
start_response('500 Internal Server Error',
[(http_runtime_constants.ERROR_CODE_HEADER, '1')])
message = httplib.HTTPMessage(cStringIO.StringIO(stdout))
return [message.fp.read()]
message = httplib.HTTPMessage(cStringIO.StringIO(stdout))
if 'Status' in message:
status = message['Status']
del message['Status']
else:
status = '200 OK'
# Ensures that we avoid merging repeat headers into a single header,
# allowing use of multiple Set-Cookie headers.
headers = []
for name in message:
for value in message.getheaders(name):
headers.append((name, value))
start_response(status, headers)
return [message.fp.read()]
def __call__(self, environ, start_response):
"""Handles an HTTP request for the runtime using a PHP executable.
Args:
environ: An environ dict for the request as defined in PEP-333.
start_response: A function with semantics defined in PEP-333.
Returns:
An iterable over strings containing the body of the HTTP response.
"""
user_environ = self.environ_template.copy()
environ_utils.propagate_environs(environ, user_environ)
user_environ['REQUEST_METHOD'] = environ.get('REQUEST_METHOD', 'GET')
user_environ['PATH_INFO'] = environ['PATH_INFO']
user_environ['QUERY_STRING'] = environ['QUERY_STRING']
# Construct the partial URL that PHP expects for REQUEST_URI
# (http://php.net/manual/en/reserved.variables.server.php) using part of
# the process described in PEP-333
# (http://www.python.org/dev/peps/pep-0333/#url-reconstruction).
user_environ['REQUEST_URI'] = urllib.quote(user_environ['PATH_INFO'])
if user_environ['QUERY_STRING']:
user_environ['REQUEST_URI'] += '?' + user_environ['QUERY_STRING']
# Modify the SCRIPT_FILENAME to specify the setup script that readies the
# PHP environment. Put the user script in REAL_SCRIPT_FILENAME.
user_environ['REAL_SCRIPT_FILENAME'] = os.path.normpath(
os.path.join(
self.config.application_root,
environ[http_runtime_constants.SCRIPT_HEADER].lstrip('/')))
user_environ['SCRIPT_FILENAME'] = SETUP_PHP_PATH
user_environ['REMOTE_REQUEST_ID'] = environ[
http_runtime_constants.REQUEST_ID_ENVIRON]
# Pass the APPLICATION_ROOT so we can use it in the setup script. We will
# remove it from the environment before we execute the user script.
user_environ['APPLICATION_ROOT'] = self.config.application_root
if 'CONTENT_TYPE' in environ:
user_environ['CONTENT_TYPE'] = environ['CONTENT_TYPE']
user_environ['HTTP_CONTENT_TYPE'] = environ['CONTENT_TYPE']
if 'CONTENT_LENGTH' in environ:
user_environ['CONTENT_LENGTH'] = environ['CONTENT_LENGTH']
user_environ['HTTP_CONTENT_LENGTH'] = environ['CONTENT_LENGTH']
content = environ['wsgi.input'].read(int(
environ['CONTENT_LENGTH']))
else:
content = ''
# On Windows, in order to run a side-by-side assembly the specified env
# must include a valid SystemRoot.
if 'SYSTEMROOT' in os.environ:
user_environ['SYSTEMROOT'] = os.environ['SYSTEMROOT']
# See http://www.php.net/manual/en/ini.core.php#ini.include-path.
include_paths = [self.config.application_root, SDK_PATH]
if sys.platform == 'win32':
# See https://bugs.php.net/bug.php?id=46034 for quoting requirements.
include_path = 'include_path="%s"' % ';'.join(include_paths)
else:
include_path = 'include_path=%s' % ':'.join(include_paths)
args = [self.config.php_config.php_executable_path, '-d', include_path]
# Load php.ini from application's root.
args.extend(['-c', self.config.application_root])
if self.config.php_config.enable_debugger:
args.extend(['-d', 'xdebug.remote_enable="1"'])
user_environ['XDEBUG_CONFIG'] = os.environ.get('XDEBUG_CONFIG', '')
request_type = environ.pop(http_runtime_constants.REQUEST_TYPE_HEADER,
None)
if request_type == 'interactive':
args.extend(['-d', 'html_errors="0"'])
user_environ[
http_runtime_constants.REQUEST_TYPE_HEADER] = request_type
try:
p = safe_subprocess.start_process(args,
input_string=content,
env=user_environ,
cwd=self.config.application_root,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
except Exception as e:
logging.exception('Failure to start PHP with: %s', args)
start_response('500 Internal Server Error',
[(http_runtime_constants.ERROR_CODE_HEADER, '1')])
return [
'Failure to start the PHP subprocess with %r:\n%s' % (args, e)
]
if p.returncode:
if request_type == 'interactive':
start_response('200 OK', [('Content-Type', 'text/plain')])
message = httplib.HTTPMessage(cStringIO.StringIO(stdout))
return [message.fp.read()]
else:
logging.error('php failure (%r) with:\nstdout:\n%sstderr:\n%s',
p.returncode, stdout, stderr)
start_response(
'500 Internal Server Error',
[(http_runtime_constants.ERROR_CODE_HEADER, '1')])
message = httplib.HTTPMessage(cStringIO.StringIO(stdout))
return [message.fp.read()]
message = httplib.HTTPMessage(cStringIO.StringIO(stdout))
if 'Status' in message:
status = message['Status']
del message['Status']
else:
status = '200 OK'
# Ensures that we avoid merging repeat headers into a single header,
# allowing use of multiple Set-Cookie headers.
headers = []
for name in message:
for value in message.getheaders(name):
headers.append((name, value))
start_response(status, headers)
return [message.fp.read()]
def start(self):
"""Starts the runtime process and waits until it is ready to serve."""
runtime_config = self._runtime_config_getter()
# TODO: Use a different process group to isolate the child process
# from signals sent to the parent. Only available in subprocess in
# Python 2.7.
assert self._start_process_flavor in self._VALID_START_PROCESS_FLAVORS
host = 'localhost'
if self._start_process_flavor == START_PROCESS:
serialized_config = base64.b64encode(runtime_config.SerializeToString())
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process(
self._args,
serialized_config,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self._env,
cwd=self._module_configuration.application_root)
port = self._process.stdout.readline()
if '\t' in port: # Split out the host if present.
host, port = port.split('\t', 1)
elif self._start_process_flavor == START_PROCESS_FILE:
serialized_config = runtime_config.SerializeToString()
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process_file(
args=self._args,
input_string=serialized_config,
env=self._env,
cwd=self._module_configuration.application_root,
stderr=subprocess.PIPE)
port = self._read_start_process_file()
_remove_retry_sharing_violation(self._process.child_out.name)
elif self._start_process_flavor == START_PROCESS_REVERSE:
serialized_config = runtime_config.SerializeToString()
with self._process_lock:
assert not self._process, 'start() can only be called once'
port = portpicker.PickUnusedPort()
self._env['PORT'] = str(port)
# If any of the strings in args contain {port}, replace that substring
# with the selected port. This allows a user-specified runtime to
# pass the port along to the subprocess as a command-line argument.
args = [arg.replace('{port}', str(port)) for arg in self._args]
self._process = safe_subprocess.start_process_file(
args=args,
input_string=serialized_config,
env=self._env,
cwd=self._module_configuration.application_root,
stderr=subprocess.PIPE)
elif self._start_process_flavor == START_PROCESS_REVERSE_NO_FILE:
serialized_config = runtime_config.SerializeToString()
with self._process_lock:
assert not self._process, 'start() can only be called once'
port = portpicker.PickUnusedPort()
if self._extra_args_getter:
self._args.append(self._extra_args_getter(port))
# If any of the strings in _args contain {port}, {api_host}, {api_port},
# replace that substring with the selected port. This allows
# a user-specified runtime to pass the port along to the subprocess
# as a command-line argument.
args = [arg.replace('{port}', str(port))
.replace('{api_port}', str(runtime_config.api_port))
.replace('{api_host}', runtime_config.api_host)
for arg in self._args]
self._process = safe_subprocess.start_process(
args=args,
input_string=serialized_config,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self._env,
cwd=self._module_configuration.application_root)
# _stderr_tee may be pre-set by unit tests.
if self._stderr_tee is None:
self._stderr_tee = tee.Tee(self._process.stderr, sys.stderr)
self._stderr_tee.start()
error = None
try:
port = int(port)
except ValueError:
error = 'bad runtime process port [%r]' % port
logging.error(error)
finally:
self._proxy = http_proxy.HttpProxy(
host=host, port=port,
instance_died_unexpectedly=self._instance_died_unexpectedly,
instance_logs_getter=self._get_instance_logs,
error_handler_file=application_configuration.get_app_error_file(
self._module_configuration),
prior_error=error)
self._proxy.wait_for_connection()
def __call__(self, environ, start_response):
"""Handles an HTTP request for the runtime using a PHP executable.
Args:
environ: An environ dict for the request as defined in PEP-333.
start_response: A function with semantics defined in PEP-333.
Returns:
An iterable over strings containing the body of the HTTP response.
"""
user_environ = self.make_php_cgi_environ(environ)
if 'CONTENT_LENGTH' in environ:
content = environ['wsgi.input'].read(int(
environ['CONTENT_LENGTH']))
else:
content = ''
args = self.make_php_cgi_args()
# Handles interactive request.
request_type = environ.pop(http_runtime_constants.REQUEST_TYPE_HEADER,
None)
if request_type == 'interactive':
args.extend(['-d', 'html_errors="0"'])
user_environ[
http_runtime_constants.REQUEST_TYPE_HEADER] = request_type
try:
# stderr is not captured here so that it propagates to the parent process
# and gets printed out to consle.
p = safe_subprocess.start_process(args,
input_string=content,
env=user_environ,
cwd=self.config.application_root,
stdout=subprocess.PIPE)
stdout, _ = p.communicate()
except Exception as e:
logging.exception('Failure to start PHP with: %s', args)
start_response('500 Internal Server Error',
[(http_runtime_constants.ERROR_CODE_HEADER, '1')])
return [
'Failure to start the PHP subprocess with %r:\n%s' % (args, e)
]
if p.returncode:
if request_type == 'interactive':
start_response('200 OK', [('Content-Type', 'text/plain')])
message = httplib.HTTPMessage(cStringIO.StringIO(stdout))
return [message.fp.read()]
else:
logging.error('php failure (%r) with:\nstdout:\n%s',
p.returncode, stdout)
start_response(
'500 Internal Server Error',
[(http_runtime_constants.ERROR_CODE_HEADER, '1')])
message = httplib.HTTPMessage(cStringIO.StringIO(stdout))
return [message.fp.read()]
message = httplib.HTTPMessage(cStringIO.StringIO(stdout))
if 'Status' in message:
status = message['Status']
del message['Status']
else:
status = '200 OK'
# Ensures that we avoid merging repeat headers into a single header,
# allowing use of multiple Set-Cookie headers.
headers = []
for name in message:
for value in message.getheaders(name):
headers.append((name, value))
start_response(status, headers)
return [message.fp.read()]
def __call__(self, environ, start_response):
"""Handles an HTTP request for the runtime using a PHP executable.
Args:
environ: An environ dict for the request as defined in PEP-333.
start_response: A function with semantics defined in PEP-333.
Returns:
An iterable over strings containing the body of the HTTP response.
"""
user_environ = self.environ_template.copy()
self.copy_headers(environ, user_environ)
user_environ['REQUEST_METHOD'] = environ.get('REQUEST_METHOD', 'GET')
user_environ['PATH_INFO'] = environ['PATH_INFO']
user_environ['QUERY_STRING'] = environ['QUERY_STRING']
# Construct the partial URL that PHP expects for REQUEST_URI
# (http://php.net/manual/en/reserved.variables.server.php) using part of
# the process described in PEP-333
# (http://www.python.org/dev/peps/pep-0333/#url-reconstruction).
user_environ['REQUEST_URI'] = urllib.quote(user_environ['PATH_INFO'])
if user_environ['QUERY_STRING']:
user_environ['REQUEST_URI'] += '?' + user_environ['QUERY_STRING']
# Modify the SCRIPT_FILENAME to specify the setup script that readies the
# PHP environment. Put the user script in REAL_SCRIPT_FILENAME.
user_environ['REAL_SCRIPT_FILENAME'] = environ[
http_runtime_constants.SCRIPT_HEADER]
user_environ['SCRIPT_FILENAME'] = SETUP_PHP_PATH
user_environ['REMOTE_REQUEST_ID'] = environ[
http_runtime_constants.REQUEST_ID_ENVIRON]
# Pass the APPLICATION_ROOT so we can use it in the setup script. We will
# remove it from the environment before we execute the user script.
user_environ['APPLICATION_ROOT'] = self.config.application_root
if 'CONTENT_TYPE' in environ:
user_environ['CONTENT_TYPE'] = environ['CONTENT_TYPE']
user_environ['HTTP_CONTENT_TYPE'] = environ['CONTENT_TYPE']
if 'CONTENT_LENGTH' in environ:
user_environ['CONTENT_LENGTH'] = environ['CONTENT_LENGTH']
user_environ['HTTP_CONTENT_LENGTH'] = environ['CONTENT_LENGTH']
content = environ['wsgi.input'].read(int(environ['CONTENT_LENGTH']))
else:
content = ''
# On Windows, in order to run a side-by-side assembly the specified env
# must include a valid SystemRoot.
if 'SYSTEMROOT' in os.environ:
user_environ['SYSTEMROOT'] = os.environ['SYSTEMROOT']
# See http://www.php.net/manual/en/ini.core.php#ini.include-path.
include_paths = [self.config.application_root, SDK_PATH]
if sys.platform == 'win32':
# See https://bugs.php.net/bug.php?id=46034 for quoting requirements.
include_path = 'include_path="%s"' % ';'.join(include_paths)
else:
include_path = 'include_path=%s' % ':'.join(include_paths)
args = [self.config.php_config.php_executable_path, '-d', include_path]
if self.config.php_config.enable_debugger:
args.extend(['-d', 'xdebug.remote_enable="1"'])
user_environ['XDEBUG_CONFIG'] = os.environ.get('XDEBUG_CONFIG', '')
request_type = environ.pop(http_runtime_constants.REQUEST_TYPE_HEADER, None)
if request_type == 'interactive':
args.extend(['-d', 'html_errors="0"'])
user_environ[http_runtime_constants.REQUEST_TYPE_HEADER] = request_type
try:
p = safe_subprocess.start_process(args,
input_string=content,
env=user_environ,
cwd=self.config.application_root,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
except Exception as e:
logging.exception('Failure to start PHP with: %s', args)
start_response('500 Internal Server Error',
[(http_runtime_constants.ERROR_CODE_HEADER, '1')])
return ['Failure to start the PHP subprocess with %r:\n%s' % (args, e)]
if p.returncode:
if request_type == 'interactive':
start_response('200 OK', [('Content-Type', 'text/plain')])
message = httplib.HTTPMessage(cStringIO.StringIO(stdout))
return [message.fp.read()]
else:
logging.error('php failure (%r) with:\nstdout:\n%sstderr:\n%s',
p.returncode, stdout, stderr)
start_response('500 Internal Server Error',
[(http_runtime_constants.ERROR_CODE_HEADER, '1')])
message = httplib.HTTPMessage(cStringIO.StringIO(stdout))
return [message.fp.read()]
message = httplib.HTTPMessage(cStringIO.StringIO(stdout))
if 'Status' in message:
status = message['Status']
del message['Status']
else:
status = '200 OK'
# Ensures that we avoid merging repeat headers into a single header,
# allowing use of multiple Set-Cookie headers.
headers = []
for name in message:
for value in message.getheaders(name):
headers.append((name, value))
start_response(status, headers)
return [message.fp.read()]
def start(self):
"""Starts the runtime process and waits until it is ready to serve."""
runtime_config = self._runtime_config_getter()
# TODO: Use a different process group to isolate the child process
# from signals sent to the parent. Only available in subprocess in
# Python 2.7.
assert self._start_process_flavor in self._VALID_START_PROCESS_FLAVORS
if self._start_process_flavor == START_PROCESS:
serialized_config = base64.b64encode(runtime_config.SerializeToString())
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process(
self._args,
serialized_config,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self._env,
cwd=self._module_configuration.application_root)
port = self._process.stdout.readline()
if '\t' in port: # Split out the host if present.
host, port = port.split('\t', 1)
elif self._start_process_flavor == START_PROCESS_FILE:
serialized_config = runtime_config.SerializeToString()
with self._process_lock:
assert not self._process, 'start() can only be called once'
self._process = safe_subprocess.start_process_file(
args=self._args,
input_string=serialized_config,
env=self._env,
cwd=self._module_configuration.application_root,
stderr=subprocess.PIPE)
port = self._read_start_process_file()
_remove_retry_sharing_violation(self._process.child_out.name)
elif self._start_process_flavor == START_PROCESS_REVERSE:
serialized_config = runtime_config.SerializeToString()
with self._process_lock:
assert not self._process, 'start() can only be called once'
port = portpicker.PickUnusedPort()
self._env['PORT'] = str(port)
# If any of the strings in args contain {port}, replace that substring
# with the selected port. This allows a user-specified runtime to
# pass the port along to the subprocess as a command-line argument.
args = [arg.replace('{port}', str(port)) for arg in self._args]
self._process = safe_subprocess.start_process_file(
args=args,
input_string=serialized_config,
env=self._env,
cwd=self._module_configuration.application_root,
stderr=subprocess.PIPE)
elif self._start_process_flavor == START_PROCESS_REVERSE_NO_FILE:
serialized_config = runtime_config.SerializeToString()
with self._process_lock:
assert not self._process, 'start() can only be called once'
port = portpicker.PickUnusedPort()
if self._extra_args_getter:
self._args.append(self._extra_args_getter(port))
# If any of the strings in _args contain {port}, {api_host}, {api_port},
# replace that substring with the selected port. This allows
# a user-specified runtime to pass the port along to the subprocess
# as a command-line argument.
args = [arg.replace('{port}', str(port))
.replace('{api_port}', str(runtime_config.api_port))
.replace('{api_host}', runtime_config.api_host)
for arg in self._args]
self._process = safe_subprocess.start_process(
args=args,
input_string=serialized_config,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self._env,
cwd=self._module_configuration.application_root)
# _stderr_tee may be pre-set by unit tests.
if self._stderr_tee is None:
self._stderr_tee = tee.Tee(self._process.stderr, sys.stderr)
self._stderr_tee.start()
error = None
try:
port = int(port)
except ValueError:
error = 'bad runtime process port [%r]' % port
logging.error(error)
finally:
self._proxy = http_proxy.HttpProxy(
host='localhost', port=port,
instance_died_unexpectedly=self._instance_died_unexpectedly,
instance_logs_getter=self._get_instance_logs,
error_handler_file=application_configuration.get_app_error_file(
self._module_configuration),
prior_error=error)
self._proxy.wait_for_connection()