def print_proc_output():
"""Prints the proc's output until it is exhausted."""
for line in iter(lambda: proc.stdout.readline() or None, None):
common.write_stdout_safe(line)
def run_tests(args):
"""Run the scripts to start end-to-end tests."""
if is_oppia_server_already_running():
sys.exit(1)
install_third_party_libraries(args.skip_install)
with contextlib.ExitStack() as stack:
dev_mode = not args.prod_env
if args.skip_build:
build.modify_constants(prod_env=args.prod_env)
else:
build_js_files(dev_mode, source_maps=args.source_maps)
stack.callback(build.set_constants_to_default)
stack.enter_context(servers.managed_redis_server())
stack.enter_context(servers.managed_elasticsearch_dev_server())
if constants.EMULATOR_MODE:
stack.enter_context(servers.managed_firebase_auth_emulator())
stack.enter_context(
servers.managed_cloud_datastore_emulator(clear_datastore=True))
app_yaml_path = 'app.yaml' if args.prod_env else 'app_dev.yaml'
stack.enter_context(
servers.managed_dev_appserver(
app_yaml_path,
port=GOOGLE_APP_ENGINE_PORT,
log_level=args.server_log_level,
# Automatic restart can be disabled since we don't expect code
# changes to happen while the e2e tests are running.
automatic_restart=False,
skip_sdk_update_check=True,
env={
**os.environ,
'PORTSERVER_ADDRESS':
common.PORTSERVER_SOCKET_FILEPATH,
}))
stack.enter_context(
servers.managed_webdriver_server(
chrome_version=args.chrome_driver_version))
proc = stack.enter_context(
servers.managed_protractor_server(
suite_name=args.suite,
dev_mode=dev_mode,
debug_mode=args.debug_mode,
sharding_instances=args.sharding_instances,
stdout=subprocess.PIPE))
print('Servers have come up.\n'
'Note: If ADD_SCREENSHOT_REPORTER is set to true in '
'core/tests/protractor.conf.js, you can view screenshots of the '
'failed tests in ../protractor-screenshots/')
output_lines = []
while True:
# Keep reading lines until an empty string is returned. Empty
# strings signal that the process has ended.
for line in iter(proc.stdout.readline, b''):
if isinstance(line, str):
# Although our unit tests always provide unicode strings,
# the actual server needs this failsafe since it can output
# non-unicode strings.
line = line.encode('utf-8') # pragma: no cover
output_lines.append(line.rstrip())
# Replaces non-ASCII characters with '?'.
common.write_stdout_safe(line.decode('ascii',
errors='replace'))
# The poll() method returns None while the process is running,
# otherwise it returns the return code of the process (an int).
if proc.poll() is not None:
break
return output_lines, proc.returncode
def managed_webpack_compiler(
config_path=None, use_prod_env=False, use_source_maps=False,
watch_mode=False, max_old_space_size=None):
"""Returns context manager to start/stop the webpack compiler gracefully.
Args:
config_path: str|None. Path to an explicit webpack config, or None to
determine it from the other args.
use_prod_env: bool. Whether to compile for use in production. Only
respected if config_path is None.
use_source_maps: bool. Whether to compile with source maps. Only
respected if config_path is None.
watch_mode: bool. Run the compiler in watch mode, which rebuilds on file
change.
max_old_space_size: int|None. Sets the max memory size of the compiler's
"old memory" section. As memory consumption approaches the limit,
the compiler will spend more time on garbage collection in an effort
to free unused memory.
Yields:
psutil.Process. The Webpack compiler process.
"""
if config_path is not None:
pass
elif use_prod_env:
config_path = (
common.WEBPACK_PROD_SOURCE_MAPS_CONFIG if use_source_maps else
common.WEBPACK_PROD_CONFIG)
else:
config_path = (
common.WEBPACK_DEV_SOURCE_MAPS_CONFIG if use_source_maps else
common.WEBPACK_DEV_CONFIG)
compiler_args = [
common.NODE_BIN_PATH, common.WEBPACK_BIN_PATH, '--config', config_path,
]
if max_old_space_size:
# NOTE: --max-old-space-size is a flag for Node.js, not the Webpack
# compiler, so we insert it immediately after NODE_BIN_PATH.
compiler_args.insert(1, '--max-old-space-size=%d' % max_old_space_size)
if watch_mode:
compiler_args.extend(['--color', '--watch', '--progress'])
with contextlib.ExitStack() as exit_stack:
# OK to use shell=True here because we are passing string literals and
# constants, so there is no risk of a shell-injection attack.
proc = exit_stack.enter_context(managed_process(
compiler_args, human_readable_name='Webpack Compiler', shell=True,
# Capture compiler's output to detect when builds have completed.
stdout=subprocess.PIPE))
if watch_mode:
for line in iter(lambda: proc.stdout.readline() or None, None):
common.write_stdout_safe(line)
# Message printed when a compilation has succeeded. We break
# after the first one to ensure the site is ready to be visited.
if b'Built at: ' in line:
break
else:
# If none of the lines contained the string 'Built at',
# raise an error because a build hasn't finished successfully.
raise IOError('First build never completed')
def print_proc_output():
"""Prints the proc's output until it is exhausted."""
for line in iter(lambda: proc.stdout.readline() or None, None):
common.write_stdout_safe(line)
# Start a thread to print the rest of the compiler's output to stdout.
printer_thread = threading.Thread(target=print_proc_output)
printer_thread.start()
exit_stack.callback(printer_thread.join)
yield proc
