def run_tests(args):
"""Run the scripts to start end-to-end tests."""
oppia_instance_is_already_running = is_oppia_server_already_running()
if oppia_instance_is_already_running:
sys.exit(1)
setup_and_install_dependencies(args.skip_install)
common.start_redis_server()
atexit.register(cleanup)
dev_mode = not args.prod_env
if args.skip_build:
build.modify_constants(prod_env=args.prod_env)
else:
build_js_files(
dev_mode, deparallelize_terser=args.deparallelize_terser,
source_maps=args.source_maps)
version = args.chrome_driver_version or get_chrome_driver_version()
python_utils.PRINT('\n\nCHROMEDRIVER VERSION: %s\n\n' % version)
start_webdriver_manager(version)
managed_dev_appserver = common.managed_dev_appserver(
'app.yaml' if args.prod_env else 'app_dev.yaml',
port=GOOGLE_APP_ENGINE_PORT, log_level=args.server_log_level,
clear_datastore=True, skip_sdk_update_check=True,
env={'PORTSERVER_ADDRESS': PORTSERVER_SOCKET_FILEPATH})
with managed_dev_appserver:
common.wait_for_port_to_be_open(WEB_DRIVER_PORT)
common.wait_for_port_to_be_open(GOOGLE_APP_ENGINE_PORT)
python_utils.PRINT(
'Note: If ADD_SCREENSHOT_REPORTER is set to true in'
'core/tests/protractor.conf.js, you can view screenshots'
'of the failed tests in ../protractor-screenshots/')
commands = [common.NODE_BIN_PATH]
if args.debug_mode:
commands.append('--inspect-brk')
# This flag ensures tests fail if waitFor calls time out.
commands.append('--unhandled-rejections=strict')
commands.append(PROTRACTOR_BIN_PATH)
commands.extend(get_e2e_test_parameters(
args.sharding_instances, args.suite, dev_mode))
p = subprocess.Popen(commands, stdout=subprocess.PIPE)
output_lines = []
while True:
nextline = p.stdout.readline()
if len(nextline) == 0 and p.poll() is not None:
break
if isinstance(nextline, str):
# This is a failsafe line in case we get non-unicode input,
# but the tests provide all strings as unicode.
nextline = nextline.decode('utf-8') # pragma: nocover
output_lines.append(nextline.rstrip())
# Replaces non-ASCII characters with '?'.
sys.stdout.write(nextline.encode('ascii', errors='replace'))
return output_lines, p.returncode
def main():
"""Runs lighthouse checks and deletes reports."""
enable_webpages()
atexit.register(cleanup)
python_utils.PRINT('Building files in production mode.')
build.main(args=['--prod_env'])
build.modify_constants(prod_env=True)
start_google_app_engine_server()
common.wait_for_port_to_be_open(GOOGLE_APP_ENGINE_PORT)
run_lighthouse_checks()
def main(args=None):
"""Run the scripts to start end-to-end tests."""
parsed_args = _PARSER.parse_args(args=args)
oppia_instance_is_already_running = is_oppia_server_already_running()
if oppia_instance_is_already_running:
sys.exit(1)
setup_and_install_dependencies(parsed_args.skip_install)
common.start_redis_server()
atexit.register(cleanup)
dev_mode = not parsed_args.prod_env
update_contributor_dashboard_status_in_feconf_file(
FECONF_FILE_PATH, parsed_args.contributor_dashboard_enabled)
if parsed_args.skip_build:
build.modify_constants(prod_env=parsed_args.prod_env)
else:
build_js_files(dev_mode,
deparallelize_terser=parsed_args.deparallelize_terser,
source_maps=parsed_args.source_maps)
version = parsed_args.chrome_driver_version or get_chrome_driver_version()
python_utils.PRINT('\n\nCHROMEDRIVER VERSION: %s\n\n' % version)
start_webdriver_manager(version)
portserver_process = start_portserver()
atexit.register(cleanup_portserver, portserver_process)
start_google_app_engine_server(dev_mode, parsed_args.server_log_level)
common.wait_for_port_to_be_open(WEB_DRIVER_PORT)
common.wait_for_port_to_be_open(GOOGLE_APP_ENGINE_PORT)
ensure_screenshots_dir_is_removed()
commands = [common.NODE_BIN_PATH]
if parsed_args.debug_mode:
commands.append('--inspect-brk')
# This flag ensures tests fail if waitFor calls time out.
commands.append('--unhandled-rejections=strict')
commands.append(PROTRACTOR_BIN_PATH)
commands.extend(
get_e2e_test_parameters(parsed_args.sharding_instances,
parsed_args.suite, dev_mode))
p = subprocess.Popen(commands)
p.communicate()
sys.exit(p.returncode)
def main():
"""Runs lighthouse checks and deletes reports."""
enable_webpages()
atexit.register(cleanup)
python_utils.PRINT('Building files in production mode.')
# We are using --source_maps here, so that we have at least one CI check
# that builds using source maps in prod env. This is to ensure that
# there are no issues while deploying oppia.
build.main(args=['--prod_env', '--source_maps'])
build.modify_constants(prod_env=True)
common.start_redis_server()
start_google_app_engine_server()
common.wait_for_port_to_be_open(GOOGLE_APP_ENGINE_PORT)
run_lighthouse_checks()
def run_tests(args):
"""Run the scripts to start end-to-end tests."""
if is_oppia_server_already_running():
sys.exit(1)
install_third_party_libraries(args.skip_install)
with contextlib.ExitStack() as stack:
dev_mode = not args.prod_env
if args.skip_build:
build.modify_constants(prod_env=args.prod_env)
else:
build_js_files(dev_mode, source_maps=args.source_maps)
stack.callback(build.set_constants_to_default)
stack.enter_context(servers.managed_redis_server())
stack.enter_context(servers.managed_elasticsearch_dev_server())
if constants.EMULATOR_MODE:
stack.enter_context(servers.managed_firebase_auth_emulator())
stack.enter_context(
servers.managed_cloud_datastore_emulator(clear_datastore=True))
app_yaml_path = 'app.yaml' if args.prod_env else 'app_dev.yaml'
stack.enter_context(
servers.managed_dev_appserver(
app_yaml_path,
port=GOOGLE_APP_ENGINE_PORT,
log_level=args.server_log_level,
# Automatic restart can be disabled since we don't expect code
# changes to happen while the e2e tests are running.
automatic_restart=False,
skip_sdk_update_check=True,
env={
**os.environ,
'PORTSERVER_ADDRESS':
common.PORTSERVER_SOCKET_FILEPATH,
}))
stack.enter_context(
servers.managed_webdriver_server(
chrome_version=args.chrome_driver_version))
proc = stack.enter_context(
servers.managed_protractor_server(
suite_name=args.suite,
dev_mode=dev_mode,
debug_mode=args.debug_mode,
sharding_instances=args.sharding_instances,
stdout=subprocess.PIPE))
print('Servers have come up.\n'
'Note: If ADD_SCREENSHOT_REPORTER is set to true in '
'core/tests/protractor.conf.js, you can view screenshots of the '
'failed tests in ../protractor-screenshots/')
output_lines = []
while True:
# Keep reading lines until an empty string is returned. Empty
# strings signal that the process has ended.
for line in iter(proc.stdout.readline, b''):
if isinstance(line, str):
# Although our unit tests always provide unicode strings,
# the actual server needs this failsafe since it can output
# non-unicode strings.
line = line.encode('utf-8') # pragma: no cover
output_lines.append(line.rstrip())
# Replaces non-ASCII characters with '?'.
common.write_stdout_safe(line.decode('ascii',
errors='replace'))
# The poll() method returns None while the process is running,
# otherwise it returns the return code of the process (an int).
if proc.poll() is not None:
break
return output_lines, proc.returncode
def run_tests(args=None):
"""Run the scripts to start end-to-end tests."""
parsed_args = _PARSER.parse_args(args=args)
oppia_instance_is_already_running = is_oppia_server_already_running()
if oppia_instance_is_already_running:
sys.exit(1)
setup_and_install_dependencies(parsed_args.skip_install)
common.start_redis_server()
atexit.register(cleanup)
dev_mode = not parsed_args.prod_env
if parsed_args.skip_build:
build.modify_constants(prod_env=parsed_args.prod_env)
else:
build_js_files(dev_mode,
deparallelize_terser=parsed_args.deparallelize_terser,
source_maps=parsed_args.source_maps)
version = parsed_args.chrome_driver_version or get_chrome_driver_version()
python_utils.PRINT('\n\nCHROMEDRIVER VERSION: %s\n\n' % version)
start_webdriver_manager(version)
portserver_process = start_portserver()
atexit.register(cleanup_portserver, portserver_process)
start_google_app_engine_server(dev_mode, parsed_args.server_log_level)
common.wait_for_port_to_be_open(WEB_DRIVER_PORT)
common.wait_for_port_to_be_open(GOOGLE_APP_ENGINE_PORT)
python_utils.PRINT(
'Note: If ADD_SCREENSHOT_REPORTER is set to true in'
'core/tests/protractor.conf.js, you can view screenshots'
'of the failed tests in ../protractor-screenshots/')
commands = [common.NODE_BIN_PATH]
if parsed_args.debug_mode:
commands.append('--inspect-brk')
# This flag ensures tests fail if waitFor calls time out.
commands.append('--unhandled-rejections=strict')
commands.append(PROTRACTOR_BIN_PATH)
commands.extend(
get_e2e_test_parameters(parsed_args.sharding_instances,
parsed_args.suite, dev_mode))
p = subprocess.Popen(commands, stdout=subprocess.PIPE)
output_lines = []
failure_seen = False
while True:
nextline = p.stdout.readline()
if len(nextline) == 0 and p.poll() is not None:
break
sys.stdout.write(nextline)
sys.stdout.flush()
new_output_line = nextline.strip()
if new_output_line.decode('utf-8') == FAILURE_OUTPUT_STRING:
failure_seen = True
output_lines.append(new_output_line)
flaky_tests_list = []
google_auth_decode_password = os.getenv('GOOGLE_AUTH_DECODE_PASSWORD')
if google_auth_decode_password is not None:
with python_utils.open_file('auth.json.enc', 'rb',
encoding=None) as enc_file:
with python_utils.open_file('auth.json', 'w') as dec_file:
ciphertext = enc_file.read()
plaintext = simplecrypt.decrypt(google_auth_decode_password,
ciphertext).decode('utf-8')
dec_file.write(plaintext)
sheets_scopes = ['https://www.googleapis.com/auth/spreadsheets']
creds = service_account.Credentials.from_service_account_file(
'auth.json', scopes=sheets_scopes)
sheet = googleapiclient.discovery.build(
'sheets', 'v4', credentials=creds).spreadsheets()
flaky_tests_list = get_flaky_tests_data_from_sheets(sheet)
suite_name = parsed_args.suite.lower()
if len(flaky_tests_list) > 0 and p.returncode != 0:
for i, line in enumerate(output_lines):
if line.decode('utf-8') == FAILURE_OUTPUT_STRING:
test_name = output_lines[i + 3][3:].strip().lower()
# Remove coloring characters.
ansi_escape = re.compile(
r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
failure_log = ansi_escape.sub('', output_lines[i + 4])
failure_log = failure_log.strip().lower()
for index, row in enumerate(flaky_tests_list):
flaky_suite_name = row[0].strip().lower()
flaky_test_message = row[1].strip().lower()
flaky_error_message = row[2].strip().lower()
if (suite_name == flaky_suite_name
or flaky_suite_name == '[general]'):
if (test_name == flaky_test_message
or flaky_test_message == 'many'):
if flaky_error_message in failure_log:
update_flaky_tests_count(sheet, index, row[3])
try:
cleanup_portserver(portserver_process)
cleanup()
except Exception: # pragma: no cover
# This is marked as no cover because the
# exception happens due to some processes
# running on the local system, which might
# interfere with the cleanup stuff. This is
# added as a failsafe to make sure that
# even when it throws an exception, the
# test is retried.
pass # pragma: no cover
return 'flake'
if failure_seen:
cleanup_portserver(portserver_process)
cleanup()
return 'fail'
return 'pass'
