def run_tests(args): """Run the scripts to start end-to-end tests.""" oppia_instance_is_already_running = is_oppia_server_already_running() if oppia_instance_is_already_running: sys.exit(1) setup_and_install_dependencies(args.skip_install) common.start_redis_server() atexit.register(cleanup) dev_mode = not args.prod_env if args.skip_build: build.modify_constants(prod_env=args.prod_env) else: build_js_files( dev_mode, deparallelize_terser=args.deparallelize_terser, source_maps=args.source_maps) version = args.chrome_driver_version or get_chrome_driver_version() python_utils.PRINT('\n\nCHROMEDRIVER VERSION: %s\n\n' % version) start_webdriver_manager(version) managed_dev_appserver = common.managed_dev_appserver( 'app.yaml' if args.prod_env else 'app_dev.yaml', port=GOOGLE_APP_ENGINE_PORT, log_level=args.server_log_level, clear_datastore=True, skip_sdk_update_check=True, env={'PORTSERVER_ADDRESS': PORTSERVER_SOCKET_FILEPATH}) with managed_dev_appserver: common.wait_for_port_to_be_open(WEB_DRIVER_PORT) common.wait_for_port_to_be_open(GOOGLE_APP_ENGINE_PORT) python_utils.PRINT( 'Note: If ADD_SCREENSHOT_REPORTER is set to true in' 'core/tests/protractor.conf.js, you can view screenshots' 'of the failed tests in ../protractor-screenshots/') commands = [common.NODE_BIN_PATH] if args.debug_mode: commands.append('--inspect-brk') # This flag ensures tests fail if waitFor calls time out. commands.append('--unhandled-rejections=strict') commands.append(PROTRACTOR_BIN_PATH) commands.extend(get_e2e_test_parameters( args.sharding_instances, args.suite, dev_mode)) p = subprocess.Popen(commands, stdout=subprocess.PIPE) output_lines = [] while True: nextline = p.stdout.readline() if len(nextline) == 0 and p.poll() is not None: break if isinstance(nextline, str): # This is a failsafe line in case we get non-unicode input, # but the tests provide all strings as unicode. nextline = nextline.decode('utf-8') # pragma: nocover output_lines.append(nextline.rstrip()) # Replaces non-ASCII characters with '?'. sys.stdout.write(nextline.encode('ascii', errors='replace')) return output_lines, p.returncode
def main(args=None): """Runs lighthouse checks and deletes reports.""" parsed_args = _PARSER.parse_args(args=args) if parsed_args.mode == LIGHTHOUSE_MODE_ACCESSIBILITY: lighthouse_mode = LIGHTHOUSE_MODE_ACCESSIBILITY server_mode = SERVER_MODE_DEV elif parsed_args.mode == LIGHTHOUSE_MODE_PERFORMANCE: lighthouse_mode = LIGHTHOUSE_MODE_PERFORMANCE server_mode = SERVER_MODE_PROD else: raise Exception('Invalid parameter passed in: \'%s\', please choose' 'from \'accessibility\' or \'performance\'' % parsed_args.mode) enable_webpages() atexit.register(cleanup) if lighthouse_mode == LIGHTHOUSE_MODE_PERFORMANCE: python_utils.PRINT('Building files in production mode.') # We are using --source_maps here, so that we have at least one CI check # that builds using source maps in prod env. This is to ensure that # there are no issues while deploying oppia. build.main(args=['--prod_env', '--source_maps']) elif lighthouse_mode == LIGHTHOUSE_MODE_ACCESSIBILITY: build.main(args=[]) run_webpack_compilation() else: raise Exception('Invalid lighthouse mode: \'%s\', please choose' 'from \'accessibility\' or \'performance\'' % lighthouse_mode) common.start_redis_server() # TODO(#11549): Move this to top of the file. import contextlib2 managed_dev_appserver = common.managed_dev_appserver( APP_YAML_FILENAMES[server_mode], port=GOOGLE_APP_ENGINE_PORT, clear_datastore=True, log_level='critical', skip_sdk_update_check=True) with contextlib2.ExitStack() as stack: stack.enter_context(common.managed_elasticsearch_dev_server()) stack.enter_context(common.managed_firebase_auth_emulator()) stack.enter_context(managed_dev_appserver) # Wait for the servers to come up. common.wait_for_port_to_be_open(feconf.ES_LOCALHOST_PORT) common.wait_for_port_to_be_open(GOOGLE_APP_ENGINE_PORT) run_lighthouse_puppeteer_script() run_lighthouse_checks(lighthouse_mode)
def main(): """Runs lighthouse checks and deletes reports.""" enable_webpages() atexit.register(cleanup) python_utils.PRINT('Building files in production mode.') build.main(args=['--prod_env']) build.modify_constants(prod_env=True) start_google_app_engine_server() common.wait_for_port_to_be_open(GOOGLE_APP_ENGINE_PORT) run_lighthouse_checks()
def main(): """Runs lighthouse checks and deletes reports.""" enable_webpages() atexit.register(cleanup) python_utils.PRINT('Building files in production mode.') # We are using --source_maps here, so that we have at least one CI check # that builds using source maps in prod env. This is to ensure that # there are no issues while deploying oppia. build.main(args=['--prod_env', '--source_maps']) common.start_redis_server() start_google_app_engine_server() common.wait_for_port_to_be_open(GOOGLE_APP_ENGINE_PORT) run_lighthouse_checks()
def main(args=None): """Run the scripts to start end-to-end tests.""" parsed_args = _PARSER.parse_args(args=args) oppia_instance_is_already_running = is_oppia_server_already_running() if oppia_instance_is_already_running: sys.exit(1) setup_and_install_dependencies(parsed_args.skip_install) common.start_redis_server() atexit.register(cleanup) dev_mode = not parsed_args.prod_env update_contributor_dashboard_status_in_feconf_file( FECONF_FILE_PATH, parsed_args.contributor_dashboard_enabled) if parsed_args.skip_build: build.modify_constants(prod_env=parsed_args.prod_env) else: build_js_files(dev_mode, deparallelize_terser=parsed_args.deparallelize_terser, source_maps=parsed_args.source_maps) version = parsed_args.chrome_driver_version or get_chrome_driver_version() python_utils.PRINT('\n\nCHROMEDRIVER VERSION: %s\n\n' % version) start_webdriver_manager(version) portserver_process = start_portserver() atexit.register(cleanup_portserver, portserver_process) start_google_app_engine_server(dev_mode, parsed_args.server_log_level) common.wait_for_port_to_be_open(WEB_DRIVER_PORT) common.wait_for_port_to_be_open(GOOGLE_APP_ENGINE_PORT) ensure_screenshots_dir_is_removed() commands = [common.NODE_BIN_PATH] if parsed_args.debug_mode: commands.append('--inspect-brk') # This flag ensures tests fail if waitFor calls time out. commands.append('--unhandled-rejections=strict') commands.append(PROTRACTOR_BIN_PATH) commands.extend( get_e2e_test_parameters(parsed_args.sharding_instances, parsed_args.suite, dev_mode)) p = subprocess.Popen(commands) p.communicate() sys.exit(p.returncode)
def main(args=None): """Runs lighthouse checks and deletes reports.""" parsed_args = _PARSER.parse_args(args=args) if parsed_args.mode == LIGHTHOUSE_MODE_ACCESSIBILITY: lighthouse_mode = LIGHTHOUSE_MODE_ACCESSIBILITY server_mode = SERVER_MODE_DEV elif parsed_args.mode == LIGHTHOUSE_MODE_PERFORMANCE: lighthouse_mode = LIGHTHOUSE_MODE_PERFORMANCE server_mode = SERVER_MODE_PROD else: raise Exception( 'Invalid parameter passed in: \'%s\', please choose' 'from \'accessibility\' or \'performance\'' % parsed_args.mode) enable_webpages() atexit.register(cleanup) if lighthouse_mode == LIGHTHOUSE_MODE_PERFORMANCE: python_utils.PRINT('Building files in production mode.') # We are using --source_maps here, so that we have at least one CI check # that builds using source maps in prod env. This is to ensure that # there are no issues while deploying oppia. build.main(args=['--prod_env', '--source_maps']) elif lighthouse_mode == LIGHTHOUSE_MODE_ACCESSIBILITY: build.main(args=[]) run_webpack_compilation() else: raise Exception( 'Invalid lighthouse mode: \'%s\', please choose' 'from \'accessibility\' or \'performance\'' % lighthouse_mode) common.start_redis_server() start_google_app_engine_server(server_mode) common.wait_for_port_to_be_open(GOOGLE_APP_ENGINE_PORT) run_lighthouse_puppeteer_script() run_lighthouse_checks(lighthouse_mode)
def run_tests(args=None): """Run the scripts to start end-to-end tests.""" parsed_args = _PARSER.parse_args(args=args) oppia_instance_is_already_running = is_oppia_server_already_running() if oppia_instance_is_already_running: sys.exit(1) setup_and_install_dependencies(parsed_args.skip_install) common.start_redis_server() atexit.register(cleanup) dev_mode = not parsed_args.prod_env if parsed_args.skip_build: build.modify_constants(prod_env=parsed_args.prod_env) else: build_js_files(dev_mode, deparallelize_terser=parsed_args.deparallelize_terser, source_maps=parsed_args.source_maps) version = parsed_args.chrome_driver_version or get_chrome_driver_version() python_utils.PRINT('\n\nCHROMEDRIVER VERSION: %s\n\n' % version) start_webdriver_manager(version) portserver_process = start_portserver() atexit.register(cleanup_portserver, portserver_process) start_google_app_engine_server(dev_mode, parsed_args.server_log_level) common.wait_for_port_to_be_open(WEB_DRIVER_PORT) common.wait_for_port_to_be_open(GOOGLE_APP_ENGINE_PORT) python_utils.PRINT( 'Note: If ADD_SCREENSHOT_REPORTER is set to true in' 'core/tests/protractor.conf.js, you can view screenshots' 'of the failed tests in ../protractor-screenshots/') commands = [common.NODE_BIN_PATH] if parsed_args.debug_mode: commands.append('--inspect-brk') # This flag ensures tests fail if waitFor calls time out. commands.append('--unhandled-rejections=strict') commands.append(PROTRACTOR_BIN_PATH) commands.extend( get_e2e_test_parameters(parsed_args.sharding_instances, parsed_args.suite, dev_mode)) p = subprocess.Popen(commands, stdout=subprocess.PIPE) output_lines = [] failure_seen = False while True: nextline = p.stdout.readline() if len(nextline) == 0 and p.poll() is not None: break sys.stdout.write(nextline) sys.stdout.flush() new_output_line = nextline.strip() if new_output_line.decode('utf-8') == FAILURE_OUTPUT_STRING: failure_seen = True output_lines.append(new_output_line) flaky_tests_list = [] google_auth_decode_password = os.getenv('GOOGLE_AUTH_DECODE_PASSWORD') if google_auth_decode_password is not None: with python_utils.open_file('auth.json.enc', 'rb', encoding=None) as enc_file: with python_utils.open_file('auth.json', 'w') as dec_file: ciphertext = enc_file.read() plaintext = simplecrypt.decrypt(google_auth_decode_password, ciphertext).decode('utf-8') dec_file.write(plaintext) sheets_scopes = ['https://www.googleapis.com/auth/spreadsheets'] creds = service_account.Credentials.from_service_account_file( 'auth.json', scopes=sheets_scopes) sheet = googleapiclient.discovery.build( 'sheets', 'v4', credentials=creds).spreadsheets() flaky_tests_list = get_flaky_tests_data_from_sheets(sheet) suite_name = parsed_args.suite.lower() if len(flaky_tests_list) > 0 and p.returncode != 0: for i, line in enumerate(output_lines): if line.decode('utf-8') == FAILURE_OUTPUT_STRING: test_name = output_lines[i + 3][3:].strip().lower() # Remove coloring characters. ansi_escape = re.compile( r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') failure_log = ansi_escape.sub('', output_lines[i + 4]) failure_log = failure_log.strip().lower() for index, row in enumerate(flaky_tests_list): flaky_suite_name = row[0].strip().lower() flaky_test_message = row[1].strip().lower() flaky_error_message = row[2].strip().lower() if (suite_name == flaky_suite_name or flaky_suite_name == '[general]'): if (test_name == flaky_test_message or flaky_test_message == 'many'): if flaky_error_message in failure_log: update_flaky_tests_count(sheet, index, row[3]) try: cleanup_portserver(portserver_process) cleanup() except Exception: # pragma: no cover # This is marked as no cover because the # exception happens due to some processes # running on the local system, which might # interfere with the cleanup stuff. This is # added as a failsafe to make sure that # even when it throws an exception, the # test is retried. pass # pragma: no cover return 'flake' if failure_seen: cleanup_portserver(portserver_process) cleanup() return 'fail' return 'pass'