def _run_fail_fast(self, fail_fast, use_app_run): """Runs the py_test binary in a subprocess. Args: fail_fast: string, the fail fast value. use_app_run: bool, whether the test helper should call `absltest.main(argv=)` inside `app.run`. Returns: (stdout, exit_code) tuple of (string, int). """ env = absltest_env.inherited_env() if fail_fast is not None: env['TESTBRIDGE_TEST_RUNNER_FAIL_FAST'] = fail_fast env['USE_APP_RUN'] = '1' if use_app_run else '0' proc = subprocess.Popen( args=[_bazelize_command.get_executable_path(self._test_name)], env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) stdout = proc.communicate()[0] logging.info('output: %s', stdout) return stdout, proc.wait()
def _run_test(self, extra_argv, extra_env): """Runs the py_test binary in a subprocess, with the given args or env. Args: extra_argv: extra args to pass to the test extra_env: extra env vars to set when running the test Returns: (stdout, test_cases, exit_code) tuple of (str, list of strs, int). """ env = dict(os.environ) # If *this* test is being run with this flag, we don't want to # automatically set it for all tests we run. env.pop('TEST_RANDOMIZE_ORDERING_SEED', '') if extra_env is not None: env.update(extra_env) command = ([_bazelize_command.get_executable_path(self._test_name)] + extra_argv) proc = subprocess.Popen(args=command, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) stdout, _ = proc.communicate() test_lines = [l for l in stdout.splitlines() if l.startswith('class ')] return stdout, test_lines, proc.wait()
def _run_test_and_get_xml(self, flag): """Runs xml_reporter_helper_test and returns an Element instance. Runs xml_reporter_helper_test in a new process so that it can exercise the entire test infrastructure, and easily test issues in the test fixture. Args: flag: flag to pass to xml_reporter_helper_test Returns: The Element instance of the XML output. """ xml_fhandle, xml_fname = tempfile.mkstemp() os.close(xml_fhandle) try: binary_name = 'absl/testing/tests/xml_reporter_helper_test' args = [ _bazelize_command.get_executable_path(binary_name), flag, '--xml_output_file=%s' % xml_fname ] ret = subprocess.call(args) self.assertNotEqual(ret, 0) xml = ElementTree.parse(xml_fname).getroot() finally: os.remove(xml_fname) return xml
def _run_fail_fast(self, fail_fast): """Runs the py_test binary in a subprocess. Args: fail_fast: string, the fail fast value. Returns: (stdout, exit_code) tuple of (string, int). """ env = {} if 'SYSTEMROOT' in os.environ: # This is used by the random module on Windows to locate crypto # libraries. env['SYSTEMROOT'] = os.environ['SYSTEMROOT'] additional_args = [] if fail_fast is not None: env['TESTBRIDGE_TEST_RUNNER_FAIL_FAST'] = fail_fast proc = subprocess.Popen( args=([_bazelize_command.get_executable_path(self._test_name)] + additional_args), env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) stdout = proc.communicate()[0] logging.info('output: %s', stdout) return stdout, proc.wait()
def _run_filtered(self, test_filter, use_env_variable): """Runs the py_test binary in a subprocess. Args: test_filter: string, the filter argument to use. use_env_variable: bool, pass the test filter as environment variable if True, otherwise pass as command line arguments. Returns: (stdout, exit_code) tuple of (string, int). """ env = {} if 'SYSTEMROOT' in os.environ: # This is used by the random module on Windows to locate crypto # libraries. env['SYSTEMROOT'] = os.environ['SYSTEMROOT'] additional_args = [] if test_filter is not None: if use_env_variable: env['TESTBRIDGE_TEST_ONLY'] = test_filter elif test_filter: additional_args.extend(test_filter.split(' ')) proc = subprocess.Popen( args=([_bazelize_command.get_executable_path(self._test_name)] + additional_args), env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) stdout = proc.communicate()[0] logging.info('output: %s', stdout) return stdout, proc.wait()
def run_helper(self, expect_success, expected_stdout_substring=None, expected_stderr_substring=None, arguments=(), env_overrides=None): env = os.environ.copy() env['APP_TEST_HELPER_TYPE'] = self.helper_type if env_overrides: env.update(env_overrides) helper = 'absl/tests/app_test_helper_{}'.format(self.helper_type) process = subprocess.Popen( [_bazelize_command.get_executable_path(helper)] + list(arguments), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, universal_newlines=True) stdout, stderr = process.communicate() message = 'returncode: {}\nstdout: {}\nstderr: {}'.format( process.returncode, stdout, stderr) if expect_success: self.assertEqual(0, process.returncode, msg=message) else: self.assertNotEqual(0, process.returncode, msg=message) if expected_stdout_substring: self.assertIn(expected_stdout_substring, stdout) if expected_stderr_substring: self.assertIn(expected_stderr_substring, stderr) return process.returncode, stdout, stderr
def _run_filtered(self, test_filter): """Runs the py_test binary in a subprocess. Args: test_filter: string, the filter argument to use. Returns: (stdout, exit_code) tuple of (string, int). """ env = {} if 'SYSTEMROOT' in os.environ: # This is used by the random module on Windows to locate crypto # libraries. env['SYSTEMROOT'] = os.environ['SYSTEMROOT'] if test_filter is not None: env['TESTBRIDGE_TEST_ONLY'] = test_filter proc = subprocess.Popen( args=[_bazelize_command.get_executable_path(self._test_path)], env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) stdout = proc.communicate()[0] logging.info('output: %s', stdout) return stdout, proc.wait()
def test_argparse_with_app_run(self, main_func_name, flags_parser_func_name, args, output_strings): env = os.environ.copy() env['MAIN_FUNC'] = main_func_name env['FLAGS_PARSER_FUNC'] = flags_parser_func_name helper = _bazelize_command.get_executable_path( 'absl/flags/tests/argparse_flags_test_helper', add_version_suffix=False) try: stdout = subprocess.check_output([helper] + args, env=env, universal_newlines=True) except subprocess.CalledProcessError as e: error_info = ('ERROR: argparse_helper failed\n' 'Command: {}\n' 'Exit code: {}\n' '----- output -----\n{}' '------------------') error_info = error_info.format( e.cmd, e.returncode, e.output + '\n' if e.output else '<empty>') print(error_info, file=sys.stderr) raise for output_string in output_strings: self.assertIn(output_string, stdout)
def _run_test(self, flag, num_errors, num_failures, suites): xml_fhandle, xml_fname = tempfile.mkstemp() os.close(xml_fhandle) try: prog = os.path.join(os.path.dirname(__file__), 'xml_reporter_helper_test') args = [ _bazelize_command.get_executable_path(prog), flag, '--xml_output_file=%s' % xml_fname ] ret = subprocess.call(args) self.assertNotEqual(ret, 0) xml = ElementTree.parse(xml_fname).getroot() logging.info('xml output is:\n%s', ElementTree.tostring(xml)) finally: os.remove(xml_fname) self.assertEqual(int(xml.attrib['errors']), num_errors) self.assertEqual(int(xml.attrib['failures']), num_failures) self.assertLen(xml, len(suites)) actual_suites = sorted(xml.findall('testsuite'), key=lambda x: x.attrib['name']) suites = sorted(suites, key=lambda x: x['name']) for actual_suite, expected_suite in zip(actual_suites, suites): self.assertEqual(actual_suite.attrib['name'], expected_suite['name']) self.assertLen(actual_suite, len(expected_suite['cases'])) actual_cases = sorted(actual_suite.findall('testcase'), key=lambda x: x.attrib['name']) expected_cases = sorted(expected_suite['cases'], key=lambda x: x['name']) for actual_case, expected_case in zip(actual_cases, expected_cases): self.assertEqual(actual_case.attrib['name'], expected_case['name']) self.assertEqual(actual_case.attrib['classname'], expected_case['classname']) if 'error' in expected_case: actual_error = actual_case.find('error') self.assertEqual(actual_error.attrib['message'], expected_case['error']) if 'failure' in expected_case: actual_failure = actual_case.find('failure') self.assertEqual(actual_failure.attrib['message'], expected_case['failure']) return xml
def run_helper(self, expect_success, expected_stdout_substring=None, expected_stderr_substring=None, arguments=(), env_overrides=None): env = os.environ.copy() env['APP_TEST_HELPER_TYPE'] = self.helper_type env['PYTHONIOENCODING'] = 'utf8' if env_overrides: env.update(env_overrides) helper = 'absl/tests/app_test_helper_{}'.format(self.helper_type) process = subprocess.Popen( [_bazelize_command.get_executable_path(helper)] + list(arguments), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, universal_newlines=False) stdout, stderr = process.communicate() # In Python 2, we can't control the encoding used by universal_newline # mode, which can cause UnicodeDecodeErrors when subprocess tries to # conver the bytes to unicode, so we have to decode it manually. stdout = _normalize_newlines(stdout.decode('utf8')) stderr = _normalize_newlines(stderr.decode('utf8')) message = (u'Command: {command}\n' 'Exit Code: {exitcode}\n' '===== stdout =====\n{stdout}' '===== stderr =====\n{stderr}' '=================='.format( command=' '.join([helper] + list(arguments)), exitcode=process.returncode, stdout=stdout or '<no output>\n', stderr=stderr or '<no output>\n')) if expect_success: self.assertEqual(0, process.returncode, msg=message) else: self.assertNotEqual(0, process.returncode, msg=message) if expected_stdout_substring: self.assertIn(expected_stdout_substring, stdout, message) if expected_stderr_substring: self.assertIn(expected_stderr_substring, stderr, message) return process.returncode, stdout, stderr
def _run_sharded(self, total_shards, shard_index, shard_file=None, env=None): """Runs the py_test binary in a subprocess. Args: total_shards: int, the total number of shards. shard_index: int, the shard index. shard_file: string, if not 'None', the path to the shard file. This method asserts it is properly created. env: Environment variables to be set for the py_test binary. Returns: (stdout, exit_code) tuple of (string, int). """ if env is None: env = {} env.update({ 'TEST_TOTAL_SHARDS': str(total_shards), 'TEST_SHARD_INDEX': str(shard_index) }) if 'SYSTEMROOT' in os.environ: # This is used by the random module on Windows to locate crypto # libraries. env['SYSTEMROOT'] = os.environ['SYSTEMROOT'] if shard_file: self._shard_file = shard_file env['TEST_SHARD_STATUS_FILE'] = shard_file if os.path.exists(shard_file): os.unlink(shard_file) proc = subprocess.Popen( args=[_bazelize_command.get_executable_path(self._test_name)], env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) stdout = proc.communicate()[0] if shard_file: self.assertTrue(os.path.exists(shard_file)) return (stdout, proc.wait())
def _run_sharded(self, total_shards, shard_index, shard_file=None, additional_env=None): """Runs the py_test binary in a subprocess. Args: total_shards: int, the total number of shards. shard_index: int, the shard index. shard_file: string, if not 'None', the path to the shard file. This method asserts it is properly created. additional_env: Additional environment variables to be set for the py_test binary. Returns: (stdout, exit_code) tuple of (string, int). """ env = absltest_env.inherited_env() if additional_env: env.update(additional_env) env.update({ 'TEST_TOTAL_SHARDS': str(total_shards), 'TEST_SHARD_INDEX': str(shard_index) }) if shard_file: self._shard_file = shard_file env['TEST_SHARD_STATUS_FILE'] = shard_file if os.path.exists(shard_file): os.unlink(shard_file) proc = subprocess.Popen( args=[_bazelize_command.get_executable_path(self._test_name)], env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) stdout = proc.communicate()[0] if shard_file: self.assertTrue(os.path.exists(shard_file)) return (stdout, proc.wait())
def _run_filtered(self, test_filter, use_env_variable, use_app_run): """Runs the py_test binary in a subprocess. Args: test_filter: string, the filter argument to use. use_env_variable: bool, pass the test filter as environment variable if True, otherwise pass as command line arguments. use_app_run: bool, whether the test helper should call `absltest.main(argv=)` inside `app.run`. Returns: (stdout, exit_code) tuple of (string, int). """ env = absltest_env.inherited_env() env['USE_APP_RUN'] = '1' if use_app_run else '0' additional_args = [] if test_filter is not None: if use_env_variable: env['TESTBRIDGE_TEST_ONLY'] = test_filter elif test_filter: if sys.version_info[:2] >= (3, 7): # The -k flags are passed as positional arguments to absl.flags. additional_args.append('--') additional_args.extend(['-k=' + f for f in test_filter.split(' ')]) else: additional_args.extend(test_filter.split(' ')) proc = subprocess.Popen( args=([_bazelize_command.get_executable_path(self._test_name)] + additional_args), env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) stdout = proc.communicate()[0] logging.info('output: %s', stdout) return stdout, proc.wait()
def _get_helper(self): binary_name = 'absl/testing/tests/xml_reporter_helper_test' return _bazelize_command.get_executable_path(binary_name)
def _get_helper(self): helper_name = 'absl/logging/tests/logging_functional_test_helper' return _bazelize_command.get_executable_path(helper_name)
def _get_helper(self): return _bazelize_command.get_executable_path( os.path.join(os.path.dirname(__file__), 'logging_functional_test_helper'))