def _run_fail_fast(self, fail_fast, use_app_run):
"""Runs the py_test binary in a subprocess.
Args:
fail_fast: string, the fail fast value.
use_app_run: bool, whether the test helper should call
`absltest.main(argv=)` inside `app.run`.
Returns:
(stdout, exit_code) tuple of (string, int).
"""
env = absltest_env.inherited_env()
if fail_fast is not None:
env['TESTBRIDGE_TEST_RUNNER_FAIL_FAST'] = fail_fast
env['USE_APP_RUN'] = '1' if use_app_run else '0'
proc = subprocess.Popen(
args=[_bazelize_command.get_executable_path(self._test_name)],
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True)
stdout = proc.communicate()[0]
logging.info('output: %s', stdout)
return stdout, proc.wait()
def _run_test(self, extra_argv, extra_env):
"""Runs the py_test binary in a subprocess, with the given args or env.
Args:
extra_argv: extra args to pass to the test
extra_env: extra env vars to set when running the test
Returns:
(stdout, test_cases, exit_code) tuple of (str, list of strs, int).
"""
env = dict(os.environ)
# If *this* test is being run with this flag, we don't want to
# automatically set it for all tests we run.
env.pop('TEST_RANDOMIZE_ORDERING_SEED', '')
if extra_env is not None:
env.update(extra_env)
command = ([_bazelize_command.get_executable_path(self._test_name)] +
extra_argv)
proc = subprocess.Popen(args=command,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True)
stdout, _ = proc.communicate()
test_lines = [l for l in stdout.splitlines() if l.startswith('class ')]
return stdout, test_lines, proc.wait()
def _run_test_and_get_xml(self, flag):
"""Runs xml_reporter_helper_test and returns an Element instance.
Runs xml_reporter_helper_test in a new process so that it can
exercise the entire test infrastructure, and easily test issues in
the test fixture.
Args:
flag: flag to pass to xml_reporter_helper_test
Returns:
The Element instance of the XML output.
"""
xml_fhandle, xml_fname = tempfile.mkstemp()
os.close(xml_fhandle)
try:
binary_name = 'absl/testing/tests/xml_reporter_helper_test'
args = [
_bazelize_command.get_executable_path(binary_name), flag,
'--xml_output_file=%s' % xml_fname
]
ret = subprocess.call(args)
self.assertNotEqual(ret, 0)
xml = ElementTree.parse(xml_fname).getroot()
finally:
os.remove(xml_fname)
return xml
def _run_fail_fast(self, fail_fast):
"""Runs the py_test binary in a subprocess.
Args:
fail_fast: string, the fail fast value.
Returns:
(stdout, exit_code) tuple of (string, int).
"""
env = {}
if 'SYSTEMROOT' in os.environ:
# This is used by the random module on Windows to locate crypto
# libraries.
env['SYSTEMROOT'] = os.environ['SYSTEMROOT']
additional_args = []
if fail_fast is not None:
env['TESTBRIDGE_TEST_RUNNER_FAIL_FAST'] = fail_fast
proc = subprocess.Popen(
args=([_bazelize_command.get_executable_path(self._test_name)] +
additional_args),
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True)
stdout = proc.communicate()[0]
logging.info('output: %s', stdout)
return stdout, proc.wait()
def _run_filtered(self, test_filter, use_env_variable):
"""Runs the py_test binary in a subprocess.
Args:
test_filter: string, the filter argument to use.
use_env_variable: bool, pass the test filter as environment variable if
True, otherwise pass as command line arguments.
Returns:
(stdout, exit_code) tuple of (string, int).
"""
env = {}
if 'SYSTEMROOT' in os.environ:
# This is used by the random module on Windows to locate crypto
# libraries.
env['SYSTEMROOT'] = os.environ['SYSTEMROOT']
additional_args = []
if test_filter is not None:
if use_env_variable:
env['TESTBRIDGE_TEST_ONLY'] = test_filter
elif test_filter:
additional_args.extend(test_filter.split(' '))
proc = subprocess.Popen(
args=([_bazelize_command.get_executable_path(self._test_name)] +
additional_args),
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True)
stdout = proc.communicate()[0]
logging.info('output: %s', stdout)
return stdout, proc.wait()
def run_helper(self,
expect_success,
expected_stdout_substring=None,
expected_stderr_substring=None,
arguments=(),
env_overrides=None):
env = os.environ.copy()
env['APP_TEST_HELPER_TYPE'] = self.helper_type
if env_overrides:
env.update(env_overrides)
helper = 'absl/tests/app_test_helper_{}'.format(self.helper_type)
process = subprocess.Popen(
[_bazelize_command.get_executable_path(helper)] + list(arguments),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env,
universal_newlines=True)
stdout, stderr = process.communicate()
message = 'returncode: {}\nstdout: {}\nstderr: {}'.format(
process.returncode, stdout, stderr)
if expect_success:
self.assertEqual(0, process.returncode, msg=message)
else:
self.assertNotEqual(0, process.returncode, msg=message)
if expected_stdout_substring:
self.assertIn(expected_stdout_substring, stdout)
if expected_stderr_substring:
self.assertIn(expected_stderr_substring, stderr)
return process.returncode, stdout, stderr
def _run_filtered(self, test_filter):
"""Runs the py_test binary in a subprocess.
Args:
test_filter: string, the filter argument to use.
Returns:
(stdout, exit_code) tuple of (string, int).
"""
env = {}
if 'SYSTEMROOT' in os.environ:
# This is used by the random module on Windows to locate crypto
# libraries.
env['SYSTEMROOT'] = os.environ['SYSTEMROOT']
if test_filter is not None:
env['TESTBRIDGE_TEST_ONLY'] = test_filter
proc = subprocess.Popen(
args=[_bazelize_command.get_executable_path(self._test_path)],
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True)
stdout = proc.communicate()[0]
logging.info('output: %s', stdout)
return stdout, proc.wait()
def test_argparse_with_app_run(self, main_func_name,
flags_parser_func_name, args,
output_strings):
env = os.environ.copy()
env['MAIN_FUNC'] = main_func_name
env['FLAGS_PARSER_FUNC'] = flags_parser_func_name
helper = _bazelize_command.get_executable_path(
'absl/flags/tests/argparse_flags_test_helper',
add_version_suffix=False)
try:
stdout = subprocess.check_output([helper] + args,
env=env,
universal_newlines=True)
except subprocess.CalledProcessError as e:
error_info = ('ERROR: argparse_helper failed\n'
'Command: {}\n'
'Exit code: {}\n'
'----- output -----\n{}'
'------------------')
error_info = error_info.format(
e.cmd, e.returncode,
e.output + '\n' if e.output else '<empty>')
print(error_info, file=sys.stderr)
raise
for output_string in output_strings:
self.assertIn(output_string, stdout)
def _run_test(self, flag, num_errors, num_failures, suites):
xml_fhandle, xml_fname = tempfile.mkstemp()
os.close(xml_fhandle)
try:
prog = os.path.join(os.path.dirname(__file__),
'xml_reporter_helper_test')
args = [
_bazelize_command.get_executable_path(prog), flag,
'--xml_output_file=%s' % xml_fname
]
ret = subprocess.call(args)
self.assertNotEqual(ret, 0)
xml = ElementTree.parse(xml_fname).getroot()
logging.info('xml output is:\n%s', ElementTree.tostring(xml))
finally:
os.remove(xml_fname)
self.assertEqual(int(xml.attrib['errors']), num_errors)
self.assertEqual(int(xml.attrib['failures']), num_failures)
self.assertLen(xml, len(suites))
actual_suites = sorted(xml.findall('testsuite'),
key=lambda x: x.attrib['name'])
suites = sorted(suites, key=lambda x: x['name'])
for actual_suite, expected_suite in zip(actual_suites, suites):
self.assertEqual(actual_suite.attrib['name'],
expected_suite['name'])
self.assertLen(actual_suite, len(expected_suite['cases']))
actual_cases = sorted(actual_suite.findall('testcase'),
key=lambda x: x.attrib['name'])
expected_cases = sorted(expected_suite['cases'],
key=lambda x: x['name'])
for actual_case, expected_case in zip(actual_cases,
expected_cases):
self.assertEqual(actual_case.attrib['name'],
expected_case['name'])
self.assertEqual(actual_case.attrib['classname'],
expected_case['classname'])
if 'error' in expected_case:
actual_error = actual_case.find('error')
self.assertEqual(actual_error.attrib['message'],
expected_case['error'])
if 'failure' in expected_case:
actual_failure = actual_case.find('failure')
self.assertEqual(actual_failure.attrib['message'],
expected_case['failure'])
return xml
def run_helper(self,
expect_success,
expected_stdout_substring=None,
expected_stderr_substring=None,
arguments=(),
env_overrides=None):
env = os.environ.copy()
env['APP_TEST_HELPER_TYPE'] = self.helper_type
env['PYTHONIOENCODING'] = 'utf8'
if env_overrides:
env.update(env_overrides)
helper = 'absl/tests/app_test_helper_{}'.format(self.helper_type)
process = subprocess.Popen(
[_bazelize_command.get_executable_path(helper)] + list(arguments),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env,
universal_newlines=False)
stdout, stderr = process.communicate()
# In Python 2, we can't control the encoding used by universal_newline
# mode, which can cause UnicodeDecodeErrors when subprocess tries to
# conver the bytes to unicode, so we have to decode it manually.
stdout = _normalize_newlines(stdout.decode('utf8'))
stderr = _normalize_newlines(stderr.decode('utf8'))
message = (u'Command: {command}\n'
'Exit Code: {exitcode}\n'
'===== stdout =====\n{stdout}'
'===== stderr =====\n{stderr}'
'=================='.format(
command=' '.join([helper] + list(arguments)),
exitcode=process.returncode,
stdout=stdout or '<no output>\n',
stderr=stderr or '<no output>\n'))
if expect_success:
self.assertEqual(0, process.returncode, msg=message)
else:
self.assertNotEqual(0, process.returncode, msg=message)
if expected_stdout_substring:
self.assertIn(expected_stdout_substring, stdout, message)
if expected_stderr_substring:
self.assertIn(expected_stderr_substring, stderr, message)
return process.returncode, stdout, stderr
def _run_sharded(self,
total_shards,
shard_index,
shard_file=None,
env=None):
"""Runs the py_test binary in a subprocess.
Args:
total_shards: int, the total number of shards.
shard_index: int, the shard index.
shard_file: string, if not 'None', the path to the shard file.
This method asserts it is properly created.
env: Environment variables to be set for the py_test binary.
Returns:
(stdout, exit_code) tuple of (string, int).
"""
if env is None:
env = {}
env.update({
'TEST_TOTAL_SHARDS': str(total_shards),
'TEST_SHARD_INDEX': str(shard_index)
})
if 'SYSTEMROOT' in os.environ:
# This is used by the random module on Windows to locate crypto
# libraries.
env['SYSTEMROOT'] = os.environ['SYSTEMROOT']
if shard_file:
self._shard_file = shard_file
env['TEST_SHARD_STATUS_FILE'] = shard_file
if os.path.exists(shard_file):
os.unlink(shard_file)
proc = subprocess.Popen(
args=[_bazelize_command.get_executable_path(self._test_name)],
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True)
stdout = proc.communicate()[0]
if shard_file:
self.assertTrue(os.path.exists(shard_file))
return (stdout, proc.wait())
def _run_sharded(self,
total_shards,
shard_index,
shard_file=None,
additional_env=None):
"""Runs the py_test binary in a subprocess.
Args:
total_shards: int, the total number of shards.
shard_index: int, the shard index.
shard_file: string, if not 'None', the path to the shard file.
This method asserts it is properly created.
additional_env: Additional environment variables to be set for the py_test
binary.
Returns:
(stdout, exit_code) tuple of (string, int).
"""
env = absltest_env.inherited_env()
if additional_env:
env.update(additional_env)
env.update({
'TEST_TOTAL_SHARDS': str(total_shards),
'TEST_SHARD_INDEX': str(shard_index)
})
if shard_file:
self._shard_file = shard_file
env['TEST_SHARD_STATUS_FILE'] = shard_file
if os.path.exists(shard_file):
os.unlink(shard_file)
proc = subprocess.Popen(
args=[_bazelize_command.get_executable_path(self._test_name)],
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True)
stdout = proc.communicate()[0]
if shard_file:
self.assertTrue(os.path.exists(shard_file))
return (stdout, proc.wait())
def _run_filtered(self, test_filter, use_env_variable, use_app_run):
"""Runs the py_test binary in a subprocess.
Args:
test_filter: string, the filter argument to use.
use_env_variable: bool, pass the test filter as environment variable if
True, otherwise pass as command line arguments.
use_app_run: bool, whether the test helper should call
`absltest.main(argv=)` inside `app.run`.
Returns:
(stdout, exit_code) tuple of (string, int).
"""
env = absltest_env.inherited_env()
env['USE_APP_RUN'] = '1' if use_app_run else '0'
additional_args = []
if test_filter is not None:
if use_env_variable:
env['TESTBRIDGE_TEST_ONLY'] = test_filter
elif test_filter:
if sys.version_info[:2] >= (3, 7):
# The -k flags are passed as positional arguments to absl.flags.
additional_args.append('--')
additional_args.extend(['-k=' + f for f in test_filter.split(' ')])
else:
additional_args.extend(test_filter.split(' '))
proc = subprocess.Popen(
args=([_bazelize_command.get_executable_path(self._test_name)] +
additional_args),
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True)
stdout = proc.communicate()[0]
logging.info('output: %s', stdout)
return stdout, proc.wait()
def _get_helper(self):
binary_name = 'absl/testing/tests/xml_reporter_helper_test'
return _bazelize_command.get_executable_path(binary_name)
def _get_helper(self):
helper_name = 'absl/logging/tests/logging_functional_test_helper'
return _bazelize_command.get_executable_path(helper_name)
def _get_helper(self):
return _bazelize_command.get_executable_path(
os.path.join(os.path.dirname(__file__),
'logging_functional_test_helper'))
