def check_test_output_files(testcase: DataDrivenTestCase,
step: int,
strip_prefix: str = '') -> None:
for path, expected_content in testcase.output_files:
if path.startswith(strip_prefix):
path = path[len(strip_prefix):]
if not os.path.exists(path):
raise AssertionError(
'Expected file {} was not produced by test case{}'.format(
path, ' on step %d' % step if testcase.output2 else ''))
with open(path, 'r', encoding='utf8') as output_file:
actual_output_content = output_file.read().splitlines()
normalized_output = normalize_file_output(
actual_output_content, os.path.abspath(test_temp_dir))
# We always normalize things like timestamp, but only handle operating-system
# specific things if requested.
if testcase.normalize_output:
if testcase.suite.native_sep and os.path.sep == '\\':
normalized_output = [
fix_cobertura_filename(line) for line in normalized_output
]
normalized_output = normalize_error_messages(normalized_output)
assert_string_arrays_equal(
expected_content.splitlines(), normalized_output,
'Output file {} did not match its expected output{}'.format(
path, ' on step %d' % step if testcase.output2 else ''))
def test_python_cmdline(testcase: DataDrivenTestCase) -> None:
assert testcase.old_cwd is not None, "test was not properly set up"
# Write the program to a file.
program = '_program.py'
program_path = os.path.join(test_temp_dir, program)
with open(program_path, 'w') as file:
for s in testcase.input:
file.write('{}\n'.format(s))
args = parse_args(testcase.input[0])
args.append('--show-traceback')
args.append('--no-site-packages')
# Type check the program.
fixed = [python3_path, '-m', 'mypy']
env = os.environ.copy()
env['PYTHONPATH'] = PREFIX
process = subprocess.Popen(fixed + args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=test_temp_dir,
env=env)
outb = process.stdout.read()
# Split output into lines.
out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
result = process.wait()
# Remove temp file.
os.remove(program_path)
# Compare actual output to expected.
if testcase.output_files:
for path, expected_content in testcase.output_files:
if not os.path.exists(path):
raise AssertionError(
'Expected file {} was not produced by test case'.format(
path))
with open(path, 'r') as output_file:
actual_output_content = output_file.read().splitlines()
normalized_output = normalize_file_output(
actual_output_content, os.path.abspath(test_temp_dir))
if testcase.native_sep and os.path.sep == '\\':
normalized_output = [
fix_cobertura_filename(line) for line in normalized_output
]
normalized_output = normalize_error_messages(normalized_output)
assert_string_arrays_equal(
expected_content.splitlines(), normalized_output,
'Output file {} did not match its expected output'.format(
path))
else:
out = normalize_error_messages(out)
obvious_result = 1 if out else 0
if obvious_result != result:
out.append('== Return code: {}'.format(result))
assert_string_arrays_equal(
testcase.output, out,
'Invalid output ({}, line {})'.format(testcase.file,
testcase.line))
def test_python_cmdline(testcase: DataDrivenTestCase) -> None:
assert testcase.old_cwd is not None, "test was not properly set up"
# Write the program to a file.
program = '_program.py'
program_path = os.path.join(test_temp_dir, program)
with open(program_path, 'w') as file:
for s in testcase.input:
file.write('{}\n'.format(s))
args = parse_args(testcase.input[0])
args.append('--show-traceback')
# Type check the program.
fixed = [python3_path,
os.path.join(testcase.old_cwd, 'scripts', 'mypy')]
process = subprocess.Popen(fixed + args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=test_temp_dir)
outb = process.stdout.read()
# Split output into lines.
out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
result = process.wait()
# Remove temp file.
os.remove(program_path)
# Compare actual output to expected.
if testcase.output_files:
for path, expected_content in testcase.output_files:
if not os.path.exists(path):
raise AssertionError(
'Expected file {} was not produced by test case'.format(path))
with open(path, 'r') as output_file:
actual_output_content = output_file.read().splitlines()
normalized_output = normalize_file_output(actual_output_content,
os.path.abspath(test_temp_dir))
if testcase.native_sep and os.path.sep == '\\':
normalized_output = [fix_cobertura_filename(line) for line in normalized_output]
normalized_output = normalize_error_messages(normalized_output)
assert_string_arrays_equal(expected_content.splitlines(), normalized_output,
'Output file {} did not match its expected output'.format(
path))
else:
out = normalize_error_messages(out)
obvious_result = 1 if out else 0
if obvious_result != result:
out.append('== Return code: {}'.format(result))
assert_string_arrays_equal(testcase.output, out,
'Invalid output ({}, line {})'.format(
testcase.file, testcase.line))
def test_python_evaluation(testcase: DataDrivenTestCase) -> None:
# Write the program to a file.
program = '_program.py'
program_path = os.path.join(test_temp_dir, program)
with open(program_path, 'w') as file:
for s in testcase.input:
file.write('{}\n'.format(s))
args = parse_args(testcase.input[0])
args.append('--show-traceback')
# Type check the program.
fixed = [python3_path, os.path.join(testcase.old_cwd, 'scripts', 'mypy')]
process = subprocess.Popen(fixed + args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=test_temp_dir)
outb = process.stdout.read()
# Split output into lines.
out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
# Remove temp file.
os.remove(program_path)
# Compare actual output to expected.
if testcase.output_files:
for path, expected_content in testcase.output_files:
if not os.path.exists(path):
raise AssertionFailure(
'Expected file {} was not produced by test case'.format(
path))
with open(path, 'r') as output_file:
actual_output_content = output_file.read().splitlines()
normalized_output = normalize_file_output(
actual_output_content, os.path.abspath(test_temp_dir))
if testcase.native_sep and os.path.sep == '\\':
normalized_output = [
fix_cobertura_filename(line) for line in normalized_output
]
normalized_output = normalize_error_messages(normalized_output)
assert_string_arrays_equal(
expected_content.splitlines(), normalized_output,
'Output file {} did not match its expected output'.format(
path))
else:
out = normalize_error_messages(out)
assert_string_arrays_equal(
testcase.output, out,
'Invalid output ({}, line {})'.format(testcase.file,
testcase.line))
def test_python_cmdline(testcase: DataDrivenTestCase) -> None:
assert testcase.old_cwd is not None, "test was not properly set up"
# Write the program to a file.
program = '_program.py'
program_path = os.path.join(test_temp_dir, program)
with open(program_path, 'w') as file:
for s in testcase.input:
file.write('{}\n'.format(s))
args = parse_args(testcase.input[0])
args.append('--show-traceback')
args.append('--no-site-packages')
# Type check the program.
fixed = [python3_path, '-m', 'mypy']
env = os.environ.copy()
env['PYTHONPATH'] = PREFIX
process = subprocess.Popen(fixed + args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=test_temp_dir,
env=env)
outb, errb = process.communicate()
result = process.returncode
# Split output into lines.
out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
err = [s.rstrip('\n\r') for s in str(errb, 'utf8').splitlines()]
if "PYCHARM_HOSTED" in os.environ:
for pos, line in enumerate(err):
if line.startswith('pydev debugger: '):
# Delete the attaching debugger message itself, plus the extra newline added.
del err[pos:pos + 2]
break
# Remove temp file.
os.remove(program_path)
# Compare actual output to expected.
if testcase.output_files:
# Ignore stdout, but we insist on empty stderr and zero status.
if err or result:
raise AssertionError(
'Expected zero status and empty stderr, got %d and\n%s' %
(result, '\n'.join(err + out)))
for path, expected_content in testcase.output_files:
if not os.path.exists(path):
raise AssertionError(
'Expected file {} was not produced by test case'.format(
path))
with open(path, 'r') as output_file:
actual_output_content = output_file.read().splitlines()
normalized_output = normalize_file_output(
actual_output_content, os.path.abspath(test_temp_dir))
# We always normalize things like timestamp, but only handle operating-system
# specific things if requested.
if testcase.normalize_output:
if testcase.suite.native_sep and os.path.sep == '\\':
normalized_output = [
fix_cobertura_filename(line)
for line in normalized_output
]
normalized_output = normalize_error_messages(normalized_output)
assert_string_arrays_equal(
expected_content.splitlines(), normalized_output,
'Output file {} did not match its expected output'.format(
path))
else:
if testcase.normalize_output:
out = normalize_error_messages(err + out)
obvious_result = 1 if out else 0
if obvious_result != result:
out.append('== Return code: {}'.format(result))
assert_string_arrays_equal(
testcase.output, out,
'Invalid output ({}, line {})'.format(testcase.file,
testcase.line))
def test_python_cmdline(testcase: DataDrivenTestCase, step: int) -> None:
assert testcase.old_cwd is not None, "test was not properly set up"
# Write the program to a file.
program = '_program.py'
program_path = os.path.join(test_temp_dir, program)
with open(program_path, 'w', encoding='utf8') as file:
for s in testcase.input:
file.write('{}\n'.format(s))
args = parse_args(testcase.input[0])
args.append('--show-traceback')
args.append('--no-site-packages')
# Type check the program.
fixed = [python3_path, '-m', 'mypy']
env = os.environ.copy()
env['PYTHONPATH'] = PREFIX
process = subprocess.Popen(fixed + args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=test_temp_dir,
env=env)
outb, errb = process.communicate()
result = process.returncode
# Split output into lines.
out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
err = [s.rstrip('\n\r') for s in str(errb, 'utf8').splitlines()]
if "PYCHARM_HOSTED" in os.environ:
for pos, line in enumerate(err):
if line.startswith('pydev debugger: '):
# Delete the attaching debugger message itself, plus the extra newline added.
del err[pos:pos + 2]
break
# Remove temp file.
os.remove(program_path)
# Compare actual output to expected.
if testcase.output_files:
# Ignore stdout, but we insist on empty stderr and zero status.
if err or result:
raise AssertionError(
'Expected zero status and empty stderr%s, got %d and\n%s' %
(' on step %d' % step if testcase.output2 else '',
result, '\n'.join(err + out)))
for path, expected_content in testcase.output_files:
if not os.path.exists(path):
raise AssertionError(
'Expected file {} was not produced by test case{}'.format(
path, ' on step %d' % step if testcase.output2 else ''))
with open(path, 'r', encoding='utf8') as output_file:
actual_output_content = output_file.read().splitlines()
normalized_output = normalize_file_output(actual_output_content,
os.path.abspath(test_temp_dir))
# We always normalize things like timestamp, but only handle operating-system
# specific things if requested.
if testcase.normalize_output:
if testcase.suite.native_sep and os.path.sep == '\\':
normalized_output = [fix_cobertura_filename(line)
for line in normalized_output]
normalized_output = normalize_error_messages(normalized_output)
assert_string_arrays_equal(expected_content.splitlines(), normalized_output,
'Output file {} did not match its expected output{}'.format(
path, ' on step %d' % step if testcase.output2 else ''))
else:
if testcase.normalize_output:
out = normalize_error_messages(err + out)
obvious_result = 1 if out else 0
if obvious_result != result:
out.append('== Return code: {}'.format(result))
expected_out = testcase.output if step == 1 else testcase.output2[step]
assert_string_arrays_equal(expected_out, out,
'Invalid output ({}, line {}){}'.format(
testcase.file, testcase.line,
' on step %d' % step if testcase.output2 else ''))
