def check_test_output_files(testcase: DataDrivenTestCase, step: int, strip_prefix: str = '') -> None: for path, expected_content in testcase.output_files: if path.startswith(strip_prefix): path = path[len(strip_prefix):] if not os.path.exists(path): raise AssertionError( 'Expected file {} was not produced by test case{}'.format( path, ' on step %d' % step if testcase.output2 else '')) with open(path, 'r', encoding='utf8') as output_file: actual_output_content = output_file.read().splitlines() normalized_output = normalize_file_output( actual_output_content, os.path.abspath(test_temp_dir)) # We always normalize things like timestamp, but only handle operating-system # specific things if requested. if testcase.normalize_output: if testcase.suite.native_sep and os.path.sep == '\\': normalized_output = [ fix_cobertura_filename(line) for line in normalized_output ] normalized_output = normalize_error_messages(normalized_output) assert_string_arrays_equal( expected_content.splitlines(), normalized_output, 'Output file {} did not match its expected output{}'.format( path, ' on step %d' % step if testcase.output2 else ''))
def test_python_cmdline(testcase: DataDrivenTestCase) -> None: assert testcase.old_cwd is not None, "test was not properly set up" # Write the program to a file. program = '_program.py' program_path = os.path.join(test_temp_dir, program) with open(program_path, 'w') as file: for s in testcase.input: file.write('{}\n'.format(s)) args = parse_args(testcase.input[0]) args.append('--show-traceback') args.append('--no-site-packages') # Type check the program. fixed = [python3_path, '-m', 'mypy'] env = os.environ.copy() env['PYTHONPATH'] = PREFIX process = subprocess.Popen(fixed + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=test_temp_dir, env=env) outb = process.stdout.read() # Split output into lines. out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()] result = process.wait() # Remove temp file. os.remove(program_path) # Compare actual output to expected. if testcase.output_files: for path, expected_content in testcase.output_files: if not os.path.exists(path): raise AssertionError( 'Expected file {} was not produced by test case'.format( path)) with open(path, 'r') as output_file: actual_output_content = output_file.read().splitlines() normalized_output = normalize_file_output( actual_output_content, os.path.abspath(test_temp_dir)) if testcase.native_sep and os.path.sep == '\\': normalized_output = [ fix_cobertura_filename(line) for line in normalized_output ] normalized_output = normalize_error_messages(normalized_output) assert_string_arrays_equal( expected_content.splitlines(), normalized_output, 'Output file {} did not match its expected output'.format( path)) else: out = normalize_error_messages(out) obvious_result = 1 if out else 0 if obvious_result != result: out.append('== Return code: {}'.format(result)) assert_string_arrays_equal( testcase.output, out, 'Invalid output ({}, line {})'.format(testcase.file, testcase.line))
def test_python_cmdline(testcase: DataDrivenTestCase) -> None: assert testcase.old_cwd is not None, "test was not properly set up" # Write the program to a file. program = '_program.py' program_path = os.path.join(test_temp_dir, program) with open(program_path, 'w') as file: for s in testcase.input: file.write('{}\n'.format(s)) args = parse_args(testcase.input[0]) args.append('--show-traceback') # Type check the program. fixed = [python3_path, os.path.join(testcase.old_cwd, 'scripts', 'mypy')] process = subprocess.Popen(fixed + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=test_temp_dir) outb = process.stdout.read() # Split output into lines. out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()] result = process.wait() # Remove temp file. os.remove(program_path) # Compare actual output to expected. if testcase.output_files: for path, expected_content in testcase.output_files: if not os.path.exists(path): raise AssertionError( 'Expected file {} was not produced by test case'.format(path)) with open(path, 'r') as output_file: actual_output_content = output_file.read().splitlines() normalized_output = normalize_file_output(actual_output_content, os.path.abspath(test_temp_dir)) if testcase.native_sep and os.path.sep == '\\': normalized_output = [fix_cobertura_filename(line) for line in normalized_output] normalized_output = normalize_error_messages(normalized_output) assert_string_arrays_equal(expected_content.splitlines(), normalized_output, 'Output file {} did not match its expected output'.format( path)) else: out = normalize_error_messages(out) obvious_result = 1 if out else 0 if obvious_result != result: out.append('== Return code: {}'.format(result)) assert_string_arrays_equal(testcase.output, out, 'Invalid output ({}, line {})'.format( testcase.file, testcase.line))
def test_python_evaluation(testcase: DataDrivenTestCase) -> None: # Write the program to a file. program = '_program.py' program_path = os.path.join(test_temp_dir, program) with open(program_path, 'w') as file: for s in testcase.input: file.write('{}\n'.format(s)) args = parse_args(testcase.input[0]) args.append('--show-traceback') # Type check the program. fixed = [python3_path, os.path.join(testcase.old_cwd, 'scripts', 'mypy')] process = subprocess.Popen(fixed + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=test_temp_dir) outb = process.stdout.read() # Split output into lines. out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()] # Remove temp file. os.remove(program_path) # Compare actual output to expected. if testcase.output_files: for path, expected_content in testcase.output_files: if not os.path.exists(path): raise AssertionFailure( 'Expected file {} was not produced by test case'.format( path)) with open(path, 'r') as output_file: actual_output_content = output_file.read().splitlines() normalized_output = normalize_file_output( actual_output_content, os.path.abspath(test_temp_dir)) if testcase.native_sep and os.path.sep == '\\': normalized_output = [ fix_cobertura_filename(line) for line in normalized_output ] normalized_output = normalize_error_messages(normalized_output) assert_string_arrays_equal( expected_content.splitlines(), normalized_output, 'Output file {} did not match its expected output'.format( path)) else: out = normalize_error_messages(out) assert_string_arrays_equal( testcase.output, out, 'Invalid output ({}, line {})'.format(testcase.file, testcase.line))
def test_python_cmdline(testcase: DataDrivenTestCase) -> None: assert testcase.old_cwd is not None, "test was not properly set up" # Write the program to a file. program = '_program.py' program_path = os.path.join(test_temp_dir, program) with open(program_path, 'w') as file: for s in testcase.input: file.write('{}\n'.format(s)) args = parse_args(testcase.input[0]) args.append('--show-traceback') args.append('--no-site-packages') # Type check the program. fixed = [python3_path, '-m', 'mypy'] env = os.environ.copy() env['PYTHONPATH'] = PREFIX process = subprocess.Popen(fixed + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=test_temp_dir, env=env) outb, errb = process.communicate() result = process.returncode # Split output into lines. out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()] err = [s.rstrip('\n\r') for s in str(errb, 'utf8').splitlines()] if "PYCHARM_HOSTED" in os.environ: for pos, line in enumerate(err): if line.startswith('pydev debugger: '): # Delete the attaching debugger message itself, plus the extra newline added. del err[pos:pos + 2] break # Remove temp file. os.remove(program_path) # Compare actual output to expected. if testcase.output_files: # Ignore stdout, but we insist on empty stderr and zero status. if err or result: raise AssertionError( 'Expected zero status and empty stderr, got %d and\n%s' % (result, '\n'.join(err + out))) for path, expected_content in testcase.output_files: if not os.path.exists(path): raise AssertionError( 'Expected file {} was not produced by test case'.format( path)) with open(path, 'r') as output_file: actual_output_content = output_file.read().splitlines() normalized_output = normalize_file_output( actual_output_content, os.path.abspath(test_temp_dir)) # We always normalize things like timestamp, but only handle operating-system # specific things if requested. if testcase.normalize_output: if testcase.suite.native_sep and os.path.sep == '\\': normalized_output = [ fix_cobertura_filename(line) for line in normalized_output ] normalized_output = normalize_error_messages(normalized_output) assert_string_arrays_equal( expected_content.splitlines(), normalized_output, 'Output file {} did not match its expected output'.format( path)) else: if testcase.normalize_output: out = normalize_error_messages(err + out) obvious_result = 1 if out else 0 if obvious_result != result: out.append('== Return code: {}'.format(result)) assert_string_arrays_equal( testcase.output, out, 'Invalid output ({}, line {})'.format(testcase.file, testcase.line))
def test_python_cmdline(testcase: DataDrivenTestCase, step: int) -> None: assert testcase.old_cwd is not None, "test was not properly set up" # Write the program to a file. program = '_program.py' program_path = os.path.join(test_temp_dir, program) with open(program_path, 'w', encoding='utf8') as file: for s in testcase.input: file.write('{}\n'.format(s)) args = parse_args(testcase.input[0]) args.append('--show-traceback') args.append('--no-site-packages') # Type check the program. fixed = [python3_path, '-m', 'mypy'] env = os.environ.copy() env['PYTHONPATH'] = PREFIX process = subprocess.Popen(fixed + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=test_temp_dir, env=env) outb, errb = process.communicate() result = process.returncode # Split output into lines. out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()] err = [s.rstrip('\n\r') for s in str(errb, 'utf8').splitlines()] if "PYCHARM_HOSTED" in os.environ: for pos, line in enumerate(err): if line.startswith('pydev debugger: '): # Delete the attaching debugger message itself, plus the extra newline added. del err[pos:pos + 2] break # Remove temp file. os.remove(program_path) # Compare actual output to expected. if testcase.output_files: # Ignore stdout, but we insist on empty stderr and zero status. if err or result: raise AssertionError( 'Expected zero status and empty stderr%s, got %d and\n%s' % (' on step %d' % step if testcase.output2 else '', result, '\n'.join(err + out))) for path, expected_content in testcase.output_files: if not os.path.exists(path): raise AssertionError( 'Expected file {} was not produced by test case{}'.format( path, ' on step %d' % step if testcase.output2 else '')) with open(path, 'r', encoding='utf8') as output_file: actual_output_content = output_file.read().splitlines() normalized_output = normalize_file_output(actual_output_content, os.path.abspath(test_temp_dir)) # We always normalize things like timestamp, but only handle operating-system # specific things if requested. if testcase.normalize_output: if testcase.suite.native_sep and os.path.sep == '\\': normalized_output = [fix_cobertura_filename(line) for line in normalized_output] normalized_output = normalize_error_messages(normalized_output) assert_string_arrays_equal(expected_content.splitlines(), normalized_output, 'Output file {} did not match its expected output{}'.format( path, ' on step %d' % step if testcase.output2 else '')) else: if testcase.normalize_output: out = normalize_error_messages(err + out) obvious_result = 1 if out else 0 if obvious_result != result: out.append('== Return code: {}'.format(result)) expected_out = testcase.output if step == 1 else testcase.output2[step] assert_string_arrays_equal(expected_out, out, 'Invalid output ({}, line {}){}'.format( testcase.file, testcase.line, ' on step %d' % step if testcase.output2 else ''))