def test_python_cmdline(testcase: DataDrivenTestCase, step: int) -> None: assert testcase.old_cwd is not None, "test was not properly set up" # Write the program to a file. program = '_program.py' program_path = os.path.join(test_temp_dir, program) with open(program_path, 'w', encoding='utf8') as file: for s in testcase.input: file.write('{}\n'.format(s)) args = parse_args(testcase.input[0]) args.append('--show-traceback') args.append('--no-site-packages') if '--error-summary' not in args: args.append('--no-error-summary') # Type check the program. fixed = [python3_path, '-m', 'mypy'] env = os.environ.copy() env['PYTHONPATH'] = PREFIX process = subprocess.Popen(fixed + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=test_temp_dir, env=env) outb, errb = process.communicate() result = process.returncode # Split output into lines. out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()] err = [s.rstrip('\n\r') for s in str(errb, 'utf8').splitlines()] if "PYCHARM_HOSTED" in os.environ: for pos, line in enumerate(err): if line.startswith('pydev debugger: '): # Delete the attaching debugger message itself, plus the extra newline added. del err[pos:pos + 2] break # Remove temp file. os.remove(program_path) # Compare actual output to expected. if testcase.output_files: # Ignore stdout, but we insist on empty stderr and zero status. if err or result: raise AssertionError( 'Expected zero status and empty stderr%s, got %d and\n%s' % (' on step %d' % step if testcase.output2 else '', result, '\n'.join(err + out))) check_test_output_files(testcase, step) else: if testcase.normalize_output: out = normalize_error_messages(err + out) obvious_result = 1 if out else 0 if obvious_result != result: out.append('== Return code: {}'.format(result)) expected_out = testcase.output if step == 1 else testcase.output2[step] # Strip "tmp/" out of the test so that # E: works... expected_out = [s.replace("tmp" + os.sep, "") for s in expected_out] assert_string_arrays_equal( expected_out, out, 'Invalid output ({}, line {}){}'.format( testcase.file, testcase.line, ' on step %d' % step if testcase.output2 else ''))
def run_case_once(self, testcase: DataDrivenTestCase, operations: List[FileOperation] = [], incremental_step: int = 0) -> None: original_program_text = '\n'.join(testcase.input) module_data = self.parse_module(original_program_text, incremental_step) # Unload already loaded plugins, they may be updated. for file, _ in testcase.files: module = module_from_path(file) if module.endswith('_plugin') and module in sys.modules: del sys.modules[module] if incremental_step == 0 or incremental_step == 1: # In run 1, copy program text to program file. for module_name, program_path, program_text in module_data: if module_name == '__main__': with open(program_path, 'w', encoding='utf8') as f: f.write(program_text) break elif incremental_step > 1: # In runs 2+, copy *.[num] files to * files. for op in operations: if isinstance(op, UpdateFile): # Modify/create file copy_and_fudge_mtime(op.source_path, op.target_path) else: # Delete file # Use retries to work around potential flakiness on Windows (AppVeyor). path = op.path retry_on_error(lambda: os.remove(path)) # Parse options after moving files (in case mypy.ini is being moved). options = parse_options(original_program_text, testcase, incremental_step) options.use_builtins_fixtures = True options.show_traceback = True # Enable some options automatically based on test file name. if 'optional' in testcase.file: options.strict_optional = True if 'columns' in testcase.file: options.show_column_numbers = True if 'errorcodes' in testcase.file: options.show_error_codes = True if incremental_step and options.incremental: # Don't overwrite # flags: --no-incremental in incremental test cases options.incremental = True else: options.incremental = False # Don't waste time writing cache unless we are specifically looking for it if not testcase.writescache: options.cache_dir = os.devnull sources = [] for module_name, program_path, program_text in module_data: # Always set to none so we're forced to reread the module in incremental mode sources.append( BuildSource(program_path, module_name, None if incremental_step else program_text)) plugin_dir = os.path.join(test_data_prefix, 'plugins') sys.path.insert(0, plugin_dir) res = None try: res = build.build(sources=sources, options=options, alt_lib_path=test_temp_dir) a = res.errors except CompileError as e: a = e.messages finally: assert sys.path[0] == plugin_dir del sys.path[0] if testcase.normalize_output: a = normalize_error_messages(a) # Make sure error messages match if incremental_step == 0: # Not incremental msg = 'Unexpected type checker output ({}, line {})' output = testcase.output elif incremental_step == 1: msg = 'Unexpected type checker output in incremental, run 1 ({}, line {})' output = testcase.output elif incremental_step > 1: msg = ('Unexpected type checker output in incremental, run {}'. format(incremental_step) + ' ({}, line {})') output = testcase.output2.get(incremental_step, []) else: raise AssertionError() if output != a and testcase.config.getoption('--update-data', False): update_testcase_output(testcase, a) assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line)) if res: if options.cache_dir != os.devnull: self.verify_cache(module_data, res.errors, res.manager, res.graph) name = 'targets' if incremental_step: name += str(incremental_step + 1) expected = testcase.expected_fine_grained_targets.get( incremental_step + 1) actual = res.manager.processed_targets # Skip the initial builtin cycle. actual = [ t for t in actual if not any( t.startswith(mod) for mod in core_modules + ['mypy_extensions']) ] if expected is not None: assert_target_equivalence(name, expected, actual) if incremental_step > 1: suffix = '' if incremental_step == 2 else str( incremental_step - 1) expected_rechecked = testcase.expected_rechecked_modules.get( incremental_step - 1) if expected_rechecked is not None: assert_module_equivalence('rechecked' + suffix, expected_rechecked, res.manager.rechecked_modules) expected_stale = testcase.expected_stale_modules.get( incremental_step - 1) if expected_stale is not None: assert_module_equivalence('stale' + suffix, expected_stale, res.manager.stale_modules) if testcase.output_files: check_test_output_files(testcase, incremental_step, strip_prefix='tmp/')