Esempio n. 1
0
def test_python_cmdline(testcase: DataDrivenTestCase) -> None:
    assert testcase.old_cwd is not None, "test was not properly set up"
    # Write the program to a file.
    program = '_program.py'
    program_path = os.path.join(test_temp_dir, program)
    with open(program_path, 'w') as file:
        for s in testcase.input:
            file.write('{}\n'.format(s))
    args = parse_args(testcase.input[0])
    args.append('--show-traceback')
    args.append('--no-site-packages')
    # Type check the program.
    fixed = [python3_path, '-m', 'mypy']
    env = os.environ.copy()
    env['PYTHONPATH'] = PREFIX
    process = subprocess.Popen(fixed + args,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.STDOUT,
                               cwd=test_temp_dir,
                               env=env)
    outb = process.stdout.read()
    # Split output into lines.
    out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
    result = process.wait()
    # Remove temp file.
    os.remove(program_path)
    # Compare actual output to expected.
    if testcase.output_files:
        for path, expected_content in testcase.output_files:
            if not os.path.exists(path):
                raise AssertionError(
                    'Expected file {} was not produced by test case'.format(
                        path))
            with open(path, 'r') as output_file:
                actual_output_content = output_file.read().splitlines()
            normalized_output = normalize_file_output(
                actual_output_content, os.path.abspath(test_temp_dir))
            if testcase.native_sep and os.path.sep == '\\':
                normalized_output = [
                    fix_cobertura_filename(line) for line in normalized_output
                ]
            normalized_output = normalize_error_messages(normalized_output)
            assert_string_arrays_equal(
                expected_content.splitlines(), normalized_output,
                'Output file {} did not match its expected output'.format(
                    path))
    else:
        out = normalize_error_messages(out)
        obvious_result = 1 if out else 0
        if obvious_result != result:
            out.append('== Return code: {}'.format(result))
        assert_string_arrays_equal(
            testcase.output, out,
            'Invalid output ({}, line {})'.format(testcase.file,
                                                  testcase.line))
Esempio n. 2
0
def test_python_cmdline(testcase: DataDrivenTestCase) -> None:
    assert testcase.old_cwd is not None, "test was not properly set up"
    # Write the program to a file.
    program = '_program.py'
    program_path = os.path.join(test_temp_dir, program)
    with open(program_path, 'w') as file:
        for s in testcase.input:
            file.write('{}\n'.format(s))
    args = parse_args(testcase.input[0])
    args.append('--show-traceback')
    # Type check the program.
    fixed = [python3_path,
             os.path.join(testcase.old_cwd, 'scripts', 'mypy')]
    process = subprocess.Popen(fixed + args,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.STDOUT,
                               cwd=test_temp_dir)
    outb = process.stdout.read()
    # Split output into lines.
    out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
    result = process.wait()
    # Remove temp file.
    os.remove(program_path)
    # Compare actual output to expected.
    if testcase.output_files:
        for path, expected_content in testcase.output_files:
            if not os.path.exists(path):
                raise AssertionError(
                    'Expected file {} was not produced by test case'.format(path))
            with open(path, 'r') as output_file:
                actual_output_content = output_file.read().splitlines()
            normalized_output = normalize_file_output(actual_output_content,
                                                      os.path.abspath(test_temp_dir))
            if testcase.native_sep and os.path.sep == '\\':
                normalized_output = [fix_cobertura_filename(line) for line in normalized_output]
            normalized_output = normalize_error_messages(normalized_output)
            assert_string_arrays_equal(expected_content.splitlines(), normalized_output,
                                       'Output file {} did not match its expected output'.format(
                                           path))
    else:
        out = normalize_error_messages(out)
        obvious_result = 1 if out else 0
        if obvious_result != result:
            out.append('== Return code: {}'.format(result))
        assert_string_arrays_equal(testcase.output, out,
                                   'Invalid output ({}, line {})'.format(
                                       testcase.file, testcase.line))
Esempio n. 3
0
def test_python_cmdline(testcase: DataDrivenTestCase, step: int) -> None:
    assert testcase.old_cwd is not None, "test was not properly set up"
    # Write the program to a file.
    program = '_program.py'
    program_path = os.path.join(test_temp_dir, program)
    with open(program_path, 'w', encoding='utf8') as file:
        for s in testcase.input:
            file.write('{}\n'.format(s))
    args = parse_args(testcase.input[0])
    args.append('--show-traceback')
    args.append('--no-site-packages')
    if '--error-summary' not in args:
        args.append('--no-error-summary')
    # Type check the program.
    fixed = [python3_path, '-m', 'mypy']
    env = os.environ.copy()
    env['PYTHONPATH'] = PREFIX
    process = subprocess.Popen(fixed + args,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               cwd=test_temp_dir,
                               env=env)
    outb, errb = process.communicate()
    result = process.returncode
    # Split output into lines.
    out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
    err = [s.rstrip('\n\r') for s in str(errb, 'utf8').splitlines()]

    if "PYCHARM_HOSTED" in os.environ:
        for pos, line in enumerate(err):
            if line.startswith('pydev debugger: '):
                # Delete the attaching debugger message itself, plus the extra newline added.
                del err[pos:pos + 2]
                break

    # Remove temp file.
    os.remove(program_path)
    # Compare actual output to expected.
    if testcase.output_files:
        # Ignore stdout, but we insist on empty stderr and zero status.
        if err or result:
            raise AssertionError(
                'Expected zero status and empty stderr%s, got %d and\n%s' %
                (' on step %d' % step if testcase.output2 else '', result,
                 '\n'.join(err + out)))
        check_test_output_files(testcase, step)
    else:
        if testcase.normalize_output:
            out = normalize_error_messages(err + out)
        obvious_result = 1 if out else 0
        if obvious_result != result:
            out.append('== Return code: {}'.format(result))
        expected_out = testcase.output if step == 1 else testcase.output2[step]
        # Strip "tmp/" out of the test so that # E: works...
        expected_out = [s.replace("tmp" + os.sep, "") for s in expected_out]
        assert_string_arrays_equal(
            expected_out, out, 'Invalid output ({}, line {}){}'.format(
                testcase.file, testcase.line,
                ' on step %d' % step if testcase.output2 else ''))
Esempio n. 4
0
def test_python_evaluation(testcase: DataDrivenTestCase) -> None:
    # Write the program to a file.
    program = '_program.py'
    program_path = os.path.join(test_temp_dir, program)
    with open(program_path, 'w') as file:
        for s in testcase.input:
            file.write('{}\n'.format(s))
    args = parse_args(testcase.input[0])
    args.append('--show-traceback')
    # Type check the program.
    fixed = [python3_path, os.path.join(testcase.old_cwd, 'scripts', 'mypy')]
    process = subprocess.Popen(fixed + args,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.STDOUT,
                               cwd=test_temp_dir)
    outb = process.stdout.read()
    # Split output into lines.
    out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
    # Remove temp file.
    os.remove(program_path)
    # Compare actual output to expected.
    if testcase.output_files:
        for path, expected_content in testcase.output_files:
            if not os.path.exists(path):
                raise AssertionFailure(
                    'Expected file {} was not produced by test case'.format(
                        path))
            with open(path, 'r') as output_file:
                actual_output_content = output_file.read().splitlines()
            normalized_output = normalize_file_output(
                actual_output_content, os.path.abspath(test_temp_dir))
            if testcase.native_sep and os.path.sep == '\\':
                normalized_output = [
                    fix_cobertura_filename(line) for line in normalized_output
                ]
            normalized_output = normalize_error_messages(normalized_output)
            assert_string_arrays_equal(
                expected_content.splitlines(), normalized_output,
                'Output file {} did not match its expected output'.format(
                    path))
    else:
        out = normalize_error_messages(out)
        assert_string_arrays_equal(
            testcase.output, out,
            'Invalid output ({}, line {})'.format(testcase.file,
                                                  testcase.line))
Esempio n. 5
0
    def run_case_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
        find_module_clear_caches()
        program_text = '\n'.join(testcase.input)
        module_name, program_name, program_text = self.parse_module(program_text)

        options = self.parse_options(program_text)
        options.use_builtins_fixtures = True
        options.python_version = testcase_pyversion(testcase.file, testcase.name)

        output = testcase.output
        if incremental:
            options.incremental = True
            if incremental == 1:
                # In run 1, copy program text to program file.
                output = []
                with open(program_name, 'w') as f:
                    f.write(program_text)
            elif incremental == 2:
                # In run 2, copy *.py.next files to *.py files.
                for dn, dirs, files in os.walk(os.curdir):
                    for file in files:
                        if file.endswith('.py.next'):
                            full = os.path.join(dn, file)
                            target = full[:-5]
                            shutil.copy(full, target)
            # Always set to none so we're forced to reread program_name
            program_text = None
        source = BuildSource(program_name, module_name, program_text)
        try:
            res = build.build(sources=[source],
                              options=options,
                              alt_lib_path=test_temp_dir)
            a = res.errors
        except CompileError as e:
            res = None
            a = e.messages
        a = normalize_error_messages(a)

        if output != a and self.update_data:
            update_testcase_output(testcase, a)

        assert_string_arrays_equal(
            output, a,
            'Invalid type checker output ({}, line {})'.format(
                testcase.file, testcase.line))

        if incremental and res:
            self.verify_cache(module_name, program_name, a, res.manager)
            if testcase.expected_stale_modules is not None and incremental == 2:
                assert_string_arrays_equal(
                    list(sorted(testcase.expected_stale_modules)),
                    list(sorted(res.manager.stale_modules.difference({"__main__"}))),
                    'Set of stale modules does not match expected set')
Esempio n. 6
0
    def run_test_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
        find_module_clear_caches()
        program_text = '\n'.join(testcase.input)
        module_name, program_name, program_text = self.parse_module(program_text)

        options = self.parse_options(program_text)
        options.use_builtins_fixtures = True
        options.python_version = testcase_pyversion(testcase.file, testcase.name)

        output = testcase.output
        if incremental:
            options.incremental = True
            if incremental == 1:
                # In run 1, copy program text to program file.
                output = []
                with open(program_name, 'w') as f:
                    f.write(program_text)
                    program_text = None
            elif incremental == 2:
                # In run 2, copy *.py.next files to *.py files.
                for dn, dirs, files in os.walk(os.curdir):
                    for file in files:
                        if file.endswith('.py.next'):
                            full = os.path.join(dn, file)
                            target = full[:-5]
                            shutil.copy(full, target)
        source = BuildSource(program_name, module_name, program_text)
        try:
            res = build.build(sources=[source],
                              options=options,
                              alt_lib_path=test_temp_dir)
            a = res.errors
        except CompileError as e:
            res = None
            a = e.messages
        a = normalize_error_messages(a)

        if output != a and mypy.myunit.UPDATE_TESTCASES:
            update_testcase_output(testcase, a, mypy.myunit.APPEND_TESTCASES)

        assert_string_arrays_equal(
            output, a,
            'Invalid type checker output ({}, line {})'.format(
                testcase.file, testcase.line))

        if incremental and res:
            self.verify_cache(module_name, program_name, a, res.manager)
            if testcase.expected_stale_modules is not None and incremental == 2:
                assert_string_arrays_equal(
                    list(sorted(testcase.expected_stale_modules)),
                    list(sorted(res.manager.stale_modules.difference({"__main__"}))),
                    'Set of stale modules does not match expected set')
Esempio n. 7
0
    def run_case(self, testcase: DataDrivenTestCase) -> None:
        name = testcase.name
        # We use the test case name to decide which data structures to dump.
        # Dumping everything would result in very verbose test cases.
        if name.endswith('_symtable'):
            kind = SYMTABLE
        elif name.endswith('_typeinfo'):
            kind = TYPEINFO
        elif name.endswith('_types'):
            kind = TYPES
        else:
            kind = AST

        main_src = '\n'.join(testcase.input)
        result = self.build(main_src)
        assert result is not None, 'cases where CompileError occurred should not be run'
        result.manager.fscache.flush()
        fine_grained_manager = FineGrainedBuildManager(result)

        a = []
        if result.errors:
            a.extend(result.errors)

        target_path = os.path.join(test_temp_dir, 'target.py')
        shutil.copy(os.path.join(test_temp_dir, 'target.py.next'), target_path)

        a.extend(self.dump(fine_grained_manager, kind))
        old_subexpr = get_subexpressions(result.manager.modules['target'])

        a.append('==>')

        new_file, new_types = self.build_increment(fine_grained_manager,
                                                   'target', target_path)
        a.extend(self.dump(fine_grained_manager, kind))

        for expr in old_subexpr:
            if isinstance(expr, TypeVarExpr):
                # These are merged so we can't perform the check.
                continue
            # Verify that old AST nodes are removed from the expression type map.
            assert expr not in new_types

        if testcase.normalize_output:
            a = normalize_error_messages(a)

        assert_string_arrays_equal(
            testcase.output, a,
            'Invalid output ({}, line {})'.format(testcase.file,
                                                  testcase.line))
Esempio n. 8
0
    def run_case(self, testcase: DataDrivenTestCase) -> None:
        name = testcase.name
        # We use the test case name to decide which data structures to dump.
        # Dumping everything would result in very verbose test cases.
        if name.endswith('_symtable'):
            kind = SYMTABLE
        elif name.endswith('_typeinfo'):
            kind = TYPEINFO
        elif name.endswith('_types'):
            kind = TYPES
        else:
            kind = AST

        main_src = '\n'.join(testcase.input)
        result = self.build(main_src)
        assert result is not None, 'cases where CompileError occurred should not be run'
        result.manager.fscache.flush()
        fine_grained_manager = FineGrainedBuildManager(result)

        a = []
        if result.errors:
            a.extend(result.errors)

        target_path = os.path.join(test_temp_dir, 'target.py')
        shutil.copy(os.path.join(test_temp_dir, 'target.py.next'), target_path)

        a.extend(self.dump(fine_grained_manager, kind))
        old_subexpr = get_subexpressions(result.manager.modules['target'])

        a.append('==>')

        new_file, new_types = self.build_increment(fine_grained_manager, 'target', target_path)
        a.extend(self.dump(fine_grained_manager, kind))

        for expr in old_subexpr:
            if isinstance(expr, TypeVarExpr):
                # These are merged so we can't perform the check.
                continue
            # Verify that old AST nodes are removed from the expression type map.
            assert expr not in new_types

        if testcase.normalize_output:
            a = normalize_error_messages(a)

        assert_string_arrays_equal(
            testcase.output, a,
            'Invalid output ({}, line {})'.format(testcase.file,
                                                  testcase.line))
Esempio n. 9
0
    def run_test_once(self,
                      testcase: DataDrivenTestCase,
                      incremental=0) -> None:
        find_module_clear_caches()
        pyversion = testcase_pyversion(testcase.file, testcase.name)
        program_text = '\n'.join(testcase.input)
        module_name, program_name, program_text = self.parse_options(
            program_text)
        flags = self.parse_flags(program_text)
        output = testcase.output
        if incremental:
            flags.append(build.INCREMENTAL)
            if incremental == 1:
                # In run 1, copy program text to program file.
                output = []
                with open(program_name, 'w') as f:
                    f.write(program_text)
                    program_text = None
            elif incremental == 2:
                # In run 2, copy *.py.next files to *.py files.
                for dn, dirs, files in os.walk(os.curdir):
                    for file in files:
                        if file.endswith('.py.next'):
                            full = os.path.join(dn, file)
                            target = full[:-5]
                            shutil.copy(full, target)
        source = BuildSource(program_name, module_name, program_text)
        try:
            res = build.build(target=build.TYPE_CHECK,
                              sources=[source],
                              pyversion=pyversion,
                              flags=flags + [build.TEST_BUILTINS],
                              alt_lib_path=test_temp_dir)
            a = res.errors
        except CompileError as e:
            res = None
            a = e.messages
        a = normalize_error_messages(a)

        if output != a and mypy.myunit.UPDATE_TESTCASES:
            update_testcase_output(testcase, a, mypy.myunit.APPEND_TESTCASES)

        assert_string_arrays_equal(
            output, a, 'Invalid type checker output ({}, line {})'.format(
                testcase.file, testcase.line))

        if incremental and res:
            self.verify_cache(module_name, program_name, a, res.manager)
Esempio n. 10
0
def test_daemon(testcase: DataDrivenTestCase) -> None:
    assert testcase.old_cwd is not None, "test was not properly set up"
    for i, step in enumerate(parse_script(testcase.input)):
        cmd = step[0]
        expected_lines = step[1:]
        assert cmd.startswith('$')
        cmd = cmd[1:].strip()
        cmd = cmd.replace('{python}', sys.executable)
        sts, output = run_cmd(cmd)
        output_lines = output.splitlines()
        output_lines = normalize_error_messages(output_lines)
        if sts:
            output_lines.append('== Return code: %d' % sts)
        assert_string_arrays_equal(
            expected_lines, output_lines,
            "Command %d (%s) did not give expected output" % (i + 1, cmd))
Esempio n. 11
0
    def run_test_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
        find_module_clear_caches()
        program_text = '\n'.join(testcase.input)
        module_name, program_name, program_text = self.parse_module(program_text)

        options = self.parse_options(program_text)
        options.use_builtins_fixtures = True
        options.python_version = testcase_pyversion(testcase.file, testcase.name)

        output = testcase.output
        if incremental:
            options.incremental = True
            if incremental == 1:
                # In run 1, copy program text to program file.
                output = []
                with open(program_name, 'w') as f:
                    f.write(program_text)
                    program_text = None
            elif incremental == 2:
                # In run 2, copy *.py.next files to *.py files.
                for dn, dirs, files in os.walk(os.curdir):
                    for file in files:
                        if file.endswith('.py.next'):
                            full = os.path.join(dn, file)
                            target = full[:-5]
                            shutil.copy(full, target)
        source = BuildSource(program_name, module_name, program_text)
        try:
            res = build.build(sources=[source],
                              options=options,
                              alt_lib_path=test_temp_dir)
            a = res.errors
        except CompileError as e:
            res = None
            a = e.messages
        a = normalize_error_messages(a)

        if output != a and mypy.myunit.UPDATE_TESTCASES:
            update_testcase_output(testcase, a, mypy.myunit.APPEND_TESTCASES)

        assert_string_arrays_equal(
            output, a,
            'Invalid type checker output ({}, line {})'.format(
                testcase.file, testcase.line))

        if incremental and res:
            self.verify_cache(module_name, program_name, a, res.manager)
Esempio n. 12
0
def test_semanal_error(testcase: DataDrivenTestCase) -> None:
    """Perform a test case."""

    try:
        src = '\n'.join(testcase.input)
        res = build.build(sources=[BuildSource('main', None, src)],
                          options=get_semanal_options(),
                          alt_lib_path=test_temp_dir)
        a = res.errors
        assert a, 'No errors reported in {}, line {}'.format(testcase.file, testcase.line)
    except CompileError as e:
        # Verify that there was a compile error and that the error messages
        # are equivalent.
        a = e.messages
    assert_string_arrays_equal(
        testcase.output, normalize_error_messages(a),
        'Invalid compiler output ({}, line {})'.format(testcase.file, testcase.line))
Esempio n. 13
0
def test_semanal_error(testcase: DataDrivenTestCase) -> None:
    """Perform a test case."""

    try:
        src = '\n'.join(testcase.input)
        res = build.build(sources=[BuildSource('main', None, src)],
                          options=get_semanal_options(),
                          alt_lib_path=test_temp_dir)
        a = res.errors
        assert a, 'No errors reported in {}, line {}'.format(testcase.file, testcase.line)
    except CompileError as e:
        # Verify that there was a compile error and that the error messages
        # are equivalent.
        a = e.messages
    assert_string_arrays_equal(
        testcase.output, normalize_error_messages(a),
        'Invalid compiler output ({}, line {})'.format(testcase.file, testcase.line))
Esempio n. 14
0
    def run_case(self, testcase: DataDrivenTestCase) -> None:
        # Parse options from test case description (arguments must not have spaces)
        text = '\n'.join(testcase.input)
        m = re.search(r'# *cmd: *(.*)', text)
        assert m is not None, 'Test case missing "# cmd: <files>" section'
        args = m.group(1).split()

        # Write main program to run (not compiled)
        program = '_%s.py' % testcase.name
        program_path = os.path.join(test_temp_dir, program)
        with open(program_path, 'w') as f:
            f.write(text)

        out = b''
        try:
            # Compile program
            cmd = subprocess.run(
                [sys.executable,
                 os.path.join(base_path, 'scripts', 'mypyc')] + args,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                cwd='tmp')
            if 'ErrorOutput' in testcase.name:
                out += cmd.stdout

            if cmd.returncode == 0:
                # Run main program
                out += subprocess.check_output([python3_path, program],
                                               cwd='tmp')
        finally:
            suffix = 'pyd' if sys.platform == 'win32' else 'so'
            so_paths = glob.glob('tmp/**/*.{}'.format(suffix), recursive=True)
            for path in so_paths:
                os.remove(path)

        # Strip out 'tmp/' from error message paths in the testcase output,
        # due to a mismatch between this test and mypy's test suite.
        expected = [x.replace('tmp/', '') for x in testcase.output]

        # Verify output
        actual = normalize_error_messages(out.decode().splitlines())
        assert_test_output(testcase,
                           actual,
                           'Invalid output',
                           expected=expected)
Esempio n. 15
0
def test_semanal_error(testcase):
    """Perform a test case."""

    try:
        src = '\n'.join(testcase.input)
        res = build.build(target=build.SEMANTIC_ANALYSIS,
                          sources=[BuildSource('main', None, src)],
                          flags=[build.TEST_BUILTINS],
                          alt_lib_path=test_temp_dir)
        a = res.errors
        assert a, 'No errors reported in {}, line {}'.format(testcase.file, testcase.line)
    except CompileError as e:
        # Verify that there was a compile error and that the error messages
        # are equivalent.
        a = e.messages
    assert_string_arrays_equal(
        testcase.output, normalize_error_messages(a),
        'Invalid compiler output ({}, line {})'.format(testcase.file, testcase.line))
Esempio n. 16
0
def test_semanal_error(testcase):
    """Perform a test case."""

    try:
        src = '\n'.join(testcase.input)
        res = build.build(target=build.SEMANTIC_ANALYSIS,
                          sources=[BuildSource('main', None, src)],
                          flags=[build.TEST_BUILTINS],
                          alt_lib_path=test_temp_dir)
        a = res.errors
        assert a, 'No errors reported in {}, line {}'.format(testcase.file, testcase.line)
    except CompileError as e:
        # Verify that there was a compile error and that the error messages
        # are equivalent.
        a = e.messages
    assert_string_arrays_equal(
        testcase.output, normalize_error_messages(a),
        'Invalid compiler output ({}, line {})'.format(testcase.file, testcase.line))
Esempio n. 17
0
def test_semanal(testcase: DataDrivenTestCase) -> None:
    """Perform a semantic analysis test case.

    The testcase argument contains a description of the test case
    (inputs and output).
    """

    try:
        src = '\n'.join(testcase.input)
        options = get_semanal_options()
        options.python_version = testfile_pyversion(testcase.file)
        result = build.build(sources=[BuildSource('main', None, src)],
                             options=options,
                             alt_lib_path=test_temp_dir)
        a = result.errors
        if a:
            raise CompileError(a)
        # Include string representations of the source files in the actual
        # output.
        for fnam in sorted(result.files.keys()):
            f = result.files[fnam]
            # Omit the builtins module and files with a special marker in the
            # path.
            # TODO the test is not reliable
            if (not f.path.endswith((os.sep + 'builtins.pyi',
                                     'typing.pyi',
                                     'mypy_extensions.pyi',
                                     'typing_extensions.pyi',
                                     'abc.pyi',
                                     'collections.pyi',
                                     'sys.pyi'))
                    and not os.path.basename(f.path).startswith('_')
                    and not os.path.splitext(
                        os.path.basename(f.path))[0].endswith('_')):
                a += str(f).split('\n')
    except CompileError as e:
        a = e.messages
    if testcase.normalize_output:
        a = normalize_error_messages(a)
    assert_string_arrays_equal(
        testcase.output, a,
        'Invalid semantic analyzer output ({}, line {})'.format(testcase.file,
                                                                testcase.line))
Esempio n. 18
0
def test_transform(testcase: DataDrivenTestCase) -> None:
    """Perform an identity transform test case."""

    try:
        src = '\n'.join(testcase.input)
        options = parse_options(src, testcase, 1)
        options.use_builtins_fixtures = True
        options.semantic_analysis_only = True
        options.enable_incomplete_features = True
        options.show_traceback = True
        result = build.build(sources=[BuildSource('main', None, src)],
                             options=options,
                             alt_lib_path=test_temp_dir)
        a = result.errors
        if a:
            raise CompileError(a)
        # Include string representations of the source files in the actual
        # output.
        for fnam in sorted(result.files.keys()):
            f = result.files[fnam]

            # Omit the builtins module and files with a special marker in the
            # path.
            # TODO the test is not reliable
            if (not f.path.endswith((os.sep + 'builtins.pyi',
                                     'typing_extensions.pyi',
                                     'typing.pyi',
                                     'abc.pyi',
                                     'sys.pyi'))
                    and not os.path.basename(f.path).startswith('_')
                    and not os.path.splitext(
                        os.path.basename(f.path))[0].endswith('_')):
                t = TypeAssertTransformVisitor()
                t.test_only = True
                f = t.mypyfile(f)
                a += str(f).split('\n')
    except CompileError as e:
        a = e.messages
    if testcase.normalize_output:
        a = normalize_error_messages(a)
    assert_string_arrays_equal(
        testcase.output, a,
        f'Invalid semantic analyzer output ({testcase.file}, line {testcase.line})')
Esempio n. 19
0
def test_semanal(testcase: DataDrivenTestCase) -> None:
    """Perform a semantic analysis test case.

    The testcase argument contains a description of the test case
    (inputs and output).
    """

    try:
        src = '\n'.join(testcase.input)
        options = get_semanal_options(src, testcase)
        options.python_version = testfile_pyversion(testcase.file)
        result = build.build(sources=[BuildSource('main', None, src)],
                             options=options,
                             alt_lib_path=test_temp_dir)
        a = result.errors
        if a:
            raise CompileError(a)
        # Include string representations of the source files in the actual
        # output.
        for fnam in sorted(result.files.keys()):
            f = result.files[fnam]
            # Omit the builtins module and files with a special marker in the
            # path.
            # TODO the test is not reliable
            if (not f.path.endswith((os.sep + 'builtins.pyi',
                                     'typing.pyi',
                                     'mypy_extensions.pyi',
                                     'typing_extensions.pyi',
                                     'abc.pyi',
                                     'collections.pyi',
                                     'sys.pyi'))
                    and not os.path.basename(f.path).startswith('_')
                    and not os.path.splitext(
                        os.path.basename(f.path))[0].endswith('_')):
                a += str(f).split('\n')
    except CompileError as e:
        a = e.messages
    if testcase.normalize_output:
        a = normalize_error_messages(a)
    assert_string_arrays_equal(
        testcase.output, a,
        'Invalid semantic analyzer output ({}, line {})'.format(testcase.file,
                                                                testcase.line))
Esempio n. 20
0
def test_transform(testcase: DataDrivenTestCase) -> None:
    """Perform an identity transform test case."""

    try:
        src = '\n'.join(testcase.input)
        options = Options()
        options.use_builtins_fixtures = True
        options.semantic_analysis_only = True
        options.show_traceback = True
        options.python_version = testfile_pyversion(testcase.file)
        result = build.build(sources=[BuildSource('main', None, src)],
                             options=options,
                             alt_lib_path=test_temp_dir)
        a = result.errors
        if a:
            raise CompileError(a)
        # Include string representations of the source files in the actual
        # output.
        for fnam in sorted(result.files.keys()):
            f = result.files[fnam]

            # Omit the builtins module and files with a special marker in the
            # path.
            # TODO the test is not reliable
            if (not f.path.endswith((os.sep + 'builtins.pyi',
                                     'typing.pyi',
                                     'abc.pyi'))
                    and not os.path.basename(f.path).startswith('_')
                    and not os.path.splitext(
                        os.path.basename(f.path))[0].endswith('_')):
                t = TypeAssertTransformVisitor()
                f = t.mypyfile(f)
                a += str(f).split('\n')
    except CompileError as e:
        a = e.messages
    if testcase.normalize_output:
        a = normalize_error_messages(a)
    assert_string_arrays_equal(
        testcase.output, a,
        'Invalid semantic analyzer output ({}, line {})'.format(testcase.file,
                                                                testcase.line))
Esempio n. 21
0
def test_python_cmdline(testcase: DataDrivenTestCase) -> None:
    assert testcase.old_cwd is not None, "test was not properly set up"
    # Write the program to a file.
    program = '_program.py'
    program_path = os.path.join(test_temp_dir, program)
    with open(program_path, 'w') as file:
        for s in testcase.input:
            file.write('{}\n'.format(s))
    args = parse_args(testcase.input[0])
    args.append('--show-traceback')
    args.append('--no-site-packages')
    # Type check the program.
    fixed = [python3_path, '-m', 'mypy']
    env = os.environ.copy()
    env['PYTHONPATH'] = PREFIX
    process = subprocess.Popen(fixed + args,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               cwd=test_temp_dir,
                               env=env)
    outb, errb = process.communicate()
    result = process.returncode
    # Split output into lines.
    out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
    err = [s.rstrip('\n\r') for s in str(errb, 'utf8').splitlines()]

    if "PYCHARM_HOSTED" in os.environ:
        for pos, line in enumerate(err):
            if line.startswith('pydev debugger: '):
                # Delete the attaching debugger message itself, plus the extra newline added.
                del err[pos:pos + 2]
                break

    # Remove temp file.
    os.remove(program_path)
    # Compare actual output to expected.
    if testcase.output_files:
        # Ignore stdout, but we insist on empty stderr and zero status.
        if err or result:
            raise AssertionError(
                'Expected zero status and empty stderr, got %d and\n%s' %
                (result, '\n'.join(err + out)))
        for path, expected_content in testcase.output_files:
            if not os.path.exists(path):
                raise AssertionError(
                    'Expected file {} was not produced by test case'.format(
                        path))
            with open(path, 'r') as output_file:
                actual_output_content = output_file.read().splitlines()
            normalized_output = normalize_file_output(
                actual_output_content, os.path.abspath(test_temp_dir))
            # We always normalize things like timestamp, but only handle operating-system
            # specific things if requested.
            if testcase.normalize_output:
                if testcase.suite.native_sep and os.path.sep == '\\':
                    normalized_output = [
                        fix_cobertura_filename(line)
                        for line in normalized_output
                    ]
                normalized_output = normalize_error_messages(normalized_output)
            assert_string_arrays_equal(
                expected_content.splitlines(), normalized_output,
                'Output file {} did not match its expected output'.format(
                    path))
    else:
        if testcase.normalize_output:
            out = normalize_error_messages(err + out)
        obvious_result = 1 if out else 0
        if obvious_result != result:
            out.append('== Return code: {}'.format(result))
        assert_string_arrays_equal(
            testcase.output, out,
            'Invalid output ({}, line {})'.format(testcase.file,
                                                  testcase.line))
Esempio n. 22
0
    def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int = 0) -> None:
        find_module_clear_caches()
        original_program_text = '\n'.join(testcase.input)
        module_data = self.parse_module(original_program_text, incremental_step)

        if incremental_step:
            if incremental_step == 1:
                # In run 1, copy program text to program file.
                for module_name, program_path, program_text in module_data:
                    if module_name == '__main__':
                        with open(program_path, 'w') as f:
                            f.write(program_text)
                        break
            elif incremental_step > 1:
                # In runs 2+, copy *.[num] files to * files.
                for dn, dirs, files in os.walk(os.curdir):
                    for file in files:
                        if file.endswith('.' + str(incremental_step)):
                            full = os.path.join(dn, file)
                            target = full[:-2]
                            # Use retries to work around potential flakiness on Windows (AppVeyor).
                            retry_on_error(lambda: shutil.copy(full, target))

                            # In some systems, mtime has a resolution of 1 second which can cause
                            # annoying-to-debug issues when a file has the same size after a
                            # change. We manually set the mtime to circumvent this.
                            new_time = os.stat(target).st_mtime + 1
                            os.utime(target, times=(new_time, new_time))
                # Delete files scheduled to be deleted in [delete <path>.num] sections.
                for path in testcase.deleted_paths.get(incremental_step, set()):
                    # Use retries to work around potential flakiness on Windows (AppVeyor).
                    retry_on_error(lambda: os.remove(path))

        # Parse options after moving files (in case mypy.ini is being moved).
        options = parse_options(original_program_text, testcase, incremental_step)
        options.use_builtins_fixtures = True
        options.show_traceback = True
        if 'optional' in testcase.file:
            options.strict_optional = True
        if incremental_step:
            options.incremental = True
        else:
            options.cache_dir = os.devnull  # Don't waste time writing cache

        sources = []
        for module_name, program_path, program_text in module_data:
            # Always set to none so we're forced to reread the module in incremental mode
            sources.append(BuildSource(program_path, module_name,
                                       None if incremental_step else program_text))

        res = None
        try:
            res = build.build(sources=sources,
                              options=options,
                              alt_lib_path=test_temp_dir)
            a = res.errors
        except CompileError as e:
            a = e.messages
        a = normalize_error_messages(a)

        # Make sure error messages match
        if incremental_step == 0:
            # Not incremental
            msg = 'Unexpected type checker output ({}, line {})'
            output = testcase.output
        elif incremental_step == 1:
            msg = 'Unexpected type checker output in incremental, run 1 ({}, line {})'
            output = testcase.output
        elif incremental_step > 1:
            msg = ('Unexpected type checker output in incremental, run {}'.format(
                incremental_step) + ' ({}, line {})')
            output = testcase.output2.get(incremental_step, [])
        else:
            raise AssertionError()

        if output != a and self.update_data:
            update_testcase_output(testcase, a)
        assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line))

        if incremental_step and res:
            if options.follow_imports == 'normal' and testcase.output is None:
                self.verify_cache(module_data, a, res.manager)
            if incremental_step > 1:
                suffix = '' if incremental_step == 2 else str(incremental_step - 1)
                self.check_module_equivalence(
                    'rechecked' + suffix,
                    testcase.expected_rechecked_modules.get(incremental_step - 1),
                    res.manager.rechecked_modules)
                self.check_module_equivalence(
                    'stale' + suffix,
                    testcase.expected_stale_modules.get(incremental_step - 1),
                    res.manager.stale_modules)
Esempio n. 23
0
    def run_case_once(self,
                      testcase: DataDrivenTestCase,
                      incremental=0) -> None:
        find_module_clear_caches()
        original_program_text = '\n'.join(testcase.input)
        module_data = self.parse_module(original_program_text, incremental)

        options = self.parse_options(original_program_text, testcase)
        options.use_builtins_fixtures = True
        options.show_traceback = True
        if 'optional' in testcase.file:
            options.strict_optional = True

        if incremental:
            options.incremental = True
            if incremental == 1:
                # In run 1, copy program text to program file.
                for module_name, program_path, program_text in module_data:
                    if module_name == '__main__':
                        with open(program_path, 'w') as f:
                            f.write(program_text)
                        break
            elif incremental == 2:
                # In run 2, copy *.py.next files to *.py files.
                for dn, dirs, files in os.walk(os.curdir):
                    for file in files:
                        if file.endswith('.py.next'):
                            full = os.path.join(dn, file)
                            target = full[:-5]
                            shutil.copy(full, target)

                            # In some systems, mtime has a resolution of 1 second which can cause
                            # annoying-to-debug issues when a file has the same size after a
                            # change. We manually set the mtime to circumvent this.
                            new_time = os.stat(target).st_mtime + 1
                            os.utime(target, times=(new_time, new_time))

        sources = []
        for module_name, program_path, program_text in module_data:
            # Always set to none so we're forced to reread the module in incremental mode
            sources.append(
                BuildSource(program_path, module_name,
                            None if incremental else program_text))
        res = None
        try:
            res = build.build(sources=sources,
                              options=options,
                              alt_lib_path=test_temp_dir)
            a = res.errors
        except CompileError as e:
            a = e.messages
        a = normalize_error_messages(a)

        # Make sure error messages match
        if incremental == 0:
            msg = 'Invalid type checker output ({}, line {})'
            output = testcase.output
        elif incremental == 1:
            msg = 'Invalid type checker output in incremental, run 1 ({}, line {})'
            output = testcase.output
        elif incremental == 2:
            msg = 'Invalid type checker output in incremental, run 2 ({}, line {})'
            output = testcase.output2
        else:
            raise AssertionError()

        if output != a and self.update_data:
            update_testcase_output(testcase, a)
        assert_string_arrays_equal(output, a,
                                   msg.format(testcase.file, testcase.line))

        if incremental and res:
            if not options.silent_imports and testcase.output is None:
                self.verify_cache(module_data, a, res.manager)
            if incremental == 2:
                self.check_module_equivalence(
                    'rechecked', testcase.expected_rechecked_modules,
                    res.manager.rechecked_modules)
                self.check_module_equivalence('stale',
                                              testcase.expected_stale_modules,
                                              res.manager.stale_modules)
Esempio n. 24
0
def test_python_cmdline(testcase: DataDrivenTestCase, step: int) -> None:
    assert testcase.old_cwd is not None, "test was not properly set up"
    # Write the program to a file.
    program = '_program.py'
    program_path = os.path.join(test_temp_dir, program)
    with open(program_path, 'w', encoding='utf8') as file:
        for s in testcase.input:
            file.write('{}\n'.format(s))
    args = parse_args(testcase.input[0])
    args.append('--show-traceback')
    args.append('--no-site-packages')
    # Type check the program.
    fixed = [python3_path, '-m', 'mypy']
    env = os.environ.copy()
    env['PYTHONPATH'] = PREFIX
    process = subprocess.Popen(fixed + args,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               cwd=test_temp_dir,
                               env=env)
    outb, errb = process.communicate()
    result = process.returncode
    # Split output into lines.
    out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
    err = [s.rstrip('\n\r') for s in str(errb, 'utf8').splitlines()]

    if "PYCHARM_HOSTED" in os.environ:
        for pos, line in enumerate(err):
            if line.startswith('pydev debugger: '):
                # Delete the attaching debugger message itself, plus the extra newline added.
                del err[pos:pos + 2]
                break

    # Remove temp file.
    os.remove(program_path)
    # Compare actual output to expected.
    if testcase.output_files:
        # Ignore stdout, but we insist on empty stderr and zero status.
        if err or result:
            raise AssertionError(
                'Expected zero status and empty stderr%s, got %d and\n%s' %
                (' on step %d' % step if testcase.output2 else '',
                 result, '\n'.join(err + out)))
        for path, expected_content in testcase.output_files:
            if not os.path.exists(path):
                raise AssertionError(
                    'Expected file {} was not produced by test case{}'.format(
                        path, ' on step %d' % step if testcase.output2 else ''))
            with open(path, 'r', encoding='utf8') as output_file:
                actual_output_content = output_file.read().splitlines()
            normalized_output = normalize_file_output(actual_output_content,
                                                      os.path.abspath(test_temp_dir))
            # We always normalize things like timestamp, but only handle operating-system
            # specific things if requested.
            if testcase.normalize_output:
                if testcase.suite.native_sep and os.path.sep == '\\':
                    normalized_output = [fix_cobertura_filename(line)
                                         for line in normalized_output]
                normalized_output = normalize_error_messages(normalized_output)
            assert_string_arrays_equal(expected_content.splitlines(), normalized_output,
                                       'Output file {} did not match its expected output{}'.format(
                                           path, ' on step %d' % step if testcase.output2 else ''))
    else:
        if testcase.normalize_output:
            out = normalize_error_messages(err + out)
        obvious_result = 1 if out else 0
        if obvious_result != result:
            out.append('== Return code: {}'.format(result))
        expected_out = testcase.output if step == 1 else testcase.output2[step]
        assert_string_arrays_equal(expected_out, out,
                                   'Invalid output ({}, line {}){}'.format(
                                       testcase.file, testcase.line,
                                       ' on step %d' % step if testcase.output2 else ''))
Esempio n. 25
0
    def run_case_once(self,
                      testcase: DataDrivenTestCase,
                      operations: List[FileOperation] = [],
                      incremental_step: int = 0) -> None:
        original_program_text = '\n'.join(testcase.input)
        module_data = self.parse_module(original_program_text,
                                        incremental_step)

        # Unload already loaded plugins, they may be updated.
        for file, _ in testcase.files:
            module = module_from_path(file)
            if module.endswith('_plugin') and module in sys.modules:
                del sys.modules[module]
        if incremental_step == 0 or incremental_step == 1:
            # In run 1, copy program text to program file.
            for module_name, program_path, program_text in module_data:
                if module_name == '__main__':
                    with open(program_path, 'w', encoding='utf8') as f:
                        f.write(program_text)
                    break
        elif incremental_step > 1:
            # In runs 2+, copy *.[num] files to * files.
            for op in operations:
                if isinstance(op, UpdateFile):
                    # Modify/create file
                    copy_and_fudge_mtime(op.source_path, op.target_path)
                else:
                    # Delete file
                    # Use retries to work around potential flakiness on Windows (AppVeyor).
                    path = op.path
                    retry_on_error(lambda: os.remove(path))

        # Parse options after moving files (in case mypy.ini is being moved).
        options = parse_options(original_program_text, testcase,
                                incremental_step)
        options.use_builtins_fixtures = True
        options.show_traceback = True

        # Enable some options automatically based on test file name.
        if 'optional' in testcase.file:
            options.strict_optional = True
        if 'columns' in testcase.file:
            options.show_column_numbers = True
        if 'errorcodes' in testcase.file:
            options.show_error_codes = True

        if incremental_step and options.incremental:
            # Don't overwrite # flags: --no-incremental in incremental test cases
            options.incremental = True
        else:
            options.incremental = False
            # Don't waste time writing cache unless we are specifically looking for it
            if not testcase.writescache:
                options.cache_dir = os.devnull

        sources = []
        for module_name, program_path, program_text in module_data:
            # Always set to none so we're forced to reread the module in incremental mode
            sources.append(
                BuildSource(program_path, module_name,
                            None if incremental_step else program_text))

        plugin_dir = os.path.join(test_data_prefix, 'plugins')
        sys.path.insert(0, plugin_dir)

        res = None
        try:
            res = build.build(sources=sources,
                              options=options,
                              alt_lib_path=test_temp_dir)
            a = res.errors
        except CompileError as e:
            a = e.messages
        finally:
            assert sys.path[0] == plugin_dir
            del sys.path[0]

        if testcase.normalize_output:
            a = normalize_error_messages(a)

        # Make sure error messages match
        if incremental_step == 0:
            # Not incremental
            msg = 'Unexpected type checker output ({}, line {})'
            output = testcase.output
        elif incremental_step == 1:
            msg = 'Unexpected type checker output in incremental, run 1 ({}, line {})'
            output = testcase.output
        elif incremental_step > 1:
            msg = ('Unexpected type checker output in incremental, run {}'.
                   format(incremental_step) + ' ({}, line {})')
            output = testcase.output2.get(incremental_step, [])
        else:
            raise AssertionError()

        if output != a and testcase.config.getoption('--update-data', False):
            update_testcase_output(testcase, a)
        assert_string_arrays_equal(output, a,
                                   msg.format(testcase.file, testcase.line))

        if res:
            if options.cache_dir != os.devnull:
                self.verify_cache(module_data, res.errors, res.manager,
                                  res.graph)

            name = 'targets'
            if incremental_step:
                name += str(incremental_step + 1)
            expected = testcase.expected_fine_grained_targets.get(
                incremental_step + 1)
            actual = res.manager.processed_targets
            # Skip the initial builtin cycle.
            actual = [
                t for t in actual if not any(
                    t.startswith(mod)
                    for mod in core_modules + ['mypy_extensions'])
            ]
            if expected is not None:
                assert_target_equivalence(name, expected, actual)
            if incremental_step > 1:
                suffix = '' if incremental_step == 2 else str(
                    incremental_step - 1)
                expected_rechecked = testcase.expected_rechecked_modules.get(
                    incremental_step - 1)
                if expected_rechecked is not None:
                    assert_module_equivalence('rechecked' + suffix,
                                              expected_rechecked,
                                              res.manager.rechecked_modules)
                expected_stale = testcase.expected_stale_modules.get(
                    incremental_step - 1)
                if expected_stale is not None:
                    assert_module_equivalence('stale' + suffix, expected_stale,
                                              res.manager.stale_modules)

        if testcase.output_files:
            check_test_output_files(testcase,
                                    incremental_step,
                                    strip_prefix='tmp/')
Esempio n. 26
0
    def run_case_once(self, testcase: DataDrivenTestCase,
                      incremental_step: int) -> None:
        assert incremental_step >= 1
        build.find_module_clear_caches()
        original_program_text = '\n'.join(testcase.input)

        if incremental_step > 1:
            # In runs 2+, copy *.[num] files to * files.
            for dn, dirs, files in os.walk(os.curdir):
                for file in files:
                    if file.endswith('.' + str(incremental_step)):
                        full = os.path.join(dn, file)
                        target = full[:-2]
                        # Use retries to work around potential flakiness on Windows (AppVeyor).
                        retry_on_error(lambda: shutil.copy(full, target))

                        # In some systems, mtime has a resolution of 1 second which can cause
                        # annoying-to-debug issues when a file has the same size after a
                        # change. We manually set the mtime to circumvent this.
                        new_time = os.stat(target).st_mtime + 1
                        os.utime(target, times=(new_time, new_time))
            # Delete files scheduled to be deleted in [delete <path>.num] sections.
            for path in testcase.deleted_paths.get(incremental_step, set()):
                # Use retries to work around potential flakiness on Windows (AppVeyor).
                retry_on_error(lambda: os.remove(path))

        module_data = self.parse_module(original_program_text,
                                        incremental_step)

        if incremental_step == 1:
            # In run 1, copy program text to program file.
            for module_name, program_path, program_text in module_data:
                if module_name == '__main__' and program_text is not None:
                    with open(program_path, 'w') as f:
                        f.write(program_text)
                    break

        # Parse options after moving files (in case mypy.ini is being moved).
        options = self.parse_options(original_program_text, testcase,
                                     incremental_step)
        if incremental_step == 1:
            server_options = []  # type: List[str]
            if 'fine-grained' in testcase.file:
                server_options.append('--experimental')
                options.fine_grained_incremental = True
            self.server = dmypy_server.Server(
                server_options)  # TODO: Fix ugly API
            self.server.options = options

        assert self.server is not None  # Set in step 1 and survives into next steps
        sources = []
        for module_name, program_path, program_text in module_data:
            # Always set to none so we're forced to reread the module in incremental mode
            sources.append(build.BuildSource(program_path, module_name, None))
        response = self.server.check(sources, alt_lib_path=test_temp_dir)
        a = (response['out'] or response['err']).splitlines()
        a = normalize_error_messages(a)

        # Make sure error messages match
        if incremental_step == 1:
            msg = 'Unexpected type checker output in incremental, run 1 ({}, line {})'
            output = testcase.output
        elif incremental_step > 1:
            msg = ('Unexpected type checker output in incremental, run {}'.
                   format(incremental_step) + ' ({}, line {})')
            output = testcase.output2.get(incremental_step, [])
        else:
            raise AssertionError()

        if output != a and self.update_data:
            update_testcase_output(testcase, a)
        assert_string_arrays_equal(output, a,
                                   msg.format(testcase.file, testcase.line))

        manager = self.server.last_manager
        if manager is not None:
            if options.follow_imports == 'normal' and testcase.output is None:
                self.verify_cache(module_data, a, manager)
            if incremental_step > 1:
                suffix = '' if incremental_step == 2 else str(
                    incremental_step - 1)
                self.check_module_equivalence(
                    'rechecked' + suffix,
                    testcase.expected_rechecked_modules.get(incremental_step -
                                                            1),
                    manager.rechecked_modules)
                self.check_module_equivalence(
                    'stale' + suffix,
                    testcase.expected_stale_modules.get(incremental_step - 1),
                    manager.stale_modules)
Esempio n. 27
0
    def run_case_once(self, testcase: DataDrivenTestCase, incremental: int = 0) -> None:
        find_module_clear_caches()
        original_program_text = '\n'.join(testcase.input)
        module_data = self.parse_module(original_program_text, incremental)

        if incremental:
            if incremental == 1:
                # In run 1, copy program text to program file.
                for module_name, program_path, program_text in module_data:
                    if module_name == '__main__':
                        with open(program_path, 'w') as f:
                            f.write(program_text)
                        break
            elif incremental == 2:
                # In run 2, copy *.next files to * files.
                for dn, dirs, files in os.walk(os.curdir):
                    for file in files:
                        if file.endswith('.next'):
                            full = os.path.join(dn, file)
                            target = full[:-5]
                            shutil.copy(full, target)

                            # In some systems, mtime has a resolution of 1 second which can cause
                            # annoying-to-debug issues when a file has the same size after a
                            # change. We manually set the mtime to circumvent this.
                            new_time = os.stat(target).st_mtime + 1
                            os.utime(target, times=(new_time, new_time))

        # Parse options after moving files (in case mypy.ini is being moved).
        options = self.parse_options(original_program_text, testcase)
        options.use_builtins_fixtures = True
        options.show_traceback = True
        if 'optional' in testcase.file:
            options.strict_optional = True
        if incremental:
            options.incremental = True
        if os.path.split(testcase.file)[1] in fast_parser_files:
            options.fast_parser = True

        sources = []
        for module_name, program_path, program_text in module_data:
            # Always set to none so we're forced to reread the module in incremental mode
            sources.append(BuildSource(program_path, module_name,
                                       None if incremental else program_text))
        res = None
        try:
            res = build.build(sources=sources,
                              options=options,
                              alt_lib_path=test_temp_dir)
            a = res.errors
        except CompileError as e:
            a = e.messages
        a = normalize_error_messages(a)

        # Make sure error messages match
        if incremental == 0:
            msg = 'Invalid type checker output ({}, line {})'
            output = testcase.output
        elif incremental == 1:
            msg = 'Invalid type checker output in incremental, run 1 ({}, line {})'
            output = testcase.output
        elif incremental == 2:
            msg = 'Invalid type checker output in incremental, run 2 ({}, line {})'
            output = testcase.output2
        else:
            raise AssertionError()

        if output != a and self.update_data:
            update_testcase_output(testcase, a)
        assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line))

        if incremental and res:
            if options.follow_imports == 'normal' and testcase.output is None:
                self.verify_cache(module_data, a, res.manager)
            if incremental == 2:
                self.check_module_equivalence(
                    'rechecked',
                    testcase.expected_rechecked_modules,
                    res.manager.rechecked_modules)
                self.check_module_equivalence(
                    'stale',
                    testcase.expected_stale_modules,
                    res.manager.stale_modules)
Esempio n. 28
0
    def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int) -> None:
        assert incremental_step >= 1
        build.find_module_clear_caches()
        original_program_text = '\n'.join(testcase.input)
        module_data = self.parse_module(original_program_text, incremental_step)

        if incremental_step == 1:
            # In run 1, copy program text to program file.
            for module_name, program_path, program_text in module_data:
                if module_name == '__main__':
                    with open(program_path, 'w') as f:
                        f.write(program_text)
                    break
        elif incremental_step > 1:
            # In runs 2+, copy *.[num] files to * files.
            for dn, dirs, files in os.walk(os.curdir):
                for file in files:
                    if file.endswith('.' + str(incremental_step)):
                        full = os.path.join(dn, file)
                        target = full[:-2]
                        # Use retries to work around potential flakiness on Windows (AppVeyor).
                        retry_on_error(lambda: shutil.copy(full, target))

                        # In some systems, mtime has a resolution of 1 second which can cause
                        # annoying-to-debug issues when a file has the same size after a
                        # change. We manually set the mtime to circumvent this.
                        new_time = os.stat(target).st_mtime + 1
                        os.utime(target, times=(new_time, new_time))
            # Delete files scheduled to be deleted in [delete <path>.num] sections.
            for path in testcase.deleted_paths.get(incremental_step, set()):
                # Use retries to work around potential flakiness on Windows (AppVeyor).
                retry_on_error(lambda: os.remove(path))

        # Parse options after moving files (in case mypy.ini is being moved).
        options = self.parse_options(original_program_text, testcase, incremental_step)
        if incremental_step == 1:
            self.server = dmypy_server.Server([])  # TODO: Fix ugly API
            self.server.options = options

        assert self.server is not None  # Set in step 1 and survives into next steps
        sources = []
        for module_name, program_path, program_text in module_data:
            # Always set to none so we're forced to reread the module in incremental mode
            sources.append(build.BuildSource(program_path, module_name, None))
        response = self.server.check(sources, alt_lib_path=test_temp_dir)
        a = (response['out'] or response['err']).splitlines()
        a = normalize_error_messages(a)

        # Make sure error messages match
        if incremental_step == 1:
            msg = 'Unexpected type checker output in incremental, run 1 ({}, line {})'
            output = testcase.output
        elif incremental_step > 1:
            msg = ('Unexpected type checker output in incremental, run {}'.format(
                incremental_step) + ' ({}, line {})')
            output = testcase.output2.get(incremental_step, [])
        else:
            raise AssertionError()

        if output != a and self.update_data:
            update_testcase_output(testcase, a)
        assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line))

        manager = self.server.last_manager
        if manager is not None:
            if options.follow_imports == 'normal' and testcase.output is None:
                self.verify_cache(module_data, a, manager)
            if incremental_step > 1:
                suffix = '' if incremental_step == 2 else str(incremental_step - 1)
                self.check_module_equivalence(
                    'rechecked' + suffix,
                    testcase.expected_rechecked_modules.get(incremental_step - 1),
                    manager.rechecked_modules)
                self.check_module_equivalence(
                    'stale' + suffix,
                    testcase.expected_stale_modules.get(incremental_step - 1),
                    manager.stale_modules)
Esempio n. 29
0
 def renormalize_error_messages(messages):
     messages = [x for x in messages if not x.endswith(' defined here')]
     return normalize_error_messages(messages)
Esempio n. 30
0
    def run_case_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
        find_module_clear_caches()
        original_program_text = '\n'.join(testcase.input)
        module_data = self.parse_module(original_program_text, incremental)

        options = self.parse_options(original_program_text)
        options.use_builtins_fixtures = True
        options.python_version = testcase_pyversion(testcase.file, testcase.name)
        set_show_tb(True)  # Show traceback on crash.

        output = testcase.output
        if incremental:
            options.incremental = True
            if incremental == 1:
                # In run 1, copy program text to program file.
                output = []
                for module_name, program_path, program_text in module_data:
                    if module_name == '__main__':
                        with open(program_path, 'w') as f:
                            f.write(program_text)
                        break
            elif incremental == 2:
                # In run 2, copy *.py.next files to *.py files.
                for dn, dirs, files in os.walk(os.curdir):
                    for file in files:
                        if file.endswith('.py.next'):
                            full = os.path.join(dn, file)
                            target = full[:-5]
                            shutil.copy(full, target)

                            # In some systems, mtime has a resolution of 1 second which can cause
                            # annoying-to-debug issues when a file has the same size after a
                            # change. We manually set the mtime to circumvent this.
                            new_time = os.stat(target).st_mtime + 1
                            os.utime(target, times=(new_time, new_time))

        sources = []
        for module_name, program_path, program_text in module_data:
            # Always set to none so we're forced to reread the module in incremental mode
            program_text = None if incremental else program_text
            sources.append(BuildSource(program_path, module_name, program_text))
        try:
            res = build.build(sources=sources,
                              options=options,
                              alt_lib_path=test_temp_dir)
            a = res.errors
        except CompileError as e:
            res = None
            a = e.messages
        a = normalize_error_messages(a)

        if output != a and self.update_data:
            update_testcase_output(testcase, a)

        assert_string_arrays_equal(
            output, a,
            'Invalid type checker output ({}, line {})'.format(
                testcase.file, testcase.line))

        if incremental and res:
            if not options.silent_imports:
                self.verify_cache(module_data, a, res.manager)
            if testcase.expected_stale_modules is not None and incremental == 2:
                assert_string_arrays_equal(
                    list(sorted(testcase.expected_stale_modules)),
                    list(sorted(res.manager.stale_modules.difference({"__main__"}))),
                    'Set of stale modules does not match expected set')
Esempio n. 31
0
    def run_case_once(self,
                      testcase: DataDrivenTestCase,
                      operations: List[FileOperation] = [],
                      incremental_step: int = 0) -> None:
        original_program_text = '\n'.join(testcase.input)
        module_data = self.parse_module(original_program_text,
                                        incremental_step)

        if incremental_step == 0 or incremental_step == 1:
            # In run 1, copy program text to program file.
            for module_name, program_path, program_text in module_data:
                if module_name == '__main__':
                    with open(program_path, 'w') as f:
                        f.write(program_text)
                    break
        elif incremental_step > 1:
            # In runs 2+, copy *.[num] files to * files.
            for op in operations:
                if isinstance(op, UpdateFile):
                    # Modify/create file
                    copy_and_fudge_mtime(op.source_path, op.target_path)
                else:
                    # Delete file
                    # Use retries to work around potential flakiness on Windows (AppVeyor).
                    path = op.path
                    retry_on_error(lambda: os.remove(path))

        # Parse options after moving files (in case mypy.ini is being moved).
        options = parse_options(original_program_text, testcase,
                                incremental_step)
        options.use_builtins_fixtures = True
        options.show_traceback = True
        if 'optional' in testcase.file:
            options.strict_optional = True
        if incremental_step and options.incremental:
            # Don't overwrite # flags: --no-incremental in incremental test cases
            options.incremental = True
        else:
            options.incremental = False
            # Don't waste time writing cache unless we are specifically looking for it
            if 'writescache' not in testcase.name.lower():
                options.cache_dir = os.devnull

        sources = []
        for module_name, program_path, program_text in module_data:
            # Always set to none so we're forced to reread the module in incremental mode
            sources.append(
                BuildSource(program_path, module_name,
                            None if incremental_step else program_text))

        res = None
        try:
            res = build.build(sources=sources,
                              options=options,
                              alt_lib_path=test_temp_dir)
            a = res.errors
        except CompileError as e:
            a = e.messages
        a = normalize_error_messages(a)

        # Make sure error messages match
        if incremental_step == 0:
            # Not incremental
            msg = 'Unexpected type checker output ({}, line {})'
            output = testcase.output
        elif incremental_step == 1:
            msg = 'Unexpected type checker output in incremental, run 1 ({}, line {})'
            output = testcase.output
        elif incremental_step > 1:
            msg = ('Unexpected type checker output in incremental, run {}'.
                   format(incremental_step) + ' ({}, line {})')
            output = testcase.output2.get(incremental_step, [])
        else:
            raise AssertionError()

        if output != a and testcase.config.getoption('--update-data', False):
            update_testcase_output(testcase, a)
        assert_string_arrays_equal(output, a,
                                   msg.format(testcase.file, testcase.line))

        if res:
            if options.cache_dir != os.devnull:
                self.verify_cache(module_data, res.errors, res.manager,
                                  res.graph)

            if incremental_step > 1:
                suffix = '' if incremental_step == 2 else str(
                    incremental_step - 1)
                assert_module_equivalence(
                    'rechecked' + suffix,
                    testcase.expected_rechecked_modules.get(incremental_step -
                                                            1),
                    res.manager.rechecked_modules)
                assert_module_equivalence(
                    'stale' + suffix,
                    testcase.expected_stale_modules.get(incremental_step - 1),
                    res.manager.stale_modules)
Esempio n. 32
0
    def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int = 0) -> None:
        original_program_text = '\n'.join(testcase.input)
        module_data = self.parse_module(original_program_text, incremental_step)

        if incremental_step == 0 or incremental_step == 1:
            # In run 1, copy program text to program file.
            for module_name, program_path, program_text in module_data:
                if module_name == '__main__':
                    with open(program_path, 'w') as f:
                        f.write(program_text)
                    break
        elif incremental_step > 1:
            # In runs 2+, copy *.[num] files to * files.
            for dn, dirs, files in os.walk(os.curdir):
                for file in files:
                    if file.endswith('.' + str(incremental_step)):
                        full = os.path.join(dn, file)
                        target = full[:-2]
                        copy_and_fudge_mtime(full, target)
            # Delete files scheduled to be deleted in [delete <path>.num] sections.
            for path in testcase.deleted_paths.get(incremental_step, set()):
                # Use retries to work around potential flakiness on Windows (AppVeyor).
                retry_on_error(lambda: os.remove(path))

        # Parse options after moving files (in case mypy.ini is being moved).
        options = parse_options(original_program_text, testcase, incremental_step)
        options.use_builtins_fixtures = True
        options.show_traceback = True
        if 'optional' in testcase.file:
            options.strict_optional = True
        if incremental_step:
            options.incremental = True

        sources = []
        for module_name, program_path, program_text in module_data:
            # Always set to none so we're forced to reread the module in incremental mode
            sources.append(BuildSource(program_path, module_name,
                                       None if incremental_step else program_text))

        res = None
        try:
            res = build.build(sources=sources,
                              options=options,
                              alt_lib_path=test_temp_dir)
            a = res.errors
        except CompileError as e:
            a = e.messages
        a = normalize_error_messages(a)

        # Make sure error messages match
        if incremental_step == 0:
            # Not incremental
            msg = 'Unexpected type checker output ({}, line {})'
            output = testcase.output
        elif incremental_step == 1:
            msg = 'Unexpected type checker output in incremental, run 1 ({}, line {})'
            output = testcase.output
        elif incremental_step > 1:
            msg = ('Unexpected type checker output in incremental, run {}'.format(
                incremental_step) + ' ({}, line {})')
            output = testcase.output2.get(incremental_step, [])
        else:
            raise AssertionError()

        if output != a and self.update_data:
            update_testcase_output(testcase, a)
        assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line))

        if res:
            self.verify_cache(module_data, res.errors, res.manager, res.graph)

            if incremental_step > 1:
                suffix = '' if incremental_step == 2 else str(incremental_step - 1)
                assert_module_equivalence(
                    'rechecked' + suffix,
                    testcase.expected_rechecked_modules.get(incremental_step - 1),
                    res.manager.rechecked_modules)
                assert_module_equivalence(
                    'stale' + suffix,
                    testcase.expected_stale_modules.get(incremental_step - 1),
                    res.manager.stale_modules)
Esempio n. 33
0
    def run_case_once(self, testcase: DataDrivenTestCase,
                      operations: List[FileOperation] = [],
                      incremental_step: int = 0) -> None:
        original_program_text = '\n'.join(testcase.input)
        module_data = self.parse_module(original_program_text, incremental_step)

        # Unload already loaded plugins, they may be updated.
        for file, _ in testcase.files:
            module = module_from_path(file)
            if module.endswith('_plugin') and module in sys.modules:
                del sys.modules[module]
        if incremental_step == 0 or incremental_step == 1:
            # In run 1, copy program text to program file.
            for module_name, program_path, program_text in module_data:
                if module_name == '__main__':
                    with open(program_path, 'w', encoding='utf8') as f:
                        f.write(program_text)
                    break
        elif incremental_step > 1:
            # In runs 2+, copy *.[num] files to * files.
            for op in operations:
                if isinstance(op, UpdateFile):
                    # Modify/create file
                    copy_and_fudge_mtime(op.source_path, op.target_path)
                else:
                    # Delete file
                    # Use retries to work around potential flakiness on Windows (AppVeyor).
                    path = op.path
                    retry_on_error(lambda: os.remove(path))

        # Parse options after moving files (in case mypy.ini is being moved).
        options = parse_options(original_program_text, testcase, incremental_step)
        options.use_builtins_fixtures = True
        options.show_traceback = True
        if 'optional' in testcase.file:
            options.strict_optional = True
        if 'newsemanal' in testcase.file:
            options.new_semantic_analyzer = True
        if incremental_step and options.incremental:
            # Don't overwrite # flags: --no-incremental in incremental test cases
            options.incremental = True
        else:
            options.incremental = False
            # Don't waste time writing cache unless we are specifically looking for it
            if not testcase.writescache:
                options.cache_dir = os.devnull

        sources = []
        for module_name, program_path, program_text in module_data:
            # Always set to none so we're forced to reread the module in incremental mode
            sources.append(BuildSource(program_path, module_name,
                                       None if incremental_step else program_text))

        plugin_dir = os.path.join(test_data_prefix, 'plugins')
        sys.path.insert(0, plugin_dir)

        res = None
        try:
            res = build.build(sources=sources,
                              options=options,
                              alt_lib_path=test_temp_dir)
            a = res.errors
        except CompileError as e:
            a = e.messages
        finally:
            assert sys.path[0] == plugin_dir
            del sys.path[0]

        if testcase.normalize_output:
            a = normalize_error_messages(a)

        # Make sure error messages match
        if incremental_step == 0:
            # Not incremental
            msg = 'Unexpected type checker output ({}, line {})'
            output = testcase.output
        elif incremental_step == 1:
            msg = 'Unexpected type checker output in incremental, run 1 ({}, line {})'
            output = testcase.output
        elif incremental_step > 1:
            msg = ('Unexpected type checker output in incremental, run {}'.format(
                incremental_step) + ' ({}, line {})')
            output = testcase.output2.get(incremental_step, [])
        else:
            raise AssertionError()

        if output != a and testcase.config.getoption('--update-data', False):
            update_testcase_output(testcase, a)
        assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line))

        if res:
            if options.cache_dir != os.devnull:
                self.verify_cache(module_data, res.errors, res.manager, res.graph)

            if incremental_step > 1:
                suffix = '' if incremental_step == 2 else str(incremental_step - 1)
                assert_module_equivalence(
                    'rechecked' + suffix,
                    testcase.expected_rechecked_modules.get(incremental_step - 1),
                    res.manager.rechecked_modules)
                assert_module_equivalence(
                    'stale' + suffix,
                    testcase.expected_stale_modules.get(incremental_step - 1),
                    res.manager.stale_modules)