def run_case_once(self,
testcase: DataDrivenTestCase,
operations: List[FileOperation] = [],
incremental_step: int = 0) -> None:
original_program_text = '\n'.join(testcase.input)
module_data = self.parse_module(original_program_text,
incremental_step)
# Unload already loaded plugins, they may be updated.
for file, _ in testcase.files:
module = module_from_path(file)
if module.endswith('_plugin') and module in sys.modules:
del sys.modules[module]
if incremental_step == 0 or incremental_step == 1:
# In run 1, copy program text to program file.
for module_name, program_path, program_text in module_data:
if module_name == '__main__':
with open(program_path, 'w', encoding='utf8') as f:
f.write(program_text)
break
elif incremental_step > 1:
# In runs 2+, copy *.[num] files to * files.
for op in operations:
if isinstance(op, UpdateFile):
# Modify/create file
copy_and_fudge_mtime(op.source_path, op.target_path)
else:
# Delete file
# Use retries to work around potential flakiness on Windows (AppVeyor).
path = op.path
retry_on_error(lambda: os.remove(path))
# Parse options after moving files (in case mypy.ini is being moved).
options = parse_options(original_program_text, testcase,
incremental_step)
options.use_builtins_fixtures = True
options.show_traceback = True
# Enable some options automatically based on test file name.
if 'optional' in testcase.file:
options.strict_optional = True
if 'columns' in testcase.file:
options.show_column_numbers = True
if 'errorcodes' in testcase.file:
options.show_error_codes = True
if incremental_step and options.incremental:
# Don't overwrite # flags: --no-incremental in incremental test cases
options.incremental = True
else:
options.incremental = False
# Don't waste time writing cache unless we are specifically looking for it
if not testcase.writescache:
options.cache_dir = os.devnull
sources = []
for module_name, program_path, program_text in module_data:
# Always set to none so we're forced to reread the module in incremental mode
sources.append(
BuildSource(program_path, module_name,
None if incremental_step else program_text))
plugin_dir = os.path.join(test_data_prefix, 'plugins')
sys.path.insert(0, plugin_dir)
res = None
try:
res = build.build(sources=sources,
options=options,
alt_lib_path=test_temp_dir)
a = res.errors
except CompileError as e:
a = e.messages
finally:
assert sys.path[0] == plugin_dir
del sys.path[0]
if testcase.normalize_output:
a = normalize_error_messages(a)
# Make sure error messages match
if incremental_step == 0:
# Not incremental
msg = 'Unexpected type checker output ({}, line {})'
output = testcase.output
elif incremental_step == 1:
msg = 'Unexpected type checker output in incremental, run 1 ({}, line {})'
output = testcase.output
elif incremental_step > 1:
msg = ('Unexpected type checker output in incremental, run {}'.
format(incremental_step) + ' ({}, line {})')
output = testcase.output2.get(incremental_step, [])
else:
raise AssertionError()
if output != a and testcase.config.getoption('--update-data', False):
update_testcase_output(testcase, a)
assert_string_arrays_equal(output, a,
msg.format(testcase.file, testcase.line))
if res:
if options.cache_dir != os.devnull:
self.verify_cache(module_data, res.errors, res.manager,
res.graph)
name = 'targets'
if incremental_step:
name += str(incremental_step + 1)
expected = testcase.expected_fine_grained_targets.get(
incremental_step + 1)
actual = res.manager.processed_targets
# Skip the initial builtin cycle.
actual = [
t for t in actual if not any(
t.startswith(mod)
for mod in core_modules + ['mypy_extensions'])
]
if expected is not None:
assert_target_equivalence(name, expected, actual)
if incremental_step > 1:
suffix = '' if incremental_step == 2 else str(
incremental_step - 1)
expected_rechecked = testcase.expected_rechecked_modules.get(
incremental_step - 1)
if expected_rechecked is not None:
assert_module_equivalence('rechecked' + suffix,
expected_rechecked,
res.manager.rechecked_modules)
expected_stale = testcase.expected_stale_modules.get(
incremental_step - 1)
if expected_stale is not None:
assert_module_equivalence('stale' + suffix, expected_stale,
res.manager.stale_modules)
if testcase.output_files:
check_test_output_files(testcase,
incremental_step,
strip_prefix='tmp/')
def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int = 0) -> None:
find_module_clear_caches()
original_program_text = '\n'.join(testcase.input)
module_data = self.parse_module(original_program_text, incremental_step)
if incremental_step:
if incremental_step == 1:
# In run 1, copy program text to program file.
for module_name, program_path, program_text in module_data:
if module_name == '__main__':
with open(program_path, 'w') as f:
f.write(program_text)
break
elif incremental_step > 1:
# In runs 2+, copy *.[num] files to * files.
for dn, dirs, files in os.walk(os.curdir):
for file in files:
if file.endswith('.' + str(incremental_step)):
full = os.path.join(dn, file)
target = full[:-2]
# Use retries to work around potential flakiness on Windows (AppVeyor).
retry_on_error(lambda: shutil.copy(full, target))
# In some systems, mtime has a resolution of 1 second which can cause
# annoying-to-debug issues when a file has the same size after a
# change. We manually set the mtime to circumvent this.
new_time = os.stat(target).st_mtime + 1
os.utime(target, times=(new_time, new_time))
# Delete files scheduled to be deleted in [delete <path>.num] sections.
for path in testcase.deleted_paths.get(incremental_step, set()):
# Use retries to work around potential flakiness on Windows (AppVeyor).
retry_on_error(lambda: os.remove(path))
# Parse options after moving files (in case mypy.ini is being moved).
options = parse_options(original_program_text, testcase, incremental_step)
options.use_builtins_fixtures = True
options.show_traceback = True
if 'optional' in testcase.file:
options.strict_optional = True
if incremental_step:
options.incremental = True
else:
options.cache_dir = os.devnull # Don't waste time writing cache
sources = []
for module_name, program_path, program_text in module_data:
# Always set to none so we're forced to reread the module in incremental mode
sources.append(BuildSource(program_path, module_name,
None if incremental_step else program_text))
res = None
try:
res = build.build(sources=sources,
options=options,
alt_lib_path=test_temp_dir)
a = res.errors
except CompileError as e:
a = e.messages
a = normalize_error_messages(a)
# Make sure error messages match
if incremental_step == 0:
# Not incremental
msg = 'Unexpected type checker output ({}, line {})'
output = testcase.output
elif incremental_step == 1:
msg = 'Unexpected type checker output in incremental, run 1 ({}, line {})'
output = testcase.output
elif incremental_step > 1:
msg = ('Unexpected type checker output in incremental, run {}'.format(
incremental_step) + ' ({}, line {})')
output = testcase.output2.get(incremental_step, [])
else:
raise AssertionError()
if output != a and self.update_data:
update_testcase_output(testcase, a)
assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line))
if incremental_step and res:
if options.follow_imports == 'normal' and testcase.output is None:
self.verify_cache(module_data, a, res.manager)
if incremental_step > 1:
suffix = '' if incremental_step == 2 else str(incremental_step - 1)
self.check_module_equivalence(
'rechecked' + suffix,
testcase.expected_rechecked_modules.get(incremental_step - 1),
res.manager.rechecked_modules)
self.check_module_equivalence(
'stale' + suffix,
testcase.expected_stale_modules.get(incremental_step - 1),
res.manager.stale_modules)
def run_case_once(self, testcase: DataDrivenTestCase,
incremental_step: int) -> None:
assert incremental_step >= 1
build.find_module_clear_caches()
original_program_text = '\n'.join(testcase.input)
if incremental_step > 1:
# In runs 2+, copy *.[num] files to * files.
for dn, dirs, files in os.walk(os.curdir):
for file in files:
if file.endswith('.' + str(incremental_step)):
full = os.path.join(dn, file)
target = full[:-2]
# Use retries to work around potential flakiness on Windows (AppVeyor).
retry_on_error(lambda: shutil.copy(full, target))
# In some systems, mtime has a resolution of 1 second which can cause
# annoying-to-debug issues when a file has the same size after a
# change. We manually set the mtime to circumvent this.
new_time = os.stat(target).st_mtime + 1
os.utime(target, times=(new_time, new_time))
# Delete files scheduled to be deleted in [delete <path>.num] sections.
for path in testcase.deleted_paths.get(incremental_step, set()):
# Use retries to work around potential flakiness on Windows (AppVeyor).
retry_on_error(lambda: os.remove(path))
module_data = self.parse_module(original_program_text,
incremental_step)
if incremental_step == 1:
# In run 1, copy program text to program file.
for module_name, program_path, program_text in module_data:
if module_name == '__main__' and program_text is not None:
with open(program_path, 'w') as f:
f.write(program_text)
break
# Parse options after moving files (in case mypy.ini is being moved).
options = self.parse_options(original_program_text, testcase,
incremental_step)
if incremental_step == 1:
server_options = [] # type: List[str]
if 'fine-grained' in testcase.file:
server_options.append('--experimental')
options.fine_grained_incremental = True
self.server = dmypy_server.Server(
server_options) # TODO: Fix ugly API
self.server.options = options
assert self.server is not None # Set in step 1 and survives into next steps
sources = []
for module_name, program_path, program_text in module_data:
# Always set to none so we're forced to reread the module in incremental mode
sources.append(build.BuildSource(program_path, module_name, None))
response = self.server.check(sources, alt_lib_path=test_temp_dir)
a = (response['out'] or response['err']).splitlines()
a = normalize_error_messages(a)
# Make sure error messages match
if incremental_step == 1:
msg = 'Unexpected type checker output in incremental, run 1 ({}, line {})'
output = testcase.output
elif incremental_step > 1:
msg = ('Unexpected type checker output in incremental, run {}'.
format(incremental_step) + ' ({}, line {})')
output = testcase.output2.get(incremental_step, [])
else:
raise AssertionError()
if output != a and self.update_data:
update_testcase_output(testcase, a)
assert_string_arrays_equal(output, a,
msg.format(testcase.file, testcase.line))
manager = self.server.last_manager
if manager is not None:
if options.follow_imports == 'normal' and testcase.output is None:
self.verify_cache(module_data, a, manager)
if incremental_step > 1:
suffix = '' if incremental_step == 2 else str(
incremental_step - 1)
self.check_module_equivalence(
'rechecked' + suffix,
testcase.expected_rechecked_modules.get(incremental_step -
1),
manager.rechecked_modules)
self.check_module_equivalence(
'stale' + suffix,
testcase.expected_stale_modules.get(incremental_step - 1),
manager.stale_modules)
def run_case_once(self,
testcase: DataDrivenTestCase,
operations: List[FileOperation] = [],
incremental_step: int = 0) -> None:
original_program_text = '\n'.join(testcase.input)
module_data = self.parse_module(original_program_text,
incremental_step)
if incremental_step == 0 or incremental_step == 1:
# In run 1, copy program text to program file.
for module_name, program_path, program_text in module_data:
if module_name == '__main__':
with open(program_path, 'w') as f:
f.write(program_text)
break
elif incremental_step > 1:
# In runs 2+, copy *.[num] files to * files.
for op in operations:
if isinstance(op, UpdateFile):
# Modify/create file
copy_and_fudge_mtime(op.source_path, op.target_path)
else:
# Delete file
# Use retries to work around potential flakiness on Windows (AppVeyor).
path = op.path
retry_on_error(lambda: os.remove(path))
# Parse options after moving files (in case mypy.ini is being moved).
options = parse_options(original_program_text, testcase,
incremental_step)
options.use_builtins_fixtures = True
options.show_traceback = True
if 'optional' in testcase.file:
options.strict_optional = True
if incremental_step and options.incremental:
# Don't overwrite # flags: --no-incremental in incremental test cases
options.incremental = True
else:
options.incremental = False
# Don't waste time writing cache unless we are specifically looking for it
if 'writescache' not in testcase.name.lower():
options.cache_dir = os.devnull
sources = []
for module_name, program_path, program_text in module_data:
# Always set to none so we're forced to reread the module in incremental mode
sources.append(
BuildSource(program_path, module_name,
None if incremental_step else program_text))
res = None
try:
res = build.build(sources=sources,
options=options,
alt_lib_path=test_temp_dir)
a = res.errors
except CompileError as e:
a = e.messages
a = normalize_error_messages(a)
# Make sure error messages match
if incremental_step == 0:
# Not incremental
msg = 'Unexpected type checker output ({}, line {})'
output = testcase.output
elif incremental_step == 1:
msg = 'Unexpected type checker output in incremental, run 1 ({}, line {})'
output = testcase.output
elif incremental_step > 1:
msg = ('Unexpected type checker output in incremental, run {}'.
format(incremental_step) + ' ({}, line {})')
output = testcase.output2.get(incremental_step, [])
else:
raise AssertionError()
if output != a and testcase.config.getoption('--update-data', False):
update_testcase_output(testcase, a)
assert_string_arrays_equal(output, a,
msg.format(testcase.file, testcase.line))
if res:
if options.cache_dir != os.devnull:
self.verify_cache(module_data, res.errors, res.manager,
res.graph)
if incremental_step > 1:
suffix = '' if incremental_step == 2 else str(
incremental_step - 1)
assert_module_equivalence(
'rechecked' + suffix,
testcase.expected_rechecked_modules.get(incremental_step -
1),
res.manager.rechecked_modules)
assert_module_equivalence(
'stale' + suffix,
testcase.expected_stale_modules.get(incremental_step - 1),
res.manager.stale_modules)
def run_case_once(self, testcase: DataDrivenTestCase,
operations: List[FileOperation] = [],
incremental_step: int = 0) -> None:
original_program_text = '\n'.join(testcase.input)
module_data = self.parse_module(original_program_text, incremental_step)
# Unload already loaded plugins, they may be updated.
for file, _ in testcase.files:
module = module_from_path(file)
if module.endswith('_plugin') and module in sys.modules:
del sys.modules[module]
if incremental_step == 0 or incremental_step == 1:
# In run 1, copy program text to program file.
for module_name, program_path, program_text in module_data:
if module_name == '__main__':
with open(program_path, 'w', encoding='utf8') as f:
f.write(program_text)
break
elif incremental_step > 1:
# In runs 2+, copy *.[num] files to * files.
for op in operations:
if isinstance(op, UpdateFile):
# Modify/create file
copy_and_fudge_mtime(op.source_path, op.target_path)
else:
# Delete file
# Use retries to work around potential flakiness on Windows (AppVeyor).
path = op.path
retry_on_error(lambda: os.remove(path))
# Parse options after moving files (in case mypy.ini is being moved).
options = parse_options(original_program_text, testcase, incremental_step)
options.use_builtins_fixtures = True
options.show_traceback = True
if 'optional' in testcase.file:
options.strict_optional = True
if 'newsemanal' in testcase.file:
options.new_semantic_analyzer = True
if incremental_step and options.incremental:
# Don't overwrite # flags: --no-incremental in incremental test cases
options.incremental = True
else:
options.incremental = False
# Don't waste time writing cache unless we are specifically looking for it
if not testcase.writescache:
options.cache_dir = os.devnull
sources = []
for module_name, program_path, program_text in module_data:
# Always set to none so we're forced to reread the module in incremental mode
sources.append(BuildSource(program_path, module_name,
None if incremental_step else program_text))
plugin_dir = os.path.join(test_data_prefix, 'plugins')
sys.path.insert(0, plugin_dir)
res = None
try:
res = build.build(sources=sources,
options=options,
alt_lib_path=test_temp_dir)
a = res.errors
except CompileError as e:
a = e.messages
finally:
assert sys.path[0] == plugin_dir
del sys.path[0]
if testcase.normalize_output:
a = normalize_error_messages(a)
# Make sure error messages match
if incremental_step == 0:
# Not incremental
msg = 'Unexpected type checker output ({}, line {})'
output = testcase.output
elif incremental_step == 1:
msg = 'Unexpected type checker output in incremental, run 1 ({}, line {})'
output = testcase.output
elif incremental_step > 1:
msg = ('Unexpected type checker output in incremental, run {}'.format(
incremental_step) + ' ({}, line {})')
output = testcase.output2.get(incremental_step, [])
else:
raise AssertionError()
if output != a and testcase.config.getoption('--update-data', False):
update_testcase_output(testcase, a)
assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line))
if res:
if options.cache_dir != os.devnull:
self.verify_cache(module_data, res.errors, res.manager, res.graph)
if incremental_step > 1:
suffix = '' if incremental_step == 2 else str(incremental_step - 1)
assert_module_equivalence(
'rechecked' + suffix,
testcase.expected_rechecked_modules.get(incremental_step - 1),
res.manager.rechecked_modules)
assert_module_equivalence(
'stale' + suffix,
testcase.expected_stale_modules.get(incremental_step - 1),
res.manager.stale_modules)
def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int) -> None:
assert incremental_step >= 1
build.find_module_clear_caches()
original_program_text = '\n'.join(testcase.input)
module_data = self.parse_module(original_program_text, incremental_step)
if incremental_step == 1:
# In run 1, copy program text to program file.
for module_name, program_path, program_text in module_data:
if module_name == '__main__':
with open(program_path, 'w') as f:
f.write(program_text)
break
elif incremental_step > 1:
# In runs 2+, copy *.[num] files to * files.
for dn, dirs, files in os.walk(os.curdir):
for file in files:
if file.endswith('.' + str(incremental_step)):
full = os.path.join(dn, file)
target = full[:-2]
# Use retries to work around potential flakiness on Windows (AppVeyor).
retry_on_error(lambda: shutil.copy(full, target))
# In some systems, mtime has a resolution of 1 second which can cause
# annoying-to-debug issues when a file has the same size after a
# change. We manually set the mtime to circumvent this.
new_time = os.stat(target).st_mtime + 1
os.utime(target, times=(new_time, new_time))
# Delete files scheduled to be deleted in [delete <path>.num] sections.
for path in testcase.deleted_paths.get(incremental_step, set()):
# Use retries to work around potential flakiness on Windows (AppVeyor).
retry_on_error(lambda: os.remove(path))
# Parse options after moving files (in case mypy.ini is being moved).
options = self.parse_options(original_program_text, testcase, incremental_step)
if incremental_step == 1:
self.server = dmypy_server.Server([]) # TODO: Fix ugly API
self.server.options = options
assert self.server is not None # Set in step 1 and survives into next steps
sources = []
for module_name, program_path, program_text in module_data:
# Always set to none so we're forced to reread the module in incremental mode
sources.append(build.BuildSource(program_path, module_name, None))
response = self.server.check(sources, alt_lib_path=test_temp_dir)
a = (response['out'] or response['err']).splitlines()
a = normalize_error_messages(a)
# Make sure error messages match
if incremental_step == 1:
msg = 'Unexpected type checker output in incremental, run 1 ({}, line {})'
output = testcase.output
elif incremental_step > 1:
msg = ('Unexpected type checker output in incremental, run {}'.format(
incremental_step) + ' ({}, line {})')
output = testcase.output2.get(incremental_step, [])
else:
raise AssertionError()
if output != a and self.update_data:
update_testcase_output(testcase, a)
assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line))
manager = self.server.last_manager
if manager is not None:
if options.follow_imports == 'normal' and testcase.output is None:
self.verify_cache(module_data, a, manager)
if incremental_step > 1:
suffix = '' if incremental_step == 2 else str(incremental_step - 1)
self.check_module_equivalence(
'rechecked' + suffix,
testcase.expected_rechecked_modules.get(incremental_step - 1),
manager.rechecked_modules)
self.check_module_equivalence(
'stale' + suffix,
testcase.expected_stale_modules.get(incremental_step - 1),
manager.stale_modules)
def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int = 0) -> None:
original_program_text = '\n'.join(testcase.input)
module_data = self.parse_module(original_program_text, incremental_step)
if incremental_step == 0 or incremental_step == 1:
# In run 1, copy program text to program file.
for module_name, program_path, program_text in module_data:
if module_name == '__main__':
with open(program_path, 'w') as f:
f.write(program_text)
break
elif incremental_step > 1:
# In runs 2+, copy *.[num] files to * files.
for dn, dirs, files in os.walk(os.curdir):
for file in files:
if file.endswith('.' + str(incremental_step)):
full = os.path.join(dn, file)
target = full[:-2]
copy_and_fudge_mtime(full, target)
# Delete files scheduled to be deleted in [delete <path>.num] sections.
for path in testcase.deleted_paths.get(incremental_step, set()):
# Use retries to work around potential flakiness on Windows (AppVeyor).
retry_on_error(lambda: os.remove(path))
# Parse options after moving files (in case mypy.ini is being moved).
options = parse_options(original_program_text, testcase, incremental_step)
options.use_builtins_fixtures = True
options.show_traceback = True
if 'optional' in testcase.file:
options.strict_optional = True
if incremental_step:
options.incremental = True
sources = []
for module_name, program_path, program_text in module_data:
# Always set to none so we're forced to reread the module in incremental mode
sources.append(BuildSource(program_path, module_name,
None if incremental_step else program_text))
res = None
try:
res = build.build(sources=sources,
options=options,
alt_lib_path=test_temp_dir)
a = res.errors
except CompileError as e:
a = e.messages
a = normalize_error_messages(a)
# Make sure error messages match
if incremental_step == 0:
# Not incremental
msg = 'Unexpected type checker output ({}, line {})'
output = testcase.output
elif incremental_step == 1:
msg = 'Unexpected type checker output in incremental, run 1 ({}, line {})'
output = testcase.output
elif incremental_step > 1:
msg = ('Unexpected type checker output in incremental, run {}'.format(
incremental_step) + ' ({}, line {})')
output = testcase.output2.get(incremental_step, [])
else:
raise AssertionError()
if output != a and self.update_data:
update_testcase_output(testcase, a)
assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line))
if res:
self.verify_cache(module_data, res.errors, res.manager, res.graph)
if incremental_step > 1:
suffix = '' if incremental_step == 2 else str(incremental_step - 1)
assert_module_equivalence(
'rechecked' + suffix,
testcase.expected_rechecked_modules.get(incremental_step - 1),
res.manager.rechecked_modules)
assert_module_equivalence(
'stale' + suffix,
testcase.expected_stale_modules.get(incremental_step - 1),
res.manager.stale_modules)