def update(self, changed_modules: List[Tuple[str, str]]) -> List[str]:
"""Update previous build result by processing changed modules.
Also propagate changes to other modules as needed, but only process
those parts of other modules that are affected by the changes. Retain
the existing ASTs and symbol tables of unaffected modules.
Create new graph with new State objects, but reuse original BuildManager.
Args:
changed_modules: Modules changed since the previous update/build; each is
a (module id, path) tuple. Includes modified, added and deleted modules.
Assume this is correct; it's not validated here.
Returns:
A list of errors.
"""
assert changed_modules, 'No changed modules'
# Reset global caches for the new build.
find_module_clear_caches()
self.triggered = []
changed_modules = dedupe_modules(changed_modules + self.stale)
initial_set = {id for id, _ in changed_modules}
self.manager.log_fine_grained(
'==== update %s ====' %
', '.join(repr(id) for id, _ in changed_modules))
if self.previous_targets_with_errors and is_verbose(self.manager):
self.manager.log_fine_grained(
'previous targets with errors: %s' %
sorted(self.previous_targets_with_errors))
if self.blocking_error:
# Handle blocking errors first. We'll exit as soon as we find a
# module that still has blocking errors.
self.manager.log_fine_grained('existing blocker: %s' %
self.blocking_error[0])
changed_modules = dedupe_modules([self.blocking_error] +
changed_modules)
self.blocking_error = None
while changed_modules:
next_id, next_path = changed_modules.pop(0)
if next_id not in self.previous_modules and next_id not in initial_set:
self.manager.log_fine_grained(
'skip %r (module not in import graph)' % next_id)
continue
result = self.update_single(next_id, next_path)
messages, remaining, (next_id, next_path), blocker = result
changed_modules = [(id, path) for id, path in changed_modules
if id != next_id]
changed_modules = dedupe_modules(remaining + changed_modules)
if blocker:
self.blocking_error = (next_id, next_path)
self.stale = changed_modules
return messages
return messages
def run_case_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
find_module_clear_caches()
program_text = '\n'.join(testcase.input)
module_name, program_name, program_text = self.parse_module(program_text)
options = self.parse_options(program_text)
options.use_builtins_fixtures = True
options.python_version = testcase_pyversion(testcase.file, testcase.name)
output = testcase.output
if incremental:
options.incremental = True
if incremental == 1:
# In run 1, copy program text to program file.
output = []
with open(program_name, 'w') as f:
f.write(program_text)
elif incremental == 2:
# In run 2, copy *.py.next files to *.py files.
for dn, dirs, files in os.walk(os.curdir):
for file in files:
if file.endswith('.py.next'):
full = os.path.join(dn, file)
target = full[:-5]
shutil.copy(full, target)
# Always set to none so we're forced to reread program_name
program_text = None
source = BuildSource(program_name, module_name, program_text)
try:
res = build.build(sources=[source],
options=options,
alt_lib_path=test_temp_dir)
a = res.errors
except CompileError as e:
res = None
a = e.messages
a = normalize_error_messages(a)
if output != a and self.update_data:
update_testcase_output(testcase, a)
assert_string_arrays_equal(
output, a,
'Invalid type checker output ({}, line {})'.format(
testcase.file, testcase.line))
if incremental and res:
self.verify_cache(module_name, program_name, a, res.manager)
if testcase.expected_stale_modules is not None and incremental == 2:
assert_string_arrays_equal(
list(sorted(testcase.expected_stale_modules)),
list(sorted(res.manager.stale_modules.difference({"__main__"}))),
'Set of stale modules does not match expected set')
def run_test_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
find_module_clear_caches()
program_text = '\n'.join(testcase.input)
module_name, program_name, program_text = self.parse_module(program_text)
options = self.parse_options(program_text)
options.use_builtins_fixtures = True
options.python_version = testcase_pyversion(testcase.file, testcase.name)
output = testcase.output
if incremental:
options.incremental = True
if incremental == 1:
# In run 1, copy program text to program file.
output = []
with open(program_name, 'w') as f:
f.write(program_text)
program_text = None
elif incremental == 2:
# In run 2, copy *.py.next files to *.py files.
for dn, dirs, files in os.walk(os.curdir):
for file in files:
if file.endswith('.py.next'):
full = os.path.join(dn, file)
target = full[:-5]
shutil.copy(full, target)
source = BuildSource(program_name, module_name, program_text)
try:
res = build.build(sources=[source],
options=options,
alt_lib_path=test_temp_dir)
a = res.errors
except CompileError as e:
res = None
a = e.messages
a = normalize_error_messages(a)
if output != a and mypy.myunit.UPDATE_TESTCASES:
update_testcase_output(testcase, a, mypy.myunit.APPEND_TESTCASES)
assert_string_arrays_equal(
output, a,
'Invalid type checker output ({}, line {})'.format(
testcase.file, testcase.line))
if incremental and res:
self.verify_cache(module_name, program_name, a, res.manager)
if testcase.expected_stale_modules is not None and incremental == 2:
assert_string_arrays_equal(
list(sorted(testcase.expected_stale_modules)),
list(sorted(res.manager.stale_modules.difference({"__main__"}))),
'Set of stale modules does not match expected set')
def run_test_once(self,
testcase: DataDrivenTestCase,
incremental=0) -> None:
find_module_clear_caches()
pyversion = testcase_pyversion(testcase.file, testcase.name)
program_text = '\n'.join(testcase.input)
module_name, program_name, program_text = self.parse_options(
program_text)
flags = self.parse_flags(program_text)
output = testcase.output
if incremental:
flags.append(build.INCREMENTAL)
if incremental == 1:
# In run 1, copy program text to program file.
output = []
with open(program_name, 'w') as f:
f.write(program_text)
program_text = None
elif incremental == 2:
# In run 2, copy *.py.next files to *.py files.
for dn, dirs, files in os.walk(os.curdir):
for file in files:
if file.endswith('.py.next'):
full = os.path.join(dn, file)
target = full[:-5]
shutil.copy(full, target)
source = BuildSource(program_name, module_name, program_text)
try:
res = build.build(target=build.TYPE_CHECK,
sources=[source],
pyversion=pyversion,
flags=flags + [build.TEST_BUILTINS],
alt_lib_path=test_temp_dir)
a = res.errors
except CompileError as e:
res = None
a = e.messages
a = normalize_error_messages(a)
if output != a and mypy.myunit.UPDATE_TESTCASES:
update_testcase_output(testcase, a, mypy.myunit.APPEND_TESTCASES)
assert_string_arrays_equal(
output, a, 'Invalid type checker output ({}, line {})'.format(
testcase.file, testcase.line))
if incremental and res:
self.verify_cache(module_name, program_name, a, res.manager)
def run_test_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
find_module_clear_caches()
program_text = '\n'.join(testcase.input)
module_name, program_name, program_text = self.parse_module(program_text)
options = self.parse_options(program_text)
options.use_builtins_fixtures = True
options.python_version = testcase_pyversion(testcase.file, testcase.name)
output = testcase.output
if incremental:
options.incremental = True
if incremental == 1:
# In run 1, copy program text to program file.
output = []
with open(program_name, 'w') as f:
f.write(program_text)
program_text = None
elif incremental == 2:
# In run 2, copy *.py.next files to *.py files.
for dn, dirs, files in os.walk(os.curdir):
for file in files:
if file.endswith('.py.next'):
full = os.path.join(dn, file)
target = full[:-5]
shutil.copy(full, target)
source = BuildSource(program_name, module_name, program_text)
try:
res = build.build(sources=[source],
options=options,
alt_lib_path=test_temp_dir)
a = res.errors
except CompileError as e:
res = None
a = e.messages
a = normalize_error_messages(a)
if output != a and mypy.myunit.UPDATE_TESTCASES:
update_testcase_output(testcase, a, mypy.myunit.APPEND_TESTCASES)
assert_string_arrays_equal(
output, a,
'Invalid type checker output ({}, line {})'.format(
testcase.file, testcase.line))
if incremental and res:
self.verify_cache(module_name, program_name, a, res.manager)
def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int = 0) -> None:
find_module_clear_caches()
original_program_text = '\n'.join(testcase.input)
module_data = self.parse_module(original_program_text, incremental_step)
if incremental_step:
if incremental_step == 1:
# In run 1, copy program text to program file.
for module_name, program_path, program_text in module_data:
if module_name == '__main__':
with open(program_path, 'w') as f:
f.write(program_text)
break
elif incremental_step > 1:
# In runs 2+, copy *.[num] files to * files.
for dn, dirs, files in os.walk(os.curdir):
for file in files:
if file.endswith('.' + str(incremental_step)):
full = os.path.join(dn, file)
target = full[:-2]
# Use retries to work around potential flakiness on Windows (AppVeyor).
retry_on_error(lambda: shutil.copy(full, target))
# In some systems, mtime has a resolution of 1 second which can cause
# annoying-to-debug issues when a file has the same size after a
# change. We manually set the mtime to circumvent this.
new_time = os.stat(target).st_mtime + 1
os.utime(target, times=(new_time, new_time))
# Delete files scheduled to be deleted in [delete <path>.num] sections.
for path in testcase.deleted_paths.get(incremental_step, set()):
# Use retries to work around potential flakiness on Windows (AppVeyor).
retry_on_error(lambda: os.remove(path))
# Parse options after moving files (in case mypy.ini is being moved).
options = parse_options(original_program_text, testcase, incremental_step)
options.use_builtins_fixtures = True
options.show_traceback = True
if 'optional' in testcase.file:
options.strict_optional = True
if incremental_step:
options.incremental = True
else:
options.cache_dir = os.devnull # Don't waste time writing cache
sources = []
for module_name, program_path, program_text in module_data:
# Always set to none so we're forced to reread the module in incremental mode
sources.append(BuildSource(program_path, module_name,
None if incremental_step else program_text))
res = None
try:
res = build.build(sources=sources,
options=options,
alt_lib_path=test_temp_dir)
a = res.errors
except CompileError as e:
a = e.messages
a = normalize_error_messages(a)
# Make sure error messages match
if incremental_step == 0:
# Not incremental
msg = 'Unexpected type checker output ({}, line {})'
output = testcase.output
elif incremental_step == 1:
msg = 'Unexpected type checker output in incremental, run 1 ({}, line {})'
output = testcase.output
elif incremental_step > 1:
msg = ('Unexpected type checker output in incremental, run {}'.format(
incremental_step) + ' ({}, line {})')
output = testcase.output2.get(incremental_step, [])
else:
raise AssertionError()
if output != a and self.update_data:
update_testcase_output(testcase, a)
assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line))
if incremental_step and res:
if options.follow_imports == 'normal' and testcase.output is None:
self.verify_cache(module_data, a, res.manager)
if incremental_step > 1:
suffix = '' if incremental_step == 2 else str(incremental_step - 1)
self.check_module_equivalence(
'rechecked' + suffix,
testcase.expected_rechecked_modules.get(incremental_step - 1),
res.manager.rechecked_modules)
self.check_module_equivalence(
'stale' + suffix,
testcase.expected_stale_modules.get(incremental_step - 1),
res.manager.stale_modules)
def run_case_once(self,
testcase: DataDrivenTestCase,
incremental=0) -> None:
find_module_clear_caches()
original_program_text = '\n'.join(testcase.input)
module_data = self.parse_module(original_program_text, incremental)
options = self.parse_options(original_program_text, testcase)
options.use_builtins_fixtures = True
options.show_traceback = True
if 'optional' in testcase.file:
options.strict_optional = True
if incremental:
options.incremental = True
if incremental == 1:
# In run 1, copy program text to program file.
for module_name, program_path, program_text in module_data:
if module_name == '__main__':
with open(program_path, 'w') as f:
f.write(program_text)
break
elif incremental == 2:
# In run 2, copy *.py.next files to *.py files.
for dn, dirs, files in os.walk(os.curdir):
for file in files:
if file.endswith('.py.next'):
full = os.path.join(dn, file)
target = full[:-5]
shutil.copy(full, target)
# In some systems, mtime has a resolution of 1 second which can cause
# annoying-to-debug issues when a file has the same size after a
# change. We manually set the mtime to circumvent this.
new_time = os.stat(target).st_mtime + 1
os.utime(target, times=(new_time, new_time))
sources = []
for module_name, program_path, program_text in module_data:
# Always set to none so we're forced to reread the module in incremental mode
sources.append(
BuildSource(program_path, module_name,
None if incremental else program_text))
res = None
try:
res = build.build(sources=sources,
options=options,
alt_lib_path=test_temp_dir)
a = res.errors
except CompileError as e:
a = e.messages
a = normalize_error_messages(a)
# Make sure error messages match
if incremental == 0:
msg = 'Invalid type checker output ({}, line {})'
output = testcase.output
elif incremental == 1:
msg = 'Invalid type checker output in incremental, run 1 ({}, line {})'
output = testcase.output
elif incremental == 2:
msg = 'Invalid type checker output in incremental, run 2 ({}, line {})'
output = testcase.output2
else:
raise AssertionError()
if output != a and self.update_data:
update_testcase_output(testcase, a)
assert_string_arrays_equal(output, a,
msg.format(testcase.file, testcase.line))
if incremental and res:
if not options.silent_imports and testcase.output is None:
self.verify_cache(module_data, a, res.manager)
if incremental == 2:
self.check_module_equivalence(
'rechecked', testcase.expected_rechecked_modules,
res.manager.rechecked_modules)
self.check_module_equivalence('stale',
testcase.expected_stale_modules,
res.manager.stale_modules)
def run_case_once(self, testcase: DataDrivenTestCase,
incremental_step: int) -> None:
assert incremental_step >= 1
build.find_module_clear_caches()
original_program_text = '\n'.join(testcase.input)
if incremental_step > 1:
# In runs 2+, copy *.[num] files to * files.
for dn, dirs, files in os.walk(os.curdir):
for file in files:
if file.endswith('.' + str(incremental_step)):
full = os.path.join(dn, file)
target = full[:-2]
# Use retries to work around potential flakiness on Windows (AppVeyor).
retry_on_error(lambda: shutil.copy(full, target))
# In some systems, mtime has a resolution of 1 second which can cause
# annoying-to-debug issues when a file has the same size after a
# change. We manually set the mtime to circumvent this.
new_time = os.stat(target).st_mtime + 1
os.utime(target, times=(new_time, new_time))
# Delete files scheduled to be deleted in [delete <path>.num] sections.
for path in testcase.deleted_paths.get(incremental_step, set()):
# Use retries to work around potential flakiness on Windows (AppVeyor).
retry_on_error(lambda: os.remove(path))
module_data = self.parse_module(original_program_text,
incremental_step)
if incremental_step == 1:
# In run 1, copy program text to program file.
for module_name, program_path, program_text in module_data:
if module_name == '__main__' and program_text is not None:
with open(program_path, 'w') as f:
f.write(program_text)
break
# Parse options after moving files (in case mypy.ini is being moved).
options = self.parse_options(original_program_text, testcase,
incremental_step)
if incremental_step == 1:
server_options = [] # type: List[str]
if 'fine-grained' in testcase.file:
server_options.append('--experimental')
options.fine_grained_incremental = True
self.server = dmypy_server.Server(
server_options) # TODO: Fix ugly API
self.server.options = options
assert self.server is not None # Set in step 1 and survives into next steps
sources = []
for module_name, program_path, program_text in module_data:
# Always set to none so we're forced to reread the module in incremental mode
sources.append(build.BuildSource(program_path, module_name, None))
response = self.server.check(sources, alt_lib_path=test_temp_dir)
a = (response['out'] or response['err']).splitlines()
a = normalize_error_messages(a)
# Make sure error messages match
if incremental_step == 1:
msg = 'Unexpected type checker output in incremental, run 1 ({}, line {})'
output = testcase.output
elif incremental_step > 1:
msg = ('Unexpected type checker output in incremental, run {}'.
format(incremental_step) + ' ({}, line {})')
output = testcase.output2.get(incremental_step, [])
else:
raise AssertionError()
if output != a and self.update_data:
update_testcase_output(testcase, a)
assert_string_arrays_equal(output, a,
msg.format(testcase.file, testcase.line))
manager = self.server.last_manager
if manager is not None:
if options.follow_imports == 'normal' and testcase.output is None:
self.verify_cache(module_data, a, manager)
if incremental_step > 1:
suffix = '' if incremental_step == 2 else str(
incremental_step - 1)
self.check_module_equivalence(
'rechecked' + suffix,
testcase.expected_rechecked_modules.get(incremental_step -
1),
manager.rechecked_modules)
self.check_module_equivalence(
'stale' + suffix,
testcase.expected_stale_modules.get(incremental_step - 1),
manager.stale_modules)
def run_case_once(self, testcase: DataDrivenTestCase, incremental: int = 0) -> None:
find_module_clear_caches()
original_program_text = '\n'.join(testcase.input)
module_data = self.parse_module(original_program_text, incremental)
if incremental:
if incremental == 1:
# In run 1, copy program text to program file.
for module_name, program_path, program_text in module_data:
if module_name == '__main__':
with open(program_path, 'w') as f:
f.write(program_text)
break
elif incremental == 2:
# In run 2, copy *.next files to * files.
for dn, dirs, files in os.walk(os.curdir):
for file in files:
if file.endswith('.next'):
full = os.path.join(dn, file)
target = full[:-5]
shutil.copy(full, target)
# In some systems, mtime has a resolution of 1 second which can cause
# annoying-to-debug issues when a file has the same size after a
# change. We manually set the mtime to circumvent this.
new_time = os.stat(target).st_mtime + 1
os.utime(target, times=(new_time, new_time))
# Parse options after moving files (in case mypy.ini is being moved).
options = self.parse_options(original_program_text, testcase)
options.use_builtins_fixtures = True
options.show_traceback = True
if 'optional' in testcase.file:
options.strict_optional = True
if incremental:
options.incremental = True
if os.path.split(testcase.file)[1] in fast_parser_files:
options.fast_parser = True
sources = []
for module_name, program_path, program_text in module_data:
# Always set to none so we're forced to reread the module in incremental mode
sources.append(BuildSource(program_path, module_name,
None if incremental else program_text))
res = None
try:
res = build.build(sources=sources,
options=options,
alt_lib_path=test_temp_dir)
a = res.errors
except CompileError as e:
a = e.messages
a = normalize_error_messages(a)
# Make sure error messages match
if incremental == 0:
msg = 'Invalid type checker output ({}, line {})'
output = testcase.output
elif incremental == 1:
msg = 'Invalid type checker output in incremental, run 1 ({}, line {})'
output = testcase.output
elif incremental == 2:
msg = 'Invalid type checker output in incremental, run 2 ({}, line {})'
output = testcase.output2
else:
raise AssertionError()
if output != a and self.update_data:
update_testcase_output(testcase, a)
assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line))
if incremental and res:
if options.follow_imports == 'normal' and testcase.output is None:
self.verify_cache(module_data, a, res.manager)
if incremental == 2:
self.check_module_equivalence(
'rechecked',
testcase.expected_rechecked_modules,
res.manager.rechecked_modules)
self.check_module_equivalence(
'stale',
testcase.expected_stale_modules,
res.manager.stale_modules)
def run_case_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
find_module_clear_caches()
original_program_text = '\n'.join(testcase.input)
module_data = self.parse_module(original_program_text, incremental)
options = self.parse_options(original_program_text)
options.use_builtins_fixtures = True
options.python_version = testcase_pyversion(testcase.file, testcase.name)
set_show_tb(True) # Show traceback on crash.
output = testcase.output
if incremental:
options.incremental = True
if incremental == 1:
# In run 1, copy program text to program file.
output = []
for module_name, program_path, program_text in module_data:
if module_name == '__main__':
with open(program_path, 'w') as f:
f.write(program_text)
break
elif incremental == 2:
# In run 2, copy *.py.next files to *.py files.
for dn, dirs, files in os.walk(os.curdir):
for file in files:
if file.endswith('.py.next'):
full = os.path.join(dn, file)
target = full[:-5]
shutil.copy(full, target)
# In some systems, mtime has a resolution of 1 second which can cause
# annoying-to-debug issues when a file has the same size after a
# change. We manually set the mtime to circumvent this.
new_time = os.stat(target).st_mtime + 1
os.utime(target, times=(new_time, new_time))
sources = []
for module_name, program_path, program_text in module_data:
# Always set to none so we're forced to reread the module in incremental mode
program_text = None if incremental else program_text
sources.append(BuildSource(program_path, module_name, program_text))
try:
res = build.build(sources=sources,
options=options,
alt_lib_path=test_temp_dir)
a = res.errors
except CompileError as e:
res = None
a = e.messages
a = normalize_error_messages(a)
if output != a and self.update_data:
update_testcase_output(testcase, a)
assert_string_arrays_equal(
output, a,
'Invalid type checker output ({}, line {})'.format(
testcase.file, testcase.line))
if incremental and res:
if not options.silent_imports:
self.verify_cache(module_data, a, res.manager)
if testcase.expected_stale_modules is not None and incremental == 2:
assert_string_arrays_equal(
list(sorted(testcase.expected_stale_modules)),
list(sorted(res.manager.stale_modules.difference({"__main__"}))),
'Set of stale modules does not match expected set')
def run_case_once(self, testcase: DataDrivenTestCase, incremental_step: int) -> None:
assert incremental_step >= 1
build.find_module_clear_caches()
original_program_text = '\n'.join(testcase.input)
module_data = self.parse_module(original_program_text, incremental_step)
if incremental_step == 1:
# In run 1, copy program text to program file.
for module_name, program_path, program_text in module_data:
if module_name == '__main__':
with open(program_path, 'w') as f:
f.write(program_text)
break
elif incremental_step > 1:
# In runs 2+, copy *.[num] files to * files.
for dn, dirs, files in os.walk(os.curdir):
for file in files:
if file.endswith('.' + str(incremental_step)):
full = os.path.join(dn, file)
target = full[:-2]
# Use retries to work around potential flakiness on Windows (AppVeyor).
retry_on_error(lambda: shutil.copy(full, target))
# In some systems, mtime has a resolution of 1 second which can cause
# annoying-to-debug issues when a file has the same size after a
# change. We manually set the mtime to circumvent this.
new_time = os.stat(target).st_mtime + 1
os.utime(target, times=(new_time, new_time))
# Delete files scheduled to be deleted in [delete <path>.num] sections.
for path in testcase.deleted_paths.get(incremental_step, set()):
# Use retries to work around potential flakiness on Windows (AppVeyor).
retry_on_error(lambda: os.remove(path))
# Parse options after moving files (in case mypy.ini is being moved).
options = self.parse_options(original_program_text, testcase, incremental_step)
if incremental_step == 1:
self.server = dmypy_server.Server([]) # TODO: Fix ugly API
self.server.options = options
assert self.server is not None # Set in step 1 and survives into next steps
sources = []
for module_name, program_path, program_text in module_data:
# Always set to none so we're forced to reread the module in incremental mode
sources.append(build.BuildSource(program_path, module_name, None))
response = self.server.check(sources, alt_lib_path=test_temp_dir)
a = (response['out'] or response['err']).splitlines()
a = normalize_error_messages(a)
# Make sure error messages match
if incremental_step == 1:
msg = 'Unexpected type checker output in incremental, run 1 ({}, line {})'
output = testcase.output
elif incremental_step > 1:
msg = ('Unexpected type checker output in incremental, run {}'.format(
incremental_step) + ' ({}, line {})')
output = testcase.output2.get(incremental_step, [])
else:
raise AssertionError()
if output != a and self.update_data:
update_testcase_output(testcase, a)
assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line))
manager = self.server.last_manager
if manager is not None:
if options.follow_imports == 'normal' and testcase.output is None:
self.verify_cache(module_data, a, manager)
if incremental_step > 1:
suffix = '' if incremental_step == 2 else str(incremental_step - 1)
self.check_module_equivalence(
'rechecked' + suffix,
testcase.expected_rechecked_modules.get(incremental_step - 1),
manager.rechecked_modules)
self.check_module_equivalence(
'stale' + suffix,
testcase.expected_stale_modules.get(incremental_step - 1),
manager.stale_modules)