def run_fts(self):
if os.path.exists(
os.path.join(self.tempdir, 'superlists', 'functional_tests')):
return self.run_command(
Command("python manage.py test functional_tests"))
else:
return self.run_command(Command("python functional_tests.py"))
def run_interactive_manage_py(self, listing):
output_before = self.listings[self.pos + 1]
assert isinstance(output_before, Output)
LIKELY_INPUTS = ('yes', 'no', '1', '2', "''")
user_input = self.listings[self.pos + 2]
if isinstance(user_input, Command) and user_input in LIKELY_INPUTS:
if user_input == 'yes':
print('yes case')
# in this case there is moar output after the yes
output_after = self.listings[self.pos + 3]
assert isinstance(output_after, Output)
expected_output = Output(wrap_long_lines(output_before + ' ' + output_after.lstrip()))
next_output = None
elif user_input == '1':
print('migrations 1 case')
# in this case there is another hop
output_after = self.listings[self.pos + 3]
assert isinstance(output_after, Output)
first_input = user_input
next_input = self.listings[self.pos + 4]
assert isinstance(next_input, Command)
next_output = self.listings[self.pos + 5]
expected_output = Output(wrap_long_lines(
output_before + '\n' + output_after + '\n' + next_output
))
user_input = Command(first_input + '\n' + next_input)
else:
expected_output = output_before
output_after = None
next_output = None
if user_input == '2':
ignore_errors = True
else:
ignore_errors = False
else:
user_input = None
expected_output = output_before
output_after = None
ignore_errors = True
next_output = None
output = self.run_command(listing, user_input=user_input, ignore_errors=ignore_errors)
self.assert_console_output_correct(output, expected_output)
listing.was_checked = True
output_before.was_checked = True
self.pos += 2
if user_input is not None:
user_input.was_run = True
self.pos += 1
if output_after is not None:
output_after.was_checked = True
self.pos += 1
if next_output is not None:
self.pos += 2
next_output.was_checked = True
first_input.was_run = True
next_input.was_run = True
def run_unit_tests(self):
if os.path.exists(
os.path.join(self.tempdir, 'superlists', 'accounts', 'tests')):
return self.run_command(
Command("python manage.py test lists accounts"))
else:
return self.run_command(Command("python manage.py test lists"))
def check_commit(self, pos):
if self.listings[pos].endswith('commit -a'):
self.listings[pos] = Command(self.listings[pos] +
'm "commit for listing %d"' %
(self.pos, ))
elif self.listings[pos].endswith('commit'):
self.listings[pos] = Command(self.listings[pos] +
' -am "commit for listing %d"' %
(self.pos, ))
commit = self.run_command(self.listings[pos])
assert 'insertion' in commit or 'changed' in commit
self.pos += 1
def check_test_code_cycle(self, pos, test_command_in_listings=True, ft=False):
self.write_to_file(self.listings[pos])
self._strip_out_any_pycs()
if test_command_in_listings:
pos += 1
self.assertIn('test', self.listings[pos])
test_run = self.run_command(self.listings[pos])
elif ft:
test_run = self.run_command(Command("python functional_tests.py"))
else:
test_run = self.run_command(Command("python manage.py test lists"))
pos += 1
self.assert_console_output_correct(test_run, self.listings[pos])
def check_final_diff(self, ignore=None, diff=None):
if diff is None:
diff = self.run_command(
Command('git diff -w repo/{}'.format(self.chapter_name)))
try:
print('checking final diff', diff)
except io.BlockingIOError:
pass
self.assertNotIn('fatal:', diff)
start_marker = 'diff --git a/\n'
commit = Commit.from_diff(start_marker + diff)
if ignore is None:
if commit.lines_to_add:
self.fail('Found lines to add in diff:\n{}'.format(
commit.lines_to_add))
if commit.lines_to_remove:
self.fail('Found lines to remove in diff:\n{}'.format(
commit.lines_to_remove))
return
if "moves" in ignore:
ignore.remove("moves")
difference_lines = commit.deleted_lines + commit.new_lines
else:
difference_lines = commit.lines_to_add + commit.lines_to_remove
for line in difference_lines:
if any(ignorable in line for ignorable in ignore):
continue
self.fail('Found divergent line in diff:\n{}'.format(line))
def check_final_diff(self, ignore=None, diff=None):
if diff is None:
diff = self.run_command(
Command('git diff -w repo/chapter_{0:02d}'.format(
self.chapter_no)))
try:
print('checking final diff', diff)
except io.BlockingIOError:
pass
self.assertNotIn('fatal:', diff)
start_marker = 'diff --git a/\n'
commit = Commit.from_diff(start_marker + diff)
error = AssertionError(
'Final diff was not empty, was:\n{}'.format(diff))
if ignore is None:
if commit.lines_to_add or commit.lines_to_remove:
raise error
return
if "moves" in ignore:
ignore.remove("moves")
difference_lines = commit.deleted_lines + commit.new_lines
else:
difference_lines = commit.lines_to_add + commit.lines_to_remove
for line in difference_lines:
if any(ignorable in line for ignorable in ignore):
continue
raise error
def check_final_diff(self,
chapter,
ignore_moves=False,
ignore_secret_key=False,
diff=None):
if diff is None:
diff = self.run_command(
Command('git diff -b repo/chapter_{0:02d}'.format(chapter)))
try:
print('checking final diff', diff)
except io.BlockingIOError:
pass
self.assertNotIn('fatal:', diff)
start_marker = 'diff --git a/\n'
commit = Commit.from_diff(start_marker + diff)
error = AssertionError(
'Final diff was not empty, was:\n{}'.format(diff))
if ignore_secret_key:
for line in commit.lines_to_add + commit.lines_to_remove:
if 'SECRET_KEY' not in line:
raise error
elif ignore_moves:
if commit.deleted_lines or commit.new_lines:
raise AssertionError(
'Found lines to delete or add in diff.\nto delete:\n{}\n\nto add:\n{}'
.format('\n- '.join(commit.deleted_lines),
'\n+'.join(commit.new_lines)))
elif commit.lines_to_add or commit.lines_to_remove:
raise error
def test_calls_sourcetree_run_command_and_marks_as_run(self):
self.sourcetree.run_command = Mock()
cmd = Command('foo')
output = self.run_command(cmd, cwd='bar', user_input='thing')
assert output == self.sourcetree.run_command.return_value
self.sourcetree.run_command.assert_called_with(
'foo', cwd='bar', user_input='thing', ignore_errors=False,
)
assert cmd.was_run
def replace_command_with_check(self, pos, old, new):
listing = self.listings[pos]
all_listings = '\n'.join(str(t) for t in enumerate(self.listings))
error = f'Could not find {old} at pos {pos}: "{listing}". Listings were:\n{all_listings}'
if old not in listing:
raise Exception(error)
assert type(listing) == Command
new_listing = Command(listing.replace(old, new))
for attr, val in vars(listing).items():
setattr(new_listing, attr, val)
self.listings[pos] = new_listing
def apply_patch(self, codelisting):
tf = tempfile.NamedTemporaryFile(delete=False)
tf.write(codelisting.contents.encode('utf8'))
tf.write('\n'.encode('utf8'))
tf.close()
print('patch:\n', codelisting.contents)
patch_output = self.run_command(
Command('patch --fuzz=3 --no-backup-if-mismatch %s %s' %
(codelisting.filename, tf.name)))
print(patch_output)
self.assertNotIn('malformed', patch_output)
self.assertNotIn('failed', patch_output.lower())
codelisting.was_checked = True
with open(os.path.join(self.tempdir, codelisting.filename)) as f:
print(f.read())
os.remove(tf.name)
self.pos += 1
codelisting.was_written = True
def restart_dev_server(self):
print('restarting dev server')
self.run_command(Command('pkill -f runserver'))
time.sleep(1)
self.start_dev_server()
time.sleep(1)
def start_dev_server(self):
self.run_command(Command('python manage.py runserver'))
self.dev_server_running = True
time.sleep(1)
def recognise_listing_and_process_it(self):
listing = self.listings[self.pos]
if listing.dofirst:
print("DOFIRST", listing.dofirst)
self.sourcetree.patch_from_commit(listing.dofirst, )
if listing.skip:
print("SKIP")
listing.was_checked = True
listing.was_written = True
self.pos += 1
elif listing.type == 'test':
print("TEST RUN")
self.run_test_and_check_result()
elif listing.type == 'bdd test':
print("BDD TEST RUN")
self.run_test_and_check_result(bdd=True)
elif listing.type == 'git diff':
print("GIT DIFF")
self.check_diff_or_status(self.pos)
elif listing.type == 'git status':
print("STATUS")
self.check_diff_or_status(self.pos)
elif listing.type == 'git commit':
print("COMMIT")
self.check_commit(self.pos)
elif listing.type == 'interactive manage.py':
print("INTERACTIVE MANAGE.PY")
output_before = self.listings[self.pos + 1]
assert isinstance(output_before, Output)
LIKELY_INPUTS = ('yes', 'no', '1', '2', "''")
user_input = self.listings[self.pos + 2]
if isinstance(user_input, Command) and user_input in LIKELY_INPUTS:
if user_input == 'yes':
print('yes case')
# in this case there is moar output after the yes
output_after = self.listings[self.pos + 3]
assert isinstance(output_after, Output)
expected_output = Output(
wrap_long_lines(output_before + ' ' +
output_after.lstrip()))
next_output = None
elif user_input == '1':
print('migrations 1 case')
# in this case there is another hop
output_after = self.listings[self.pos + 3]
assert isinstance(output_after, Output)
first_input = user_input
next_input = self.listings[self.pos + 4]
assert isinstance(next_input, Command)
next_output = self.listings[self.pos + 5]
expected_output = Output(
wrap_long_lines(output_before + '\n' + output_after +
'\n' + next_output))
user_input = Command(first_input + '\n' + next_input)
else:
expected_output = output_before
output_after = None
next_output = None
if user_input == '2':
ignore_errors = True
else:
ignore_errors = False
else:
user_input = None
expected_output = output_before
output_after = None
ignore_errors = True
next_output = None
output = self.run_command(listing,
user_input=user_input,
ignore_errors=ignore_errors)
self.assert_console_output_correct(output, expected_output)
listing.was_checked = True
output_before.was_checked = True
self.pos += 2
if user_input is not None:
user_input.was_run = True
self.pos += 1
if output_after is not None:
output_after.was_checked = True
self.pos += 1
if next_output is not None:
self.pos += 2
next_output.was_checked = True
first_input.was_run = True
next_input.was_run = True
elif listing.type == 'tree':
print("TREE")
self.assert_directory_tree_correct(listing)
self.pos += 1
elif listing.type == 'server command':
if DO_SERVER_COMMANDS:
server_output = self.run_server_command(listing)
listing.was_run = True
self.pos += 1
next_listing = self.listings[self.pos]
if next_listing.type == 'output' and not next_listing.skip:
if DO_SERVER_COMMANDS:
for line in next_listing.split('\n'):
assert line.strip() in server_output
next_listing.was_checked = True
self.pos += 1
elif listing.type == 'other command':
print("A COMMAND")
output = self.run_command(listing)
next_listing = self.listings[self.pos + 1]
if next_listing.type == 'output' and not next_listing.skip:
ls = listing.startswith('ls')
self.assert_console_output_correct(output, next_listing, ls=ls)
next_listing.was_checked = True
listing.was_checked = True
self.pos += 2
elif 'tree' in listing and next_listing.type == 'tree':
self.assert_console_output_correct(output, next_listing)
next_listing.was_checked = True
listing.was_checked = True
self.pos += 2
else:
listing.was_checked = True
self.pos += 1
elif listing.type == 'diff':
print("DIFF")
self.apply_patch(listing)
elif listing.type == 'code listing currentcontents':
actual_contents = self.sourcetree.get_contents(listing.filename)
self.check_current_contents(listing, actual_contents)
self.pos += 1
elif listing.type == 'code listing':
print("CODE")
self.write_to_file(listing)
self.pos += 1
elif listing.type == 'code listing with git ref':
print("CODE FROM GIT REF")
self.sourcetree.apply_listing_from_commit(listing)
self.pos += 1
elif listing.type == 'server code listing':
print("SERVER CODE")
self.write_file_on_server(listing.filename, listing.contents)
listing.was_written = True
self.pos += 1
elif listing.type == 'qunit output':
self.check_qunit_output(listing)
self.pos += 1
elif listing.type == 'output':
self._strip_out_any_pycs()
test_run = self.run_unit_tests()
if 'OK' in test_run and 'OK' not in listing:
print('unit tests pass, must be an FT:\n', test_run)
test_run = self.run_fts()
try:
self.assert_console_output_correct(test_run, listing)
except AssertionError as e:
if 'OK' in test_run and 'OK' in listing:
print('got error when checking unit tests', e)
test_run = self.run_fts()
self.assert_console_output_correct(test_run, listing)
else:
raise
self.pos += 1
else:
self.fail('not implemented for ' + str(listing))
def recognise_listing_and_process_it(self):
listing = self.listings[self.pos]
if listing.dofirst:
print("DOFIRST", listing.dofirst)
self.sourcetree.patch_from_commit(
listing.dofirst,
)
if listing.skip:
print("SKIP")
listing.was_checked = True
listing.was_written = True
self.pos += 1
elif listing.type == 'test':
print("TEST RUN")
self.run_test_and_check_result()
elif listing.type == 'git diff':
print("GIT DIFF")
self.check_diff_or_status(self.pos)
elif listing.type == 'git status':
print("STATUS")
self.check_diff_or_status(self.pos)
elif listing.type == 'git commit':
print("COMMIT")
self.check_commit(self.pos)
elif listing.type == 'interactive manage.py':
print("INTERACTIVE MANAGE.PY")
output_before = self.listings[self.pos + 1]
assert isinstance(output_before, Output)
LIKELY_INPUTS = ('yes', 'no', '1', '2', "''")
user_input = self.listings[self.pos + 2]
if isinstance(user_input, Command) and user_input in LIKELY_INPUTS:
if user_input == 'yes':
print('yes case')
# in this case there is moar output after the yes
output_after = self.listings[self.pos + 3]
assert isinstance(output_after, Output)
expected_output = Output(wrap_long_lines(output_before + ' ' + output_after.lstrip()))
next_output = None
elif user_input == '1':
print('migrations 1 case')
# in this case there is another hop
output_after = self.listings[self.pos + 3]
assert isinstance(output_after, Output)
first_input = user_input
next_input = self.listings[self.pos + 4]
assert isinstance(next_input, Command)
next_output = self.listings[self.pos + 5]
expected_output = Output(wrap_long_lines(
output_before + '\n' + output_after + '\n' + next_output
))
user_input = Command(first_input + '\n' + next_input)
else:
expected_output = output_before
output_after = None
next_output = None
if user_input == '2':
ignore_errors = True
else:
ignore_errors = False
else:
user_input = None
expected_output = output_before
output_after = None
ignore_errors = True
next_output = None
output = self.run_command(listing, user_input=user_input, ignore_errors=ignore_errors)
self.assert_console_output_correct(output, expected_output)
listing.was_checked = True
output_before.was_checked = True
self.pos += 2
if user_input is not None:
user_input.was_run = True
self.pos += 1
if output_after is not None:
output_after.was_checked = True
self.pos += 1
if next_output is not None:
self.pos += 2
next_output.was_checked = True
first_input.was_run = True
next_input.was_run = True
elif listing.type == 'tree':
print("TREE")
self.assert_directory_tree_correct(listing)
self.pos += 1
elif listing.type == 'server command':
server_output = self.run_server_command(listing)
listing.was_run = True
self.pos += 1
next_listing = self.listings[self.pos]
if next_listing.type == 'output' and not next_listing.skip:
for line in next_listing.split('\n'):
assert line.strip() in server_output
next_listing.was_checked = True
self.pos += 1
elif listing.type == 'other command':
print("A COMMAND")
output = self.run_command(listing)
next_listing = self.listings[self.pos + 1]
if next_listing.type == 'output' and not next_listing.skip:
ls = listing.startswith('ls')
self.assert_console_output_correct(output, next_listing, ls=ls)
next_listing.was_checked = True
listing.was_checked = True
self.pos += 2
elif 'tree' in listing and next_listing.type == 'tree':
self.assert_console_output_correct(output, next_listing)
next_listing.was_checked = True
listing.was_checked = True
self.pos += 2
else:
listing.was_checked = True
self.pos += 1
elif listing.type == 'diff':
print("DIFF")
self.apply_patch(listing)
elif listing.type == 'code listing currentcontents':
actual_contents = self.sourcetree.get_contents(
listing.filename
)
print("CHECK CURRENT CONTENTS")
stripped_actual_lines = [l.strip() for l in actual_contents.split('\n')]
listing_contents = re.sub(r' +#$', '', listing.contents, flags=re.MULTILINE)
for line in listing_contents.split('\n'):
if line and not '[...]' in line:
self.assertIn(line.strip(), stripped_actual_lines)
listing.was_written = True
self.pos += 1
elif listing.type == 'code listing':
print("CODE")
self.write_to_file(listing)
self.pos += 1
elif listing.type == 'code listing with git ref':
print("CODE FROM GIT REF")
self.sourcetree.apply_listing_from_commit(listing)
self.pos += 1
elif listing.type == 'server code listing':
print("SERVER CODE")
self.write_file_on_server(listing.filename, listing.contents)
self.pos += 1
elif listing.type == 'qunit output':
self.check_qunit_output(listing)
self.pos += 1
elif listing.type == 'output':
self._strip_out_any_pycs()
test_run = self.run_unit_tests()
if 'OK' in test_run and not 'OK' in listing:
print('unit tests pass, must be an FT:\n', test_run)
test_run = self.run_fts()
try:
self.assert_console_output_correct(test_run, listing)
except AssertionError as e:
if 'OK' in test_run and 'OK' in listing:
print('got error when checking unit tests', e)
test_run = self.run_fts()
self.assert_console_output_correct(test_run, listing)
else:
raise
self.pos += 1
else:
self.fail('not implemented for ' + str(listing))
