def test_success_result(self):
"""
Will run the tests and detect a success result.
"""
plugin_dir = create_plugin(self.plugindir, 'bundle', 'plugin')
self.add_timeline(plugin_dir, [('a.txt', 'a\n')])
self.add_timeline(plugin_dir, [('a.txt', 'aa\n')])
self.add_expectation(
plugin_dir, u'''
.. expectation::
:from: 01
:to: 02
▾ plugin
✓ line 1: a.txt
a is -
✓ line 1: a.txt
aa is +''')
ptr = PluginTestRunner(plugin_dir)
results = ptr.run()
self.assertEqual(1, len(results))
self.assertIsInstance(results[0], SuccessResult)
self.assertEqual(results[0].actual.strip(),
results[0].expectation.output.strip())
self.assertEqual((1, 2), results[0].expectation.range)
def test_specific_range(self):
"""
Will run only a specific range of tests.
"""
plugin_dir = create_plugin(self.plugindir, 'bundle', 'plugin')
self.add_timeline(plugin_dir, [('a.txt', 'a\n')])
self.add_timeline(plugin_dir, [('a.txt', 'aa\n')])
self.add_expectation(plugin_dir, u'''
.. expectation::
:from: 01
:to: 02
▾ plugin
✓ line 1: a.txt
a is -
✓ line 1: a.txt
aa is +''')
ptr = PluginTestRunner(plugin_dir)
# An expectation exists for the following range
self.assertEqual(
1,
len(ptr.run(test_range=[(1, 2)])))
# An expectation does not exist for this one
self.assertEqual(
0,
len(ptr.run(test_range=[(2, 3)])))
def test_can_change_settings(self):
"""
Altering the settings will be used correctly.
"""
plugin_dir = create_plugin(self.plugindir,
'bundle',
'plugin',
settings={'verbose': 'yes'})
self.add_timeline(plugin_dir, [('a.txt', 'a\n')])
self.add_timeline(plugin_dir, [('a.txt', 'aa\n')])
self.add_expectation(
plugin_dir, u'''
.. plugin-settings::
verbose = no
.. expectation::
:from: 01
:to: 02
▾ plugin
✓ a.txt
File has been modified''')
ptr = PluginTestRunner(plugin_dir)
results = ptr.run()
self.assertIsInstance(results[0], SuccessResult)
def test(self, argv):
"""
Run the tests for a plugin.
"""
plugin = argv.plugin
test_range = argv.range
verbose = argv.verbose
with self.out() as out:
if test_range:
test_range = parse_range(test_range)
try:
ptr = PluginTestRunner(plugin)
results = ptr.run(test_range=test_range)
reporter = PluginTestReporter(results)
test_results = reporter.dumps(verbose=verbose).splitlines()
failures = [i for i in results if isinstance(i, FailureResult)]
if failures:
# Raise as an error so the status code will be non-zero
raise CommandError('\n'.join(test_results))
# No failures, ok to send this to stdout
out.extend(test_results)
except ExpectationError as e:
raise CommandError(str(e))
def test_non_json_stdout(self):
"""
Still processes if the plugin returns something other than JSON data.
"""
plugin_dir = create_plugin(self.plugindir, 'bundle', 'plugin')
self.add_timeline(plugin_dir, [('a.txt', 'a\n')])
self.add_timeline(plugin_dir, [('a.txt', 'aa\n')])
self.add_expectation(
plugin_dir, u'''
.. expectation::
:from: 01
:to: 02
Output''')
ptr = PluginTestRunner(plugin_dir)
with patch.object(Plugin, 'pre_commit'):
Plugin.pre_commit.return_value = (0, 'Non-JSON', '')
results = ptr.run()
self.assertResults(
u'''
▾ plugin
✓ Non-JSON''', results[0].actual)
def test_non_zero_exit_code(self):
"""
If the exit code is non-zero, gets stderr instead.
"""
plugin_dir = create_plugin(self.plugindir, 'bundle', 'plugin')
self.add_timeline(plugin_dir, [('a.txt', 'a\n')])
self.add_timeline(plugin_dir, [('a.txt', 'aa\n')])
self.add_expectation(plugin_dir, u'''
.. expectation::
:from: 01
:to: 02
Output''')
ptr = PluginTestRunner(plugin_dir)
with patch.object(Plugin, 'pre_commit'):
Plugin.pre_commit.return_value = (1, '', 'Error')
results = ptr.run()
self.assertResults(u'''
Exit code: 1
Std out:
(none)
Std err:
Error''', results[0].actual)
def test_non_json_stdout(self):
"""
Still processes if the plugin returns something other than JSON data.
"""
plugin_dir = create_plugin(self.plugindir, 'bundle', 'plugin')
self.add_timeline(plugin_dir, [('a.txt', 'a\n')])
self.add_timeline(plugin_dir, [('a.txt', 'aa\n')])
self.add_expectation(plugin_dir, u'''
.. expectation::
:from: 01
:to: 02
Output''')
ptr = PluginTestRunner(plugin_dir)
with patch.object(Plugin, 'pre_commit'):
Plugin.pre_commit.return_value = (0, 'Non-JSON', '')
results = ptr.run()
self.assertResults(u'''
▾ plugin
✓ Non-JSON''', results[0].actual)
def test_can_change_settings(self):
"""
Altering the settings will be used correctly.
"""
plugin_dir = create_plugin(
self.plugindir, 'bundle', 'plugin',
settings={'verbose': 'yes'})
self.add_timeline(plugin_dir, [('a.txt', 'a\n')])
self.add_timeline(plugin_dir, [('a.txt', 'aa\n')])
self.add_expectation(plugin_dir, u'''
.. plugin-settings::
verbose = no
.. expectation::
:from: 01
:to: 02
▾ plugin
✓ a.txt
File has been modified''')
ptr = PluginTestRunner(plugin_dir)
results = ptr.run()
self.assertIsInstance(results[0], SuccessResult)
def test_non_zero_exit_code(self):
"""
If the exit code is non-zero, gets stderr instead.
"""
plugin_dir = create_plugin(self.plugindir, 'bundle', 'plugin')
self.add_timeline(plugin_dir, [('a.txt', 'a\n')])
self.add_timeline(plugin_dir, [('a.txt', 'aa\n')])
self.add_expectation(
plugin_dir, u'''
.. expectation::
:from: 01
:to: 02
Output''')
ptr = PluginTestRunner(plugin_dir)
with patch.object(Plugin, 'pre_commit'):
Plugin.pre_commit.return_value = (1, '', 'Error')
results = ptr.run()
self.assertResults(
u'''
Exit code: 1
Std out:
(none)
Std err:
Error''', results[0].actual)
def test_specific_range(self):
"""
Will run only a specific range of tests.
"""
plugin_dir = create_plugin(self.plugindir, 'bundle', 'plugin')
self.add_timeline(plugin_dir, [('a.txt', 'a\n')])
self.add_timeline(plugin_dir, [('a.txt', 'aa\n')])
self.add_expectation(
plugin_dir, u'''
.. expectation::
:from: 01
:to: 02
▾ plugin
✓ line 1: a.txt
a is -
✓ line 1: a.txt
aa is +''')
ptr = PluginTestRunner(plugin_dir)
# An expectation exists for the following range
self.assertEqual(1, len(ptr.run(test_range=[(1, 2)])))
# An expectation does not exist for this one
self.assertEqual(0, len(ptr.run(test_range=[(2, 3)])))
def test_success_result(self):
"""
Will run the tests and detect a success result.
"""
plugin_dir = create_plugin(self.plugindir, 'bundle', 'plugin')
self.add_timeline(plugin_dir, [('a.txt', 'a\n')])
self.add_timeline(plugin_dir, [('a.txt', 'aa\n')])
self.add_expectation(plugin_dir, u'''
.. expectation::
:from: 01
:to: 02
▾ plugin
✓ line 1: a.txt
a is -
✓ line 1: a.txt
aa is +''')
ptr = PluginTestRunner(plugin_dir)
results = ptr.run()
self.assertEqual(1, len(results))
self.assertIsInstance(results[0], SuccessResult)
self.assertEqual(
results[0].actual.strip(),
results[0].expectation.output.strip())
self.assertEqual((1, 2), results[0].expectation.range)
def test_multiple_expectations(self):
"""
Multiple tests can be ran.
"""
plugin_dir = create_plugin(
self.plugindir, 'bundle', 'plugin',
settings={'verbose': 'no'})
self.add_timeline(
plugin_dir, [('src/a.txt', 'a\n')])
self.add_timeline(
plugin_dir, [('src/a.txt', 'aa\n')])
self.add_timeline(
plugin_dir, [('src/a.txt', 'aaa\n')])
self.add_timeline(
plugin_dir, [
('src/a.txt', 'aaa\n'),
('src/b.txt', 'bbb\n')])
self.add_expectation(
plugin_dir, u'''
.. expectation::
:from: 01
:to: 02
▾ plugin
✓ src/a.txt
File has been modified
.. expectation::
:from: 02
:to: 03
▾ plugin
✓ src/a.txt
File has been modified
.. expectation::
:from: 03
:to: 04
▾ plugin
✓ src/b.txt
File has been modified''')
ptr = PluginTestRunner(plugin_dir)
results = ptr.run()
success = [isinstance(i, SuccessResult) for i in results]
self.assertTrue(all(success))
def test_multiple_expectations(self):
"""
Multiple tests can be ran.
"""
plugin_dir = create_plugin(self.plugindir,
'bundle',
'plugin',
settings={'verbose': 'no'})
self.add_timeline(plugin_dir, [('src/a.txt', 'a\n')])
self.add_timeline(plugin_dir, [('src/a.txt', 'aa\n')])
self.add_timeline(plugin_dir, [('src/a.txt', 'aaa\n')])
self.add_timeline(plugin_dir, [('src/a.txt', 'aaa\n'),
('src/b.txt', 'bbb\n')])
self.add_expectation(
plugin_dir, u'''
.. expectation::
:from: 01
:to: 02
▾ plugin
✓ src/a.txt
File has been modified
.. expectation::
:from: 02
:to: 03
▾ plugin
✓ src/a.txt
File has been modified
.. expectation::
:from: 03
:to: 04
▾ plugin
✓ src/b.txt
File has been modified''')
ptr = PluginTestRunner(plugin_dir)
results = ptr.run()
success = [isinstance(i, SuccessResult) for i in results]
self.assertTrue(all(success))
def test_no_tests_to_run(self):
"""
Plugin has no tests to run.
"""
plugin_dir = create_plugin(self.plugindir, 'bundle', 'plugin')
# Since our newly created plugin has no tests, this should be empty
with self.assertRaises(ExpectationNoTests) as ec:
PluginTestRunner(plugin_dir)
self.assertIn('Could not find any tests', str(ec.exception))
def test_no_expectations(self):
"""
We have a test timeline without any expectations.
"""
plugin_dir = create_plugin(self.plugindir, 'bundle', 'plugin')
self.add_timeline(plugin_dir, [('a.txt', 'a\n')])
self.add_timeline(plugin_dir, [('a.txt', 'aa\n')])
# Since our newly created plugin has no tests, this should be empty
with self.assertRaises(ExpectationFileNotFound) as ec:
PluginTestRunner(plugin_dir)
self.assertIn('Missing expectation file', str(ec.exception))
def test_failure_result(self):
"""
Will run the tests and detect a failure result.
"""
plugin_dir = create_plugin(self.plugindir, 'bundle', 'plugin')
self.add_timeline(plugin_dir, [('a.txt', 'a\n')])
self.add_timeline(plugin_dir, [('a.txt', 'aa\n')])
self.add_expectation(plugin_dir, u'''
.. expectation::
:from: 01
:to: 02
▾ plugin
✓ Gobbildy gook''')
ptr = PluginTestRunner(plugin_dir)
results = ptr.run()
self.assertEqual(1, len(results))
self.assertIsInstance(results[0], FailureResult)
def test_failure_result(self):
"""
Will run the tests and detect a failure result.
"""
plugin_dir = create_plugin(self.plugindir, 'bundle', 'plugin')
self.add_timeline(plugin_dir, [('a.txt', 'a\n')])
self.add_timeline(plugin_dir, [('a.txt', 'aa\n')])
self.add_expectation(
plugin_dir, u'''
.. expectation::
:from: 01
:to: 02
▾ plugin
✓ Gobbildy gook''')
ptr = PluginTestRunner(plugin_dir)
results = ptr.run()
self.assertEqual(1, len(results))
self.assertIsInstance(results[0], FailureResult)
def test_loads_tests_and_expectations(self):
"""
Will load the timeline and expecatations when created.
"""
plugin_dir = create_plugin(self.plugindir, 'bundle', 'plugin')
self.add_timeline(plugin_dir, [('a.txt', 'a\n')])
self.add_timeline(plugin_dir, [('a.txt', 'aa\n')])
self.add_expectation(
plugin_dir, '''
.. expectation::
:from: 01
:to: 02
Test output''')
ptr = PluginTestRunner(plugin_dir)
# Just make sure that we have the expected objects
self.assertIsInstance(ptr.timeline, NumberedDirectoriesToGit)
self.assertIsInstance(ptr.expectations[0], Expectation)