def test_help(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['dev', 'special', '--help'])
self.assertEqual(call.exception.args, (0, ))
self.maxDiff = None
expected = """
usage: plainbox dev special [-h] (-j | -J | -e | -d) [--dot-resources]
[-T TEST-PLAN-ID] [-i PATTERN] [-x PATTERN]
optional arguments:
-h, --help show this help message and exit
-j, --list-jobs list jobs instead of running them
-J, --list-job-hashes
list jobs with cheksums instead of running them
-e, --list-expressions
list all unique resource expressions
-d, --dot print a graph of jobs instead of running them
--dot-resources show resource relationships (for --dot)
test selection options:
-T TEST-PLAN-ID, --test-plan TEST-PLAN-ID
load the specified test plan
-i PATTERN, --include-pattern PATTERN
include jobs matching the given regular expression
-x PATTERN, --exclude-pattern PATTERN
exclude jobs matching the given regular expression
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_help(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['--help'])
self.assertEqual(call.exception.args, (0, ))
self.maxDiff = None
expected = """
usage: plainbox [--help] [--version] | [options] <command> ...
positional arguments:
{session,dev}
session session management commands
dev development commands
optional arguments:
-h, --help show this help message and exit
--version show program's version number and exit
logging and debugging:
-v, --verbose be more verbose (same as --log-level=INFO)
-D, --debug enable DEBUG messages on the root logger
-C, --debug-console display DEBUG messages in the console
-T LOGGER, --trace LOGGER
enable DEBUG messages on the specified logger (can be
used multiple times)
-P, --pdb jump into pdb (python debugger) when a command crashes
-I, --debug-interrupt
crash on SIGINT/KeyboardInterrupt, useful with --pdb
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_help(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['--help'])
self.assertEqual(call.exception.args, (0,))
self.maxDiff = None
expected = """
usage: plainbox [-h] [-v] [-l {DEBUG,INFO,WARNING,ERROR,CRITICAL}]
[-u {headless,text,graphics}] [--load-extra FILE] [-r PATTERN]
[--list-jobs]
optional arguments:
-h, --help show this help message and exit
-v, --version show program's version number and exit
-l {DEBUG,INFO,WARNING,ERROR,CRITICAL}, --log-level {DEBUG,INFO,WARNING,ERROR,CRITICAL}
Set logging level
user interface options:
-u {headless,text,graphics}, --ui {headless,text,graphics}
select the UI front-end (defaults to auto)
job definition options:
--load-extra FILE Load extra job definitions from FILE
-r PATTERN, --run-pattern PATTERN
Run jobs matching the given pattern
--list-jobs List all jobs
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_run_list_jobs(self):
with TestIO() as io:
with self.assertRaises(SystemExit) as call:
main(['-c', 'stub', 'dev', 'special', '--list-jobs'])
self.assertEqual(call.exception.args, (0,))
self.assertIn("stub/false", io.stdout.splitlines())
self.assertIn("stub/true", io.stdout.splitlines())
def test_help(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['--help'])
self.assertEqual(call.exception.args, (0, ))
self.maxDiff = None
expected = """
usage: plainbox [-h] [-v] [-l {DEBUG,INFO,WARNING,ERROR,CRITICAL}]
[-u {headless,text,graphics}] [--load-extra FILE] [-r PATTERN]
[--list-jobs]
optional arguments:
-h, --help show this help message and exit
-v, --version show program's version number and exit
-l {DEBUG,INFO,WARNING,ERROR,CRITICAL}, --log-level {DEBUG,INFO,WARNING,ERROR,CRITICAL}
Set logging level
user interface options:
-u {headless,text,graphics}, --ui {headless,text,graphics}
select the UI front-end (defaults to auto)
job definition options:
--load-extra FILE Load extra job definitions from FILE
-r PATTERN, --run-pattern PATTERN
Run jobs matching the given pattern
--list-jobs List all jobs
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_run_list_expressions(self):
with TestIO() as io:
with self.assertRaises(SystemExit) as call:
main(['-c', 'stub', 'dev', 'special', '--list-expressions'])
self.assertEqual(call.exception.args, (0,))
self.assertIn(
'stub_package.name == "checkbox"', io.stdout.splitlines())
def test_job_result(self):
# Create a scratch directory so that we can save results there. The
# shared directory is also used for running tests as some test jobs
# leave junk around the current directory.
with TemporaryDirectory() as scratch_dir:
# Save results to results.json in the scratch directory
actual_results_path = os.path.join(scratch_dir, 'results.json')
# Redirect all standard IO so that the test is silent.
# Run the script, having relocated to the scratch directory
# Capture SystemExit that is always raised by main() so that
# we can observe the return code as well.
with TestIO(), TestCwd(scratch_dir),\
self.assertRaises(SystemExit) as call:
main(['run', '-r', self.parameters.job_name,
'--output-format=json', '-o', actual_results_path])
# Check the return code for correctness
self.assertEqual(call.exception.args, (0,))
# Load the actual results and keep them in memory
with open(actual_results_path, encoding='UTF-8') as stream:
actual_result = json.load(stream)
# [ At this time TestIO and TemporaryDirectory are gone ]
# Load the expected results and keep them in memory
reference_path = resource_filename(
"plainbox", "data/{}.json".format(self.parameters.job_name))
with open(reference_path, encoding='UTF-8') as stream:
expected_result = json.load(stream)
# Check that results match expected values
self.assertEqual(actual_result, expected_result)
def test_run_without_args(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
with patch('plainbox.impl.commands.run.authenticate_warmup'
) as mock_warmup:
mock_warmup.return_value = 0
main(['run'])
self.assertEqual(call.exception.args, (0, ))
expected1 = """
===============================[ Analyzing Jobs ]===============================
Estimated duration cannot be determined for automated jobs.
Estimated duration cannot be determined for manual jobs.
==============================[ Running All Jobs ]==============================
==================================[ Results ]===================================
"""
expected2 = """
===============================[ Authentication ]===============================
===============================[ Analyzing Jobs ]===============================
Estimated duration cannot be determined for automated jobs.
Estimated duration cannot be determined for manual jobs.
==============================[ Running All Jobs ]==============================
==================================[ Results ]===================================
"""
self.assertIn(io.combined,
[cleandoc(expected1) + "\n",
cleandoc(expected2) + "\n"])
def test_help(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['special', '--help'])
self.assertEqual(call.exception.args, (0,))
self.maxDiff = None
expected = """
usage: plainbox special [-h] (-j | -e | -d) [--dot-resources]
[--load-extra FILE] [-r PATTERN] [-W WHITELIST]
optional arguments:
-h, --help show this help message and exit
-j, --list-jobs List jobs instead of running them
-e, --list-expressions
List all unique resource expressions
-d, --dot Print a graph of jobs instead of running them
--dot-resources Render resource relationships (for --dot)
job definition options:
--load-extra FILE Load extra job definitions from FILE
-r PATTERN, --run-pattern PATTERN
Run jobs matching the given pattern
-W WHITELIST, --whitelist WHITELIST
Load whitelist containing run patterns
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_version(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['--version'])
self.assertEqual(call.exception.args, (0,))
self.assertEqual(io.combined, "{}\n".format(
ToolBase.format_version_tuple(version)))
def test_run_list_expressions(self):
with TestIO() as io:
with self.assertRaises(SystemExit) as call:
main(['dev', 'special', '--list-expressions'])
self.assertEqual(call.exception.args, (0, ))
# See comment in test_run_list_jobs
self.assertIn("package.name == 'samba'", io.stdout.splitlines())
def test_run_list_expressions(self):
with TestIO() as io:
with self.assertRaises(SystemExit) as call:
main(['special', '--list-expressions'])
self.assertEqual(call.exception.args, (0,))
# See comment in test_run_list_jobs
self.assertIn("package.name == 'samba'", io.stdout.splitlines())
def test_help(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['sru', '--help'])
self.assertEqual(call.exception.args, (0,))
self.maxDiff = None
expected = """
usage: plainbox sru [-h] [--check-config] --secure-id SECURE-ID
[--fallback FILE] [--destination URL] [-n]
optional arguments:
-h, --help show this help message and exit
--check-config Run plainbox check-config before starting
sru-specific options:
--secure-id SECURE-ID
Associate submission with a machine using this SECURE-
ID (unset)
--fallback FILE If submission fails save the test report as FILE
(unset)
--destination URL POST the test report XML to this URL (https://certific
ation.canonical.com/submissions/submit/)
execution options:
-n, --dry-run Skip all usual jobs. Only local, resource and
attachment jobs are started
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_run_without_args(self, mock_check_output):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
with patch('plainbox.impl.commands.run.authenticate_warmup') as mock_warmup:
mock_warmup.return_value = 0
main(['run'])
self.assertEqual(call.exception.args, (0,))
expected1 = """
===============================[ Analyzing Jobs ]===============================
Estimated duration cannot be determined for automated jobs.
Estimated duration cannot be determined for manual jobs.
==============================[ Running All Jobs ]==============================
==================================[ Results ]===================================
"""
expected2 = """
===============================[ Authentication ]===============================
===============================[ Analyzing Jobs ]===============================
Estimated duration cannot be determined for automated jobs.
Estimated duration cannot be determined for manual jobs.
==============================[ Running All Jobs ]==============================
==================================[ Results ]===================================
"""
self.assertIn(io.combined, [
cleandoc(expected1) + "\n",
cleandoc(expected2) + "\n"])
def test_help(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['dev', 'special', '--help'])
self.assertEqual(call.exception.args, (0,))
self.maxDiff = None
expected = """
usage: plainbox dev special [-h] (-j | -J | -e | -d) [--dot-resources]
[-i PATTERN] [-x PATTERN] [-w WHITELIST]
optional arguments:
-h, --help show this help message and exit
-j, --list-jobs List jobs instead of running them
-J, --list-job-hashes
List jobs with hashes instead of running them
-e, --list-expressions
List all unique resource expressions
-d, --dot Print a graph of jobs instead of running them
--dot-resources Render resource relationships (for --dot)
job definition options:
-i PATTERN, --include-pattern PATTERN
Run jobs matching the given regular expression.
Matches from the start to the end of the line.
-x PATTERN, --exclude-pattern PATTERN
Do not run jobs matching the given regular expression.
Matches from the start to the end of the line.
-w WHITELIST, --whitelist WHITELIST
Load whitelist containing run patterns
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_version(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['--version'])
self.assertEqual(call.exception.args, (0, ))
self.assertEqual(io.combined,
"{}\n".format(ToolBase.format_version_tuple(version)))
def test_help(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['dev', 'special', '--help'])
self.assertEqual(call.exception.args, (0, ))
self.maxDiff = None
expected = """
usage: plainbox dev special [-h] (-j | -J | -e | -d) [--dot-resources]
[-i PATTERN] [-x PATTERN] [-w WHITELIST]
optional arguments:
-h, --help show this help message and exit
-j, --list-jobs List jobs instead of running them
-J, --list-job-hashes
List jobs with hashes instead of running them
-e, --list-expressions
List all unique resource expressions
-d, --dot Print a graph of jobs instead of running them
--dot-resources Render resource relationships (for --dot)
job definition options:
-i PATTERN, --include-pattern PATTERN
Run jobs matching the given regular expression.
Matches from the start to the end of the line.
-x PATTERN, --exclude-pattern PATTERN
Do not run jobs matching the given regular expression.
Matches from the start to the end of the line.
-w WHITELIST, --whitelist WHITELIST
Load whitelist containing run patterns
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_run_list_jobs_with_filtering(self):
with TestIO() as io:
with self.assertRaises(SystemExit) as call:
main(['special', '--include-pattern=usb3.+', '--list-jobs'])
self.assertEqual(call.exception.args, (0,))
# Test that usb3 insertion test was listed but the usb (2.0) test was not
self.assertIn("usb3/insert", io.stdout.splitlines())
self.assertNotIn("usb/insert", io.stdout.splitlines())
def test_dummy(self):
with TestIO(combined=True) as io:
main([])
expected = """
===============================[ Analyzing Jobs ]===============================
==============================[ Running All Jobs ]==============================
==================================[ Results ]===================================
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_run_list_jobs(self):
with TestIO() as io:
with self.assertRaises(SystemExit) as call:
main(['--providers', 'stub', 'dev', 'special', '--list-jobs'])
self.assertEqual(call.exception.args, (0, ))
self.assertIn("2013.com.canonical.plainbox::stub/false",
io.stdout.splitlines())
self.assertIn("2013.com.canonical.plainbox::stub/true",
io.stdout.splitlines())
def test_output_format_list(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['run', '--output-format=?'])
self.assertEqual(call.exception.args, (0,))
expected = """
Available output formats: json, rfc822, text, xml
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_run_dot(self):
with TestIO() as io:
with self.assertRaises(SystemExit) as call:
main(['special', '--dot'])
self.assertEqual(call.exception.args, (0,))
# See comment in test_run_list_jobs
self.assertIn('\t"usb/insert" [color=orange];', io.stdout.splitlines())
# Do basic graph checks
self._check_digraph_sanity(io)
def test_run_dot(self):
with TestIO() as io:
with self.assertRaises(SystemExit) as call:
main(['--providers', 'stub', 'dev', 'special', '--dot'])
self.assertEqual(call.exception.args, (0, ))
self.assertIn('\t"2013.com.canonical.plainbox::stub/true" [];',
io.stdout.splitlines())
# Do basic graph checks
self._check_digraph_sanity(io)
def test_run_dot(self):
with TestIO() as io:
with self.assertRaises(SystemExit) as call:
main(['-c', 'stub', 'dev', 'special', '--dot'])
self.assertEqual(call.exception.args, (0,))
self.assertIn(
'\t"stub/true" [];', io.stdout.splitlines())
# Do basic graph checks
self._check_digraph_sanity(io)
def test_run_dot(self):
with TestIO() as io:
with self.assertRaises(SystemExit) as call:
main(['dev', 'special', '--dot'])
self.assertEqual(call.exception.args, (0, ))
# See comment in test_run_list_jobs
self.assertIn('\t"usb/insert" [color=orange];', io.stdout.splitlines())
# Do basic graph checks
self._check_digraph_sanity(io)
def test_run_list_expressions(self):
with TestIO() as io:
with self.assertRaises(SystemExit) as call:
main([
'--providers', 'stub', 'dev', 'special',
'--list-expressions'
])
self.assertEqual(call.exception.args, (0, ))
self.assertIn('stub_package.name == "checkbox"',
io.stdout.splitlines())
def test_run_without_args(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main([])
self.assertEqual(call.exception.args, (2,))
expected = """
usage: plainbox [-h] [-v] {run,special,self-test,sru,check-config} ...
plainbox: error: too few arguments
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_run_without_args(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main([])
self.assertEqual(call.exception.args, (2, ))
expected = """
usage: plainbox [--help] [--version] | [options] <command> ...
plainbox: error: too few arguments
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_run_list_jobs_with_filtering(self):
with TestIO() as io:
with self.assertRaises(SystemExit) as call:
main([
'dev', 'special', '--include-pattern=usb3.+', '--list-jobs'
])
self.assertEqual(call.exception.args, (0, ))
# Test that usb3 insertion test was listed but the usb (2.0) test was not
self.assertIn("usb3/insert", io.stdout.splitlines())
self.assertNotIn("usb/insert", io.stdout.splitlines())
def test_run_without_args(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['run'])
self.assertEqual(call.exception.args, (0,))
expected = """
===============================[ Analyzing Jobs ]===============================
==============================[ Running All Jobs ]==============================
==================================[ Results ]===================================
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_run_without_args(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['dev', 'special'])
self.assertEqual(call.exception.args, (2,))
expected = """
usage: plainbox dev special [-h] (-j | -J | -e | -d) [--dot-resources]
[-i PATTERN] [-x PATTERN] [-w WHITELIST]
plainbox dev special: error: one of the arguments -j/--list-jobs -J/--list-job-hashes -e/--list-expressions -d/--dot is required
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_run_without_args(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['dev', 'special'])
self.assertEqual(call.exception.args, (2, ))
expected = """
usage: plainbox dev special [-h] (-j | -J | -e | -d) [--dot-resources]
[-T TEST-PLAN-ID] [-i PATTERN] [-x PATTERN]
plainbox dev special: error: one of the arguments -j/--list-jobs -J/--list-job-hashes -e/--list-expressions -d/--dot is required
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_run_list_jobs(self):
with TestIO() as io:
with self.assertRaises(SystemExit) as call:
main(['dev', 'special', '--list-jobs'])
self.assertEqual(call.exception.args, (0, ))
# This is pretty shoddy, ideally we'd load a special job and test for
# that but it's something that keeps the test more valid than
# otherwise.
self.assertIn("mediacard/mmc-insert", io.stdout.splitlines())
self.assertIn("usb3/insert", io.stdout.splitlines())
self.assertIn("usb/insert", io.stdout.splitlines())
def test_output_format_list(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
with patch('plainbox.impl.commands.run.get_all_exporters') as mock_get_all_exporters:
mock_get_all_exporters.return_value = self._exporters
main(['run', '--output-format=?'])
self.assertEqual(call.exception.args, (0,))
expected = """
Available output formats: json, rfc822, text, xml
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_run_list_jobs(self):
with TestIO() as io:
with self.assertRaises(SystemExit) as call:
main(['special', '--list-jobs'])
self.assertEqual(call.exception.args, (0,))
# This is pretty shoddy, ideally we'd load a special job and test for
# that but it's something that keeps the test more valid than
# otherwise.
self.assertIn("mediacard/mmc-insert", io.stdout.splitlines())
self.assertIn("usb3/insert", io.stdout.splitlines())
self.assertIn("usb/insert", io.stdout.splitlines())
def test_output_format_list(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
with patch('plainbox.impl.commands.run.get_all_exporters'
) as mock_get_all_exporters:
mock_get_all_exporters.return_value = self._exporters
main(['run', '--output-format=?'])
self.assertEqual(call.exception.args, (0, ))
expected = """
Available output formats: json, rfc822, text, xml
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_run_list_jobs_with_filtering(self):
with TestIO() as io:
with self.assertRaises(SystemExit) as call:
main(['--providers', 'stub', 'dev', 'special',
('--include-pattern='
'2013.com.canonical.plainbox::stub/false'),
'--list-jobs'])
self.assertEqual(call.exception.args, (0,))
self.assertIn(
"2013.com.canonical.plainbox::stub/false", io.stdout.splitlines())
self.assertNotIn(
"2013.com.canonical.plainbox::stub/true", io.stdout.splitlines())
def test_run_without_args(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main([])
self.assertEqual(call.exception.args, (2,))
expected = """
usage: plainbox [-h] [--version] [-c {src,deb,auto,stub,ihv}] [-v] [-D] [-C]
[-T LOGGER] [-P] [-I]
{run,self-test,check-config,dev,startprovider} ...
plainbox: error: too few arguments
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_run_without_args(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main([])
self.assertEqual(call.exception.args, (2, ))
expected = """
usage: plainbox [-h] [--version] [-c {src,deb,auto,stub,ihv}] [-v] [-D] [-C]
[-T LOGGER] [-P] [-I]
{run,self-test,sru,check-config,dev,service} ...
plainbox: error: too few arguments
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_run_dot_with_resources(self):
with TestIO() as io:
with self.assertRaises(SystemExit) as call:
main(['dev', 'special', '--dot', '--dot-resources'])
self.assertEqual(call.exception.args, (0, ))
# See comment in test_run_list_jobs
self.assertIn('\t"usb/insert" [color=orange];', io.stdout.splitlines())
self.assertIn(
'\t"daemons/smbd" -> "package" [style=dashed, label="package.name == \'samba\'"];',
io.stdout.splitlines())
# Do basic graph checks
self._check_digraph_sanity(io)
def test_output_option_list(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['run', '--output-option=?'])
self.assertEqual(call.exception.args, (0,))
expected = """
Each format may support a different set of options
json: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, machine-json
rfc822: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments
text: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments
xml:
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_run_dot_with_resources(self):
with TestIO() as io:
with self.assertRaises(SystemExit) as call:
main(['special', '--dot', '--dot-resources'])
self.assertEqual(call.exception.args, (0,))
# See comment in test_run_list_jobs
self.assertIn(
'\t"usb/insert" [color=orange];', io.stdout.splitlines())
self.assertIn(
'\t"daemons/smbd" -> "package" [style=dashed, label="package.name == \'samba\'"];',
io.stdout.splitlines())
# Do basic graph checks
self._check_digraph_sanity(io)
def test_help(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['run', '--help'])
self.assertEqual(call.exception.args, (0, ))
self.maxDiff = None
expected = """
usage: plainbox run [-h] [--non-interactive] [-n] [--dont-suppress-output]
[-f FORMAT] [-p OPTIONS] [-o FILE] [-t TRANSPORT]
[--transport-where WHERE] [--transport-options OPTIONS]
[-T TEST-PLAN-ID] [-i PATTERN] [-x PATTERN] [-w WHITELIST]
optional arguments:
-h, --help show this help message and exit
user interface options:
--non-interactive skip tests that require interactivity
-n, --dry-run don't really run most jobs
--dont-suppress-output
don't suppress the output of certain job plugin types
output options:
-f FORMAT, --output-format FORMAT
save test results in the specified FORMAT (pass ? for
a list of choices)
-p OPTIONS, --output-options OPTIONS
comma-separated list of options for the export
mechanism (pass ? for a list of choices)
-o FILE, --output-file FILE
save test results to the specified FILE (or to stdout
if FILE is -)
-t TRANSPORT, --transport TRANSPORT
use TRANSPORT to send results somewhere (pass ? for a
list of choices)
--transport-where WHERE
where to send data using the selected transport
--transport-options OPTIONS
comma-separated list of key-value options (k=v) to be
passed to the transport
test selection options:
-T TEST-PLAN-ID, --test-plan TEST-PLAN-ID
load the specified test plan
-i PATTERN, --include-pattern PATTERN
include jobs matching the given regular expression
-x PATTERN, --exclude-pattern PATTERN
exclude jobs matching the given regular expression
-w WHITELIST, --whitelist WHITELIST
load whitelist containing run patterns
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_run_dot_with_resources(self):
with TestIO() as io:
with self.assertRaises(SystemExit) as call:
main(['-c', 'stub', 'dev', 'special', '--dot',
'--dot-resources'])
self.assertEqual(call.exception.args, (0,))
self.assertIn(
'\t"stub/true" [];', io.stdout.splitlines())
self.assertIn(
('\t"stub/requirement/good" -> "stub_package" [style=dashed, label'
'="stub_package.name == \'checkbox\'"];'),
io.stdout.splitlines())
# Do basic graph checks
self._check_digraph_sanity(io)
def test_help(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['run', '--help'])
self.assertEqual(call.exception.args, (0,))
self.maxDiff = None
expected = """
usage: plainbox run [-h] [--not-interactive] [-n] [-f FORMAT] [-p OPTIONS]
[-o FILE] [-t TRANSPORT] [--transport-where WHERE]
[--transport-options OPTIONS] [-i PATTERN] [-x PATTERN]
[-w WHITELIST]
optional arguments:
-h, --help show this help message and exit
user interface options:
--not-interactive Skip tests that require interactivity
-n, --dry-run Don't actually run any jobs
output options:
-f FORMAT, --output-format FORMAT
Save test results in the specified FORMAT (pass ? for
a list of choices)
-p OPTIONS, --output-options OPTIONS
Comma-separated list of options for the export
mechanism (pass ? for a list of choices)
-o FILE, --output-file FILE
Save test results to the specified FILE (or to stdout
if FILE is -)
-t TRANSPORT, --transport TRANSPORT
use TRANSPORT to send results somewhere (pass ? for a
list of choices)
--transport-where WHERE
Where to send data using the selected transport. This
is passed as-is and is transport-dependent.
--transport-options OPTIONS
Comma-separated list of key-value options (k=v) to be
passed to the transport.
job definition options:
-i PATTERN, --include-pattern PATTERN
Run jobs matching the given regular expression.
Matches from the start to the end of the line.
-x PATTERN, --exclude-pattern PATTERN
Do not run jobs matching the given regular expression.
Matches from the start to the end of the line.
-w WHITELIST, --whitelist WHITELIST
Load whitelist containing run patterns
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_dummy(self):
with TestIO(combined=True) as io:
main([])
expected = """
========================[ Searching for Matching Jobs ]=========================
Matching jobs:
===============================[ Analyzing Jobs ]===============================
Required resource jobs:
============================[ Gathering Resources ]=============================
No resource jobs required
==================================[ Testing ]===================================
No jobs selected
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_dummy(self):
with TestIO(combined=True) as io:
main([])
expected = """
========================[ Searching for Matching Jobs ]=========================
Matching jobs:
===============================[ Analyzing Jobs ]===============================
Required resource jobs:
============================[ Gathering Resources ]=============================
No resource jobs required
==================================[ Testing ]===================================
No jobs selected
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_help(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['run', '--help'])
self.assertEqual(call.exception.args, (0, ))
self.maxDiff = None
expected = """
usage: plainbox run [-h] [--not-interactive] [-n] [-f FORMAT] [-p OPTIONS]
[-o FILE] [-t TRANSPORT] [--transport-where WHERE]
[--transport-options OPTIONS] [-i PATTERN] [-x PATTERN]
[-w WHITELIST]
optional arguments:
-h, --help show this help message and exit
user interface options:
--not-interactive Skip tests that require interactivity
-n, --dry-run Don't actually run any jobs
output options:
-f FORMAT, --output-format FORMAT
Save test results in the specified FORMAT (pass ? for
a list of choices)
-p OPTIONS, --output-options OPTIONS
Comma-separated list of options for the export
mechanism (pass ? for a list of choices)
-o FILE, --output-file FILE
Save test results to the specified FILE (or to stdout
if FILE is -)
-t TRANSPORT, --transport TRANSPORT
use TRANSPORT to send results somewhere (pass ? for a
list of choices)
--transport-where WHERE
Where to send data using the selected transport. This
is passed as-is and is transport-dependent.
--transport-options OPTIONS
Comma-separated list of key-value options (k=v) to be
passed to the transport.
job definition options:
-i PATTERN, --include-pattern PATTERN
Run jobs matching the given regular expression.
Matches from the start to the end of the line.
-x PATTERN, --exclude-pattern PATTERN
Do not run jobs matching the given regular expression.
Matches from the start to the end of the line.
-w WHITELIST, --whitelist WHITELIST
Load whitelist containing run patterns
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_run_without_args(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
with patch('plainbox.impl.commands.run.authenticate_warmup') as mock_warmup:
mock_warmup.return_value = 0
main(['run'])
self.assertEqual(call.exception.args, (0,))
expected = """
===============================[ Authentication ]===============================
===============================[ Analyzing Jobs ]===============================
==============================[ Running All Jobs ]==============================
==================================[ Results ]===================================
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_output_format_list(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['run', '--output-format=?'])
self.assertEqual(call.exception.args, (0, ))
expected = """
Available output formats:
2013.com.canonical.plainbox::hexr - Generate XML (for certification)
2013.com.canonical.plainbox::html - Generate a standalone HTML
2013.com.canonical.plainbox::json - Generate JSON output
2013.com.canonical.plainbox::rfc822 - Generate RCF822 output
2013.com.canonical.plainbox::text - Generate plain text output
2013.com.canonical.plainbox::xlsx - Generate an Excel 2007+ XLSX document
"""
self.assertIn(cleandoc(expected) + "\n", io.combined)
def test_output_option_list(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['run', '--output-option=?'])
self.assertEqual(call.exception.args, (0, ))
expected = """
Each format may support a different set of options
2013.com.canonical.plainbox::hexr:
2013.com.canonical.plainbox::html:
2013.com.canonical.plainbox::json: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash, with-category-map, with-certification-status, machine-json
2013.com.canonical.plainbox::rfc822: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash, with-category-map, with-certification-status
2013.com.canonical.plainbox::text: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash, with-category-map, with-certification-status
2013.com.canonical.plainbox::xlsx: with-sys-info, with-summary, with-job-description, with-text-attachments, with-unit-categories
"""
self.assertIn(cleandoc(expected) + "\n", io.combined)
def test_output_option_list(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
with patch('plainbox.impl.commands.run.get_all_exporters') as mock_get_all_exporters:
mock_get_all_exporters.return_value = self._exporters
main(['run', '--output-option=?'])
self.assertEqual(call.exception.args, (0,))
expected = """
Each format may support a different set of options
json: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash, machine-json
rfc822: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash
text: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash
xml:
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_output_format_list(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['run', '--output-format=?'])
self.assertEqual(call.exception.args, (0,))
expected = """
Available output formats:
com.canonical.plainbox::html - Generate a standalone HTML
com.canonical.plainbox::json - Generate JSON output
com.canonical.plainbox::text - Generate plain text output
com.canonical.plainbox::tar - Generate a tar.xz archive
com.canonical.plainbox::xlsx - Generate an Excel 2007+ XLSX document
com.canonical.plainbox::global - Generate a text file containing only the test run global result
"""
self.assertIn(cleandoc(expected) + "\n", io.combined)
def test_output_option_list(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
with patch('plainbox.impl.commands.run.get_all_exporters'
) as mock_get_all_exporters:
mock_get_all_exporters.return_value = self._exporters
main(['run', '--output-option=?'])
self.assertEqual(call.exception.args, (0, ))
expected = """
Each format may support a different set of options
json: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash, machine-json
rfc822: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash
text: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash
xml:
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_run_dot_with_resources(self):
with TestIO() as io:
with self.assertRaises(SystemExit) as call:
main([
'--providers', 'stub', 'dev', 'special', '--dot',
'--dot-resources'
])
self.assertEqual(call.exception.args, (0, ))
self.assertIn('\t"2013.com.canonical.plainbox::stub/true" [];',
io.stdout.splitlines())
self.assertIn(
('\t"2013.com.canonical.plainbox::stub/requirement/good" -> '
'"2013.com.canonical.plainbox::stub_package" [style=dashed, label'
'="stub_package.name == \'checkbox\'"];'), io.stdout.splitlines())
# Do basic graph checks
self._check_digraph_sanity(io)
def test_run_without_args(self, mock_check_output):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['run', '--no-color'])
self.assertEqual(call.exception.args, (0, ))
expected = """
===============================[ Analyzing Jobs ]===============================
=============================[ Session Statistics ]=============================
This session is about 0.00% complete
Estimated duration cannot be determined for automated jobs.
Estimated duration cannot be determined for manual jobs.
Size of the desired job list: 0
Size of the effective execution plan: 0
===========================[ Running Selected Jobs ]============================
==================================[ Results ]===================================
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_help(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['--help'])
self.assertEqual(call.exception.args, (0,))
self.maxDiff = None
expected = """
usage: plainbox [-h] [-v] {run,special,self-test} ...
positional arguments:
{run,special,self-test}
run run a test job
special special/internal commands
self-test run integration tests
optional arguments:
-h, --help show this help message and exit
-v, --version show program's version number and exit
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_help(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['sru', '--help'])
self.assertEqual(call.exception.args, (0, ))
self.maxDiff = None
expected = """
usage: plainbox sru [-h] [--check-config] --secure-id SECURE-ID
[--fallback FILE] [--destination URL] [--staging] [-n]
[-i PATTERN] [-x PATTERN] [-w WHITELIST]
optional arguments:
-h, --help show this help message and exit
--check-config Run plainbox check-config before starting
sru-specific options:
--secure-id SECURE-ID
Associate submission with a machine using this SECURE-
ID (unset)
--fallback FILE If submission fails save the test report as FILE
(unset)
--destination URL POST the test report XML to this URL (https://certific
ation.canonical.com/submissions/submit/)
--staging Override --destination to use the staging
certification website
execution options:
-n, --dry-run Skip all usual jobs. Only local, resource and
attachment jobs are started
job definition options:
-i PATTERN, --include-pattern PATTERN
Run jobs matching the given regular expression.
Matches from the start to the end of the line.
-x PATTERN, --exclude-pattern PATTERN
Do not run jobs matching the given regular expression.
Matches from the start to the end of the line.
-w WHITELIST, --whitelist WHITELIST
Load whitelist containing run patterns
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def execute_job(job_name):
"""
Execute the specified job.
The job is invoked using a high-level interface from box so the test will
actually execute the same way as the UI would execute it. It will
create/tear-down appropriate session objects as well.
Returns (result, return_code) where result is the deserialized JSON saved
at the end of the job.
"""
# Create a scratch directory so that we can save results there. The
# shared directory is also used for running tests as some test jobs
# leave junk around the current directory.
with TemporaryDirectory() as scratch_dir:
# Save results to results.json in the scratch directory
pathname = os.path.join(scratch_dir, 'results.json')
# Redirect all standard IO so that the test is silent.
# Run the script, having relocated to the scratch directory
with TestIO() as io, TestCwd(scratch_dir):
try:
main([
'run', '-i', job_name, '--output-format=json', '-o',
pathname
])
except SystemExit as exc:
# Capture SystemExit that is always raised by main() so that we
# can observe the return code as well.
job_return_code = exc.args[0]
else:
job_return_code = None
# Load the actual results and keep them in memory
with open(pathname, encoding='UTF-8') as stream:
job_result = json.load(stream)
job_outcome = job_result['result_map'][job_name]['outcome']
job_execution_duration = job_result['result_map'][job_name] \
['execution_duration']
# [ At this time TestIO and TemporaryDirectory are gone ]
return (job_name, job_outcome, job_execution_duration, job_return_code,
io.stdout, io.stderr)
def test_help(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['--help'])
self.assertEqual(call.exception.args, (0, ))
self.maxDiff = None
expected = """
usage: plainbox [-h] [--version] [--providers {all,stub}] [-v] [-D] [-C]
[-T LOGGER] [-P] [-I]
{run,self-test,check-config,dev,startprovider} ...
positional arguments:
{run,self-test,check-config,dev,startprovider}
run run a test job
self-test run unit and integration tests
check-config check and display plainbox configuration
dev development commands
startprovider create a new provider (directory)
optional arguments:
-h, --help show this help message and exit
--version show program's version number and exit
provider list and development:
--providers {all,stub}
which providers to load
logging and debugging:
-v, --verbose be more verbose (same as --log-level=INFO)
-D, --debug enable DEBUG messages on the root logger
-C, --debug-console display DEBUG messages in the console
-T LOGGER, --trace LOGGER
enable DEBUG messages on the specified logger (can be
used multiple times)
-P, --pdb jump into pdb (python debugger) when a command crashes
-I, --debug-interrupt
crash on SIGINT/KeyboardInterrupt, useful with --pdb
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
