def test_invalid_pattern_excluding(self):
ns = Mock(name="ns")
ns.include_pattern_list = ['fo.*']
ns.exclude_pattern_list = ['\[bar']
observed = self.obj._get_matching_job_list(
ns, [self.job_foo, self.job_bar])
self.assertEqual(observed, [self.job_foo])
def MockJobDefinition(name, *args, **kwargs):
"""
Mock for JobDefinition class
"""
job = Mock(*args, spec_set=JobDefinition, **kwargs)
job.name = name
return job
def MockJobDefinition(id, *args, **kwargs):
"""
Mock for JobDefinition class
"""
job = Mock(*args, name="job-with-id:{}".format(id),
spec_set=JobDefinition, **kwargs)
job.id = id
return job
def test_no_prefix_matching_excluding(self):
# Exclude patterns should only match whole job name
ns = Mock(name="ns")
ns.include_pattern_list = ['.+']
ns.exclude_pattern_list = ['fo', 'ba.+']
observed = self.obj._get_matching_job_list(
ns, [self.job_foo, self.job_bar])
self.assertEqual(observed, [self.job_foo])
def test_matching_job_list(self):
# Nothing gets selected automatically
ns = Mock(name="ns")
ns.include_pattern_list = []
ns.exclude_pattern_list = []
observed = self.obj._get_matching_job_list(
ns, [self.job_foo, self.job_bar])
self.assertEqual(observed, [])
def test_matching_job_list_excluding(self):
# Excluding jobs with glob pattern works
ns = Mock(name="ns")
ns.include_pattern_list = ['.+']
ns.exclude_pattern_list = ['f.+']
observed = self.obj._get_matching_job_list(
ns, [self.job_foo, self.job_bar])
self.assertEqual(observed, [self.job_bar])
def test_invalid_pattern_excluding(self):
ns = Mock()
ns.whitelist = None
ns.include_pattern_list = ['fo.*']
ns.exclude_pattern_list = ['[bar']
observed = self.obj._get_matching_job_list(
ns, [self.job_foo, self.job_bar])
self.assertEqual(observed, [self.job_foo])
def test_invalid_pattern_including(self):
ns = Mock(name="ns")
ns.whitelist = []
ns.include_pattern_list = ['?']
ns.exclude_pattern_list = []
observed = self.obj._get_matching_job_list(
ns, [self.job_foo, self.job_bar])
self.assertEqual(observed, [])
def setUp(self):
self.job_foo = MockJobDefinition(id='foo')
self.job_bar = MockJobDefinition(id='bar')
self.job_baz = MockJobDefinition(id='baz')
self.provider1 = Mock(IProvider1)
self.config = Mock(name='config')
self.provider1.get_builtin_whitelists.return_value = []
self.provider_list = [self.provider1]
self.obj = CheckBoxInvocationMixIn(self.provider_list, self.config)
def test_matching_job_list(self):
# Nothing gets selected automatically
ns = Mock()
ns.whitelist = None
ns.include_pattern_list = []
ns.exclude_pattern_list = []
observed = self.obj._get_matching_job_list(ns, [
self.job_foo, self.job_bar])
self.assertEqual(observed, [])
def test_matching_job_list_excluding(self):
# Excluding jobs with glob pattern works
ns = Mock()
ns.whitelist = None
ns.include_pattern_list = ['.+']
ns.exclude_pattern_list = ['f.+']
observed = self.obj._get_matching_job_list(ns, [
self.job_foo, self.job_bar])
self.assertEqual(observed, [self.job_bar])
def test_no_prefix_matching_excluding(self):
# Exclude patterns should only match whole job name
ns = Mock()
ns.whitelist = None
ns.include_pattern_list = ['.+']
ns.exclude_pattern_list = ['fo', 'ba.+']
observed = self.obj._get_matching_job_list(ns, [self.job_foo,
self.job_bar])
self.assertEqual(observed, [self.job_foo])
def MockJobDefinition(id, *args, **kwargs):
"""
Mock for JobDefinition class
"""
job = Mock(*args,
name="job-with-id:{}".format(id),
spec_set=JobDefinition,
**kwargs)
job.id = id
return job
def test_matching_job_list_multiple_whitelists(self):
ns = Mock(name="ns")
ns.whitelist = [
mock_whitelist("whitelist_a", "foo", "a.whitelist"),
mock_whitelist("whitelist_b", "baz", "b.whitelist"),
]
ns.include_pattern_list = []
ns.exclude_pattern_list = []
observed = self.obj._get_matching_job_list(
ns, [self.job_foo, self.job_bar, self.job_baz])
self.assertEqual(observed, [self.job_foo, self.job_baz])
def test_matching_job_list_multiple_whitelists(self):
ns = Mock(name="ns")
ns.whitelist = [
mock_whitelist("whitelist_a", "foo", "a.whitelist"),
mock_whitelist("whitelist_b", "baz", "b.whitelist"),
]
ns.include_pattern_list = []
ns.exclude_pattern_list = []
observed = self.obj._get_matching_job_list(ns, [
self.job_foo, self.job_bar, self.job_baz])
self.assertEqual(observed, [self.job_foo, self.job_baz])
def test_matching_job_list_whitelist(self):
# whitelists contain list of include patterns
# that are read and interpreted as usual
ns = Mock(name="ns")
ns.whitelist = [
mock_whitelist("foo_whitelist", "foo", "foo.whitelist")]
ns.include_pattern_list = []
ns.exclude_pattern_list = []
observed = self.obj._get_matching_job_list(ns, [
self.job_foo, self.job_bar])
self.assertEqual(observed, [self.job_foo])
def test_matching_job_list_whitelist(self):
# whitelists contain list of include patterns
# that are read and interpreted as usual
ns = Mock(name="ns")
ns.whitelist = [
mock_whitelist("foo_whitelist", "foo", "foo.whitelist")]
ns.include_pattern_list = []
ns.exclude_pattern_list = []
observed = self.obj._get_matching_job_list(ns, [
self.job_foo, self.job_bar])
self.assertEqual(observed, [self.job_foo])
def test_no_prefix_matching_including(self):
# Include patterns should only match whole job name
ns = Mock(name="ns")
ns.whitelist = [
mock_whitelist("whitelist_a", "fo", "a.whitelist"),
mock_whitelist("whitelist_b", "ba.+", "b.whitelist"),
]
ns.include_pattern_list = ['fo', 'ba.+']
ns.exclude_pattern_list = []
observed = self.obj._get_matching_job_list(
ns, [self.job_foo, self.job_bar])
self.assertEqual(observed, [self.job_bar])
def test_no_prefix_matching_including(self):
# Include patterns should only match whole job name
ns = Mock(name="ns")
ns.whitelist = [
mock_whitelist("whitelist_a", "fo", "a.whitelist"),
mock_whitelist("whitelist_b", "ba.+", "b.whitelist"),
]
ns.include_pattern_list = ['fo', 'ba.+']
ns.exclude_pattern_list = []
observed = self.obj._get_matching_job_list(ns, [self.job_foo,
self.job_bar])
self.assertEqual(observed, [self.job_bar])
def setUp(self):
self.provider1 = Mock(spec=IProvider1)
self.job_foo = MockJobDefinition(id='foo', provider=self.provider1)
self.job_bar = MockJobDefinition(id='bar', provider=self.provider1)
self.job_baz = MockJobDefinition(id='baz', provider=self.provider1)
self.provider1.whitelist_list = []
self.provider1.id_map = defaultdict(
list, foo=[self.job_foo], bar=[self.job_bar], baz=[self.job_baz])
self.provider1.unit_list = [self.job_foo, self.job_bar, self.job_baz]
self.config = Mock(name='config')
self.provider_loader = lambda: [self.provider1]
self.obj = CheckBoxInvocationMixIn(self.provider_loader, self.config)
def test_matching_job_list_whitelist(self):
# whitelists contain list of include patterns
# that are read and interpreted as usual
whitelist = Mock()
whitelist.readlines.return_value = ['foo']
whitelists = [whitelist]
ns = Mock()
ns.whitelist = whitelists
ns.include_pattern_list = []
ns.exclude_pattern_list = []
observed = self.obj._get_matching_job_list(ns, [
self.job_foo, self.job_bar])
self.assertEqual(observed, [self.job_foo])
def test_matching_job_list_multiple_whitelists(self):
whitelist_a = Mock()
whitelist_a.readlines.return_value = ['foo']
whitelist_b = Mock()
whitelist_b.readlines.return_value = ['baz']
whitelists = [whitelist_a, whitelist_b]
ns = Mock()
ns.whitelist = whitelists
ns.include_pattern_list = []
ns.exclude_pattern_list = []
observed = self.obj._get_matching_job_list(ns, [
self.job_foo, self.job_bar, self.job_baz])
self.assertEqual(observed, [self.job_foo, self.job_baz])
def mock_whitelist(name, text, filename):
"""
Create a mocked whitelist for
CheckBoxInvocationMixIn._get_matching_job_list(). Specifically
for ``ns.whitelists`` as passed to that function.
:param name:
Name of the mocked object, helps in debugging
:param text:
Full text of the whitelist
:param filename:
Filename of the whitelist file
"""
whitelist = Mock(spec=TextIOWrapper, name=name)
whitelist.name = filename
whitelist.read.return_value = text
return whitelist
def mock_whitelist(name, text, filename):
"""
Create a mocked whitelist for
CheckBoxInvocationMixIn._get_matching_job_list(). Specifically
for ``ns.whitelists`` as passed to that function.
:param name:
Name of the mocked object, helps in debugging
:param text:
Full text of the whitelist
:param filename:
Filename of the whitelist file
"""
whitelist = Mock(spec=TextIOWrapper, name=name)
whitelist.name = filename
whitelist.read.return_value = text
return whitelist
def test_get_warm_up_sequence(self):
# create a mocked execution controller
ctrl = Mock(spec_set=IExecutionController, name='ctrl')
# create a fake warm up function
warm_up_func = Mock(name='warm_up_func')
# make the execution controller accept any job
ctrl.get_score.return_value = 1
# make the execution controller return warm_up_func as warm-up
ctrl.get_warm_up_for_job.return_value = warm_up_func
# make a pair of mock jobs for our controller to see
job1 = Mock(spec_set=IJobDefinition, name='job1')
job2 = Mock(spec_set=IJobDefinition, name='job2')
with TemporaryDirectory() as session_dir:
# Create a real runner with a fake execution controller, empty list
# of providers and fake io-log directory.
runner = JobRunner(session_dir,
provider_list=[],
jobs_io_log_dir=os.path.join(
session_dir, 'io-log'),
execution_ctrl_list=[ctrl])
# Ensure that we got the warm up function we expected
self.assertEqual(runner.get_warm_up_sequence([job1, job2]),
[warm_up_func])
class TestRun(TestCase):
@patch.dict('sys.modules', {'concurrent': Mock()})
def setUp(self):
warnings.filterwarnings('ignore',
'validate is deprecated since version 0.11')
# session data are kept in XDG_CACHE_HOME/plainbox/.session
# To avoid resuming a real session, we have to select a temporary
# location instead
self._sandbox = tempfile.mkdtemp()
self._env = os.environ
os.environ['XDG_CACHE_HOME'] = self._sandbox
def test_help(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['run', '--help'])
self.assertEqual(call.exception.args, (0, ))
self.maxDiff = None
expected = """
usage: plainbox run [-h] [--non-interactive] [-n] [--dont-suppress-output]
[-f FORMAT] [-p OPTIONS] [-o FILE] [-t TRANSPORT]
[--transport-where WHERE] [--transport-options OPTIONS]
[-T TEST-PLAN-ID] [-i PATTERN] [-x PATTERN] [-w WHITELIST]
optional arguments:
-h, --help show this help message and exit
user interface options:
--non-interactive skip tests that require interactivity
-n, --dry-run don't really run most jobs
--dont-suppress-output
don't suppress the output of certain job plugin types
output options:
-f FORMAT, --output-format FORMAT
save test results in the specified FORMAT (pass ? for
a list of choices)
-p OPTIONS, --output-options OPTIONS
comma-separated list of options for the export
mechanism (pass ? for a list of choices)
-o FILE, --output-file FILE
save test results to the specified FILE (or to stdout
if FILE is -)
-t TRANSPORT, --transport TRANSPORT
use TRANSPORT to send results somewhere (pass ? for a
list of choices)
--transport-where WHERE
where to send data using the selected transport
--transport-options OPTIONS
comma-separated list of key-value options (k=v) to be
passed to the transport
test selection options:
-T TEST-PLAN-ID, --test-plan TEST-PLAN-ID
load the specified test plan
-i PATTERN, --include-pattern PATTERN
include jobs matching the given regular expression
-x PATTERN, --exclude-pattern PATTERN
exclude jobs matching the given regular expression
-w WHITELIST, --whitelist WHITELIST
load whitelist containing run patterns
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
@patch('plainbox.impl.ctrl.check_output')
def test_run_without_args(self, mock_check_output):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['run', '--no-color'])
self.assertEqual(call.exception.args, (0, ))
expected = """
===============================[ Analyzing Jobs ]===============================
=============================[ Session Statistics ]=============================
This session is about 0.00% complete
Estimated duration cannot be determined for automated jobs.
Estimated duration cannot be determined for manual jobs.
Size of the desired job list: 0
Size of the effective execution plan: 0
===========================[ Running Selected Jobs ]============================
==================================[ Results ]===================================
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_output_format_list(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['run', '--output-format=?'])
self.assertEqual(call.exception.args, (0, ))
expected = """
Available output formats:
2013.com.canonical.plainbox::hexr - Generate XML (for certification)
2013.com.canonical.plainbox::html - Generate a standalone HTML
2013.com.canonical.plainbox::json - Generate JSON output
2013.com.canonical.plainbox::rfc822 - Generate RCF822 output
2013.com.canonical.plainbox::text - Generate plain text output
2013.com.canonical.plainbox::tar - Generate a tar.xz archive
2013.com.canonical.plainbox::xlsx - Generate an Excel 2007+ XLSX document
2013.com.canonical.plainbox::global - Generate a text file containing only the test run global result
"""
self.assertIn(cleandoc(expected) + "\n", io.combined)
def test_output_option_list(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['run', '--output-option=?'])
self.assertEqual(call.exception.args, (0, ))
expected = """
Each format may support a different set of options
2013.com.canonical.plainbox::hexr:
2013.com.canonical.plainbox::html:
2013.com.canonical.plainbox::json: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash, with-category-map, with-certification-status, machine-json
2013.com.canonical.plainbox::rfc822: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash, with-category-map, with-certification-status
2013.com.canonical.plainbox::text: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash, with-category-map, with-certification-status
2013.com.canonical.plainbox::tar:
2013.com.canonical.plainbox::xlsx: with-sys-info, with-summary, with-job-description, with-text-attachments, with-unit-categories
2013.com.canonical.plainbox::global:
"""
self.assertIn(cleandoc(expected) + "\n", io.combined)
def tearDown(self):
shutil.rmtree(self._sandbox)
os.environ = self._env
warnings.resetwarnings()
class TestRun(TestCase):
@patch.dict('sys.modules', {'concurrent': Mock()})
def setUp(self):
# session data are kept in XDG_CACHE_HOME/plainbox/.session
# To avoid resuming a real session, we have to select a temporary
# location instead
self._sandbox = tempfile.mkdtemp()
self._env = os.environ
os.environ['XDG_CACHE_HOME'] = self._sandbox
self._exporters = OrderedDict([
('json', JSONSessionStateExporter),
('rfc822', RFC822SessionStateExporter),
('text', TextSessionStateExporter),
('xml', XMLSessionStateExporter),
])
def test_help(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
main(['run', '--help'])
self.assertEqual(call.exception.args, (0, ))
self.maxDiff = None
expected = """
usage: plainbox run [-h] [--not-interactive] [-n] [-f FORMAT] [-p OPTIONS]
[-o FILE] [-t TRANSPORT] [--transport-where WHERE]
[--transport-options OPTIONS] [-i PATTERN] [-x PATTERN]
[-w WHITELIST]
optional arguments:
-h, --help show this help message and exit
user interface options:
--not-interactive skip tests that require interactivity
-n, --dry-run don't really run most jobs
output options:
-f FORMAT, --output-format FORMAT
save test results in the specified FORMAT (pass ? for
a list of choices)
-p OPTIONS, --output-options OPTIONS
comma-separated list of options for the export
mechanism (pass ? for a list of choices)
-o FILE, --output-file FILE
save test results to the specified FILE (or to stdout
if FILE is -)
-t TRANSPORT, --transport TRANSPORT
use TRANSPORT to send results somewhere (pass ? for a
list of choices)
--transport-where WHERE
where to send data using the selected transport
--transport-options OPTIONS
comma-separated list of key-value options (k=v) to be
passed to the transport
job definition options:
-i PATTERN, --include-pattern PATTERN
include jobs matching the given regular expression
-x PATTERN, --exclude-pattern PATTERN
exclude jobs matching the given regular expression
-w WHITELIST, --whitelist WHITELIST
load whitelist containing run patterns
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
@patch('plainbox.impl.ctrl.check_output')
def test_run_without_args(self, mock_check_output):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
with patch('plainbox.impl.commands.run.authenticate_warmup'
) as mock_warmup:
mock_warmup.return_value = 0
main(['run'])
self.assertEqual(call.exception.args, (0, ))
expected1 = """
===============================[ Analyzing Jobs ]===============================
Estimated duration cannot be determined for automated jobs.
Estimated duration cannot be determined for manual jobs.
==============================[ Running All Jobs ]==============================
==================================[ Results ]===================================
"""
expected2 = """
===============================[ Authentication ]===============================
===============================[ Analyzing Jobs ]===============================
Estimated duration cannot be determined for automated jobs.
Estimated duration cannot be determined for manual jobs.
==============================[ Running All Jobs ]==============================
==================================[ Results ]===================================
"""
self.assertIn(io.combined,
[cleandoc(expected1) + "\n",
cleandoc(expected2) + "\n"])
def test_output_format_list(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
with patch('plainbox.impl.commands.run.get_all_exporters'
) as mock_get_all_exporters:
mock_get_all_exporters.return_value = self._exporters
main(['run', '--output-format=?'])
self.assertEqual(call.exception.args, (0, ))
expected = """
Available output formats: json, rfc822, text, xml
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def test_output_option_list(self):
with TestIO(combined=True) as io:
with self.assertRaises(SystemExit) as call:
with patch('plainbox.impl.commands.run.get_all_exporters'
) as mock_get_all_exporters:
mock_get_all_exporters.return_value = self._exporters
main(['run', '--output-option=?'])
self.assertEqual(call.exception.args, (0, ))
expected = """
Each format may support a different set of options
json: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash, machine-json
rfc822: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash
text: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash
xml:
"""
self.assertEqual(io.combined, cleandoc(expected) + "\n")
def tearDown(self):
shutil.rmtree(self._sandbox)
os.environ = self._env
class IntegrationTests(TestCaseWithParameters):
"""
Test cases for checking execution and outcome of checkbox jobs.
Each test case is parametrized by the job id and execution "profile".
The profile is simply a string that somehow characterizes where this test
is applicable.
"""
# XXX: we cannot use weak resource cache here because test parameters
# iterate over methods first and then over actual scenarios so our cache
# would constantly loose data. This might be fixable with a different
# implementation of test parameters but that's not a low hanging fruit.
cache = ResourceCache(weak=False)
parameter_names = ('scenario_pathname',)
@patch.dict('sys.modules', {'concurrent': Mock()})
def setUp(self):
# session data are kept in XDG_CACHE_HOME/plainbox/.session
# To avoid resuming a real session, we have to select a temporary
# location instead
self._sandbox = tempfile.mkdtemp()
self._env = os.environ
os.environ['XDG_CACHE_HOME'] = self._sandbox
# Load the expected results and keep them in memory
self.scenario_data = self.cache.get(
key=('scenario_data', self.parameters.scenario_pathname),
operation=lambda: load_scenario_data(
self.parameters.scenario_pathname))
# Skip tests that are not applicable for the current system
self.skip_if_incompatible()
# Execute the job and remember the results.
(self.job_id, self.job_outcome, self.job_execution_duration,
self.job_return_code, self.job_stdout,
self.job_stderr) = self.cache.get(
key=('job-run-artifacts', self.parameters.scenario_pathname),
operation=lambda: execute_job(self.scenario_data['job_name']))
def test_job_outcome(self):
# Check that results match expected values
self.assertEqual(self.job_outcome,
self.scenario_data['result']['result_map'] \
[self.job_id]['outcome'])
def test_job_return_code(self):
# Check the return code for correctness
self.assertEqual(self.job_return_code,
self.scenario_data.get("return_code", 0))
def skip_if_incompatible(self):
"""
Skip a job if it is incompatible with the current environment
"""
if self.scenario_data.get('profile') != 'default':
self.skipTest("not applicable for current profile")
@classmethod
def _discover_test_scenarios(cls, package='plainbox',
dirname="/test-data/integration-tests/",
extension=".json"):
"""
Discover test scenarios.
Generates special absolute pathnames to scenario files. All those paths
are really relative to the plainbox package. Those pathnames are
suitable for pkg_resources.resource_ functions.
All reference data should be dropped to
``plainbox/test-data/integration-tests/`` as a json file
"""
for name in resource_listdir(package, dirname):
resource_pathname = os.path.join(dirname, name)
if resource_isdir(package, resource_pathname):
for item in cls._discover_test_scenarios(package,
resource_pathname,
extension):
yield item
elif resource_pathname.endswith(extension):
yield resource_pathname
@classmethod
def get_parameter_values(cls):
"""
Implementation detail of TestCaseWithParameters
Creates subsequent tuples for each job that has reference data
"""
for scenario_pathname in cls._discover_test_scenarios():
yield (scenario_pathname,)
def tearDown(self):
shutil.rmtree(self._sandbox)
os.environ = self._env