Beispiel #1
0
 def test_invalid_pattern_excluding(self):
     ns = Mock(name="ns")
     ns.include_pattern_list = ['fo.*']
     ns.exclude_pattern_list = ['\[bar']
     observed = self.obj._get_matching_job_list(
         ns, [self.job_foo, self.job_bar])
     self.assertEqual(observed, [self.job_foo])
def MockJobDefinition(name, *args, **kwargs):
    """
    Mock for JobDefinition class
    """
    job = Mock(*args, spec_set=JobDefinition, **kwargs)
    job.name = name
    return job
Beispiel #3
0
def MockJobDefinition(id, *args, **kwargs):
    """
    Mock for JobDefinition class
    """
    job = Mock(*args, name="job-with-id:{}".format(id),
               spec_set=JobDefinition, **kwargs)
    job.id = id
    return job
Beispiel #4
0
 def test_no_prefix_matching_excluding(self):
     # Exclude patterns should only match whole job name
     ns = Mock(name="ns")
     ns.include_pattern_list = ['.+']
     ns.exclude_pattern_list = ['fo', 'ba.+']
     observed = self.obj._get_matching_job_list(
         ns, [self.job_foo, self.job_bar])
     self.assertEqual(observed, [self.job_foo])
Beispiel #5
0
 def test_matching_job_list(self):
     # Nothing gets selected automatically
     ns = Mock(name="ns")
     ns.include_pattern_list = []
     ns.exclude_pattern_list = []
     observed = self.obj._get_matching_job_list(
         ns, [self.job_foo, self.job_bar])
     self.assertEqual(observed, [])
Beispiel #6
0
 def test_matching_job_list_excluding(self):
     # Excluding jobs with glob pattern works
     ns = Mock(name="ns")
     ns.include_pattern_list = ['.+']
     ns.exclude_pattern_list = ['f.+']
     observed = self.obj._get_matching_job_list(
         ns, [self.job_foo, self.job_bar])
     self.assertEqual(observed, [self.job_bar])
Beispiel #7
0
 def test_invalid_pattern_excluding(self):
     ns = Mock()
     ns.whitelist = None
     ns.include_pattern_list = ['fo.*']
     ns.exclude_pattern_list = ['[bar']
     observed = self.obj._get_matching_job_list(
         ns, [self.job_foo, self.job_bar])
     self.assertEqual(observed, [self.job_foo])
Beispiel #8
0
 def test_invalid_pattern_including(self):
     ns = Mock(name="ns")
     ns.whitelist = []
     ns.include_pattern_list = ['?']
     ns.exclude_pattern_list = []
     observed = self.obj._get_matching_job_list(
         ns, [self.job_foo, self.job_bar])
     self.assertEqual(observed, [])
Beispiel #9
0
 def setUp(self):
     self.job_foo = MockJobDefinition(id='foo')
     self.job_bar = MockJobDefinition(id='bar')
     self.job_baz = MockJobDefinition(id='baz')
     self.provider1 = Mock(IProvider1)
     self.config = Mock(name='config')
     self.provider1.get_builtin_whitelists.return_value = []
     self.provider_list = [self.provider1]
     self.obj = CheckBoxInvocationMixIn(self.provider_list, self.config)
Beispiel #10
0
 def test_matching_job_list(self):
     # Nothing gets selected automatically
     ns = Mock()
     ns.whitelist = None
     ns.include_pattern_list = []
     ns.exclude_pattern_list = []
     observed = self.obj._get_matching_job_list(ns, [
         self.job_foo, self.job_bar])
     self.assertEqual(observed, [])
Beispiel #11
0
 def test_matching_job_list_excluding(self):
     # Excluding jobs with glob pattern works
     ns = Mock()
     ns.whitelist = None
     ns.include_pattern_list = ['.+']
     ns.exclude_pattern_list = ['f.+']
     observed = self.obj._get_matching_job_list(ns, [
         self.job_foo, self.job_bar])
     self.assertEqual(observed, [self.job_bar])
Beispiel #12
0
 def test_no_prefix_matching_excluding(self):
     # Exclude patterns should only match whole job name
     ns = Mock()
     ns.whitelist = None
     ns.include_pattern_list = ['.+']
     ns.exclude_pattern_list = ['fo', 'ba.+']
     observed = self.obj._get_matching_job_list(ns, [self.job_foo,
                                                     self.job_bar])
     self.assertEqual(observed, [self.job_foo])
Beispiel #13
0
def MockJobDefinition(id, *args, **kwargs):
    """
    Mock for JobDefinition class
    """
    job = Mock(*args,
               name="job-with-id:{}".format(id),
               spec_set=JobDefinition,
               **kwargs)
    job.id = id
    return job
Beispiel #14
0
 def test_matching_job_list_multiple_whitelists(self):
     ns = Mock(name="ns")
     ns.whitelist = [
         mock_whitelist("whitelist_a", "foo", "a.whitelist"),
         mock_whitelist("whitelist_b", "baz", "b.whitelist"),
     ]
     ns.include_pattern_list = []
     ns.exclude_pattern_list = []
     observed = self.obj._get_matching_job_list(
         ns, [self.job_foo, self.job_bar, self.job_baz])
     self.assertEqual(observed, [self.job_foo, self.job_baz])
Beispiel #15
0
 def test_matching_job_list_multiple_whitelists(self):
     ns = Mock(name="ns")
     ns.whitelist = [
         mock_whitelist("whitelist_a", "foo", "a.whitelist"),
         mock_whitelist("whitelist_b", "baz", "b.whitelist"),
     ]
     ns.include_pattern_list = []
     ns.exclude_pattern_list = []
     observed = self.obj._get_matching_job_list(ns, [
         self.job_foo, self.job_bar, self.job_baz])
     self.assertEqual(observed, [self.job_foo, self.job_baz])
Beispiel #16
0
 def test_matching_job_list_whitelist(self):
     # whitelists contain list of include patterns
     # that are read and interpreted as usual
     ns = Mock(name="ns")
     ns.whitelist = [
         mock_whitelist("foo_whitelist", "foo", "foo.whitelist")]
     ns.include_pattern_list = []
     ns.exclude_pattern_list = []
     observed = self.obj._get_matching_job_list(ns, [
         self.job_foo, self.job_bar])
     self.assertEqual(observed, [self.job_foo])
 def test_matching_job_list_whitelist(self):
     # whitelists contain list of include patterns
     # that are read and interpreted as usual
     ns = Mock(name="ns")
     ns.whitelist = [
         mock_whitelist("foo_whitelist", "foo", "foo.whitelist")]
     ns.include_pattern_list = []
     ns.exclude_pattern_list = []
     observed = self.obj._get_matching_job_list(ns, [
         self.job_foo, self.job_bar])
     self.assertEqual(observed, [self.job_foo])
Beispiel #18
0
 def test_no_prefix_matching_including(self):
     # Include patterns should only match whole job name
     ns = Mock(name="ns")
     ns.whitelist = [
         mock_whitelist("whitelist_a", "fo", "a.whitelist"),
         mock_whitelist("whitelist_b", "ba.+", "b.whitelist"),
     ]
     ns.include_pattern_list = ['fo', 'ba.+']
     ns.exclude_pattern_list = []
     observed = self.obj._get_matching_job_list(
         ns, [self.job_foo, self.job_bar])
     self.assertEqual(observed, [self.job_bar])
Beispiel #19
0
 def test_no_prefix_matching_including(self):
     # Include patterns should only match whole job name
     ns = Mock(name="ns")
     ns.whitelist = [
         mock_whitelist("whitelist_a", "fo", "a.whitelist"),
         mock_whitelist("whitelist_b", "ba.+", "b.whitelist"),
     ]
     ns.include_pattern_list = ['fo', 'ba.+']
     ns.exclude_pattern_list = []
     observed = self.obj._get_matching_job_list(ns, [self.job_foo,
                                                     self.job_bar])
     self.assertEqual(observed, [self.job_bar])
 def setUp(self):
     self.provider1 = Mock(spec=IProvider1)
     self.job_foo = MockJobDefinition(id='foo', provider=self.provider1)
     self.job_bar = MockJobDefinition(id='bar', provider=self.provider1)
     self.job_baz = MockJobDefinition(id='baz', provider=self.provider1)
     self.provider1.whitelist_list = []
     self.provider1.id_map = defaultdict(
         list, foo=[self.job_foo], bar=[self.job_bar], baz=[self.job_baz])
     self.provider1.unit_list = [self.job_foo, self.job_bar, self.job_baz]
     self.config = Mock(name='config')
     self.provider_loader = lambda: [self.provider1]
     self.obj = CheckBoxInvocationMixIn(self.provider_loader, self.config)
Beispiel #21
0
 def test_matching_job_list_whitelist(self):
     # whitelists contain list of include patterns
     # that are read and interpreted as usual
     whitelist = Mock()
     whitelist.readlines.return_value = ['foo']
     whitelists = [whitelist]
     ns = Mock()
     ns.whitelist = whitelists
     ns.include_pattern_list = []
     ns.exclude_pattern_list = []
     observed = self.obj._get_matching_job_list(ns, [
         self.job_foo, self.job_bar])
     self.assertEqual(observed, [self.job_foo])
Beispiel #22
0
 def test_matching_job_list_multiple_whitelists(self):
     whitelist_a = Mock()
     whitelist_a.readlines.return_value = ['foo']
     whitelist_b = Mock()
     whitelist_b.readlines.return_value = ['baz']
     whitelists = [whitelist_a, whitelist_b]
     ns = Mock()
     ns.whitelist = whitelists
     ns.include_pattern_list = []
     ns.exclude_pattern_list = []
     observed = self.obj._get_matching_job_list(ns, [
         self.job_foo, self.job_bar, self.job_baz])
     self.assertEqual(observed, [self.job_foo, self.job_baz])
Beispiel #23
0
def mock_whitelist(name, text, filename):
    """
    Create a mocked whitelist for
    CheckBoxInvocationMixIn._get_matching_job_list(). Specifically
    for ``ns.whitelists`` as passed to that function.

    :param name:
        Name of the mocked object, helps in debugging
    :param text:
        Full text of the whitelist
    :param filename:
        Filename of the whitelist file
    """
    whitelist = Mock(spec=TextIOWrapper, name=name)
    whitelist.name = filename
    whitelist.read.return_value = text
    return whitelist
Beispiel #24
0
def mock_whitelist(name, text, filename):
    """
    Create a mocked whitelist for
    CheckBoxInvocationMixIn._get_matching_job_list(). Specifically
    for ``ns.whitelists`` as passed to that function.

    :param name:
        Name of the mocked object, helps in debugging
    :param text:
        Full text of the whitelist
    :param filename:
        Filename of the whitelist file
    """
    whitelist = Mock(spec=TextIOWrapper, name=name)
    whitelist.name = filename
    whitelist.read.return_value = text
    return whitelist
 def test_get_warm_up_sequence(self):
     # create a mocked execution controller
     ctrl = Mock(spec_set=IExecutionController, name='ctrl')
     # create a fake warm up function
     warm_up_func = Mock(name='warm_up_func')
     # make the execution controller accept any job
     ctrl.get_score.return_value = 1
     # make the execution controller return warm_up_func as warm-up
     ctrl.get_warm_up_for_job.return_value = warm_up_func
     # make a pair of mock jobs for our controller to see
     job1 = Mock(spec_set=IJobDefinition, name='job1')
     job2 = Mock(spec_set=IJobDefinition, name='job2')
     with TemporaryDirectory() as session_dir:
         # Create a real runner with a fake execution controller, empty list
         # of providers and fake io-log directory.
         runner = JobRunner(session_dir,
                            provider_list=[],
                            jobs_io_log_dir=os.path.join(
                                session_dir, 'io-log'),
                            execution_ctrl_list=[ctrl])
         # Ensure that we got the warm up function we expected
         self.assertEqual(runner.get_warm_up_sequence([job1, job2]),
                          [warm_up_func])
Beispiel #26
0
class TestRun(TestCase):
    @patch.dict('sys.modules', {'concurrent': Mock()})
    def setUp(self):
        warnings.filterwarnings('ignore',
                                'validate is deprecated since version 0.11')
        # session data are kept in XDG_CACHE_HOME/plainbox/.session
        # To avoid resuming a real session, we have to select a temporary
        # location instead
        self._sandbox = tempfile.mkdtemp()
        self._env = os.environ
        os.environ['XDG_CACHE_HOME'] = self._sandbox

    def test_help(self):
        with TestIO(combined=True) as io:
            with self.assertRaises(SystemExit) as call:
                main(['run', '--help'])
            self.assertEqual(call.exception.args, (0, ))
        self.maxDiff = None
        expected = """
        usage: plainbox run [-h] [--non-interactive] [-n] [--dont-suppress-output]
                            [-f FORMAT] [-p OPTIONS] [-o FILE] [-t TRANSPORT]
                            [--transport-where WHERE] [--transport-options OPTIONS]
                            [-T TEST-PLAN-ID] [-i PATTERN] [-x PATTERN] [-w WHITELIST]

        optional arguments:
          -h, --help            show this help message and exit

        user interface options:
          --non-interactive     skip tests that require interactivity
          -n, --dry-run         don't really run most jobs
          --dont-suppress-output
                                don't suppress the output of certain job plugin types

        output options:
          -f FORMAT, --output-format FORMAT
                                save test results in the specified FORMAT (pass ? for
                                a list of choices)
          -p OPTIONS, --output-options OPTIONS
                                comma-separated list of options for the export
                                mechanism (pass ? for a list of choices)
          -o FILE, --output-file FILE
                                save test results to the specified FILE (or to stdout
                                if FILE is -)
          -t TRANSPORT, --transport TRANSPORT
                                use TRANSPORT to send results somewhere (pass ? for a
                                list of choices)
          --transport-where WHERE
                                where to send data using the selected transport
          --transport-options OPTIONS
                                comma-separated list of key-value options (k=v) to be
                                passed to the transport

        test selection options:
          -T TEST-PLAN-ID, --test-plan TEST-PLAN-ID
                                load the specified test plan
          -i PATTERN, --include-pattern PATTERN
                                include jobs matching the given regular expression
          -x PATTERN, --exclude-pattern PATTERN
                                exclude jobs matching the given regular expression
          -w WHITELIST, --whitelist WHITELIST
                                load whitelist containing run patterns
        """
        self.assertEqual(io.combined, cleandoc(expected) + "\n")

    @patch('plainbox.impl.ctrl.check_output')
    def test_run_without_args(self, mock_check_output):
        with TestIO(combined=True) as io:
            with self.assertRaises(SystemExit) as call:
                main(['run', '--no-color'])
            self.assertEqual(call.exception.args, (0, ))
        expected = """
        ===============================[ Analyzing Jobs ]===============================
        =============================[ Session Statistics ]=============================
        This session is about 0.00% complete
        Estimated duration cannot be determined for automated jobs.
        Estimated duration cannot be determined for manual jobs.
        Size of the desired job list: 0
        Size of the effective execution plan: 0
        ===========================[ Running Selected Jobs ]============================
        ==================================[ Results ]===================================
        """
        self.assertEqual(io.combined, cleandoc(expected) + "\n")

    def test_output_format_list(self):
        with TestIO(combined=True) as io:
            with self.assertRaises(SystemExit) as call:
                main(['run', '--output-format=?'])
            self.assertEqual(call.exception.args, (0, ))
        expected = """
        Available output formats:
        2013.com.canonical.plainbox::hexr - Generate XML (for certification)
        2013.com.canonical.plainbox::html - Generate a standalone HTML
        2013.com.canonical.plainbox::json - Generate JSON output
        2013.com.canonical.plainbox::rfc822 - Generate RCF822 output
        2013.com.canonical.plainbox::text - Generate plain text output
        2013.com.canonical.plainbox::tar - Generate a tar.xz archive
        2013.com.canonical.plainbox::xlsx - Generate an Excel 2007+ XLSX document
        2013.com.canonical.plainbox::global - Generate a text file containing only the test run global result
        """
        self.assertIn(cleandoc(expected) + "\n", io.combined)

    def test_output_option_list(self):
        with TestIO(combined=True) as io:
            with self.assertRaises(SystemExit) as call:
                main(['run', '--output-option=?'])
            self.assertEqual(call.exception.args, (0, ))
        expected = """
        Each format may support a different set of options
        2013.com.canonical.plainbox::hexr: 
        2013.com.canonical.plainbox::html: 
        2013.com.canonical.plainbox::json: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash, with-category-map, with-certification-status, machine-json
        2013.com.canonical.plainbox::rfc822: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash, with-category-map, with-certification-status
        2013.com.canonical.plainbox::text: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash, with-category-map, with-certification-status
        2013.com.canonical.plainbox::tar: 
        2013.com.canonical.plainbox::xlsx: with-sys-info, with-summary, with-job-description, with-text-attachments, with-unit-categories
        2013.com.canonical.plainbox::global: 
        """
        self.assertIn(cleandoc(expected) + "\n", io.combined)

    def tearDown(self):
        shutil.rmtree(self._sandbox)
        os.environ = self._env
        warnings.resetwarnings()
Beispiel #27
0
class TestRun(TestCase):
    @patch.dict('sys.modules', {'concurrent': Mock()})
    def setUp(self):
        # session data are kept in XDG_CACHE_HOME/plainbox/.session
        # To avoid resuming a real session, we have to select a temporary
        # location instead
        self._sandbox = tempfile.mkdtemp()
        self._env = os.environ
        os.environ['XDG_CACHE_HOME'] = self._sandbox
        self._exporters = OrderedDict([
            ('json', JSONSessionStateExporter),
            ('rfc822', RFC822SessionStateExporter),
            ('text', TextSessionStateExporter),
            ('xml', XMLSessionStateExporter),
        ])

    def test_help(self):
        with TestIO(combined=True) as io:
            with self.assertRaises(SystemExit) as call:
                main(['run', '--help'])
            self.assertEqual(call.exception.args, (0, ))
        self.maxDiff = None
        expected = """
        usage: plainbox run [-h] [--not-interactive] [-n] [-f FORMAT] [-p OPTIONS]
                            [-o FILE] [-t TRANSPORT] [--transport-where WHERE]
                            [--transport-options OPTIONS] [-i PATTERN] [-x PATTERN]
                            [-w WHITELIST]

        optional arguments:
          -h, --help            show this help message and exit

        user interface options:
          --not-interactive     skip tests that require interactivity
          -n, --dry-run         don't really run most jobs

        output options:
          -f FORMAT, --output-format FORMAT
                                save test results in the specified FORMAT (pass ? for
                                a list of choices)
          -p OPTIONS, --output-options OPTIONS
                                comma-separated list of options for the export
                                mechanism (pass ? for a list of choices)
          -o FILE, --output-file FILE
                                save test results to the specified FILE (or to stdout
                                if FILE is -)
          -t TRANSPORT, --transport TRANSPORT
                                use TRANSPORT to send results somewhere (pass ? for a
                                list of choices)
          --transport-where WHERE
                                where to send data using the selected transport
          --transport-options OPTIONS
                                comma-separated list of key-value options (k=v) to be
                                passed to the transport

        job definition options:
          -i PATTERN, --include-pattern PATTERN
                                include jobs matching the given regular expression
          -x PATTERN, --exclude-pattern PATTERN
                                exclude jobs matching the given regular expression
          -w WHITELIST, --whitelist WHITELIST
                                load whitelist containing run patterns
        """
        self.assertEqual(io.combined, cleandoc(expected) + "\n")

    @patch('plainbox.impl.ctrl.check_output')
    def test_run_without_args(self, mock_check_output):
        with TestIO(combined=True) as io:
            with self.assertRaises(SystemExit) as call:
                with patch('plainbox.impl.commands.run.authenticate_warmup'
                           ) as mock_warmup:
                    mock_warmup.return_value = 0
                    main(['run'])
            self.assertEqual(call.exception.args, (0, ))
        expected1 = """
        ===============================[ Analyzing Jobs ]===============================
        Estimated duration cannot be determined for automated jobs.
        Estimated duration cannot be determined for manual jobs.
        ==============================[ Running All Jobs ]==============================
        ==================================[ Results ]===================================
        """
        expected2 = """
        ===============================[ Authentication ]===============================
        ===============================[ Analyzing Jobs ]===============================
        Estimated duration cannot be determined for automated jobs.
        Estimated duration cannot be determined for manual jobs.
        ==============================[ Running All Jobs ]==============================
        ==================================[ Results ]===================================
        """
        self.assertIn(io.combined,
                      [cleandoc(expected1) + "\n",
                       cleandoc(expected2) + "\n"])

    def test_output_format_list(self):
        with TestIO(combined=True) as io:
            with self.assertRaises(SystemExit) as call:
                with patch('plainbox.impl.commands.run.get_all_exporters'
                           ) as mock_get_all_exporters:
                    mock_get_all_exporters.return_value = self._exporters
                    main(['run', '--output-format=?'])
            self.assertEqual(call.exception.args, (0, ))
        expected = """
        Available output formats: json, rfc822, text, xml
        """
        self.assertEqual(io.combined, cleandoc(expected) + "\n")

    def test_output_option_list(self):
        with TestIO(combined=True) as io:
            with self.assertRaises(SystemExit) as call:
                with patch('plainbox.impl.commands.run.get_all_exporters'
                           ) as mock_get_all_exporters:
                    mock_get_all_exporters.return_value = self._exporters
                    main(['run', '--output-option=?'])
            self.assertEqual(call.exception.args, (0, ))
        expected = """
        Each format may support a different set of options
        json: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash, machine-json
        rfc822: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash
        text: with-io-log, squash-io-log, flatten-io-log, with-run-list, with-job-list, with-resource-map, with-job-defs, with-attachments, with-comments, with-job-via, with-job-hash
        xml: 
        """
        self.assertEqual(io.combined, cleandoc(expected) + "\n")

    def tearDown(self):
        shutil.rmtree(self._sandbox)
        os.environ = self._env
Beispiel #28
0
class IntegrationTests(TestCaseWithParameters):
    """
    Test cases for checking execution and outcome of checkbox jobs.
    Each test case is parametrized by the job id and execution "profile".

    The profile is simply a string that somehow characterizes where this test
    is applicable.
    """

    # XXX: we cannot use weak resource cache here because test parameters
    # iterate over methods first and then over actual scenarios so our cache
    # would constantly loose data. This might be fixable with a different
    # implementation of test parameters but that's not a low hanging fruit.
    cache = ResourceCache(weak=False)

    parameter_names = ('scenario_pathname',)

    @patch.dict('sys.modules', {'concurrent': Mock()})
    def setUp(self):
        # session data are kept in XDG_CACHE_HOME/plainbox/.session
        # To avoid resuming a real session, we have to select a temporary
        # location instead
        self._sandbox = tempfile.mkdtemp()
        self._env = os.environ
        os.environ['XDG_CACHE_HOME'] = self._sandbox
        # Load the expected results and keep them in memory
        self.scenario_data = self.cache.get(
            key=('scenario_data', self.parameters.scenario_pathname),
            operation=lambda: load_scenario_data(
                self.parameters.scenario_pathname))
        # Skip tests that are not applicable for the current system
        self.skip_if_incompatible()
        # Execute the job and remember the results.
        (self.job_id, self.job_outcome, self.job_execution_duration,
         self.job_return_code, self.job_stdout,
         self.job_stderr) = self.cache.get(
             key=('job-run-artifacts', self.parameters.scenario_pathname),
             operation=lambda: execute_job(self.scenario_data['job_name']))

    def test_job_outcome(self):
        # Check that results match expected values
        self.assertEqual(self.job_outcome,
                         self.scenario_data['result']['result_map'] \
                                           [self.job_id]['outcome'])

    def test_job_return_code(self):
        # Check the return code for correctness
        self.assertEqual(self.job_return_code,
                         self.scenario_data.get("return_code", 0))

    def skip_if_incompatible(self):
        """
        Skip a job if it is incompatible with the current environment
        """
        if self.scenario_data.get('profile') != 'default':
            self.skipTest("not applicable for current profile")

    @classmethod
    def _discover_test_scenarios(cls, package='plainbox',
                                 dirname="/test-data/integration-tests/",
                                 extension=".json"):
        """
        Discover test scenarios.

        Generates special absolute pathnames to scenario files. All those paths
        are really relative to the plainbox package. Those pathnames are
        suitable for pkg_resources.resource_ functions.

        All reference data should be dropped to
        ``plainbox/test-data/integration-tests/`` as a json file
        """
        for name in resource_listdir(package, dirname):
            resource_pathname = os.path.join(dirname, name)
            if resource_isdir(package, resource_pathname):
                for item in cls._discover_test_scenarios(package,
                                                         resource_pathname,
                                                         extension):
                    yield item
            elif resource_pathname.endswith(extension):
                yield resource_pathname

    @classmethod
    def get_parameter_values(cls):
        """
        Implementation detail of TestCaseWithParameters

        Creates subsequent tuples for each job that has reference data
        """
        for scenario_pathname in cls._discover_test_scenarios():
            yield (scenario_pathname,)

    def tearDown(self):
        shutil.rmtree(self._sandbox)
        os.environ = self._env