def test_upper_dict_required(self): test_yaml = [{'name': 'house', 'url': '/'}] with self.assertRaises(exception.GabbiFormatError) as failure: suitemaker.test_suite_from_dict(self.loader, 'foo', test_yaml, '.', 'localhost', 80, None, None) self.assertEqual('malformed test file, invalid format', str(failure.exception))
def test_method_url_pair_format_error(self): test_yaml = {'defaults': {'GET': '/foo'}, 'tests': []} with self.assertRaises(exception.GabbiFormatError) as failure: suitemaker.test_suite_from_dict(self.loader, 'foo', test_yaml, '.', 'localhost', 80, None, None) self.assertIn('"METHOD: url" pairs not allowed in defaults', str(failure.exception))
def test_inner_list_required(self): test_yaml = {'tests': {'name': 'house', 'url': '/'}} with self.assertRaises(exception.GabbiFormatError) as failure: suitemaker.test_suite_from_dict(self.loader, 'foo', test_yaml, '.', 'localhost', 80, None, None) self.assertIn('test chunk is not a dict at', str(failure.exception))
def test_url_key_required(self): test_yaml = {'tests': [{'name': 'missing url'}]} with self.assertRaises(exception.GabbiFormatError) as failure: suitemaker.test_suite_from_dict(self.loader, 'foo', test_yaml, '.', 'localhost', 80, None, None) self.assertEqual('Test url missing in test foo_missing_url.', str(failure.exception))
def test_unsupported_key_errors(self): test_yaml = {'tests': [{ 'url': '/', 'name': 'simple', 'bad_key': 'wow', }]} with self.assertRaises(exception.GabbiFormatError) as failure: suitemaker.test_suite_from_dict(self.loader, 'foo', test_yaml, '.', 'localhost', 80, None, None) self.assertIn("Invalid test keys used in test foo_simple:", str(failure.exception))
def test_method_url_pair_duplication_format_error(self): test_yaml = {'tests': [{ 'GET': '/', 'POST': '/', 'name': 'duplicate methods', }]} with self.assertRaises(exception.GabbiFormatError) as failure: suitemaker.test_suite_from_dict(self.loader, 'foo', test_yaml, '.', 'localhost', 80, None, None) self.assertIn( 'duplicate method/URL directive in "foo_duplicate_methods"', str(failure.exception) )
def test_suite_catches_fixture_fail(self): """Verify fixture failure. When a fixture fails in start_fixture it should fail the first test in the suite and skip the others. """ loader = unittest.defaultTestLoader result = unittest.TestResult() test_data = { 'fixtures': ['FakeFixture'], 'tests': [{ 'name': 'alpha', 'GET': '/' }, { 'name': 'beta', 'GET': '/' }] } test_suite = suitemaker.test_suite_from_dict(loader, 'foo', test_data, '.', 'localhost', 80, sys.modules[__name__], None) test_suite.run(result) self.assertEqual(2, len(result.skipped)) self.assertEqual(1, len(result.errors)) errored_test, trace = result.errors[0] self.assertIn('foo_alpha', str(errored_test)) self.assertIn(VALUE_ERROR, trace)
def run_test(test_class_instance, test_dir, filename): d = utils.load_yaml(yaml_file=os.path.join(test_dir, filename)) test_suite = suitemaker.test_suite_from_dict( loader=unittest.defaultTestLoader, test_base_name="gabbi", suite_dict=d, test_directory=test_dir, host='example.com', port=None, fixture_module=None, intercept=None, handlers=runner.initialize_handlers([]), test_loader_name="tempest") # NOTE(sileht): We hide stdout/stderr and reraise the failure # manually, tempest will print it ittest_class. with open(os.devnull, 'w') as stream: result = unittest.TextTestRunner( stream=stream, verbosity=0, failfast=True, ).run(test_suite) if not result.wasSuccessful(): failures = (result.errors + result.failures + result.unexpectedSuccesses) if failures: test, bt = failures[0] name = test.test_data.get('name', test.id()) msg = 'From test "%s" :\n%s' % (name, bt) test_class_instance.fail(msg) test_class_instance.assertTrue(result.wasSuccessful())
def run_suite(handle, handler_objects, host, port, prefix, force_ssl=False, failfast=False, data_dir='.', verbosity=False, name='input', safe_yaml=True, quiet=False): """Run the tests from the YAML in handle.""" data = utils.load_yaml(handle, safe=safe_yaml) if force_ssl: if 'defaults' in data: data['defaults']['ssl'] = True else: data['defaults'] = {'ssl': True} if verbosity: if 'defaults' in data: data['defaults']['verbose'] = verbosity else: data['defaults'] = {'verbose': verbosity} loader = unittest.defaultTestLoader test_suite = suitemaker.test_suite_from_dict( loader, name, data, data_dir, host, port, None, None, prefix=prefix, handlers=handler_objects, test_loader_name='gabbi-runner') # The default runner stream is stderr. stream = sys.stderr if quiet: # We want to swallow the output that the runner is # producing. stream = open(os.devnull, 'w') result = ConciseTestRunner( stream=stream, verbosity=2, failfast=failfast).run(test_suite) return result.wasSuccessful()
def _do_test(self, filename): token = self.os_admin.auth_provider.get_token() url = self.os_admin.auth_provider.base_url({ 'service': CONF.metric.catalog_type, 'endpoint_type': CONF.metric.endpoint_type }) parsed_url = urlparse.urlsplit(url) prefix = parsed_url.path.rstrip('/') # turn it into a prefix if parsed_url.scheme == 'https': port = 443 require_ssl = True else: port = 80 require_ssl = False host = parsed_url.hostname if parsed_url.port: port = parsed_url.port os.environ["GNOCCHI_SERVICE_TOKEN"] = token os.environ["GNOCCHI_AUTHORIZATION"] = "not used" with open(os.path.join(TEST_DIR, filename)) as f: suite_dict = utils.load_yaml(f) suite_dict.setdefault('defaults', {})['ssl'] = require_ssl test_suite = suitemaker.test_suite_from_dict( loader=unittest.defaultTestLoader, test_base_name="gabbi", suite_dict=suite_dict, test_directory=TEST_DIR, host=host, port=port, fixture_module=None, intercept=None, prefix=prefix, handlers=runner.initialize_handlers([]), test_loader_name="tempest") # NOTE(sileht): We hide stdout/stderr and reraise the failure # manually, tempest will print it itself. with open(os.devnull, 'w') as stream: result = unittest.TextTestRunner( stream=stream, verbosity=0, failfast=True, ).run(test_suite) if not result.wasSuccessful(): failures = (result.errors + result.failures + result.unexpectedSuccesses) if failures: test, bt = failures[0] name = test.test_data.get('name', test.id()) msg = 'From test "%s" :\n%s' % (name, bt) self.fail(msg) self.assertTrue(result.wasSuccessful())
def test_dict_on_invalid_key(self): test_yaml = {'tests': [{ 'name': '...', 'GET': '/', 'response_html': { 'foo': 'hello', 'bar': 'world', } }]} with self.assertRaises(exception.GabbiFormatError) as failure: suitemaker.test_suite_from_dict(self.loader, 'foo', test_yaml, '.', 'localhost', 80, None, None) self.assertIn( "invalid key in test: 'response_html'", str(failure.exception) )
def test_suite_from_yaml(loader, test_base_name, test_yaml, test_directory, host, port, fixture_module, intercept, prefix=''): """Legacy wrapper retained for backwards compatibility.""" with warnings.catch_warnings(): # ensures warnings filter is restored warnings.simplefilter('default', DeprecationWarning) warnings.warn('test_suite_from_yaml has been renamed to ' 'test_suite_from_dict', DeprecationWarning, stacklevel=2) return suitemaker.test_suite_from_dict( loader, test_base_name, test_yaml, test_directory, host, port, fixture_module, intercept, prefix)
def _do_test(self, filename): admin_auth = self.os_admin.auth_provider.get_auth() auth = self.os_primary.auth_provider.get_auth() networks = self.os_primary.networks_client.list_networks( **{'router:external': False, 'fields': 'id'})['networks'] os.environ.update({ "ADMIN_TOKEN": admin_auth[0], "USER_TOKEN": auth[0], "AODH_GRANULARITY": str(config.CONF.telemetry.alarm_granularity), "AODH_SERVICE_URL": self._get_endpoint(auth, "alarming_plugin"), "GNOCCHI_SERVICE_URL": self._get_endpoint(auth, "metric"), "PANKO_SERVICE_URL": self._get_endpoint(auth, "event"), "HEAT_SERVICE_URL": self._get_endpoint(auth, "orchestration"), "NOVA_SERVICE_URL": self._get_endpoint(auth, "compute"), "GLANCE_SERVICE_URL": self._get_endpoint(auth, "image"), "GLANCE_IMAGE_NAME": self.glance_image_create(), "NOVA_FLAVOR_REF": config.CONF.compute.flavor_ref, "NEUTRON_NETWORK": networks[0].get('id'), }) with open(os.path.join(TEST_DIR, filename)) as f: test_suite = suitemaker.test_suite_from_dict( loader=unittest.defaultTestLoader, test_base_name="gabbi", suite_dict=utils.load_yaml(f), test_directory=TEST_DIR, host=None, port=None, fixture_module=None, intercept=None, handlers=runner.initialize_handlers([]), test_loader_name="tempest") # NOTE(sileht): We hide stdout/stderr and reraise the failure # manually, tempest will print it itself. with open(os.devnull, 'w') as stream: result = unittest.TextTestRunner( stream=stream, verbosity=0, failfast=True, ).run(test_suite) if not result.wasSuccessful(): failures = (result.errors + result.failures + result.unexpectedSuccesses) if failures: test, bt = failures[0] name = test.test_data.get('name', test.id()) msg = 'From test "%s" :\n%s' % (name, bt) self.fail(msg) self.assertTrue(result.wasSuccessful())
def test_response_handlers_same_test_key_yaml_first(self): test_yaml = {'tests': [{ 'name': '...', 'GET': '/', 'response_json_paths': { 'foo': 'hello', 'bar': 'world', } }]} ydlj_handler_object = ydlj_handler.YAMLDiskLoadingJSONHandler() handler_objects = [ydlj_handler_object] for handler in handlers.RESPONSE_HANDLERS: handler_objects.append(handler()) file_suite = suitemaker.test_suite_from_dict( self.loader, 'foo', test_yaml, '.', 'localhost', 80, None, None, handlers=handler_objects) response_handlers = file_suite._tests[0].response_handlers self.assertIn(ydlj_handler_object, response_handlers)
def run_suite(handle, handler_objects, host, port, prefix, force_ssl=False, failfast=False, data_dir='.', verbosity=False, name='input', safe_yaml=True): """Run the tests from the YAML in handle.""" data = utils.load_yaml(handle, safe=safe_yaml) if force_ssl: if 'defaults' in data: data['defaults']['ssl'] = True else: data['defaults'] = {'ssl': True} if verbosity: if 'defaults' in data: data['defaults']['verbose'] = verbosity else: data['defaults'] = {'verbose': verbosity} loader = unittest.defaultTestLoader test_suite = suitemaker.test_suite_from_dict( loader, name, data, data_dir, host, port, None, None, prefix=prefix, handlers=handler_objects, test_loader_name='gabbi-runner') result = ConciseTestRunner(verbosity=2, failfast=failfast).run(test_suite) return result.wasSuccessful()
def run_gabbi(self, gabbi_declaration): # Take only the host name and port from the live server: _, authority = self.live_server_url.split('://') host, port = authority.split(':') # Use Gabbi to create the test suite from our declaration: suite = test_suite_from_dict( loader=defaultTestLoader, test_base_name=self.id(), suite_dict=gabbi_declaration, test_directory='.', host=host, port=port, fixture_module=None, intercept=None, handlers=[ handler() for handler in RESPONSE_HANDLERS ], ) # Run the test. We store the the output into a custom stream so that # Hypothesis can display only the simple case test result on failure # rather than every failing case: s = StringIO() result = ConciseTestRunner( stream=s, verbosity=0, ).run(suite) # If we weren't successful we need to fail the test case with the error # string from Gabbi: if not result.wasSuccessful(): self.fail( s.getvalue(), )
def test_suite_catches_fixture_fail(self): """Verify fixture failure. When a fixture fails in start_fixture it should fail the first test in the suite and skip the others. """ loader = unittest.defaultTestLoader result = unittest.TestResult() test_data = {'fixtures': ['FakeFixture'], 'tests': [{'name': 'alpha', 'GET': '/'}, {'name': 'beta', 'GET': '/'}]} test_suite = suitemaker.test_suite_from_dict( loader, 'foo', test_data, '.', 'localhost', 80, sys.modules[__name__], None) test_suite.run(result) self.assertEqual(2, len(result.skipped)) self.assertEqual(1, len(result.errors)) errored_test, trace = result.errors[0] self.assertIn('foo_alpha', str(errored_test)) self.assertIn(VALUE_ERROR, trace)
def run_suite(handle, handler_objects, host, port, prefix, force_ssl=False, failfast=False, data_dir='.', verbosity=False, name='input', safe_yaml=True): """Run the tests from the YAML in handle.""" data = utils.load_yaml(handle, safe=safe_yaml) if force_ssl: if 'defaults' in data: data['defaults']['ssl'] = True else: data['defaults'] = {'ssl': True} if verbosity: if 'defaults' in data: data['defaults']['verbose'] = verbosity else: data['defaults'] = {'verbose': verbosity} loader = unittest.defaultTestLoader test_suite = suitemaker.test_suite_from_dict( loader, name, data, data_dir, host, port, None, None, prefix=prefix, handlers=handler_objects, test_loader_name='gabbi-runner') result = ConciseTestRunner( verbosity=2, failfast=failfast).run(test_suite) return result.wasSuccessful()
def build_tests(path, loader, host=None, port=8001, intercept=None, test_loader_name=None, fixture_module=None, response_handlers=None, content_handlers=None, prefix='', require_ssl=False, url=None, inner_fixtures=None, verbose=False, use_prior_test=True, safe_yaml=True): """Read YAML files from a directory to create tests. Each YAML file represents a list of HTTP requests. :param path: The directory where yaml files are located. :param loader: The TestLoader. :param host: The host to test against. Do not use with ``intercept``. :param port: The port to test against. Used with ``host``. :param intercept: WSGI app factory for wsgi-intercept. :param test_loader_name: Base name for test classes. Use this to align the naming of the tests with other tests in a system. :param fixture_module: Python module containing fixture classes. :param response_handers: :class:`~gabbi.handlers.ResponseHandler` classes. :type response_handlers: List of ResponseHandler classes. :param content_handlers: ContentHandler classes. :type content_handlers: List of ContentHandler classes. :param prefix: A URL prefix for all URLs that are not fully qualified. :param url: A full URL to test against. Replaces host, port and prefix. :param require_ssl: If ``True``, make all tests default to using SSL. :param inner_fixtures: A list of ``Fixtures`` to use with each individual test request. :param verbose: If ``True`` or ``'all'``, make tests verbose by default ``'headers'`` and ``'body'`` are also accepted. :param use_prior_test: If ``True``, uses prior test to create ordered sequence of tests :param safe_yaml: If ``True``, recognizes only standard YAML tags and not Python object :type inner_fixtures: List of fixtures.Fixture classes. :rtype: TestSuite containing multiple TestSuites (one for each YAML file). """ # If url is being used, reset host, port and prefix. if url: host, port, prefix, force_ssl = utils.host_info_from_target(url) if force_ssl and not require_ssl: require_ssl = force_ssl # Exit immediately if we have no host to access, either via a real host # or an intercept. if not ((host is not None) ^ bool(intercept)): raise AssertionError( 'must specify exactly one of host or url, or intercept') # If the client has not provided a name to use as our base, # create one so that tests are effectively namespaced. if test_loader_name is None: all_test_base_name = inspect.stack()[1] all_test_base_name = os.path.splitext( os.path.basename(all_test_base_name[1]))[0] else: all_test_base_name = None # Initialize response and content handlers. This is effectively # duplication of effort but not results. This allows for # backwards compatibility for existing callers. response_handlers = response_handlers or [] content_handlers = content_handlers or [] handler_objects = [] for handler in (content_handlers + response_handlers + handlers.RESPONSE_HANDLERS): handler_objects.append(handler()) top_suite = suite.TestSuite() for test_file in glob.iglob('%s/*.yaml' % path): if '_' in os.path.basename(test_file): warnings.warn(exception.GabbiSyntaxWarning( "'_' in test filename %s. This can break suite grouping." % test_file)) if intercept: host = str(uuid.uuid4()) suite_dict = utils.load_yaml(yaml_file=test_file, safe=safe_yaml) test_base_name = os.path.splitext(os.path.basename(test_file))[0] if all_test_base_name: test_base_name = '%s_%s' % (all_test_base_name, test_base_name) if require_ssl: if 'defaults' in suite_dict: suite_dict['defaults']['ssl'] = True else: suite_dict['defaults'] = {'ssl': True} if any((verbose == opt for opt in [True, 'all', 'headers', 'body'])): if 'defaults' in suite_dict: suite_dict['defaults']['verbose'] = verbose else: suite_dict['defaults'] = {'verbose': verbose} if not use_prior_test: if 'defaults' in suite_dict: suite_dict['defaults']['use_prior_test'] = use_prior_test else: suite_dict['defaults'] = {'use_prior_test': use_prior_test} file_suite = suitemaker.test_suite_from_dict( loader, test_base_name, suite_dict, path, host, port, fixture_module, intercept, prefix=prefix, test_loader_name=test_loader_name, handlers=handler_objects, inner_fixtures=inner_fixtures) top_suite.addTest(file_suite) return top_suite
def build_tests(path, loader, host=None, port=8001, intercept=None, test_loader_name=None, fixture_module=None, response_handlers=None, content_handlers=None, prefix='', require_ssl=False, cert_validate=True, url=None, inner_fixtures=None, verbose=False, use_prior_test=True, safe_yaml=True): """Read YAML files from a directory to create tests. Each YAML file represents a list of HTTP requests. :param path: The directory where yaml files are located. :param loader: The TestLoader. :param host: The host to test against. Do not use with ``intercept``. :param port: The port to test against. Used with ``host``. :param intercept: WSGI app factory for wsgi-intercept. :param test_loader_name: Base name for test classes. Use this to align the naming of the tests with other tests in a system. :param fixture_module: Python module containing fixture classes. :param response_handers: :class:`~gabbi.handlers.ResponseHandler` classes. :type response_handlers: List of ResponseHandler classes. :param content_handlers: ContentHandler classes. :type content_handlers: List of ContentHandler classes. :param prefix: A URL prefix for all URLs that are not fully qualified. :param url: A full URL to test against. Replaces host, port and prefix. :param require_ssl: If ``True``, make all tests default to using SSL. :param inner_fixtures: A list of ``Fixtures`` to use with each individual test request. :type inner_fixtures: List of classes with setUp and cleanUp methods to be used as fixtures. :param verbose: If ``True`` or ``'all'``, make tests verbose by default ``'headers'`` and ``'body'`` are also accepted. :param use_prior_test: If ``True``, uses prior test to create ordered sequence of tests :param safe_yaml: If ``True``, recognizes only standard YAML tags and not Python object :param cert_validate: If ``False`` ssl server certificate will be ignored, further it will not be validated if provided (set cert_reqs=CERT_NONE to the Http object) :rtype: TestSuite containing multiple TestSuites (one for each YAML file). """ # If url is being used, reset host, port and prefix. if url: host, port, prefix, force_ssl = utils.host_info_from_target(url) if force_ssl and not require_ssl: require_ssl = force_ssl # Exit immediately if we have no host to access, either via a real host # or an intercept. if not ((host is not None) ^ bool(intercept)): raise AssertionError( 'must specify exactly one of host or url, or intercept') # If the client has not provided a name to use as our base, # create one so that tests are effectively namespaced. if test_loader_name is None: all_test_base_name = inspect.stack()[1] all_test_base_name = os.path.splitext( os.path.basename(all_test_base_name[1]))[0] else: all_test_base_name = None # Initialize response and content handlers. This is effectively # duplication of effort but not results. This allows for # backwards compatibility for existing callers. response_handlers = response_handlers or [] content_handlers = content_handlers or [] handler_objects = [] for handler in (content_handlers + response_handlers + handlers.RESPONSE_HANDLERS): handler_objects.append(handler()) top_suite = suite.TestSuite() for test_file in glob.iglob('%s/*.yaml' % path): if '_' in os.path.basename(test_file): warnings.warn( exception.GabbiSyntaxWarning( "'_' in test filename %s. This can break suite grouping." % test_file)) if intercept: host = str(uuid.uuid4()) suite_dict = utils.load_yaml(yaml_file=test_file, safe=safe_yaml) test_base_name = os.path.splitext(os.path.basename(test_file))[0] if all_test_base_name: test_base_name = '%s_%s' % (all_test_base_name, test_base_name) if require_ssl: if 'defaults' in suite_dict: suite_dict['defaults']['ssl'] = True else: suite_dict['defaults'] = {'ssl': True} if any((verbose == opt for opt in [True, 'all', 'headers', 'body'])): if 'defaults' in suite_dict: suite_dict['defaults']['verbose'] = verbose else: suite_dict['defaults'] = {'verbose': verbose} if not cert_validate: if 'defaults' in suite_dict: suite_dict['defaults']['cert_validate'] = False else: suite_dict['defaults'] = {'cert_validate': False} if not use_prior_test: if 'defaults' in suite_dict: suite_dict['defaults']['use_prior_test'] = use_prior_test else: suite_dict['defaults'] = {'use_prior_test': use_prior_test} file_suite = suitemaker.test_suite_from_dict( loader, test_base_name, suite_dict, path, host, port, fixture_module, intercept, prefix=prefix, test_loader_name=test_loader_name, handlers=handler_objects, inner_fixtures=inner_fixtures) top_suite.addTest(file_suite) return top_suite