def test_upper_dict_required(self):
test_yaml = [{'name': 'house', 'url': '/'}]
with self.assertRaises(exception.GabbiFormatError) as failure:
suitemaker.test_suite_from_dict(self.loader, 'foo', test_yaml, '.',
'localhost', 80, None, None)
self.assertEqual('malformed test file, invalid format',
str(failure.exception))
def test_method_url_pair_format_error(self):
test_yaml = {'defaults': {'GET': '/foo'}, 'tests': []}
with self.assertRaises(exception.GabbiFormatError) as failure:
suitemaker.test_suite_from_dict(self.loader, 'foo', test_yaml, '.',
'localhost', 80, None, None)
self.assertIn('"METHOD: url" pairs not allowed in defaults',
str(failure.exception))
def test_inner_list_required(self):
test_yaml = {'tests': {'name': 'house', 'url': '/'}}
with self.assertRaises(exception.GabbiFormatError) as failure:
suitemaker.test_suite_from_dict(self.loader, 'foo', test_yaml, '.',
'localhost', 80, None, None)
self.assertIn('test chunk is not a dict at',
str(failure.exception))
def test_url_key_required(self):
test_yaml = {'tests': [{'name': 'missing url'}]}
with self.assertRaises(exception.GabbiFormatError) as failure:
suitemaker.test_suite_from_dict(self.loader, 'foo', test_yaml, '.',
'localhost', 80, None, None)
self.assertEqual('Test url missing in test foo_missing_url.',
str(failure.exception))
def test_unsupported_key_errors(self):
test_yaml = {'tests': [{
'url': '/',
'name': 'simple',
'bad_key': 'wow',
}]}
with self.assertRaises(exception.GabbiFormatError) as failure:
suitemaker.test_suite_from_dict(self.loader, 'foo', test_yaml, '.',
'localhost', 80, None, None)
self.assertIn("Invalid test keys used in test foo_simple:",
str(failure.exception))
def test_method_url_pair_duplication_format_error(self):
test_yaml = {'tests': [{
'GET': '/',
'POST': '/',
'name': 'duplicate methods',
}]}
with self.assertRaises(exception.GabbiFormatError) as failure:
suitemaker.test_suite_from_dict(self.loader, 'foo', test_yaml, '.',
'localhost', 80, None, None)
self.assertIn(
'duplicate method/URL directive in "foo_duplicate_methods"',
str(failure.exception)
)
def test_suite_catches_fixture_fail(self):
"""Verify fixture failure.
When a fixture fails in start_fixture it should fail
the first test in the suite and skip the others.
"""
loader = unittest.defaultTestLoader
result = unittest.TestResult()
test_data = {
'fixtures': ['FakeFixture'],
'tests': [{
'name': 'alpha',
'GET': '/'
}, {
'name': 'beta',
'GET': '/'
}]
}
test_suite = suitemaker.test_suite_from_dict(loader, 'foo', test_data,
'.', 'localhost', 80,
sys.modules[__name__],
None)
test_suite.run(result)
self.assertEqual(2, len(result.skipped))
self.assertEqual(1, len(result.errors))
errored_test, trace = result.errors[0]
self.assertIn('foo_alpha', str(errored_test))
self.assertIn(VALUE_ERROR, trace)
def run_test(test_class_instance, test_dir, filename):
d = utils.load_yaml(yaml_file=os.path.join(test_dir, filename))
test_suite = suitemaker.test_suite_from_dict(
loader=unittest.defaultTestLoader,
test_base_name="gabbi",
suite_dict=d,
test_directory=test_dir,
host='example.com', port=None,
fixture_module=None,
intercept=None,
handlers=runner.initialize_handlers([]),
test_loader_name="tempest")
# NOTE(sileht): We hide stdout/stderr and reraise the failure
# manually, tempest will print it ittest_class.
with open(os.devnull, 'w') as stream:
result = unittest.TextTestRunner(
stream=stream, verbosity=0, failfast=True,
).run(test_suite)
if not result.wasSuccessful():
failures = (result.errors + result.failures +
result.unexpectedSuccesses)
if failures:
test, bt = failures[0]
name = test.test_data.get('name', test.id())
msg = 'From test "%s" :\n%s' % (name, bt)
test_class_instance.fail(msg)
test_class_instance.assertTrue(result.wasSuccessful())
def run_suite(handle, handler_objects, host, port, prefix, force_ssl=False,
failfast=False, data_dir='.', verbosity=False, name='input',
safe_yaml=True, quiet=False):
"""Run the tests from the YAML in handle."""
data = utils.load_yaml(handle, safe=safe_yaml)
if force_ssl:
if 'defaults' in data:
data['defaults']['ssl'] = True
else:
data['defaults'] = {'ssl': True}
if verbosity:
if 'defaults' in data:
data['defaults']['verbose'] = verbosity
else:
data['defaults'] = {'verbose': verbosity}
loader = unittest.defaultTestLoader
test_suite = suitemaker.test_suite_from_dict(
loader, name, data, data_dir, host, port, None, None, prefix=prefix,
handlers=handler_objects, test_loader_name='gabbi-runner')
# The default runner stream is stderr.
stream = sys.stderr
if quiet:
# We want to swallow the output that the runner is
# producing.
stream = open(os.devnull, 'w')
result = ConciseTestRunner(
stream=stream, verbosity=2, failfast=failfast).run(test_suite)
return result.wasSuccessful()
def _do_test(self, filename):
token = self.os_admin.auth_provider.get_token()
url = self.os_admin.auth_provider.base_url({
'service':
CONF.metric.catalog_type,
'endpoint_type':
CONF.metric.endpoint_type
})
parsed_url = urlparse.urlsplit(url)
prefix = parsed_url.path.rstrip('/') # turn it into a prefix
if parsed_url.scheme == 'https':
port = 443
require_ssl = True
else:
port = 80
require_ssl = False
host = parsed_url.hostname
if parsed_url.port:
port = parsed_url.port
os.environ["GNOCCHI_SERVICE_TOKEN"] = token
os.environ["GNOCCHI_AUTHORIZATION"] = "not used"
with open(os.path.join(TEST_DIR, filename)) as f:
suite_dict = utils.load_yaml(f)
suite_dict.setdefault('defaults', {})['ssl'] = require_ssl
test_suite = suitemaker.test_suite_from_dict(
loader=unittest.defaultTestLoader,
test_base_name="gabbi",
suite_dict=suite_dict,
test_directory=TEST_DIR,
host=host,
port=port,
fixture_module=None,
intercept=None,
prefix=prefix,
handlers=runner.initialize_handlers([]),
test_loader_name="tempest")
# NOTE(sileht): We hide stdout/stderr and reraise the failure
# manually, tempest will print it itself.
with open(os.devnull, 'w') as stream:
result = unittest.TextTestRunner(
stream=stream,
verbosity=0,
failfast=True,
).run(test_suite)
if not result.wasSuccessful():
failures = (result.errors + result.failures +
result.unexpectedSuccesses)
if failures:
test, bt = failures[0]
name = test.test_data.get('name', test.id())
msg = 'From test "%s" :\n%s' % (name, bt)
self.fail(msg)
self.assertTrue(result.wasSuccessful())
def test_dict_on_invalid_key(self):
test_yaml = {'tests': [{
'name': '...',
'GET': '/',
'response_html': {
'foo': 'hello',
'bar': 'world',
}
}]}
with self.assertRaises(exception.GabbiFormatError) as failure:
suitemaker.test_suite_from_dict(self.loader, 'foo', test_yaml, '.',
'localhost', 80, None, None)
self.assertIn(
"invalid key in test: 'response_html'",
str(failure.exception)
)
def test_suite_from_yaml(loader, test_base_name, test_yaml, test_directory,
host, port, fixture_module, intercept, prefix=''):
"""Legacy wrapper retained for backwards compatibility."""
with warnings.catch_warnings(): # ensures warnings filter is restored
warnings.simplefilter('default', DeprecationWarning)
warnings.warn('test_suite_from_yaml has been renamed to '
'test_suite_from_dict', DeprecationWarning, stacklevel=2)
return suitemaker.test_suite_from_dict(
loader, test_base_name, test_yaml, test_directory, host, port,
fixture_module, intercept, prefix)
def test_suite_from_yaml(loader, test_base_name, test_yaml, test_directory,
host, port, fixture_module, intercept, prefix=''):
"""Legacy wrapper retained for backwards compatibility."""
with warnings.catch_warnings(): # ensures warnings filter is restored
warnings.simplefilter('default', DeprecationWarning)
warnings.warn('test_suite_from_yaml has been renamed to '
'test_suite_from_dict', DeprecationWarning, stacklevel=2)
return suitemaker.test_suite_from_dict(
loader, test_base_name, test_yaml, test_directory, host, port,
fixture_module, intercept, prefix)
def _do_test(self, filename):
admin_auth = self.os_admin.auth_provider.get_auth()
auth = self.os_primary.auth_provider.get_auth()
networks = self.os_primary.networks_client.list_networks(
**{'router:external': False, 'fields': 'id'})['networks']
os.environ.update({
"ADMIN_TOKEN": admin_auth[0],
"USER_TOKEN": auth[0],
"AODH_GRANULARITY": str(config.CONF.telemetry.alarm_granularity),
"AODH_SERVICE_URL": self._get_endpoint(auth, "alarming_plugin"),
"GNOCCHI_SERVICE_URL": self._get_endpoint(auth, "metric"),
"PANKO_SERVICE_URL": self._get_endpoint(auth, "event"),
"HEAT_SERVICE_URL": self._get_endpoint(auth, "orchestration"),
"NOVA_SERVICE_URL": self._get_endpoint(auth, "compute"),
"GLANCE_SERVICE_URL": self._get_endpoint(auth, "image"),
"GLANCE_IMAGE_NAME": self.glance_image_create(),
"NOVA_FLAVOR_REF": config.CONF.compute.flavor_ref,
"NEUTRON_NETWORK": networks[0].get('id'),
})
with open(os.path.join(TEST_DIR, filename)) as f:
test_suite = suitemaker.test_suite_from_dict(
loader=unittest.defaultTestLoader,
test_base_name="gabbi",
suite_dict=utils.load_yaml(f),
test_directory=TEST_DIR,
host=None, port=None,
fixture_module=None,
intercept=None,
handlers=runner.initialize_handlers([]),
test_loader_name="tempest")
# NOTE(sileht): We hide stdout/stderr and reraise the failure
# manually, tempest will print it itself.
with open(os.devnull, 'w') as stream:
result = unittest.TextTestRunner(
stream=stream, verbosity=0, failfast=True,
).run(test_suite)
if not result.wasSuccessful():
failures = (result.errors + result.failures +
result.unexpectedSuccesses)
if failures:
test, bt = failures[0]
name = test.test_data.get('name', test.id())
msg = 'From test "%s" :\n%s' % (name, bt)
self.fail(msg)
self.assertTrue(result.wasSuccessful())
def test_response_handlers_same_test_key_yaml_first(self):
test_yaml = {'tests': [{
'name': '...',
'GET': '/',
'response_json_paths': {
'foo': 'hello',
'bar': 'world',
}
}]}
ydlj_handler_object = ydlj_handler.YAMLDiskLoadingJSONHandler()
handler_objects = [ydlj_handler_object]
for handler in handlers.RESPONSE_HANDLERS:
handler_objects.append(handler())
file_suite = suitemaker.test_suite_from_dict(
self.loader, 'foo', test_yaml, '.', 'localhost', 80, None, None,
handlers=handler_objects)
response_handlers = file_suite._tests[0].response_handlers
self.assertIn(ydlj_handler_object, response_handlers)
def run_suite(handle,
handler_objects,
host,
port,
prefix,
force_ssl=False,
failfast=False,
data_dir='.',
verbosity=False,
name='input',
safe_yaml=True):
"""Run the tests from the YAML in handle."""
data = utils.load_yaml(handle, safe=safe_yaml)
if force_ssl:
if 'defaults' in data:
data['defaults']['ssl'] = True
else:
data['defaults'] = {'ssl': True}
if verbosity:
if 'defaults' in data:
data['defaults']['verbose'] = verbosity
else:
data['defaults'] = {'verbose': verbosity}
loader = unittest.defaultTestLoader
test_suite = suitemaker.test_suite_from_dict(
loader,
name,
data,
data_dir,
host,
port,
None,
None,
prefix=prefix,
handlers=handler_objects,
test_loader_name='gabbi-runner')
result = ConciseTestRunner(verbosity=2, failfast=failfast).run(test_suite)
return result.wasSuccessful()
def run_gabbi(self, gabbi_declaration):
# Take only the host name and port from the live server:
_, authority = self.live_server_url.split('://')
host, port = authority.split(':')
# Use Gabbi to create the test suite from our declaration:
suite = test_suite_from_dict(
loader=defaultTestLoader,
test_base_name=self.id(),
suite_dict=gabbi_declaration,
test_directory='.',
host=host,
port=port,
fixture_module=None,
intercept=None,
handlers=[
handler()
for handler in RESPONSE_HANDLERS
],
)
# Run the test. We store the the output into a custom stream so that
# Hypothesis can display only the simple case test result on failure
# rather than every failing case:
s = StringIO()
result = ConciseTestRunner(
stream=s,
verbosity=0,
).run(suite)
# If we weren't successful we need to fail the test case with the error
# string from Gabbi:
if not result.wasSuccessful():
self.fail(
s.getvalue(),
)
def test_suite_catches_fixture_fail(self):
"""Verify fixture failure.
When a fixture fails in start_fixture it should fail
the first test in the suite and skip the others.
"""
loader = unittest.defaultTestLoader
result = unittest.TestResult()
test_data = {'fixtures': ['FakeFixture'],
'tests': [{'name': 'alpha', 'GET': '/'},
{'name': 'beta', 'GET': '/'}]}
test_suite = suitemaker.test_suite_from_dict(
loader, 'foo', test_data, '.', 'localhost',
80, sys.modules[__name__], None)
test_suite.run(result)
self.assertEqual(2, len(result.skipped))
self.assertEqual(1, len(result.errors))
errored_test, trace = result.errors[0]
self.assertIn('foo_alpha', str(errored_test))
self.assertIn(VALUE_ERROR, trace)
def run_suite(handle, handler_objects, host, port, prefix, force_ssl=False,
failfast=False, data_dir='.', verbosity=False, name='input',
safe_yaml=True):
"""Run the tests from the YAML in handle."""
data = utils.load_yaml(handle, safe=safe_yaml)
if force_ssl:
if 'defaults' in data:
data['defaults']['ssl'] = True
else:
data['defaults'] = {'ssl': True}
if verbosity:
if 'defaults' in data:
data['defaults']['verbose'] = verbosity
else:
data['defaults'] = {'verbose': verbosity}
loader = unittest.defaultTestLoader
test_suite = suitemaker.test_suite_from_dict(
loader, name, data, data_dir, host, port, None, None, prefix=prefix,
handlers=handler_objects, test_loader_name='gabbi-runner')
result = ConciseTestRunner(
verbosity=2, failfast=failfast).run(test_suite)
return result.wasSuccessful()
def build_tests(path, loader, host=None, port=8001, intercept=None,
test_loader_name=None, fixture_module=None,
response_handlers=None, content_handlers=None,
prefix='', require_ssl=False, url=None,
inner_fixtures=None, verbose=False,
use_prior_test=True, safe_yaml=True):
"""Read YAML files from a directory to create tests.
Each YAML file represents a list of HTTP requests.
:param path: The directory where yaml files are located.
:param loader: The TestLoader.
:param host: The host to test against. Do not use with ``intercept``.
:param port: The port to test against. Used with ``host``.
:param intercept: WSGI app factory for wsgi-intercept.
:param test_loader_name: Base name for test classes. Use this to align the
naming of the tests with other tests in a system.
:param fixture_module: Python module containing fixture classes.
:param response_handers: :class:`~gabbi.handlers.ResponseHandler` classes.
:type response_handlers: List of ResponseHandler classes.
:param content_handlers: ContentHandler classes.
:type content_handlers: List of ContentHandler classes.
:param prefix: A URL prefix for all URLs that are not fully qualified.
:param url: A full URL to test against. Replaces host, port and prefix.
:param require_ssl: If ``True``, make all tests default to using SSL.
:param inner_fixtures: A list of ``Fixtures`` to use with each
individual test request.
:param verbose: If ``True`` or ``'all'``, make tests verbose by default
``'headers'`` and ``'body'`` are also accepted.
:param use_prior_test: If ``True``, uses prior test to create ordered
sequence of tests
:param safe_yaml: If ``True``, recognizes only standard YAML tags and not
Python object
:type inner_fixtures: List of fixtures.Fixture classes.
:rtype: TestSuite containing multiple TestSuites (one for each YAML file).
"""
# If url is being used, reset host, port and prefix.
if url:
host, port, prefix, force_ssl = utils.host_info_from_target(url)
if force_ssl and not require_ssl:
require_ssl = force_ssl
# Exit immediately if we have no host to access, either via a real host
# or an intercept.
if not ((host is not None) ^ bool(intercept)):
raise AssertionError(
'must specify exactly one of host or url, or intercept')
# If the client has not provided a name to use as our base,
# create one so that tests are effectively namespaced.
if test_loader_name is None:
all_test_base_name = inspect.stack()[1]
all_test_base_name = os.path.splitext(
os.path.basename(all_test_base_name[1]))[0]
else:
all_test_base_name = None
# Initialize response and content handlers. This is effectively
# duplication of effort but not results. This allows for
# backwards compatibility for existing callers.
response_handlers = response_handlers or []
content_handlers = content_handlers or []
handler_objects = []
for handler in (content_handlers + response_handlers +
handlers.RESPONSE_HANDLERS):
handler_objects.append(handler())
top_suite = suite.TestSuite()
for test_file in glob.iglob('%s/*.yaml' % path):
if '_' in os.path.basename(test_file):
warnings.warn(exception.GabbiSyntaxWarning(
"'_' in test filename %s. This can break suite grouping."
% test_file))
if intercept:
host = str(uuid.uuid4())
suite_dict = utils.load_yaml(yaml_file=test_file,
safe=safe_yaml)
test_base_name = os.path.splitext(os.path.basename(test_file))[0]
if all_test_base_name:
test_base_name = '%s_%s' % (all_test_base_name, test_base_name)
if require_ssl:
if 'defaults' in suite_dict:
suite_dict['defaults']['ssl'] = True
else:
suite_dict['defaults'] = {'ssl': True}
if any((verbose == opt for opt in [True, 'all', 'headers', 'body'])):
if 'defaults' in suite_dict:
suite_dict['defaults']['verbose'] = verbose
else:
suite_dict['defaults'] = {'verbose': verbose}
if not use_prior_test:
if 'defaults' in suite_dict:
suite_dict['defaults']['use_prior_test'] = use_prior_test
else:
suite_dict['defaults'] = {'use_prior_test': use_prior_test}
file_suite = suitemaker.test_suite_from_dict(
loader, test_base_name, suite_dict, path, host, port,
fixture_module, intercept, prefix=prefix,
test_loader_name=test_loader_name, handlers=handler_objects,
inner_fixtures=inner_fixtures)
top_suite.addTest(file_suite)
return top_suite
def build_tests(path,
loader,
host=None,
port=8001,
intercept=None,
test_loader_name=None,
fixture_module=None,
response_handlers=None,
content_handlers=None,
prefix='',
require_ssl=False,
cert_validate=True,
url=None,
inner_fixtures=None,
verbose=False,
use_prior_test=True,
safe_yaml=True):
"""Read YAML files from a directory to create tests.
Each YAML file represents a list of HTTP requests.
:param path: The directory where yaml files are located.
:param loader: The TestLoader.
:param host: The host to test against. Do not use with ``intercept``.
:param port: The port to test against. Used with ``host``.
:param intercept: WSGI app factory for wsgi-intercept.
:param test_loader_name: Base name for test classes. Use this to align the
naming of the tests with other tests in a system.
:param fixture_module: Python module containing fixture classes.
:param response_handers: :class:`~gabbi.handlers.ResponseHandler` classes.
:type response_handlers: List of ResponseHandler classes.
:param content_handlers: ContentHandler classes.
:type content_handlers: List of ContentHandler classes.
:param prefix: A URL prefix for all URLs that are not fully qualified.
:param url: A full URL to test against. Replaces host, port and prefix.
:param require_ssl: If ``True``, make all tests default to using SSL.
:param inner_fixtures: A list of ``Fixtures`` to use with each
individual test request.
:type inner_fixtures: List of classes with setUp and cleanUp methods to
be used as fixtures.
:param verbose: If ``True`` or ``'all'``, make tests verbose by default
``'headers'`` and ``'body'`` are also accepted.
:param use_prior_test: If ``True``, uses prior test to create ordered
sequence of tests
:param safe_yaml: If ``True``, recognizes only standard YAML tags and not
Python object
:param cert_validate: If ``False`` ssl server certificate will be ignored,
further it will not be validated if provided
(set cert_reqs=CERT_NONE to the Http object)
:rtype: TestSuite containing multiple TestSuites (one for each YAML file).
"""
# If url is being used, reset host, port and prefix.
if url:
host, port, prefix, force_ssl = utils.host_info_from_target(url)
if force_ssl and not require_ssl:
require_ssl = force_ssl
# Exit immediately if we have no host to access, either via a real host
# or an intercept.
if not ((host is not None) ^ bool(intercept)):
raise AssertionError(
'must specify exactly one of host or url, or intercept')
# If the client has not provided a name to use as our base,
# create one so that tests are effectively namespaced.
if test_loader_name is None:
all_test_base_name = inspect.stack()[1]
all_test_base_name = os.path.splitext(
os.path.basename(all_test_base_name[1]))[0]
else:
all_test_base_name = None
# Initialize response and content handlers. This is effectively
# duplication of effort but not results. This allows for
# backwards compatibility for existing callers.
response_handlers = response_handlers or []
content_handlers = content_handlers or []
handler_objects = []
for handler in (content_handlers + response_handlers +
handlers.RESPONSE_HANDLERS):
handler_objects.append(handler())
top_suite = suite.TestSuite()
for test_file in glob.iglob('%s/*.yaml' % path):
if '_' in os.path.basename(test_file):
warnings.warn(
exception.GabbiSyntaxWarning(
"'_' in test filename %s. This can break suite grouping." %
test_file))
if intercept:
host = str(uuid.uuid4())
suite_dict = utils.load_yaml(yaml_file=test_file, safe=safe_yaml)
test_base_name = os.path.splitext(os.path.basename(test_file))[0]
if all_test_base_name:
test_base_name = '%s_%s' % (all_test_base_name, test_base_name)
if require_ssl:
if 'defaults' in suite_dict:
suite_dict['defaults']['ssl'] = True
else:
suite_dict['defaults'] = {'ssl': True}
if any((verbose == opt for opt in [True, 'all', 'headers', 'body'])):
if 'defaults' in suite_dict:
suite_dict['defaults']['verbose'] = verbose
else:
suite_dict['defaults'] = {'verbose': verbose}
if not cert_validate:
if 'defaults' in suite_dict:
suite_dict['defaults']['cert_validate'] = False
else:
suite_dict['defaults'] = {'cert_validate': False}
if not use_prior_test:
if 'defaults' in suite_dict:
suite_dict['defaults']['use_prior_test'] = use_prior_test
else:
suite_dict['defaults'] = {'use_prior_test': use_prior_test}
file_suite = suitemaker.test_suite_from_dict(
loader,
test_base_name,
suite_dict,
path,
host,
port,
fixture_module,
intercept,
prefix=prefix,
test_loader_name=test_loader_name,
handlers=handler_objects,
inner_fixtures=inner_fixtures)
top_suite.addTest(file_suite)
return top_suite
