def __init__(self, context):
super(UserGenerator, self).__init__(context)
creds = self.env["platforms"]["openstack"]
if creds.get("admin"):
admin_cred = copy.deepcopy(creds["admin"])
api_info = copy.deepcopy(creds.get("api_info", {}))
if "api_info" in admin_cred:
api_info.update(creds["admin"]["api_info"])
admin_cred["api_info"] = api_info
context["admin"] = {
"credential": credential.OpenStackCredential(**admin_cred)
}
if creds["users"] and not (set(self.config) - {"user_choice_method"}):
self.existing_users = creds["users"]
else:
self.existing_users = []
self.credential = context["admin"]["credential"]
project_domain = (self.credential["project_domain_name"]
or cfg.CONF.openstack.project_domain)
user_domain = (self.credential["user_domain_name"]
or cfg.CONF.openstack.user_domain)
self.DEFAULT_FOR_NEW_USERS["project_domain"] = project_domain
self.DEFAULT_FOR_NEW_USERS["user_domain"] = user_domain
with self.config.unlocked():
for key, value in self.DEFAULT_FOR_NEW_USERS.items():
self.config.setdefault(key, value)
def test_users_contains_default_endpoint_type(self, mock_identity):
credential = oscredential.OpenStackCredential("foo_url", "foo",
"foo_pass")
config = {
"config": {
"users": {
"tenants": 1,
"users_per_tenant": 2,
"resource_management_workers": 1
}
},
"env": {
"platforms": {
"openstack": {
"admin": credential,
"users": []
}
}
},
"task": {
"uuid": "task_id",
"deployment_uuid": "deployment_id"
}
}
user_generator = users.UserGenerator(config)
users_ = user_generator._create_users(2)
for user in users_:
self.assertEqual("public", user["credential"].endpoint_type)
def consume(cache, args):
username, password, project_dom, user_dom, tenant_id = args
if "client" not in cache:
clients = osclients.Clients(self.credential)
cache["client"] = identity.Identity(
clients, name_generator=self.generate_random_name)
client = cache["client"]
user = client.create_user(username,
password=password,
project_id=tenant_id,
domain_name=user_dom,
default_role=default_role)
user_credential = credential.OpenStackCredential(
auth_url=self.credential["auth_url"],
username=user.name,
password=password,
tenant_name=self.context["tenants"][tenant_id]["name"],
permission=consts.EndpointPermission.USER,
project_domain_name=project_dom,
user_domain_name=user_dom,
endpoint_type=self.credential["endpoint_type"],
https_insecure=self.credential["https_insecure"],
https_cacert=self.credential["https_cacert"],
region_name=self.credential["region_name"],
profiler_hmac_key=self.credential["profiler_hmac_key"],
profiler_conn_str=self.credential["profiler_conn_str"],
api_info=self.credential["api_info"])
users.append({
"id": user.id,
"credential": user_credential,
"tenant_id": tenant_id
})
def use_existing_users(self):
LOG.debug("Using existing users for OpenStack platform.")
api_info = copy.deepcopy(self.env["platforms"]["openstack"].get(
"api_info", {}))
for user_credential in self.existing_users:
user_credential = copy.deepcopy(user_credential)
if "api_info" in user_credential:
api_info.update(user_credential["api_info"])
user_credential["api_info"] = api_info
user_credential = credential.OpenStackCredential(**user_credential)
user_clients = osclients.Clients(user_credential)
user_id = user_clients.keystone.auth_ref.user_id
tenant_id = user_clients.keystone.auth_ref.project_id
if tenant_id not in self.context["tenants"]:
self.context["tenants"][tenant_id] = {
"id": tenant_id,
"name": user_credential.tenant_name
}
self.context["users"].append({
"credential": user_credential,
"id": user_id,
"tenant_id": tenant_id
})
def _get_context(self):
context = test.get_test_context()
cred = credential.OpenStackCredential(auth_url="url",
username="******",
password="******")
context.update({
"admin": {
"id": "fake_user_id",
"credential": cred
},
"user": {
"id": "fake_user_id",
"credential": cred
},
"tenant": {
"id": "fake",
"name": "fake",
"volumes": [{
"id": "uuid",
"size": 1
}],
"servers": [1]
}
})
return context
def test_keystone_get_session(self, auth_url):
credential = oscredential.OpenStackCredential(auth_url, "user", "pass",
"tenant")
self.set_up_keystone_mocks()
keystone = osclients.Keystone(credential, {})
version_data = mock.Mock(return_value=[{"version": (1, 0)}])
self.ksa_auth.discover.Discover.return_value = (mock.Mock(
version_data=version_data))
self.assertEqual(
(self.ksa_session.Session.return_value, self.ksa_identity_plugin),
keystone.get_session())
if auth_url.endswith("v2.0"):
self.ksa_password.assert_called_once_with(auth_url=auth_url,
password="******",
tenant_name="tenant",
username="******")
else:
self.ksa_password.assert_called_once_with(auth_url=auth_url,
password="******",
tenant_name="tenant",
username="******",
domain_name=None,
project_domain_name=None,
user_domain_name=None)
self.assertEqual([
mock.call(timeout=180.0, verify=True, cert=None),
mock.call(auth=self.ksa_identity_plugin,
timeout=180.0,
verify=True,
cert=None)
], self.ksa_session.Session.call_args_list)
def test_users_and_tenants_in_context(self, mock_identity):
identity_service = mock_identity.Identity.return_value
credential = oscredential.OpenStackCredential("foo_url",
"foo",
"foo_pass",
https_insecure=True,
https_cacert="cacert")
tmp_context = dict(self.context)
tmp_context["config"]["users"] = {
"tenants": 1,
"users_per_tenant": 2,
"resource_management_workers": 1
}
tmp_context["env"]["platforms"]["openstack"]["admin"] = credential
credential_dict = credential.to_dict()
user_list = [
mock.MagicMock(id="id_%d" % i) for i in range(self.users_num)
]
identity_service.create_user.side_effect = user_list
with users.UserGenerator(tmp_context) as ctx:
ctx.generate_random_name = mock.Mock()
ctx.setup()
create_tenant_calls = []
for i, t in enumerate(ctx.context["tenants"]):
create_tenant_calls.append(
mock.call(ctx.generate_random_name.return_value,
ctx.config["project_domain"]))
for user in ctx.context["users"]:
self.assertEqual(set(["id", "credential", "tenant_id"]),
set(user.keys()))
user_credential_dict = user["credential"].to_dict()
excluded_keys = [
"auth_url", "username", "password", "tenant_name",
"region_name", "project_domain_name", "user_domain_name",
"permission"
]
for key in (set(credential_dict.keys()) - set(excluded_keys)):
self.assertEqual(credential_dict[key],
user_credential_dict[key],
"The key '%s' differs." % key)
tenants_ids = []
for t in ctx.context["tenants"].keys():
tenants_ids.append(t)
for (user, tenant_id, orig_user) in zip(ctx.context["users"],
tenants_ids, user_list):
self.assertEqual(orig_user.id, user["id"])
self.assertEqual(tenant_id, user["tenant_id"])
def fetch_parent_env_and_admin_creds(env_name):
"""Fetch parent environment spec and openstack admin creds from it."""
env_data = env_mgr.EnvManager.get(env_name).data
openstack_platform = env_data["platforms"]["openstack"]
admin_creds = credential.OpenStackCredential(
permission=consts.EndpointPermission.ADMIN,
**openstack_platform["platform_data"]["admin"])
return env_data["spec"], admin_creds
def __init__(self, env):
openstack_platform = env.data["platforms"]["openstack"]
self.credential = credential.OpenStackCredential(
permission=consts.EndpointPermission.ADMIN,
**openstack_platform["platform_data"]["admin"])
if not self.credential:
raise exceptions.ValidationError(
f"Failed to configure 'tempest' for '{env} since "
"admin credentials for OpenStack platform is missed there."
)
self.clients = self.credential.clients()
self.available_services = self.clients.services().values()
self.conf = configparser.ConfigParser(allow_no_value=True)
def setUp(self):
super(NeutronServiceTestCase, self).setUp()
self.clients = mock.MagicMock(
credential=credential.OpenStackCredential(
auth_url="example.com", username="******", password="******"))
self.nc = self.clients.neutron.return_value
self.atomic_inst = []
self.name_generator_count = 0
def name_generator():
self.name_generator_count += 1
return f"s-{self.name_generator_count}"
self.neutron = neutron.NeutronService(clients=self.clients,
name_generator=name_generator,
atomic_inst=self.atomic_inst)
def setUp(self):
super(OSClientsTestCase, self).setUp()
self.credential = oscredential.OpenStackCredential(
"http://auth_url/v2.0", "user", "pass", "tenant")
self.clients = osclients.Clients(self.credential, {})
self.fake_keystone = fakes.FakeKeystoneClient()
keystone_patcher = mock.patch("%s.Keystone.create_client" % PATH,
return_value=self.fake_keystone)
self.mock_create_keystone_client = keystone_patcher.start()
self.auth_ref_patcher = mock.patch("%s.Keystone.auth_ref" % PATH)
self.auth_ref = self.auth_ref_patcher.start()
self.service_catalog = self.auth_ref.service_catalog
self.service_catalog.url_for = mock.MagicMock()
def __init__(self, ctx):
super(TempestContext, self).__init__(ctx)
openstack_platform = self.verifier.env.data["platforms"]["openstack"]
admin_creds = credential.OpenStackCredential(
permission=consts.EndpointPermission.ADMIN,
**openstack_platform["platform_data"]["admin"])
self.clients = admin_creds.clients()
self.available_services = self.clients.services().values()
self.conf = configparser.ConfigParser(allow_no_value=True)
self.conf_path = self.verifier.manager.configfile
self.data_dir = self.verifier.manager.home_dir
self.image_name = "tempest-image"
self._created_roles = []
self._created_images = []
self._created_flavors = []
self._created_networks = []
def create_from_env(cls):
from rally_openstack.common import credential
from rally_openstack.environment.platforms import existing
spec = existing.OpenStack.create_spec_from_sys_environ(os.environ)
if not spec["available"]:
raise ValueError(spec["message"]) from None
creds = spec["spec"]
oscred = credential.OpenStackCredential(
auth_url=creds["auth_url"],
username=creds["admin"]["username"],
password=creds["admin"]["password"],
tenant_name=creds["admin"].get(
"tenant_name", creds["admin"].get("project_name")),
endpoint_type=creds["endpoint_type"],
user_domain_name=creds["admin"].get("user_domain_name"),
project_domain_name=creds["admin"].get("project_domain_name"),
region_name=creds["region_name"],
https_cacert=creds["https_cacert"],
https_insecure=creds["https_insecure"])
return cls(oscred)
def setUp(self):
super(SaharaOutputDataSourcesTestCase, self).setUp()
fake_dict = oscredential.OpenStackCredential(
"http://fake.example.org:5000/v2.0/", "user", "passwd")
self.tenants_num = 2
self.users_per_tenant = 2
self.users = self.tenants_num * self.users_per_tenant
self.task = mock.MagicMock()
self.tenants = {}
self.users_key = []
for i in range(self.tenants_num):
self.tenants[str(i)] = {"id": str(i), "name": str(i),
"sahara": {"image": "42"}}
for j in range(self.users_per_tenant):
self.users_key.append({"id": "%s_%s" % (str(i), str(j)),
"tenant_id": str(i),
"credential": fake_dict})
self.user_key = [{"id": i, "tenant_id": j, "credential": "credential"}
for j in range(self.tenants_num)
for i in range(self.users_per_tenant)]
self.context.update({
"config": {
"users": {
"tenants": self.tenants_num,
"users_per_tenant": self.users_per_tenant,
},
"sahara_output_data_sources": {
"output_type": "hdfs",
"output_url_prefix": "hdfs://test_host/",
},
},
"admin": {"credential": mock.MagicMock()},
"task": mock.MagicMock(),
"users": self.users_key,
"tenants": self.tenants
})
def test__get_endpoint(self, mock_keystone_service_catalog, endpoint_type,
service_type, region_name):
credential = oscredential.OpenStackCredential(
"http://auth_url/v2.0",
"user",
"pass",
endpoint_type=endpoint_type,
region_name=region_name)
mock_choose_service_type = mock.MagicMock()
osclient = osclients.OSClient(credential, mock.MagicMock())
osclient.choose_service_type = mock_choose_service_type
mock_url_for = mock_keystone_service_catalog.url_for
self.assertEqual(mock_url_for.return_value,
osclient._get_endpoint(service_type))
call_args = {
"service_type": mock_choose_service_type.return_value,
"region_name": region_name
}
if endpoint_type:
call_args["interface"] = endpoint_type
mock_url_for.assert_called_once_with(**call_args)
mock_choose_service_type.assert_called_once_with(service_type)
def __init__(self, **kwargs):
self.clients = credential.OpenStackCredential(**kwargs).clients()
def __init__(self, credential, cache_obj=None):
self.credential = credential
if not isinstance(self.credential, oscred.OpenStackCredential):
self.credential = oscred.OpenStackCredential(**self.credential)
self.cache = cache_obj if cache_obj is not None else {}
def run(self):
if not os.path.exists("%s/extra" % self.BASE_DIR):
os.makedirs("%s/extra" % self.BASE_DIR)
# ensure that environment exit and check it
env = env_mgr.EnvManager.get(self.ENV_NAME)
for p_name, status in env.check_health().items():
if not status["available"]:
self.result["status"] = Status.ERROR
return
try:
subprocess.check_call(
["rally", "env", "use", "--env", self.ENV_NAME],
stdout=sys.stdout)
except subprocess.CalledProcessError:
self.result["status"] = Status.ERROR
return
openstack_platform = env.data["platforms"]["openstack"]
admin_creds = credential.OpenStackCredential(
permission=consts.EndpointPermission.ADMIN,
**openstack_platform["platform_data"]["admin"])
clients = admin_creds.clients()
if self.args.ctx_create_resources:
# If the 'ctx-create-resources' arg is provided, delete images and
# flavors, and also create a shared network to make Tempest context
# create needed resources.
LOG.info("The 'ctx-create-resources' arg is provided. Deleting "
"images and flavors, and also creating a shared network "
"to make Tempest context create needed resources.")
LOG.info("Deleting images.")
for image in clients.glance().images.list():
clients.glance().images.delete(image.id)
LOG.info("Deleting flavors.")
for flavor in clients.nova().flavors.list():
clients.nova().flavors.delete(flavor.id)
LOG.info("Creating a shared network.")
net_body = {
"network": {
"name": "shared-net-%s" % str(uuid.uuid4()),
"tenant_id": clients.keystone.auth_ref.project_id,
"shared": True
}
}
clients.neutron().create_network(net_body)
else:
# Otherwise, just in case create only flavors with the following
# properties: RAM = 64MB and 128MB, VCPUs = 1, disk = 0GB to make
# Tempest context discover them.
LOG.info("The 'ctx-create-resources' arg is not provided. "
"Creating flavors to make Tempest context discover them.")
for flv_ram in [64, 128]:
params = {
"name": "flavor-%s" % str(uuid.uuid4()),
"ram": flv_ram,
"vcpus": 1,
"disk": 0
}
LOG.info("Creating flavor '%s' with the following properties: "
"RAM = %dMB, VCPUs = 1, disk = 0GB" %
(params["name"], flv_ram))
clients.nova().flavors.create(**params)
def setUp(self):
super(TestCreateKeystoneClient, self).setUp()
self.credential = oscredential.OpenStackCredential(
"http://auth_url/v2.0", "user", "pass", "tenant")
def test__remove_url_version(self, original, cropped):
credential = oscredential.OpenStackCredential(original, "user", "pass",
"tenant")
keystone = osclients.Keystone(credential, {})
self.assertEqual(cropped, keystone._remove_url_version())
def test_task_samples_are_valid(self):
from rally_openstack.task.contexts.keystone import users
rally = utils.Rally(force_new_db=True)
# let's use pre-created users to make TestTaskSamples quicker
rapi = api.API(config_file=rally.config_filename)
deployment = rapi.deployment._get("MAIN")
openstack_platform = deployment.env_obj.data["platforms"]["openstack"]
admin_creds = credential.OpenStackCredential(
permission=consts.EndpointPermission.ADMIN,
**openstack_platform["platform_data"]["admin"])
ctx = {
"env": {
"platforms": {
"openstack": {
"admin": admin_creds.to_dict(),
"users": []
}
}
},
"task": {
"uuid": self.__class__.__name__,
"deployment_uuid": deployment["uuid"]
}
}
user_ctx = users.UserGenerator(ctx)
user_ctx.setup()
self.addCleanup(user_ctx.cleanup)
os_creds = deployment["config"]["openstack"]
user = copy.copy(os_creds["admin"])
user["username"] = ctx["users"][0]["credential"].username
user["password"] = ctx["users"][0]["credential"].password
if "project_name" in os_creds["admin"]:
# it is Keystone
user["project_name"] = ctx["users"][0]["credential"].tenant_name
else:
user["tenant_name"] = ctx["users"][0]["credential"].tenant_name
os_creds["users"] = [user]
rally("deployment destroy MAIN", write_report=False)
deployment_cfg = os.path.join(rally.tmp_dir, "new_deployment.json")
with open(deployment_cfg, "w") as f:
f.write(json.dumps({"openstack": os_creds}))
rally("deployment create --name MAIN --filename %s" % deployment_cfg,
write_report=False)
# store all failures and print them at once
failed_samples = {}
def publisher(queue):
"""List all samples and render task configs"""
samples_path = os.path.join(
os.path.dirname(rally_openstack_module.__file__), os.pardir,
"samples", "tasks")
for dirname, dirnames, filenames in os.walk(samples_path):
# NOTE(rvasilets): Skip by suggest of boris-42 because in
# future we don't what to maintain this dir
if dirname.find("tempest-do-not-run-against-production") != -1:
continue
for filename in filenames:
full_path = os.path.join(dirname, filename)
# NOTE(hughsaunders): Skip non config files
# (bug https://bugs.launchpad.net/rally/+bug/1314369)
if os.path.splitext(filename)[1] != ".json":
continue
with open(full_path) as task_file:
input_task = task_file.read()
rendered_task = rapi.task.render_template(
task_template=input_task)
queue.append((full_path, rendered_task))
def consumer(_cache, sample):
"""Validate one sample"""
full_path, rendered_task = sample
task_config = yaml.safe_load(rendered_task)
try:
rapi.task.validate(deployment="MAIN", config=task_config)
except Exception as e:
if not self._skip(str(e)):
failed_samples[full_path] = traceback.format_exc()
broker.run(publisher, consumer, self.NUMBER_OF_THREADS)
if failed_samples:
self.fail("Validation failed on the one or several samples. "
"See details below:\n%s" % "".join([
"\n======\n%s\n\n%s\n" % (k, v)
for k, v in failed_samples.items()
]))
