def _remove_default_security_group(self):
"""Delete default security group for tenants."""
clients = osclients.Clients(self.credential)
if consts.Service.NEUTRON not in clients.services().values():
return
use_sg, msg = network.wrap(clients,
self).supports_extension("security-group")
if not use_sg:
LOG.debug("Security group context is disabled: %s" % msg)
return
for user, tenant_id in self._iterate_per_tenants():
with logging.ExceptionLogger(
LOG, "Unable to delete default security group"):
uclients = osclients.Clients(user["credential"])
security_groups = uclients.neutron()\
.list_security_groups(tenant_id=tenant_id)
default = [
sg for sg in security_groups["security_groups"]
if sg["name"] == "default"
]
if default:
clients.neutron().delete_security_group(default[0]["id"])
def __init__(self, context=None, cache=None):
super(OpenStackResourceType, self).__init__(context, cache)
self._clients = None
if self._context.get("admin"):
self._clients = osclients.Clients(
self._context["admin"]["credential"])
elif self._context.get("users"):
self._clients = osclients.Clients(
self._context["users"][0]["credential"])
def setup(self):
nc = osclients.Clients(self.context["admin"]["credential"]).neutron()
agents = nc.list_agents()["agents"]
# NOTE(bence romsics): If you ever add input parameters to this context
# beware that here we use the same key in self.context as is used for
# parameter passing, so we'll overwrite it.
self.context["networking_agents"] = agents
def consume(cache, args):
username, password, project_dom, user_dom, tenant_id = args
if "client" not in cache:
clients = osclients.Clients(self.credential)
cache["client"] = identity.Identity(
clients, name_generator=self.generate_random_name)
client = cache["client"]
user = client.create_user(username,
password=password,
project_id=tenant_id,
domain_name=user_dom,
default_role=default_role)
user_credential = credential.OpenStackCredential(
auth_url=self.credential["auth_url"],
username=user.name,
password=password,
tenant_name=self.context["tenants"][tenant_id]["name"],
permission=consts.EndpointPermission.USER,
project_domain_name=project_dom,
user_domain_name=user_dom,
endpoint_type=self.credential["endpoint_type"],
https_insecure=self.credential["https_insecure"],
https_cacert=self.credential["https_cacert"],
region_name=self.credential["region_name"],
profiler_hmac_key=self.credential["profiler_hmac_key"],
profiler_conn_str=self.credential["profiler_conn_str"],
api_info=self.credential["api_info"])
users.append({
"id": user.id,
"credential": user_credential,
"tenant_id": tenant_id
})
def setup(self):
is_config_app_dir = False
pckg_path = os.path.expanduser(self.config["app_package"])
if zipfile.is_zipfile(pckg_path):
zip_name = pckg_path
elif os.path.isdir(pckg_path):
is_config_app_dir = True
zip_name = mutils.pack_dir(pckg_path)
else:
msg = "There is no zip archive or directory by this path: %s"
raise exceptions.ContextSetupFailure(msg=msg % pckg_path,
ctx_name=self.get_name())
for user, tenant_id in self._iterate_per_tenants():
clients = osclients.Clients(user["credential"])
self.context["tenants"][tenant_id]["packages"] = []
if is_config_app_dir:
self.context["tenants"][tenant_id]["murano_ctx"] = zip_name
# TODO(astudenov): use self.generate_random_name()
with open(zip_name, "rb") as f:
file = io.BytesIO(f.read())
package = clients.murano().packages.create(
{
"categories": ["Web"],
"tags": ["tag"]
}, {"file": file})
self.context["tenants"][tenant_id]["packages"].append(package)
def test_cached(self):
clients = osclients.Clients({
"auth_url": "url",
"username": "******",
"password": "******"
})
@osclients.configure(self.id())
class SomeClient(osclients.OSClient):
pass
fake_client = SomeClient(clients.credential, clients.cache)
fake_client.create_client = mock.MagicMock()
self.assertEqual({}, clients.cache)
fake_client()
self.assertEqual({self.id(): fake_client.create_client.return_value},
clients.cache)
fake_client.create_client.assert_called_once_with()
fake_client()
fake_client.create_client.assert_called_once_with()
fake_client("2")
self.assertEqual(
{
self.id(): fake_client.create_client.return_value,
"%s('2',)" % self.id(): fake_client.create_client.return_value
}, clients.cache)
clients.clear()
self.assertEqual({}, clients.cache)
def use_existing_users(self):
LOG.debug("Using existing users for OpenStack platform.")
api_info = copy.deepcopy(self.env["platforms"]["openstack"].get(
"api_info", {}))
for user_credential in self.existing_users:
user_credential = copy.deepcopy(user_credential)
if "api_info" in user_credential:
api_info.update(user_credential["api_info"])
user_credential["api_info"] = api_info
user_credential = credential.OpenStackCredential(**user_credential)
user_clients = osclients.Clients(user_credential)
user_id = user_clients.keystone.auth_ref.user_id
tenant_id = user_clients.keystone.auth_ref.project_id
if tenant_id not in self.context["tenants"]:
self.context["tenants"][tenant_id] = {
"id": tenant_id,
"name": user_credential.tenant_name
}
self.context["users"].append({
"credential": user_credential,
"id": user_id,
"tenant_id": tenant_id
})
def cleanup(self):
for user, tenant_id in self._iterate_per_tenants():
with logging.ExceptionLogger(
LOG, "Unable to delete security group: %s." %
user["secgroup"]["name"]):
clients = osclients.Clients(user["credential"])
clients.neutron().delete_security_group(user["secgroup"]["id"])
def setup(self):
# NOTE(rkiran): Some clients are not thread-safe. Thus during
# multithreading/multiprocessing, it is likely the
# sockets are left open. This problem is eliminated by
# creating a connection in setup and cleanup separately.
net_wrapper = network_wrapper.wrap(osclients.Clients(
self.context["admin"]["credential"]),
self,
config=self.config)
kwargs = {}
if self.config["dns_nameservers"] is not None:
kwargs["dns_nameservers"] = self.config["dns_nameservers"]
for user, tenant_id in self._iterate_per_tenants():
self.context["tenants"][tenant_id]["networks"] = []
self.context["tenants"][tenant_id]["subnets"] = []
for i in range(self.config["networks_per_tenant"]):
network_create_args = self.config["network_create_args"].copy()
net_infra = net_wrapper._create_network_infrastructure(
tenant_id,
dualstack=self.config["dualstack"],
subnets_num=self.config["subnets_per_network"],
network_create_args=network_create_args,
router_create_args=self.config["router"],
**kwargs)
self.context["tenants"][tenant_id]["networks"].append(
net_infra["network"])
self.context["tenants"][tenant_id]["subnets"].extend(
net_infra["subnets"])
def setup(self):
"""Create list of flavors."""
from novaclient import exceptions as nova_exceptions
self.context["flavors"] = {}
clients = osclients.Clients(self.context["admin"]["credential"])
for flavor_config in self.config:
extra_specs = flavor_config.get("extra_specs")
flavor_config = FlavorConfig(**flavor_config)
try:
flavor = clients.nova().flavors.create(**flavor_config)
except nova_exceptions.Conflict:
msg = "Using existing flavor %s" % flavor_config["name"]
if logging.is_debug():
LOG.exception(msg)
else:
LOG.warning(msg)
continue
if extra_specs:
flavor.set_keys(extra_specs)
self.context["flavors"][flavor_config["name"]] = flavor.to_dict()
LOG.debug("Created flavor with id '%s'" % flavor.id)
def create_one_image(self, user, **kwargs):
"""Create one image for the user."""
clients = osclients.Clients(user["credential"])
image_id = types.GlanceImage(self.context).pre_process(
resource_spec=self.config["image"], config={})
flavor_id = types.Flavor(self.context).pre_process(
resource_spec=self.config["flavor"], config={})
vm_scenario = vmtasks.BootRuncommandDelete(self.context,
clients=clients)
server, fip = vm_scenario._boot_server_with_fip(
image=image_id,
flavor=flavor_id,
floating_network=self.config.get("floating_network"),
userdata=self.config.get("userdata"),
key_name=user["keypair"]["name"],
security_groups=[user["secgroup"]["name"]],
**kwargs)
try:
LOG.debug("Installing tools on %r %s" % (server, fip["ip"]))
self.customize_image(server, fip, user)
LOG.debug("Stopping server %r" % server)
vm_scenario._stop_server(server)
LOG.debug("Creating snapshot for %r" % server)
custom_image = vm_scenario._create_image(server)
finally:
vm_scenario._delete_server_with_fip(server, fip)
return custom_image
def fetch_token(self):
"""Authenticate user token."""
aname = "keystone_v%s.fetch_token" % self.version
with atomic.ActionTimer(self, aname):
# use another instance of osclients.Clients to avoid usage of
# cached keystone session
clients = osclients.Clients(credential=self._clients.credential)
return clients.keystone.auth_ref.auth_token
def consume(cache, args):
role_id, user_id, project_id = args
if "client" not in cache:
clients = osclients.Clients(self.credential)
cache["client"] = identity.Identity(clients)
getattr(cache["client"], func_name)(role_id=role_id,
user_id=user_id,
project_id=project_id)
def setup(self):
for user, tenant_id in self._iterate_per_tenants():
clients = osclients.Clients(user["credential"])
self.context["tenants"][tenant_id]["networks"] = (
clients.neutron().list_networks()["networks"])
self.context["tenants"][tenant_id]["subnets"] = (
clients.neutron().list_subnets()["subnets"])
def consume(cache, args):
domain, task_id, i = args
if "client" not in cache:
clients = osclients.Clients(self.credential)
cache["client"] = identity.Identity(
clients, name_generator=self.generate_random_name)
tenant = cache["client"].create_project(domain_name=domain)
tenant_dict = {"id": tenant.id, "name": tenant.name, "users": []}
tenants.append(tenant_dict)
def setup(self):
# FIXME(andreykurilin): move all checks to validate method.
# use admin only when `service_name` is presented
admin_clients = osclients.Clients(
self.context.get("admin", {}).get("credential"))
clients = osclients.Clients(random.choice(
self.context["users"])["credential"])
services = clients.keystone.service_catalog.get_endpoints()
services_from_admin = None
for client_name, conf in self.config.items():
if "service_type" in conf and conf["service_type"] not in services:
raise exceptions.ValidationError(
"There is no service with '%s' type in your environment."
% conf["service_type"])
elif "service_name" in conf:
if not self.context.get("admin", {}).get("credential"):
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(),
msg="Setting 'service_name' is admin only operation.")
if not services_from_admin:
services_from_admin = dict(
[(s.name, s.type)
for s in admin_clients.keystone().services.list()])
if conf["service_name"] not in services_from_admin:
raise exceptions.ValidationError(
"There is no '%s' service in your environment"
% conf["service_name"])
# TODO(boris-42): Use separate key ["openstack"]["versions"]
self.context["config"]["api_versions@openstack"][client_name][
"service_type"] = services_from_admin[conf["service_name"]]
admin_cred = self.context.get("admin", {}).get("credential")
if admin_cred:
admin_cred["api_info"].update(
self.context["config"]["api_versions@openstack"]
)
for user in self.context["users"]:
user["credential"]["api_info"].update(
self.context["config"]["api_versions@openstack"]
)
def cleanup(self):
net_wrapper = network_wrapper.wrap(osclients.Clients(
self.context["admin"]["credential"]),
self,
config=self.config)
for tenant_id, tenant_ctx in self.context["tenants"].items():
for network in tenant_ctx.get("networks", []):
with logging.ExceptionLogger(
LOG,
"Failed to delete network for tenant %s" % tenant_id):
net_wrapper.delete_network(network)
def __init__(self, context=None, admin_clients=None, clients=None):
super(OpenStackScenario, self).__init__(context)
if context:
if admin_clients is None and "admin" in context:
self._admin_clients = osclients.Clients(
context["admin"]["credential"])
if clients is None:
if "users" in context and "user" not in context:
self._choose_user(context)
if "user" in context:
self._clients = osclients.Clients(
context["user"]["credential"])
if admin_clients:
self._admin_clients = admin_clients
if clients:
self._clients = clients
self._init_profiler(context)
def __init__(self, ctx):
super(Quotas, self).__init__(ctx)
self.clients = osclients.Clients(self.context["admin"]["credential"])
self.manager = {
"nova": nova_quotas.NovaQuotas(self.clients),
"cinder": cinder_quotas.CinderQuotas(self.clients),
"manila": manila_quotas.ManilaQuotas(self.clients),
"designate": designate_quotas.DesignateQuotas(self.clients),
"neutron": neutron_quotas.NeutronQuotas(self.clients)
}
self.original_quotas = []
def _prepare_open_secgroup(credential, secgroup_name):
"""Generate secgroup allowing all tcp/udp/icmp access.
In order to run tests on instances it is necessary to have SSH access.
This function generates a secgroup which allows all tcp/udp/icmp access.
:param credential: clients credential
:param secgroup_name: security group name
:returns: dict with security group details
"""
neutron = osclients.Clients(credential).neutron()
security_groups = neutron.list_security_groups()["security_groups"]
rally_open = [sg for sg in security_groups if sg["name"] == secgroup_name]
if not rally_open:
descr = "Allow ssh access to VMs created by Rally"
rally_open = neutron.create_security_group(
{"security_group": {
"name": secgroup_name,
"description": descr
}})["security_group"]
else:
rally_open = rally_open[0]
rules_to_add = [{
"protocol": "tcp",
"port_range_max": 65535,
"port_range_min": 1,
"remote_ip_prefix": "0.0.0.0/0",
"direction": "ingress",
"security_group_id": rally_open["id"]
}, {
"protocol": "udp",
"port_range_max": 65535,
"port_range_min": 1,
"remote_ip_prefix": "0.0.0.0/0",
"direction": "ingress",
"security_group_id": rally_open["id"]
}, {
"protocol": "icmp",
"remote_ip_prefix": "0.0.0.0/0",
"direction": "ingress",
"security_group_id": rally_open["id"]
}]
existing_rules = set(
_rule_to_key(r) for r in rally_open.get("security_group_rules", []))
for new_rule in rules_to_add:
if _rule_to_key(new_rule) not in existing_rules:
neutron.create_security_group_rule(
{"security_group_rule": new_rule})
return rally_open
def _get_role_object(self, context_role):
"""Check if role exists.
:param context_role: name of existing role.
"""
keystone = identity.Identity(osclients.Clients(self.credential))
default_roles = keystone.list_roles()
for def_role in default_roles:
if str(def_role.name) == context_role:
return def_role
else:
raise exceptions.NotFoundException(
"There is no role with name `%s`" % context_role)
def _create_image(self, hadoop_version, image_url, plugin_name, user,
user_name):
clients = osclients.Clients(user["credential"])
image_service = image_services.Image(
clients, name_generator=self.generate_random_name)
image = image_service.create_image(container_format="bare",
image_location=image_url,
disk_format="qcow2")
clients.sahara().images.update_image(image_id=image.id,
user_name=user_name,
desc="")
clients.sahara().images.update_tags(
image_id=image.id, new_tags=[plugin_name, hadoop_version])
return image.id
def setup(self):
admin_clients = osclients.Clients(
self.context.get("admin", {}).get("credential"))
cinder_service = block.BlockStorage(
admin_clients,
name_generator=self.generate_random_name,
atomic_inst=self.atomic_actions())
self.context["volume_types"] = []
for vtype_name in self.config:
LOG.debug("Creating Cinder volume type %s" % vtype_name)
vtype = cinder_service.create_volume_type(vtype_name)
self.context["volume_types"].append({
"id": vtype.id,
"name": vtype_name
})
def test_services(self, mock_keystone_service_catalog):
available_services = {
consts.ServiceType.IDENTITY: {},
consts.ServiceType.COMPUTE: {},
"some_service": {}
}
mock_get_endpoints = mock_keystone_service_catalog.get_endpoints
mock_get_endpoints.return_value = available_services
clients = osclients.Clients(self.credential)
self.assertEqual(
{
consts.ServiceType.IDENTITY: consts.Service.KEYSTONE,
consts.ServiceType.COMPUTE: consts.Service.NOVA,
"some_service": "__unknown__"
}, clients.services())
def _remove_default_security_group(self):
"""Delete default security group for tenants."""
admin_client = neutron.NeutronService(
clients=osclients.Clients(self.credential),
atomic_inst=self.atomic_actions())
if not admin_client.supports_extension("security-group", silent=True):
LOG.debug("Security group context is disabled.")
return
security_groups = admin_client.list_security_groups(name="default")
for security_group in security_groups:
if security_group["tenant_id"] not in self.context["tenants"]:
continue
admin_client.delete_security_group(security_group["id"])
def cleanup(self):
net_wrapper = network_wrapper.wrap(osclients.Clients(
self.context["admin"]["credential"]),
self,
config=self.config)
for tenant_id, tenant_ctx in self.context["tenants"].items():
for network in tenant_ctx.get("networks", []):
for pool in network.get("lb_pools", []):
with logging.ExceptionLogger(
LOG, "Failed to delete pool %(pool)s for tenant "
"%(tenant)s" % {
"pool": pool["pool"]["id"],
"tenant": tenant_id
}):
if self.config["lbaas_version"] == 1:
net_wrapper.delete_v1_pool(pool["pool"]["id"])
def setUp(self):
super(OSClientsTestCase, self).setUp()
self.credential = oscredential.OpenStackCredential(
"http://auth_url/v2.0", "user", "pass", "tenant")
self.clients = osclients.Clients(self.credential, {})
self.fake_keystone = fakes.FakeKeystoneClient()
keystone_patcher = mock.patch("%s.Keystone.create_client" % PATH,
return_value=self.fake_keystone)
self.mock_create_keystone_client = keystone_patcher.start()
self.auth_ref_patcher = mock.patch("%s.Keystone.auth_ref" % PATH)
self.auth_ref = self.auth_ref_patcher.start()
self.service_catalog = self.auth_ref.service_catalog
self.service_catalog.url_for = mock.MagicMock()
def setup(self):
size = self.config["size"]
volume_type = self.config.get("type", None)
volumes_per_tenant = self.config["volumes_per_tenant"]
for user, tenant_id in self._iterate_per_tenants():
self.context["tenants"][tenant_id].setdefault("volumes", [])
clients = osclients.Clients(user["credential"])
cinder_service = block.BlockStorage(
clients,
name_generator=self.generate_random_name,
atomic_inst=self.atomic_actions())
for i in range(volumes_per_tenant):
vol = cinder_service.create_volume(size,
volume_type=volume_type)
self.context["tenants"][tenant_id]["volumes"].append(
vol._as_dict())
def setup(self):
utils.init_sahara_context(self)
self.context["sahara"]["swift_objects"] = []
self.context["sahara"]["container_name"] = None
for user, tenant_id in self._iterate_per_tenants():
clients = osclients.Clients(user["credential"])
if self.config["input_type"] == "swift":
self.setup_inputs_swift(clients, tenant_id,
self.config["input_url"],
self.config["swift_files"],
user["credential"].username,
user["credential"].password)
else:
self.setup_inputs(clients, tenant_id,
self.config["input_type"],
self.config["input_url"])
def setup(self):
admin_or_user = (self.context.get("admin")
or self.context.get("users")[0])
net_wrapper = network.wrap(osclients.Clients(
admin_or_user["credential"]),
self,
config=self.config)
use_sg, msg = net_wrapper.supports_extension("security-group")
if not use_sg:
LOG.info("Security group context is disabled: %s" % msg)
return
secgroup_name = self.generate_random_name()
for user in self.context["users"]:
user["secgroup"] = _prepare_open_secgroup(user["credential"],
secgroup_name)
