def __init__(self, context=None, admin_clients=None, clients=None):
super(OpenStackScenario, self).__init__(context)
if context:
api_info = {}
if "api_versions@openstack" in context.get("config", {}):
api_versions = context["config"]["api_versions@openstack"]
for service in api_versions:
api_info[service] = {
"version": api_versions[service].get("version"),
"service_type":
api_versions[service].get("service_type")
}
if admin_clients is None and "admin" in context:
self._admin_clients = osclients.Clients(
context["admin"]["credential"], api_info)
if clients is None:
if "users" in context and "user" not in context:
self._choose_user(context)
if "user" in context:
self._clients = osclients.Clients(
context["user"]["credential"], api_info)
if admin_clients:
self._admin_clients = admin_clients
if clients:
self._clients = clients
self._init_profiler(context)
def check_health(self):
"""Check whatever platform is alive."""
if self.platform_data["admin"]:
try:
osclients.Clients(
self.platform_data["admin"]).verified_keystone()
except Exception:
d = copy.deepcopy(self.platform_data["admin"])
d["password"] = "******"
return {
"available":
False,
"message": ("Bad admin creds: \n%s" %
json.dumps(d, indent=2, sort_keys=True)),
"traceback":
traceback.format_exc()
}
for user in self.platform_data["users"]:
try:
osclients.Clients(user).keystone()
except Exception:
d = copy.deepcopy(user)
d["password"] = "******"
return {
"available":
False,
"message": ("Bad user creds: \n%s" %
json.dumps(d, indent=2, sort_keys=True)),
"traceback":
traceback.format_exc()
}
return {"available": True}
def _remove_default_security_group(self):
"""Delete default security group for tenants."""
clients = osclients.Clients(self.credential)
if consts.Service.NEUTRON not in clients.services().values():
return
use_sg, msg = network.wrap(clients,
self).supports_extension("security-group")
if not use_sg:
LOG.debug("Security group context is disabled: %s" % msg)
return
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
with logging.ExceptionLogger(
LOG, "Unable to delete default security group"):
uclients = osclients.Clients(user["credential"])
security_groups = uclients.neutron().list_security_groups()
default = [
sg for sg in security_groups["security_groups"]
if sg["name"] == "default"
]
if default:
clients.neutron().delete_security_group(default[0]["id"])
def preprocess(name, context, args):
"""Run preprocessor on scenario arguments.
:param name: Plugin name
:param context: dict with contexts data
:param args: args section of input task file
:returns processed_args: dictionary object with additional client
and resource configuration
"""
preprocessors = scenario.Scenario.get(name)._meta_get("preprocessors",
default={})
clients = None
if context.get("admin"):
clients = osclients.Clients(context["admin"]["credential"])
elif context.get("users"):
clients = osclients.Clients(context["users"][0]["credential"])
processed_args = copy.deepcopy(args)
for src, preprocessor in preprocessors.items():
resource_cfg = processed_args.get(src)
if resource_cfg:
processed_args[src] = preprocessor.transform(
clients=clients, resource_config=resource_cfg)
return processed_args
def __init__(self, context, cache=None):
super(OpenStackResourceType, self).__init__(context, cache)
self._clients = None
if self._context.get("admin"):
self._clients = osclients.Clients(
self._context["admin"]["credential"])
elif self._context.get("users"):
self._clients = osclients.Clients(
self._context["users"][0]["credential"])
def setup(self):
"""Create list of flavors."""
from novaclient import exceptions as nova_exceptions
self.context["flavors"] = {}
clients = osclients.Clients(self.context["admin"]["credential"])
for flavor_config in self.config:
extra_specs = flavor_config.get("extra_specs")
flavor_config = FlavorConfig(**flavor_config)
try:
flavor = clients.nova().flavors.create(**flavor_config)
except nova_exceptions.Conflict:
msg = "Using existing flavor %s" % flavor_config["name"]
if logging.is_debug():
LOG.exception(msg)
else:
LOG.warning(msg)
continue
if extra_specs:
flavor.set_keys(extra_specs)
self.context["flavors"][flavor_config["name"]] = flavor.to_dict()
LOG.debug("Created flavor with id '%s'" % flavor.id)
def consume(cache, args):
username, password, project_dom, user_dom, tenant_id = args
if "client" not in cache:
clients = osclients.Clients(self.credential)
cache["client"] = identity.Identity(
clients, name_generator=self.generate_random_name)
client = cache["client"]
user = client.create_user(username, password=password,
project_id=tenant_id,
domain_name=user_dom,
default_role=default_role)
user_credential = credential.OpenStackCredential(
auth_url=self.credential.auth_url,
username=user.name,
password=password,
tenant_name=self.context["tenants"][tenant_id]["name"],
permission=consts.EndpointPermission.USER,
project_domain_name=project_dom,
user_domain_name=user_dom,
endpoint_type=self.credential.endpoint_type,
https_insecure=self.credential.https_insecure,
https_cacert=self.credential.https_cacert,
region_name=self.credential.region_name,
profiler_hmac_key=self.credential.profiler_hmac_key,
profiler_conn_str=self.credential.profiler_conn_str)
users.append({"id": user.id,
"credential": user_credential,
"tenant_id": tenant_id})
def setup(self):
is_config_app_dir = False
pckg_path = os.path.expanduser(self.config["app_package"])
if zipfile.is_zipfile(pckg_path):
zip_name = pckg_path
elif os.path.isdir(pckg_path):
is_config_app_dir = True
zip_name = fileutils.pack_dir(pckg_path)
else:
msg = "There is no zip archive or directory by this path: %s"
raise exceptions.ContextSetupFailure(msg=msg % pckg_path,
ctx_name=self.get_name())
for user, tenant_id in utils.iterate_per_tenants(
self.context["users"]):
clients = osclients.Clients(user["credential"])
self.context["tenants"][tenant_id]["packages"] = []
if is_config_app_dir:
self.context["tenants"][tenant_id]["murano_ctx"] = zip_name
# TODO(astudenov): use self.generate_random_name()
package = clients.murano().packages.create(
{
"categories": ["Web"],
"tags": ["tag"]
}, {"file": open(zip_name)})
self.context["tenants"][tenant_id]["packages"].append(package)
def create_one_image(self, user, **kwargs):
"""Create one image for the user."""
clients = osclients.Clients(user["credential"])
image_id = types.GlanceImage.transform(
clients=clients, resource_config=self.config["image"])
flavor_id = types.Flavor.transform(
clients=clients, resource_config=self.config["flavor"])
vm_scenario = vmtasks.BootRuncommandDelete(self.context,
clients=clients)
server, fip = vm_scenario._boot_server_with_fip(
image=image_id,
flavor=flavor_id,
floating_network=self.config.get("floating_network"),
userdata=self.config.get("userdata"),
key_name=user["keypair"]["name"],
security_groups=[user["secgroup"]["name"]],
**kwargs)
try:
LOG.debug("Installing tools on %r %s" % (server, fip["ip"]))
self.customize_image(server, fip, user)
LOG.debug("Stopping server %r" % server)
vm_scenario._stop_server(server)
LOG.debug("Creating snapshot for %r" % server)
custom_image = vm_scenario._create_image(server)
finally:
vm_scenario._delete_server_with_fip(server, fip)
return custom_image
def setup(self):
# NOTE(rkiran): Some clients are not thread-safe. Thus during
# multithreading/multiprocessing, it is likely the
# sockets are left open. This problem is eliminated by
# creating a connection in setup and cleanup separately.
net_wrapper = network_wrapper.wrap(osclients.Clients(
self.context["admin"]["credential"]),
self,
config=self.config)
kwargs = {}
if self.config["dns_nameservers"] is not None:
kwargs["dns_nameservers"] = self.config["dns_nameservers"]
for user, tenant_id in (utils.iterate_per_tenants(
self.context.get("users", []))):
self.context["tenants"][tenant_id]["networks"] = []
for i in range(self.config["networks_per_tenant"]):
# NOTE(amaretskiy): router_create_args and subnets_num take
# effect for Neutron only.
network_create_args = self.config["network_create_args"].copy()
network = net_wrapper.create_network(
tenant_id,
dualstack=self.config["dualstack"],
subnets_num=self.config["subnets_per_network"],
network_create_args=network_create_args,
router_create_args=self.config["router"],
**kwargs)
self.context["tenants"][tenant_id]["networks"].append(network)
def setup(self):
watcher_scenario = watcher_utils.WatcherScenario({
"admin":
self.context["admin"],
"task":
self.context["task"],
"owner_id":
self.context["owner_id"],
"config": {
"api_versions": self.context["config"].get("api_versions", [])
}
})
clients = osclients.Clients(self.context["admin"]["credential"])
self.context["audit_templates"] = []
for i in six.moves.range(self.config["audit_templates_per_admin"]):
cfg_size = len(self.config["params"])
if self.config["fill_strategy"] == "round_robin":
audit_params = self.config["params"][i % cfg_size]
elif self.config["fill_strategy"] == "random":
audit_params = random.choice(self.config["params"])
goal_id = types.WatcherGoal.transform(
clients=clients, resource_config=audit_params["goal"])
strategy_id = types.WatcherStrategy.transform(
clients=clients, resource_config=audit_params["strategy"])
audit_template = watcher_scenario._create_audit_template(
goal_id, strategy_id)
self.context["audit_templates"].append(audit_template.uuid)
def setup(self):
image = self.config["image"]
flavor = self.config["flavor"]
clients = osclients.Clients(self.context["users"][0]["credential"])
image_id = types.EC2Image.transform(clients=clients,
resource_config=image)
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
LOG.debug("Booting servers for tenant %s " % user["tenant_id"])
ec2_scenario = ec2_utils.EC2Scenario({
"user":
user,
"task":
self.context["task"],
"owner_id":
self.context["owner_id"]
})
LOG.debug(
"Calling _boot_servers with "
"image_id=%(image_id)s flavor_name=%(flavor_name)s "
"servers_per_tenant=%(servers_per_tenant)s" % {
"image_id": image_id,
"flavor_name": flavor["name"],
"servers_per_tenant": self.config["servers_per_tenant"]
})
servers = ec2_scenario._boot_servers(
image_id, flavor["name"], self.config["servers_per_tenant"])
current_servers = [server.id for server in servers]
self.context["tenants"][tenant_id]["ec2_servers"] = current_servers
def consume(cache, args):
role_id, user_id, project_id = args
if "client" not in cache:
clients = osclients.Clients(self.credential)
cache["client"] = identity.Identity(clients)
getattr(cache["client"], func_name)(role_id=role_id,
user_id=user_id,
project_id=project_id)
def fetch_token(self):
"""Authenticate user token."""
cred = self._clients.credential
aname = "keystone_v%s.fetch_token" % self.version
with atomic.ActionTimer(self, aname):
clients = osclients.Clients(credential=cred,
api_info=self._clients.api_info)
return clients.keystone.auth_ref.auth_token
def setup(self):
for user, tenant_id in utils.iterate_per_tenants(
self.context.get("users", [])):
net_wrapper = network_wrapper.wrap(osclients.Clients(
user["credential"]),
self,
config=self.config)
self.context["tenants"][tenant_id]["networks"] = (
net_wrapper.list_networks())
def consume(cache, args):
domain, task_id, i = args
if "client" not in cache:
clients = osclients.Clients(self.credential)
cache["client"] = identity.Identity(
clients, name_generator=self.generate_random_name)
tenant = cache["client"].create_project(domain_name=domain)
tenant_dict = {"id": tenant.id, "name": tenant.name, "users": []}
tenants.append(tenant_dict)
def cleanup(self):
for user, tenant_id in utils.iterate_per_tenants(
self.context["users"]):
with logging.ExceptionLogger(
LOG,
"Unable to delete security group: %s."
% user["secgroup"]["name"]):
clients = osclients.Clients(user["credential"])
clients.neutron().delete_security_group(user["secgroup"]["id"])
def setUp(self):
super(CinderScenarioTestCase, self).setUp()
wrap = mock.patch("rally.plugins.openstack.wrappers.cinder.wrap")
self.mock_wrap = wrap.start()
self.addCleanup(self.mock_wrap.stop)
self.scenario = utils.CinderScenario(
self.context,
clients=osclients.Clients(
fakes.FakeUserContext.user["credential"]))
def _prepare_open_secgroup(credential, secgroup_name):
"""Generate secgroup allowing all tcp/udp/icmp access.
In order to run tests on instances it is necessary to have SSH access.
This function generates a secgroup which allows all tcp/udp/icmp access.
:param credential: clients credential
:param secgroup_name: security group name
:returns: dict with security group details
"""
neutron = osclients.Clients(credential).neutron()
security_groups = neutron.list_security_groups()["security_groups"]
rally_open = [sg for sg in security_groups if sg["name"] == secgroup_name]
if not rally_open:
descr = "Allow ssh access to VMs created by Rally"
rally_open = neutron.create_security_group(
{"security_group": {"name": secgroup_name,
"description": descr}})["security_group"]
else:
rally_open = rally_open[0]
rules_to_add = [
{
"protocol": "tcp",
"port_range_max": 65535,
"port_range_min": 1,
"remote_ip_prefix": "0.0.0.0/0",
"direction": "ingress"
},
{
"protocol": "udp",
"port_range_max": 65535,
"port_range_min": 1,
"remote_ip_prefix": "0.0.0.0/0",
"direction": "ingress"
},
{
"protocol": "icmp",
"remote_ip_prefix": "0.0.0.0/0",
"direction": "ingress"
}
]
def rule_match(criteria, existing_rule):
return all(existing_rule[key] == value
for key, value in criteria.items())
for new_rule in rules_to_add:
if not any(rule_match(new_rule, existing_rule) for existing_rule
in rally_open.get("security_group_rules", [])):
new_rule["security_group_id"] = rally_open["id"]
neutron.create_security_group_rule(
{"security_group_rule": new_rule})
return rally_open
def cleanup(self):
net_wrapper = network_wrapper.wrap(
osclients.Clients(self.context["admin"]["credential"]),
self, config=self.config)
for tenant_id, tenant_ctx in self.context["tenants"].items():
for network in tenant_ctx.get("networks", []):
with logging.ExceptionLogger(
LOG,
"Failed to delete network for tenant %s" % tenant_id):
net_wrapper.delete_network(network)
def _pre_process_method(self, resource_spec, config):
"""pre_process to transform adapter.
Adopts a call for a new style pre_process instance method if ResourceType
to old style(deprecated way) call to classmethod transform.
"""
if resource_spec is None:
# previously, such arguments were skipped
return
from rally.plugins.openstack import osclients
clients = None
if self._context.get("admin"):
clients = osclients.Clients(self._context["admin"]["credential"])
elif self._context.get("users"):
clients = osclients.Clients(self._context["users"][0]["credential"])
return self.__class__.transform(clients=clients,
resource_config=resource_spec)
def _get_role_object(self, context_role):
"""Check if role exists.
:param context_role: name of existing role.
"""
keystone = identity.Identity(osclients.Clients(self.credential))
default_roles = keystone.list_roles()
for def_role in default_roles:
if str(def_role.name) == context_role:
return def_role
else:
raise exceptions.NotFoundException(
"There is no role with name `%s`" % context_role)
def __init__(self, ctx):
super(Quotas, self).__init__(ctx)
self.clients = osclients.Clients(
self.context["admin"]["credential"],
api_info=self.context["config"].get("api_versions"))
self.manager = {
"nova": nova_quotas.NovaQuotas(self.clients),
"cinder": cinder_quotas.CinderQuotas(self.clients),
"manila": manila_quotas.ManilaQuotas(self.clients),
"designate": designate_quotas.DesignateQuotas(self.clients),
"neutron": neutron_quotas.NeutronQuotas(self.clients)
}
self.original_quotas = []
def setup(self):
admin_clients = osclients.Clients(
self.context.get("admin", {}).get("credential"),
api_info=self.context["config"].get("api_versions"))
cinder_service = block.BlockStorage(
admin_clients, name_generator=self.generate_random_name)
self.context["volume_types"] = []
for vtype_name in self.config:
LOG.debug("Creating Cinder volume type %s" % vtype_name)
vtype = cinder_service.create_volume_type(vtype_name)
self.context["volume_types"].append({
"id": vtype.id,
"name": vtype_name
})
def setup(self):
# FIXME(andreykurilin): move all checks to validate method.
# use admin only when `service_name` is presented
admin_clients = osclients.Clients(
self.context.get("admin", {}).get("credential"))
clients = osclients.Clients(
random.choice(self.context["users"])["credential"])
services = clients.keystone.service_catalog.get_endpoints()
services_from_admin = None
for client_name, conf in self.config.items():
if "service_type" in conf and conf["service_type"] not in services:
raise exceptions.ValidationError(
"There is no service with '%s' type in your environment." %
conf["service_type"])
elif "service_name" in conf:
if not self.context.get("admin", {}).get("credential"):
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(),
msg="Setting 'service_name' is admin only operation.")
if not services_from_admin:
services_from_admin = dict([
(s.name, s.type)
for s in admin_clients.keystone().services.list()
])
if conf["service_name"] not in services_from_admin:
raise exceptions.ValidationError(
"There is no '%s' service in your environment" %
conf["service_name"])
# TODO(boris-42): Use separate key ["openstack"]["versions"]
self.context["config"]["api_versions@openstack"][client_name][
"service_type"] = services_from_admin[conf["service_name"]]
# NOTE(boris-42): Required to be backward compatible
self.context["config"]["api_versions"] = (
self.context["config"]["api_versions@openstack"])
def setup(self):
image = self.config["image"]
flavor = self.config["flavor"]
auto_nic = self.config["auto_assign_nic"]
servers_per_tenant = self.config["servers_per_tenant"]
kwargs = {}
if self.config.get("nics"):
if isinstance(self.config["nics"][0], dict):
# it is a format that Nova API expects
kwargs["nics"] = list(self.config["nics"])
else:
kwargs["nics"] = [{"net-id": nic}
for nic in self.config["nics"]]
clients = osclients.Clients(self.context["users"][0]["credential"])
image_id = types.GlanceImage.transform(clients=clients,
resource_config=image)
flavor_id = types.Flavor.transform(clients=clients,
resource_config=flavor)
for iter_, (user, tenant_id) in enumerate(rutils.iterate_per_tenants(
self.context["users"])):
LOG.debug("Booting servers for user tenant %s" % user["tenant_id"])
tmp_context = {"user": user,
"tenant": self.context["tenants"][tenant_id],
"task": self.context["task"],
"owner_id": self.context["owner_id"],
"iteration": iter_}
nova_scenario = nova_utils.NovaScenario(tmp_context)
LOG.debug("Calling _boot_servers with image_id=%(image_id)s "
"flavor_id=%(flavor_id)s "
"servers_per_tenant=%(servers_per_tenant)s"
% {"image_id": image_id,
"flavor_id": flavor_id,
"servers_per_tenant": servers_per_tenant})
servers = nova_scenario._boot_servers(image_id, flavor_id,
requests=servers_per_tenant,
auto_assign_nic=auto_nic,
**kwargs)
current_servers = [server.id for server in servers]
LOG.debug("Adding booted servers %s to context" % current_servers)
self.context["tenants"][tenant_id][
"servers"] = current_servers
def cleanup(self):
net_wrapper = network_wrapper.wrap(osclients.Clients(
self.context["admin"]["credential"]),
self,
config=self.config)
for tenant_id, tenant_ctx in self.context["tenants"].items():
for network in tenant_ctx.get("networks", []):
for pool in network.get("lb_pools", []):
with logging.ExceptionLogger(
LOG, "Failed to delete pool %(pool)s for tenant "
"%(tenant)s" % {
"pool": pool["pool"]["id"],
"tenant": tenant_id
}):
if self.config["lbaas_version"] == 1:
net_wrapper.delete_v1_pool(pool["pool"]["id"])
def setup(self):
admin_or_user = (self.context.get("admin") or
self.context.get("users")[0])
net_wrapper = network.wrap(
osclients.Clients(admin_or_user["credential"]),
self, config=self.config)
use_sg, msg = net_wrapper.supports_extension("security-group")
if not use_sg:
LOG.info("Security group context is disabled: %s" % msg)
return
secgroup_name = self.generate_random_name()
for user in self.context["users"]:
user["secgroup"] = _prepare_open_secgroup(user["credential"],
secgroup_name)
def _create_image(self, hadoop_version, image_url, plugin_name, user,
user_name):
clients = osclients.Clients(
user["credential"],
api_info=self.context["config"].get("api_versions"))
image_service = image_services.Image(
clients, name_generator=self.generate_random_name)
image = image_service.create_image(container_format="bare",
image_location=image_url,
disk_format="qcow2")
clients.sahara().images.update_image(image_id=image.id,
user_name=user_name,
desc="")
clients.sahara().images.update_tags(
image_id=image.id, new_tags=[plugin_name, hadoop_version])
return image.id
def test_services(self, mock_keystone_service_catalog):
available_services = {
consts.ServiceType.IDENTITY: {},
consts.ServiceType.COMPUTE: {},
"some_service": {}
}
mock_get_endpoints = mock_keystone_service_catalog.get_endpoints
mock_get_endpoints.return_value = available_services
clients = osclients.Clients(self.credential)
self.assertEqual(
{
consts.ServiceType.IDENTITY: consts.Service.KEYSTONE,
consts.ServiceType.COMPUTE: consts.Service.NOVA,
"some_service": "__unknown__"
}, clients.services())