def setup(self):
# The user may want to use the existing image. In this case he should
# make sure that the image is public and has all required metadata.
image_uuid = self.config.get("image_uuid")
self.context["need_sahara_image_cleanup"] = not image_uuid
if image_uuid:
# Using the first user to check the existing image.
user = self.context["users"][0]
clients = osclients.Clients(user["endpoint"])
image = clients.glance().images.get(image_uuid)
if not image.is_public:
raise exceptions.BenchmarkSetupFailure(
"Image provided in the Sahara context should be public.")
image_id = image_uuid
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
self.context["tenants"][tenant_id]["sahara_image"] = image_id
else:
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
image_id = self._create_image(
hadoop_version=self.config["hadoop_version"],
image_url=self.config["image_url"],
plugin_name=self.config["plugin_name"],
user=user,
user_name=self.config["username"])
self.context["tenants"][tenant_id]["sahara_image"] = image_id
def setup(self):
image = self.config["image"]
flavor = self.config["flavor"]
clients = osclients.Clients(self.context["users"][0]["credential"])
image_id = types.EC2ImageResourceType.transform(clients=clients,
resource_config=image)
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
LOG.debug("Booting servers for tenant %s "
% (user["tenant_id"]))
user_clients = osclients.Clients(user["credential"])
ec2_scenario = ec2_utils.EC2Scenario(clients=user_clients)
LOG.debug(
"Calling _boot_servers with "
"image_id={image_id} flavor_name={flavor_name} "
"servers_per_tenant={servers_per_tenant}".format(
image_id=image_id, flavor_name=flavor["name"],
servers_per_tenant=self.config["servers_per_tenant"]))
servers = ec2_scenario._boot_servers(
image_id, flavor["name"], self.config["servers_per_tenant"])
current_servers = [server.id for server in servers]
self.context["tenants"][tenant_id]["ec2_servers"] = current_servers
def setup(self):
image = self.config["image"]
flavor = self.config["flavor"]
clients = osclients.Clients(self.context["users"][0]["credential"])
image_id = types.EC2Image.transform(clients=clients,
resource_config=image)
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
LOG.debug("Booting servers for tenant %s " % user["tenant_id"])
ec2_scenario = ec2_utils.EC2Scenario({
"user": user,
"task": self.context["task"],
"owner_id": self.context["owner_id"]})
LOG.debug(
"Calling _boot_servers with "
"image_id=%(image_id)s flavor_name=%(flavor_name)s "
"servers_per_tenant=%(servers_per_tenant)s"
% {"image_id": image_id,
"flavor_name": flavor["name"],
"servers_per_tenant": self.config["servers_per_tenant"]})
servers = ec2_scenario._boot_servers(
image_id, flavor["name"], self.config["servers_per_tenant"])
current_servers = [server.id for server in servers]
self.context["tenants"][tenant_id]["ec2_servers"] = current_servers
def cleanup(self):
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
args = user['ec2args']
client = botocoreclient.get_ec2_client(
args['url'], args['region'], args['access'], args['secret'])
self.cleanup_networks(tenant_id, client)
def setup(self):
image = self.config["image"]
image_id = None
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
LOG.info("Booting servers for user tenant %s "
% (user["tenant_id"]))
args = user['ec2args']
client = botocoreclient.APIClientEC2(
args['url'], args['region'], args['access'], args['secret'])
if image_id is None:
resp, data = client.DescribeImages(
Filters=[{'Name': 'name', 'Values': [image]},
{'Name': 'image-type', 'Values': ['machine']}])
if resp.status_code != 200:
LOG.error(ec2_tests_base.EC2ErrorConverter(data))
assert 200 == resp.status_code
image_id = data['Images'][0]['ImageId']
self.context["tenants"][tenant_id]["servers"] = list()
self.context["tenants"][tenant_id]["networks"] = list()
self.run_instances(tenant_id, client, image_id)
self.wait_for_instances(tenant_id, client)
self.assign_floating_ips(tenant_id, client)
def setup(self):
image_url = self.config["image_url"]
image_type = self.config["image_type"]
image_container = self.config["image_container"]
images_per_tenant = self.config["images_per_tenant"]
image_name = self.config.get("image_name")
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
current_images = []
clients = osclients.Clients(user["endpoint"])
glance_scenario = glance_utils.GlanceScenario(
clients=clients)
for i in range(images_per_tenant):
if image_name and i > 0:
cur_name = image_name + str(i)
elif image_name:
cur_name = image_name
else:
cur_name = glance_scenario._generate_random_name(
prefix="rally_ctx_image_")
image = glance_scenario._create_image(
cur_name, image_container, image_url, image_type,
min_ram=self.config.get("min_ram", 0),
min_disk=self.config.get("min_disk", 0))
current_images.append(image.id)
self.context["tenants"][tenant_id]["images"] = current_images
def setup(self):
# NOTE(rkiran): Some clients are not thread-safe. Thus during
# multithreading/multiprocessing, it is likely the
# sockets are left open. This problem is eliminated by
# creating a connection in setup and cleanup separately.
net_wrapper = network_wrapper.wrap(
osclients.Clients(self.context["admin"]["credential"]),
self, config=self.config)
kwargs = {}
if self.config["dns_nameservers"] is not None:
kwargs["dns_nameservers"] = self.config["dns_nameservers"]
for user, tenant_id in (utils.iterate_per_tenants(
self.context.get("users", []))):
self.context["tenants"][tenant_id]["networks"] = []
for i in range(self.config["networks_per_tenant"]):
# NOTE(amaretskiy): add_router and subnets_num take effect
# for Neutron only.
network_create_args = self.config["network_create_args"].copy()
network = net_wrapper.create_network(
tenant_id,
add_router=True,
subnets_num=self.config["subnets_per_network"],
network_create_args=network_create_args,
**kwargs)
self.context["tenants"][tenant_id]["networks"].append(network)
def setup(self):
is_config_app_dir = False
pckg_path = os.path.expanduser(self.config["app_package"])
if zipfile.is_zipfile(pckg_path):
zip_name = pckg_path
elif os.path.isdir(pckg_path):
is_config_app_dir = True
zip_name = fileutils.pack_dir(pckg_path)
else:
msg = "There is no zip archive or directory by this path: %s"
raise exceptions.ContextSetupFailure(msg=msg % pckg_path,
ctx_name=self.get_name())
for user, tenant_id in utils.iterate_per_tenants(
self.context["users"]):
clients = osclients.Clients(user["credential"])
self.context["tenants"][tenant_id]["packages"] = []
if is_config_app_dir:
self.context["tenants"][tenant_id]["murano_ctx"] = zip_name
# TODO(astudenov): use self.generate_random_name()
package = clients.murano().packages.create(
{"categories": ["Web"], "tags": ["tag"]},
{"file": open(zip_name)})
self.context["tenants"][tenant_id]["packages"].append(package)
def setup(self):
utils.init_sahara_context(self)
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
clients = osclients.Clients(user["endpoint"])
sahara = clients.sahara()
if self.config["output_type"] == "swift":
swift = swift_utils.SwiftScenario(clients=clients,
context=self.context)
container_name = rutils.generate_random_name(
prefix=self.config["output_url_prefix"])
self.context["tenants"][tenant_id]["sahara"]["container"] = {
"name": swift._create_container(
container_name=container_name),
"output_swift_objects": []
}
self.setup_outputs_swift(swift, sahara, tenant_id,
container_name,
user["endpoint"].username,
user["endpoint"].password)
else:
self.setup_outputs_hdfs(sahara, tenant_id,
self.config["output_url_prefix"])
def setup(self):
"""This method is called before the task start."""
try:
for user in self.context['users']:
osclient = osclients.Clients(user['credential'])
keystone = osclient.keystone()
creds = keystone.ec2.list(user['id'])
if not creds:
creds = keystone.ec2.create(user['id'], user['tenant_id'])
else:
creds = creds[0]
url = keystone.service_catalog.url_for(service_type='ec2')
user['ec2args'] = {
'region': 'RegionOne',
'url': url,
'access': creds.access,
'secret': creds.secret
}
if self.net_wrapper.SERVICE_IMPL == consts.Service.NEUTRON:
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
body = {"quota": {"router": -1, "floatingip": -1}}
self.net_wrapper.client.update_quota(tenant_id, body)
network = self.net_wrapper.create_network(
tenant_id, add_router=True, subnets_num=1)
self.context["tenants"][tenant_id]["network"] = network
except Exception as e:
msg = "Can't prepare ec2 client: %s" % e.message
if logging.is_debug():
LOG.exception(msg)
else:
LOG.warning(msg)
def setup(self):
template = get_data(self.config["template"])
files = {}
for key, filename in self.config.get("files", {}).items():
files[key] = get_data(filename)
parameters = self.config.get("parameters", rutils.LockedDict())
with parameters.unlocked():
if "network_id" not in parameters:
parameters["network_id"] = self._get_public_network_id()
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
for name, path in self.config.get("context_parameters",
{}).items():
parameters[name] = self._get_context_parameter(user,
tenant_id,
path)
if "router_id" not in parameters:
networks = self.context["tenants"][tenant_id]["networks"]
parameters["router_id"] = networks[0]["router_id"]
if "key_name" not in parameters:
parameters["key_name"] = user["keypair"]["name"]
heat_scenario = heat_utils.HeatScenario(
{"user": user, "task": self.context["task"],
"owner_id": self.context["owner_id"]})
self.context["tenants"][tenant_id]["stack_dataplane"] = []
for i in range(self.config["stacks_per_tenant"]):
stack = heat_scenario._create_stack(template, files=files,
parameters=parameters)
tenant_data = self.context["tenants"][tenant_id]
tenant_data["stack_dataplane"].append([stack.id, template,
files, parameters])
def setup(self):
new_sample = {
"counter_name": self.config["counter_name"],
"counter_type": self.config["counter_type"],
"counter_unit": self.config["counter_unit"],
"counter_volume": self.config["counter_volume"],
}
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
self.context["tenants"][tenant_id]["samples"] = []
self.context["tenants"][tenant_id]["resources"] = []
scenario = ceilo_utils.CeilometerScenario(
context={"user": user, "task": self.context["task"]}
)
for i in moves.xrange(self.config["resources_per_tenant"]):
samples_to_create = scenario._make_samples(
count=self.config["samples_per_resource"],
interval=self.config["timestamp_interval"],
metadata_list=self.config.get("metadata_list"),
batch_size=self.config.get("batch_size"),
**new_sample)
samples = self._store_batch_samples(
scenario, samples_to_create,
self.config.get("batches_allow_lose")
)
for sample in samples:
self.context["tenants"][tenant_id]["samples"].append(
sample.to_dict())
self.context["tenants"][tenant_id]["resources"].append(
samples[0].resource_id)
def setup(self):
for user, tenant_id in utils.iterate_per_tenants(
self.context.get("users", [])):
net_wrapper = network_wrapper.wrap(
osclients.Clients(user["endpoint"]), self, config=self.config)
self.context["tenants"][tenant_id]["networks"] = (
net_wrapper.list_networks())
def setup(self):
net_wrapper = network_wrapper.wrap(
osclients.Clients(self.context["admin"]["endpoint"]),
self.context["task"],
config=self.config)
use_lb, msg = net_wrapper.supports_extension("lbaas")
if not use_lb:
LOG.info(msg)
return
# Creates a lb-pool for every subnet created in network context.
for user, tenant_id in (utils.iterate_per_tenants(
self.context.get("users", []))):
for network in self.context["tenants"][tenant_id]["networks"]:
for subnet in network.get("subnets", []):
if self.config["lbaas_version"] == 1:
network.setdefault("lb_pools", []).append(
net_wrapper.create_v1_pool(
tenant_id,
subnet,
**self.config["pool"]))
else:
raise NotImplementedError(
"Context for LBaaS version %s not implemented."
% self.config["lbaas_version"])
def setup(self):
input_type = self.config["input_type"]
input_url = self.config["input_url"]
mains = self.config.get("mains", [])
libs = self.config.get("libs", [])
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
clients = osclients.Clients(user["endpoint"])
sahara = clients.sahara()
self.setup_inputs(sahara, tenant_id, input_type, input_url)
self.context["tenants"][tenant_id]["sahara_mains"] = []
self.context["tenants"][tenant_id]["sahara_libs"] = []
for main in mains:
self.download_and_save_lib(
sahara=sahara,
lib_type="sahara_mains",
name=main["name"],
download_url=main["download_url"],
tenant_id=tenant_id)
for lib in libs:
self.download_and_save_lib(
sahara=sahara,
lib_type="sahara_libs",
name=lib["name"],
download_url=lib["download_url"],
tenant_id=tenant_id)
def setup(self):
image_url = self.config["image_url"]
plugin_name = self.config["plugin_name"]
hadoop_version = self.config["hadoop_version"]
user_name = self.config["username"]
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
clients = osclients.Clients(user["endpoint"])
glance_util_class = glance_utils.GlanceScenario(
clients=clients)
image_name = scenarios_base.Scenario._generate_random_name(
prefix="sahara_image_", length=15)
image = glance_util_class._create_image(image_name,
"bare",
image_url,
"qcow2")
clients.sahara().images.update_image(image_id=image.id,
user_name=user_name,
desc="")
clients.sahara().images.update_tags(image_id=image.id,
new_tags=[plugin_name,
hadoop_version])
self.context["tenants"][tenant_id]["sahara_image"] = image.id
def init_sahara_context(context_instance):
context_instance.context["sahara"] = context_instance.context.get("sahara",
{})
for user, tenant_id in rutils.iterate_per_tenants(
context_instance.context["users"]):
context_instance.context["tenants"][tenant_id]["sahara"] = (
context_instance.context["tenants"][tenant_id].get("sahara", {}))
def publish(queue):
for user, tenant_id in (rutils.iterate_per_tenants(
context.get("users", []))):
context["tenants"][tenant_id]["containers"] = []
for i in range(containers_per_tenant):
args = (user, context["tenants"][tenant_id]["containers"])
queue.append(args)
def cleanup(self):
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
magnum_scenario = magnum_utils.MagnumScenario({
"user": user,
"task": self.context["task"],
"config": {"api_versions": self.context["config"].get(
"api_versions", [])}
})
# get the cluster and cluster_template
cluster_uuid = str(self.context["tenants"][tenant_id]["cluster"])
cluster = magnum_scenario._get_cluster(cluster_uuid)
cluster_template = magnum_scenario._get_cluster_template(
cluster.cluster_template_id)
if not cluster_template.tls_disabled:
dir = self.context["ca_certs_directory"]
fname = os.path.join(dir, cluster_uuid + ".key")
os.remove(fname)
fname = os.path.join(dir, cluster_uuid + "_ca.crt")
os.remove(fname)
fname = os.path.join(dir, cluster_uuid + ".crt")
os.remove(fname)
def setup(self):
image_url = self.config["image_url"]
image_type = self.config["image_type"]
image_container = self.config["image_container"]
images_per_tenant = self.config["images_per_tenant"]
image_name = self.config.get("image_name")
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
current_images = []
glance_scenario = glance_utils.GlanceScenario(
{"user": user, "task": self.context["task"]})
for i in range(images_per_tenant):
if image_name and i > 0:
cur_name = image_name + str(i)
elif image_name:
cur_name = image_name
else:
cur_name = None
image = glance_scenario._create_image(
image_container, image_url, image_type,
name=cur_name, min_ram=self.config.get("min_ram", 0),
min_disk=self.config.get("min_disk", 0))
current_images.append(image.id)
self.context["tenants"][tenant_id]["images"] = current_images
def setup(self):
image = self.config["image"]
flavor = self.config["flavor"]
servers_per_tenant = self.config["servers_per_tenant"]
clients = osclients.Clients(self.context["users"][0]["endpoint"])
image_id = types.ImageResourceType.transform(clients=clients, resource_config=image)
flavor_id = types.FlavorResourceType.transform(clients=clients, resource_config=flavor)
for user, tenant_id in rutils.iterate_per_tenants(self.context["users"]):
LOG.debug("Booting servers for user tenant %s " % (user["tenant_id"]))
nova_scenario = nova_utils.NovaScenario({"user": user})
LOG.debug(
"Calling _boot_servers with image_id=%(image_id)s "
"flavor_id=%(flavor_id)s "
"servers_per_tenant=%(servers_per_tenant)s"
% {"image_id": image_id, "flavor_id": flavor_id, "servers_per_tenant": servers_per_tenant}
)
servers = nova_scenario._boot_servers(image_id, flavor_id, servers_per_tenant)
current_servers = [server.id for server in servers]
LOG.debug("Adding booted servers %s to context" % current_servers)
self.context["tenants"][tenant_id]["servers"] = current_servers
def setup(self):
disk_format = self.config["disk_format"]
container_format = self.config["container_format"]
images_per_tenant = self.config["images_per_tenant"]
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
glance = osclients.Clients(user["credential"]).glance().images
current_images = []
for i in range(images_per_tenant):
kw = {
"name": "image-" + tenant_id[0:8] + "-" + str(i),
"container_format": container_format,
"disk_format": disk_format,
"size": 1000000,
}
image = glance.create(**kw)
current_images.append(image.id)
self.context["tenants"][tenant_id]["images"] = current_images
# NOTE(andrey-mp): call ec2 api to initialize it
args = user['ec2args']
client = botocoreclient.get_ec2_client(
args['url'], args['region'], args['access'], args['secret'])
data = client.describe_images()
def setup(self):
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
nova_scenario = nova_utils.NovaScenario({
"user": user,
"task": self.context["task"],
"config": {"api_versions": self.context["config"].get(
"api_versions", [])}
})
keypair = nova_scenario._create_keypair()
magnum_scenario = magnum_utils.MagnumScenario({
"user": user,
"task": self.context["task"],
"owner_id": self.context["owner_id"],
"config": {"api_versions": self.context["config"].get(
"api_versions", [])}
})
# create a cluster
ct_uuid = self.config.get("cluster_template_uuid", None)
if ct_uuid is None:
ctx = self.context["tenants"][tenant_id]
ct_uuid = ctx.get("cluster_template")
cluster = magnum_scenario._create_cluster(
cluster_template=ct_uuid,
node_count=self.config.get("node_count"), keypair=keypair)
self.context["tenants"][tenant_id]["cluster"] = cluster.uuid
def setup(self):
counter_name = self.config["counter_name"]
counter_type = self.config["counter_type"]
counter_unit = self.config["counter_unit"]
counter_volume = self.config["counter_volume"]
resources_per_tenant = self.config["resources_per_tenant"]
samples_per_resource = self.config["samples_per_resource"]
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
self.context["tenants"][tenant_id]["samples"] = []
self.context["tenants"][tenant_id]["resources"] = []
clients = osclients.Clients(user["endpoint"])
scenario = ceilo_utils.CeilometerScenario(
clients=clients)
for i in range(resources_per_tenant):
for j in range(samples_per_resource):
try:
sample = scenario._create_sample(counter_name,
counter_type,
counter_unit,
counter_volume)
self.context["tenants"][tenant_id]["samples"].append(
sample[0].to_dict())
except Exception as e:
LOG.error("Creating a sample failed: %(error)s"
% {"error": e})
self.context["tenants"][tenant_id]["resources"].append(
sample[0].resource_id)
def _cleanup_tenant_resources(self, resources_plural_name,
resources_singular_name):
"""Cleans up tenant resources.
:param resources_plural_name: plural name for resources
:param resources_singular_name: singular name for resource. Expected
to be part of resource deletion method name (obj._delete_%s)
"""
for user, tenant_id in (utils.iterate_per_tenants(
self.context.get("users", []))):
manila_scenario = manila_utils.ManilaScenario({
"user": user,
"config": {
"api_versions": self.context["config"].get(
"api_versions", [])}
})
resources = self.context["tenants"][tenant_id][CONTEXT_NAME].get(
resources_plural_name, [])
for resource in resources:
logger = logging.ExceptionLogger(
LOG,
_("Failed to delete %(name)s %(id)s for tenant %(t)s.") % {
"id": resource, "t": tenant_id,
"name": resources_singular_name})
with logger:
delete_func = getattr(
manila_scenario,
"_delete_%s" % resources_singular_name)
delete_func(resource)
def setup(self):
utils.init_sahara_context(self)
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
clients = osclients.Clients(user["credential"])
sahara = clients.sahara()
self.context["tenants"][tenant_id]["sahara"]["mains"] = []
self.context["tenants"][tenant_id]["sahara"]["libs"] = []
for main in self.config.get("mains", []):
self.download_and_save_lib(
sahara=sahara,
lib_type="mains",
name=main["name"],
download_url=main["download_url"],
tenant_id=tenant_id)
for lib in self.config.get("libs", []):
self.download_and_save_lib(
sahara=sahara,
lib_type="libs",
name=lib["name"],
download_url=lib["download_url"],
tenant_id=tenant_id)
def _cleanup_tenant_resources(self, resources_plural_name,
resources_singular_name):
"""Cleans up tenant resources.
:param resources_plural_name: plural name for resources, should be
one of "shares", "share_networks" or "security_services".
:param resources_singular_name: singular name for resource. Expected
to be part of resource deletion method name (obj._delete_%s)
"""
for user, tenant_id in (utils.iterate_per_tenants(
self.context.get("users", []))):
clients = osclients.Clients(user["endpoint"])
manila_scenario = manila_utils.ManilaScenario(clients=clients)
resources = self.context["tenants"][tenant_id].get(
resources_plural_name, [])
for resource in resources:
logger = log.ExceptionLogger(
LOG,
_("Failed to delete %(name)s %(id)s for tenant %(t)s.") % {
"id": resource, "t": tenant_id,
"name": resources_singular_name})
with logger:
delete_func = getattr(
manila_scenario,
"_delete_%s" % resources_singular_name)
delete_func(resource)
def setup(self):
utils.init_sahara_context(self)
self.context["sahara"]["images"] = {}
# The user may want to use the existing image. In this case he should
# make sure that the image is public and has all required metadata.
image_uuid = self.config.get("image_uuid")
self.context["sahara"]["need_image_cleanup"] = not image_uuid
if image_uuid:
# Using the first user to check the existing image.
user = self.context["users"][0]
clients = osclients.Clients(user["credential"])
image = clients.glance().images.get(image_uuid)
visibility = None
if hasattr(image, "is_public"):
visibility = "public" if image.is_public else "private"
else:
visibility = image["visibility"]
if visibility != "public":
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(),
msg="Use only public image for sahara_image context"
)
image_id = image_uuid
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
self.context["tenants"][tenant_id]["sahara"]["image"] = (
image_id)
else:
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
image_id = self._create_image(
hadoop_version=self.config["hadoop_version"],
image_url=self.config["image_url"],
plugin_name=self.config["plugin_name"],
user=user,
user_name=self.config["username"])
self.context["tenants"][tenant_id]["sahara"]["image"] = (
image_id)
def setup(self):
template = self._prepare_stack_template(self.config["resources_per_stack"])
for user, tenant_id in rutils.iterate_per_tenants(self.context["users"]):
heat_scenario = heat_utils.HeatScenario(clients=osclients.Clients(user["endpoint"]))
self.context["tenants"][tenant_id]["stacks"] = []
for i in range(self.config["stacks_per_tenant"]):
stack = heat_scenario._create_stack(template)
self.context["tenants"][tenant_id]["stacks"].append(stack.id)
def cleanup(self):
for user, tenant_id in utils.iterate_per_tenants(
self.context["users"]):
with logging.ExceptionLogger(
LOG, _("Unable to delete secgroup: %s.") %
user["secgroup"]["name"]):
clients = osclients.Clients(user["endpoint"])
clients.nova().security_groups.get(
user["secgroup"]["id"]).delete()
def setup(self):
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
nova_scenario = nova_utils.NovaScenario({
"user":
user,
"task":
self.context["task"],
"config": {
"api_versions":
self.context["config"].get("api_versions", [])
}
})
keypair = nova_scenario._create_keypair()
magnum_scenario = magnum_utils.MagnumScenario({
"user":
user,
"task":
self.context["task"],
"owner_id":
self.context["owner_id"],
"config": {
"api_versions":
self.context["config"].get("api_versions", [])
}
})
# create a cluster
ct_uuid = self.config.get("cluster_template_uuid", None)
if ct_uuid is None:
ctx = self.context["tenants"][tenant_id]
ct_uuid = ctx.get("cluster_template")
cluster = magnum_scenario._create_cluster(
cluster_template=ct_uuid,
node_count=self.config.get("node_count"),
keypair=keypair)
self.context["tenants"][tenant_id]["cluster"] = cluster.uuid
def setup(self):
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
magnum_scenario = magnum_utils.MagnumScenario({
"user":
user,
"task":
self.context["task"],
"owner_id":
self.context["owner_id"],
"config": {
"api_versions":
self.context["config"].get("api_versions", [])
}
})
cluster_template = magnum_scenario._create_cluster_template(
**self.config)
ct_uuid = cluster_template.uuid
self.context["tenants"][tenant_id]["cluster_template"] = ct_uuid
def setup(self):
"""This method is called before the task start."""
try:
for user in self.context['users']:
osclient = osclients.Clients(user['credential'])
keystone = osclient.keystone()
creds = keystone.ec2.list(user['id'])
if not creds:
creds = keystone.ec2.create(user['id'], user['tenant_id'])
else:
creds = creds[0]
url = keystone.service_catalog.url_for(service_type='ec2')
url_parts = url.rpartition(':')
nova_url = (url_parts[0] + ':8773/' +
url_parts[2].partition('/')[2])
self.context['users'][0]['ec2args'] = {
'region': 'RegionOne',
'url': url,
'nova_url': nova_url,
'access': creds.access,
'secret': creds.secret
}
if self.net_wrapper.SERVICE_IMPL == consts.Service.NEUTRON:
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
body = {"quota": {"router": -1, "floatingip": -1}}
self.net_wrapper.client.update_quota(tenant_id, body)
network = self.net_wrapper.create_network(tenant_id,
add_router=True,
subnets_num=1)
self.context["tenants"][tenant_id]["network"] = network
except Exception as e:
msg = "Can't prepare ec2 client: %s" % e.message
if logging.is_debug():
LOG.exception(msg)
else:
LOG.warning(msg)
def setup(self):
image = self.config["image"]
flavor = self.config["flavor"]
auto_nic = self.config["auto_assign_nic"]
servers_per_tenant = self.config["servers_per_tenant"]
clients = osclients.Clients(self.context["users"][0]["endpoint"])
image_id = types.ImageResourceType.transform(clients=clients,
resource_config=image)
flavor_id = types.FlavorResourceType.transform(clients=clients,
resource_config=flavor)
for iter_, (user, tenant_id) in enumerate(rutils.iterate_per_tenants(
self.context["users"])):
LOG.debug("Booting servers for user tenant %s "
% (user["tenant_id"]))
tenant = self.context["tenants"][tenant_id]
nova_scenario = nova_utils.NovaScenario({"user": user,
"tenant": tenant,
"iteration": iter_})
LOG.debug("Calling _boot_servers with image_id=%(image_id)s "
"flavor_id=%(flavor_id)s "
"servers_per_tenant=%(servers_per_tenant)s"
% {"image_id": image_id,
"flavor_id": flavor_id,
"servers_per_tenant": servers_per_tenant})
servers = nova_scenario._boot_servers(image_id, flavor_id,
requests=servers_per_tenant,
auto_assign_nic=auto_nic)
current_servers = [server.id for server in servers]
LOG.debug("Adding booted servers %s to context"
% current_servers)
self.context["tenants"][tenant_id][
"servers"] = current_servers
def setup(self):
for user, tenant_id in (utils.iterate_per_tenants(
self.context.get("users", []))):
manila_scenario = manila_utils.ManilaScenario({
"task":
self.task,
"owner_id":
self.context["owner_id"],
"user":
user,
"config": {
"api_versions":
self.context["config"].get("api_versions", [])
}
})
self._create_shares(
manila_scenario,
tenant_id,
self.config["share_proto"],
self.config["size"],
self.config["share_type"],
)
def setup(self):
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
magnum_scenario = magnum_utils.MagnumScenario({
"user": user,
"task": self.context["task"],
"config": {"api_versions": self.context["config"].get(
"api_versions", [])}
})
# get the cluster and cluster_template
cluster_uuid = str(self.context["tenants"][tenant_id]["cluster"])
cluster = magnum_scenario._get_cluster(cluster_uuid)
cluster_template = magnum_scenario._get_cluster_template(
cluster.cluster_template_id)
if not cluster_template.tls_disabled:
tls = self._generate_csr_and_key()
dir = ""
if self.config.get("directory") is not None:
dir = self.config.get("directory")
self.context["ca_certs_directory"] = dir
fname = os.path.join(dir, cluster_uuid + ".key")
with open(fname, "w") as key_file:
key_file.write(tls["key"])
# get CA certificate for this cluster
ca_cert = magnum_scenario._get_ca_certificate(cluster_uuid)
fname = os.path.join(dir, cluster_uuid + "_ca.crt")
with open(fname, "w") as ca_cert_file:
ca_cert_file.write(ca_cert.pem)
# send csr to Magnum to have it signed
csr_req = {"cluster_uuid": cluster_uuid,
"csr": tls["csr"]}
cert = magnum_scenario._create_ca_certificate(csr_req)
fname = os.path.join(dir, cluster_uuid + ".crt")
with open(fname, "w") as cert_file:
cert_file.write(cert.pem)
def _remove_default_security_group(self):
"""Delete default security group for tenants."""
clients = osclients.Clients(self.credential)
if consts.Service.NEUTRON not in clients.services().values():
return
use_sg, msg = network.wrap(clients, self).supports_extension(
"security-group")
if not use_sg:
LOG.debug("Security group context is disabled: %s" % msg)
return
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
with logging.ExceptionLogger(
LOG, "Unable to delete default security group"):
uclients = osclients.Clients(user["credential"])
security_groups = uclients.neutron().list_security_groups()
default = [sg for sg in security_groups["security_groups"]
if sg["name"] == "default"]
if default:
clients.neutron().delete_security_group(default[0]["id"])
def setup(self):
self.context["sahara_output_conf"] = {
"output_type": self.config["output_type"],
"output_url_prefix": self.config["output_url_prefix"]
}
self.context["sahara_mains"] = {}
self.context["sahara_libs"] = {}
input_type = self.config["input_type"]
input_url = self.config["input_url"]
mains = self.config.get("mains", [])
libs = self.config.get("libs", [])
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
clients = osclients.Clients(user["endpoint"])
sahara = clients.sahara()
self.setup_inputs(sahara, tenant_id, input_type, input_url)
self.context["tenants"][tenant_id]["sahara_mains"] = []
self.context["tenants"][tenant_id]["sahara_libs"] = []
for main in mains:
self.download_and_save_lib(sahara=sahara,
lib_type="sahara_mains",
name=main["name"],
download_url=main["download_url"],
tenant_id=tenant_id)
for lib in libs:
self.download_and_save_lib(sahara=sahara,
lib_type="sahara_libs",
name=lib["name"],
download_url=lib["download_url"],
tenant_id=tenant_id)
def setup(self):
image = self.config["image"]
image_id = None
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
LOG.info("Booting servers for user tenant %s "
% (user["tenant_id"]))
args = user['ec2args']
client = botocoreclient.get_ec2_client(
args['url'], args['region'], args['access'], args['secret'])
if image_id is None:
data = client.describe_images(
Filters=[{'Name': 'name', 'Values': [image]},
{'Name': 'image-type', 'Values': ['machine']}])
image_id = data['Images'][0]['ImageId']
self.context["tenants"][tenant_id]["servers"] = list()
self.context["tenants"][tenant_id]["networks"] = list()
self.run_instances(tenant_id, client, image_id)
self.wait_for_instances(tenant_id, client)
self.assign_floating_ips(tenant_id, client)
def _setup_for_autocreated_users(self):
# Create share network for each network of tenant
for user, tenant_id in (utils.iterate_per_tenants(
self.context.get("users", []))):
networks = self.context["tenants"][tenant_id].get("networks")
manila_scenario = manila_utils.ManilaScenario({
"task": self.task,
"user": user,
"config": {
"api_versions": self.context["config"].get(
"api_versions", [])}
})
self.context["tenants"][tenant_id][CONTEXT_NAME] = {
"share_networks": []}
data = {}
def _setup_share_network(tenant_id, data):
share_network = manila_scenario._create_share_network(
**data).to_dict()
self.context["tenants"][tenant_id][CONTEXT_NAME][
"share_networks"].append(share_network)
if networks:
for network in networks:
if network.get("cidr"):
data["nova_net_id"] = network["id"]
elif network.get("subnets"):
data["neutron_net_id"] = network["id"]
data["neutron_subnet_id"] = network["subnets"][0]
else:
LOG.warning(_(
"Can not determine network service provider. "
"Share network will have no data."))
_setup_share_network(tenant_id, data)
else:
_setup_share_network(tenant_id, data)
def setup(self):
utils.init_sahara_context(self)
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
clients = osclients.Clients(user["credential"])
sahara = clients.sahara()
self.context["tenants"][tenant_id]["sahara"]["mains"] = []
self.context["tenants"][tenant_id]["sahara"]["libs"] = []
for main in self.config.get("mains", []):
self.download_and_save_lib(sahara=sahara,
lib_type="mains",
name=main["name"],
download_url=main["download_url"],
tenant_id=tenant_id)
for lib in self.config.get("libs", []):
self.download_and_save_lib(sahara=sahara,
lib_type="libs",
name=lib["name"],
download_url=lib["download_url"],
tenant_id=tenant_id)
def setup(self):
utils.init_sahara_context(self)
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
clients = osclients.Clients(user["credential"])
sahara = clients.sahara()
if self.config["output_type"] == "swift":
swift = swift_utils.SwiftScenario(clients=clients,
context=self.context)
container_name = self.generate_random_name()
self.context["tenants"][tenant_id]["sahara"]["container"] = {
"name":
swift._create_container(container_name=container_name),
"output_swift_objects": []
}
self.setup_outputs_swift(swift, sahara, tenant_id,
container_name,
user["credential"].username,
user["credential"].password)
else:
self.setup_outputs_hdfs(sahara, tenant_id,
self.config["output_url_prefix"])
def setup(self):
image = self.config["image"]
flavor = self.config["flavor"]
servers_per_tenant = self.config["servers_per_tenant"]
clients = osclients.Clients(self.context["users"][0]["endpoint"])
image_id = types.ImageResourceType.transform(clients=clients,
resource_config=image)
flavor_id = types.FlavorResourceType.transform(clients=clients,
resource_config=flavor)
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
LOG.debug("Booting servers for user tenant %s "
% (user["tenant_id"]))
clients = osclients.Clients(user["endpoint"])
nova_scenario = nova_utils.NovaScenario(clients=clients)
server_name_prefix = nova_scenario._generate_random_name(
prefix="ctx_rally_server_")
LOG.debug("Calling _boot_servers with server_name_prefix=%s "
"image_id=%s flavor_id=%s servers_per_tenant=%s"
% (server_name_prefix, image_id,
flavor_id, flavor_id))
servers = nova_scenario._boot_servers(
server_name_prefix, image_id, flavor_id,
servers_per_tenant)
current_servers = [server.id for server in servers]
LOG.debug("Adding booted servers %s to context"
% current_servers)
self.context["tenants"][tenant_id][
"servers"] = current_servers
def setup(self):
for user, tenant_id in (utils.iterate_per_tenants(
self.context.get("users", []))):
self.context["tenants"][tenant_id][CONTEXT_NAME] = {
"security_services": [],
}
if self.config["security_services"]:
manila_scenario = manila_utils.ManilaScenario({
"task":
self.task,
"owner_id":
self.context["owner_id"],
"user":
user,
"config": {
"api_versions":
self.context["config"].get("api_versions", [])
}
})
for ss in self.config["security_services"]:
inst = manila_scenario._create_security_service(
**ss).to_dict()
self.context["tenants"][tenant_id][CONTEXT_NAME][
"security_services"].append(inst)
def setup(self):
template = get_data(self.config["template"])
files = {}
for key, filename in self.config.get("files", {}).items():
files[key] = get_data(filename)
parameters = self.config.get("parameters", rutils.LockedDict())
with parameters.unlocked():
if "network_id" not in parameters:
parameters["network_id"] = self._get_public_network_id()
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
for name, path in self.config.get("context_parameters",
{}).items():
parameters[name] = self._get_context_parameter(
user, tenant_id, path)
if "router_id" not in parameters:
networks = self.context["tenants"][tenant_id]["networks"]
parameters["router_id"] = networks[0]["router_id"]
if "key_name" not in parameters:
parameters["key_name"] = user["keypair"]["name"]
heat_scenario = heat_utils.HeatScenario({
"user":
user,
"task":
self.context["task"],
"owner_id":
self.context["owner_id"]
})
self.context["tenants"][tenant_id]["stack_dataplane"] = []
for i in range(self.config["stacks_per_tenant"]):
stack = heat_scenario._create_stack(template,
files=files,
parameters=parameters)
tenant_data = self.context["tenants"][tenant_id]
tenant_data["stack_dataplane"].append(
[stack.id, template, files, parameters])
def setup(self):
net_wrapper = network_wrapper.wrap(osclients.Clients(
self.context["admin"]["credential"]),
self,
config=self.config)
use_lb, msg = net_wrapper.supports_extension("lbaas")
if not use_lb:
LOG.info(msg)
return
# Creates a lb-pool for every subnet created in network context.
for user, tenant_id in (utils.iterate_per_tenants(
self.context.get("users", []))):
for network in self.context["tenants"][tenant_id]["networks"]:
for subnet in network.get("subnets", []):
if self.config["lbaas_version"] == 1:
network.setdefault("lb_pools", []).append(
net_wrapper.create_v1_pool(tenant_id, subnet,
**self.config["pool"]))
else:
raise NotImplementedError(
"Context for LBaaS version %s not implemented." %
self.config["lbaas_version"])
def setup(self):
image_url = self.config.get("image_url")
image_type = self.config.get("image_type")
disk_format = self.config.get("disk_format")
image_container = self.config.get("image_container")
container_format = self.config.get("container_format")
images_per_tenant = self.config.get("images_per_tenant")
image_name = self.config.get("image_name")
visibility = self.config.get("visibility", "private")
min_disk = self.config.get("min_disk", 0)
min_ram = self.config.get("min_ram", 0)
image_args = self.config.get("image_args", {})
is_public = image_args.get("is_public")
if is_public:
LOG.warning(
_("The 'is_public' argument is deprecated "
"since Rally 0.10.0; specify visibility "
"arguments instead"))
if "visibility" not in self.config:
visibility = "public" if is_public else "private"
if image_type:
LOG.warning(
_("The 'image_type' argument is deprecated "
"since Rally 0.10.0; specify disk_format "
"arguments instead"))
disk_format = image_type
if image_container:
LOG.warning(
_("The 'image_container' argument is deprecated "
"since Rally 0.10.0; specify container_format "
"arguments instead"))
container_format = image_container
if image_args:
LOG.warning(
_("The 'kwargs' argument is deprecated since "
"Rally 0.10.0; specify exact arguments instead"))
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
current_images = []
clients = osclients.Clients(
user["credential"],
api_info=self.context["config"].get("api_versions"))
image_service = image.Image(
clients, name_generator=self.generate_random_name)
for i in range(images_per_tenant):
if image_name and i > 0:
cur_name = image_name + str(i)
elif image_name:
cur_name = image_name
else:
cur_name = self.generate_random_name()
image_obj = image_service.create_image(
image_name=cur_name,
container_format=container_format,
image_location=image_url,
disk_format=disk_format,
visibility=visibility,
min_disk=min_disk,
min_ram=min_ram)
current_images.append(image_obj.id)
self.context["tenants"][tenant_id]["images"] = current_images
def publish(queue):
users = self.context.get("users", [])
for user, tenant_id in utils.iterate_per_tenants(users):
queue.append((user, tenant_id))
def setup(self):
image_url = self.config.get("image_url")
disk_format = self.config.get("disk_format")
container_format = self.config.get("container_format")
images_per_tenant = self.config.get("images_per_tenant")
visibility = self.config.get("visibility", "private")
min_disk = self.config.get("min_disk", 0)
min_ram = self.config.get("min_ram", 0)
image_args = self.config.get("image_args", {})
if "image_type" in self.config:
LOG.warning("The 'image_type' argument is deprecated since "
"Rally 0.10.0, use disk_format argument instead")
if not disk_format:
disk_format = self.config["image_type"]
if "image_container" in self.config:
LOG.warning("The 'image_container' argument is deprecated since "
"Rally 0.10.0; use container_format argument instead")
if not container_format:
container_format = self.config["image_container"]
if image_args:
LOG.warning(
"The 'image_args' argument is deprecated since Rally 0.10.0; "
"specify arguments in a root section of context instead")
if "is_public" in image_args:
if "visibility" not in self.config:
visibility = ("public" if image_args["is_public"]
else "private")
if "min_ram" in image_args:
if "min_ram" not in self.config:
min_ram = image_args["min_ram"]
if "min_disk" in image_args:
if "min_disk" not in self.config:
min_disk = image_args["min_disk"]
# None image_name means that image.Image will generate a random name
image_name = None
if "image_name" in self.config and images_per_tenant == 1:
image_name = self.config["image_name"]
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
current_images = []
clients = osclients.Clients(
user["credential"],
api_info=self.context["config"].get("api_versions"))
image_service = image.Image(
clients, name_generator=self.generate_random_name)
for i in range(images_per_tenant):
image_obj = image_service.create_image(
image_name=image_name,
container_format=container_format,
image_location=image_url,
disk_format=disk_format,
visibility=visibility,
min_disk=min_disk,
min_ram=min_ram)
current_images.append(image_obj.id)
self.context["tenants"][tenant_id]["images"] = current_images