def setup(self):
"""Create containers and objects, using the broker pattern."""
threads = self.config["resource_management_workers"]
containers_per_tenant = self.config["containers_per_tenant"]
containers_num = len(self.context["tenants"]) * containers_per_tenant
LOG.debug("Creating %d containers using %d threads." %
(containers_num, threads))
containers_count = len(
self._create_containers(self.context, containers_per_tenant,
threads))
if containers_count != containers_num:
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(),
msg=_("Failed to create the requested number of containers, "
"expected %(expected)s but got %(actual)s.") % {
"expected": containers_num,
"actual": containers_count
})
objects_per_container = self.config["objects_per_container"]
objects_num = containers_num * objects_per_container
LOG.debug("Creating %d objects using %d threads." %
(objects_num, threads))
objects_count = len(
self._create_objects(self.context, objects_per_container,
self.config["object_size"], threads))
if objects_count != objects_num:
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(),
msg=_("Failed to create the requested number of objects, "
"expected %(expected)s but got %(actual)s.") % {
"expected": objects_num,
"actual": objects_count
})
def setup(self):
"""Create tenants and users, using the broker pattern."""
threads = self.config["resource_management_workers"]
LOG.debug("Creating %(tenants)d tenants using %(threads)s threads" % {
"tenants": self.config["tenants"],
"threads": threads
})
self.context["tenants"] = self._create_tenants()
if len(self.context["tenants"]) < self.config["tenants"]:
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(),
msg=_("Failed to create the requested number of tenants."))
users_num = self.config["users_per_tenant"] * self.config["tenants"]
LOG.debug("Creating %(users)d users using %(threads)s threads" % {
"users": users_num,
"threads": threads
})
self.context["users"] = self._create_users()
if len(self.context["users"]) < users_num:
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(),
msg=_("Failed to create the requested number of users."))
def _setup_for_existing_users(self):
if (self.config["use_share_networks"]
and not self.config["share_networks"]):
msg = _("Usage of share networks was enabled but for deployment "
"with existing users share networks also should be "
"specified via arg 'share_networks'")
raise exceptions.ContextSetupFailure(ctx_name=self.get_name(),
msg=msg)
# Set flag that says we will not delete/cleanup share networks
self.context[CONTEXT_NAME]["delete_share_networks"] = False
for tenant_name_or_id, share_networks in self.config[
"share_networks"].items():
# Verify project existence
for tenant in self.context["tenants"].values():
if tenant_name_or_id in (tenant["id"], tenant["name"]):
tenant_id = tenant["id"]
existing_user = None
for user in self.context["users"]:
if user["tenant_id"] == tenant_id:
existing_user = user
break
break
else:
msg = _("Provided tenant Name or ID '%s' was not found in "
"existing tenants.") % tenant_name_or_id
raise exceptions.ContextSetupFailure(ctx_name=self.get_name(),
msg=msg)
self.context["tenants"][tenant_id][CONTEXT_NAME] = {}
self.context["tenants"][tenant_id][CONTEXT_NAME][
"share_networks"] = []
manila_scenario = manila_utils.ManilaScenario(
{"user": existing_user})
existing_sns = manila_scenario._list_share_networks(
detailed=False, search_opts={"project_id": tenant_id})
for sn_name_or_id in share_networks:
# Verify share network existence
for sn in existing_sns:
if sn_name_or_id in (sn.id, sn.name):
break
else:
msg = _("Specified share network '%(sn)s' does not "
"exist for tenant '%(tenant_id)s'") % {
"sn": sn_name_or_id,
"tenant_id": tenant_id
}
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(), msg=msg)
# Set share network for project
self.context["tenants"][tenant_id][CONTEXT_NAME][
"share_networks"].append(sn)
# Add shared integer var per project that will be used as index
# for list with share networks. It is required for balancing.
self.context["tenants"][tenant_id][CONTEXT_NAME]["sn_iterator"] = (
utils.RAMInt())
def setup(self):
"""Create tenants and users, using the broker pattern."""
super(UserGenerator, self).setup()
self.context["users"] = []
self.context["tenants"] = {}
self.context["user_choice_method"] = self.config["user_choice_method"]
threads = self.config["resource_management_workers"]
LOG.debug("Creating %(tenants)d tenants using %(threads)s threads" % {
"tenants": self.config["tenants"],
"threads": threads
})
self.context["tenants"] = self._create_tenants()
if len(self.context["tenants"]) < self.config["tenants"]:
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(),
msg=_("Failed to create the requested number of tenants."))
users_num = self.config["users_per_tenant"] * self.config["tenants"]
LOG.debug("Creating %(users)d users using %(threads)s threads" % {
"users": users_num,
"threads": threads
})
self.context["users"] = self._create_users()
for user in self.context["users"]:
self.context["tenants"][user["tenant_id"]]["users"].append(user)
if len(self.context["users"]) < users_num:
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(),
msg=_("Failed to create the requested number of users."))
def _setup_for_existing_users(self):
if (self.config["use_share_networks"]
and not self.config["share_networks"]):
msg = _("Usage of share networks was enabled but for deployment "
"with existing users share networks also should be "
"specified via arg 'share_networks'")
raise exceptions.ContextSetupFailure(ctx_name=self.get_name(),
msg=msg)
for tenant_name_or_id, share_networks in self.config[
"share_networks"].items():
# Verify project existence
for tenant in self.context["tenants"].values():
if tenant_name_or_id in (tenant["id"], tenant["name"]):
tenant_id = tenant["id"]
existing_user = None
for user in self.context["users"]:
if user["tenant_id"] == tenant_id:
existing_user = user
break
break
else:
msg = _("Provided tenant Name or ID '%s' was not found in "
"existing tenants.") % tenant_name_or_id
raise exceptions.ContextSetupFailure(ctx_name=self.get_name(),
msg=msg)
self.context["tenants"][tenant_id][CONTEXT_NAME] = {}
self.context["tenants"][tenant_id][CONTEXT_NAME][
"share_networks"] = []
manila_scenario = manila_utils.ManilaScenario({
"user": existing_user,
"config": {
"api_versions":
self.context["config"].get("api_versions", [])
}
})
existing_sns = manila_scenario._list_share_networks(
detailed=False, search_opts={"project_id": tenant_id})
for sn_name_or_id in share_networks:
# Verify share network existence
for sn in existing_sns:
if sn_name_or_id in (sn.id, sn.name):
break
else:
msg = _("Specified share network '%(sn)s' does not "
"exist for tenant '%(tenant_id)s'") % {
"sn": sn_name_or_id,
"tenant_id": tenant_id
}
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(), msg=msg)
# Set share network for project
self.context["tenants"][tenant_id][CONTEXT_NAME][
"share_networks"].append(sn.to_dict())
def setup(self):
is_config_app_dir = False
pckg_path = os.path.expanduser(self.config["app_package"])
if zipfile.is_zipfile(pckg_path):
zip_name = pckg_path
elif os.path.isdir(pckg_path):
is_config_app_dir = True
zip_name = fileutils.pack_dir(pckg_path)
else:
msg = (_LE("There is no zip archive or directory by this path:"
" %s") % pckg_path)
raise exceptions.ContextSetupFailure(msg=msg,
ctx_name=self.get_name())
for user, tenant_id in utils.iterate_per_tenants(
self.context["users"]):
clients = osclients.Clients(user["endpoint"])
self.context["tenants"][tenant_id]["packages"] = []
if is_config_app_dir:
self.context["tenants"][tenant_id]["murano_ctx"] = zip_name
package = clients.murano().packages.create(
{
"categories": ["Web"],
"tags": ["tag"]
}, {"file": open(zip_name)})
self.context["tenants"][tenant_id]["packages"].append(package)
def setup(self):
is_config_app_dir = False
pckg_path = os.path.expanduser(self.config["app_package"])
if zipfile.is_zipfile(pckg_path):
zip_name = pckg_path
elif os.path.isdir(pckg_path):
is_config_app_dir = True
zip_name = mutils.pack_dir(pckg_path)
else:
msg = "There is no zip archive or directory by this path: %s"
raise exceptions.ContextSetupFailure(msg=msg % pckg_path,
ctx_name=self.get_name())
for user, tenant_id in self._iterate_per_tenants():
clients = osclients.Clients(user["credential"])
self.context["tenants"][tenant_id]["packages"] = []
if is_config_app_dir:
self.context["tenants"][tenant_id]["murano_ctx"] = zip_name
# TODO(astudenov): use self.generate_random_name()
with open(zip_name, "rb") as f:
file = io.BytesIO(f.read())
package = clients.murano().packages.create(
{
"categories": ["Web"],
"tags": ["tag"]
}, {"file": file})
self.context["tenants"][tenant_id]["packages"].append(package)
def setup(self):
# FIXME(andreykurilin): move all checks to validate method.
# use admin only when `service_name` is presented
admin_clients = osclients.Clients(
self.context.get("admin", {}).get("credential"))
clients = osclients.Clients(
random.choice(self.context["users"])["credential"])
services = clients.keystone.service_catalog.get_endpoints()
services_from_admin = None
for client_name, conf in self.config.items():
if "service_type" in conf and conf["service_type"] not in services:
raise exceptions.ValidationError(
_("There is no service with '%s' type in your environment."
) % conf["service_type"])
elif "service_name" in conf:
if not self.context.get("admin", {}).get("credential"):
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(),
msg=_("Setting 'service_name' is allowed"
" only for 'admin' user."))
if not services_from_admin:
services_from_admin = dict([
(s.name, s.type)
for s in admin_clients.keystone().services.list()
])
if conf["service_name"] not in services_from_admin:
raise exceptions.ValidationError(
_("There is no '%s' service in your environment") %
conf["service_name"])
self.context["config"]["api_versions"][client_name][
"service_type"] = services_from_admin[conf["service_name"]]
def setup(self):
new_sample = {
"counter_name": self.config["counter_name"],
"counter_type": self.config["counter_type"],
"counter_unit": self.config["counter_unit"],
"counter_volume": self.config["counter_volume"],
}
resources = []
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
self.context["tenants"][tenant_id]["samples"] = []
self.context["tenants"][tenant_id]["resources"] = []
scenario = ceilo_utils.CeilometerScenario(
context={
"user": user,
"task": self.context["task"]
})
for i in moves.xrange(self.config["resources_per_tenant"]):
samples_to_create = scenario._make_samples(
count=self.config["samples_per_resource"],
interval=self.config["timestamp_interval"],
metadata_list=self.config.get("metadata_list"),
batch_size=self.config.get("batch_size"),
**new_sample)
samples = self._store_batch_samples(
scenario, samples_to_create,
self.config.get("batches_allow_lose"))
for sample in samples:
self.context["tenants"][tenant_id]["samples"].append(
sample.to_dict())
self.context["tenants"][tenant_id]["resources"].append(
samples[0].resource_id)
resources.append((user, samples[0].resource_id))
# NOTE(boris-42): Context should wait until samples are processed
from ceilometerclient import exc
for user, resource_id in resources:
scenario = ceilo_utils.CeilometerScenario(
context={
"user": user,
"task": self.context["task"]
})
success = False
for i in range(60):
try:
scenario._get_resource(resource_id)
success = True
break
except exc.HTTPNotFound:
time.sleep(3)
if not success:
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(),
msg="Ceilometer Resource %s is not found" % resource_id)
def all_clusters_active(self, dct):
for cluster, client in dct.items():
cluster_status = cluster.status.lower()
if cluster_status == "error":
msg = ("Sahara cluster %(name)s has failed to"
" %(action)s. Reason: '%(reason)s'"
% {"name": cluster.name, "action": "start",
"reason": cluster.status_description})
raise exceptions.ContextSetupFailure(ctx_name=self.get_name(),
msg=msg)
elif cluster_status != "active":
return False
return True
def _store_batch_samples(self, scenario, batches, batches_allow_lose):
batches_allow_lose = batches_allow_lose or 0
unsuccess = 0
for i, batch in enumerate(batches, start=1):
try:
samples = scenario._create_samples(batch)
except Exception:
unsuccess += 1
LOG.warning(_("Failed to store batch %d of Ceilometer samples"
" during context creation") % i)
if unsuccess > batches_allow_lose:
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(),
msg=_("Context failed to store too many batches of samples"))
return samples
def setup(self):
utils.init_sahara_context(self)
self.context["sahara"]["images"] = {}
# The user may want to use the existing image. In this case he should
# make sure that the image is public and has all required metadata.
image_uuid = self.config.get("image_uuid")
self.context["sahara"]["need_image_cleanup"] = not image_uuid
if image_uuid:
# Using the first user to check the existing image.
user = self.context["users"][0]
clients = osclients.Clients(user["credential"])
image = clients.glance().images.get(image_uuid)
visibility = None
if hasattr(image, "is_public"):
visibility = "public" if image.is_public else "private"
else:
visibility = image["visibility"]
if visibility != "public":
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(),
msg=_("Image provided in the Sahara context"
" should be public.")
)
image_id = image_uuid
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
self.context["tenants"][tenant_id]["sahara"]["image"] = (
image_id)
else:
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
image_id = self._create_image(
hadoop_version=self.config["hadoop_version"],
image_url=self.config["image_url"],
plugin_name=self.config["plugin_name"],
user=user,
user_name=self.config["username"])
self.context["tenants"][tenant_id]["sahara"]["image"] = (
image_id)
def setup(self):
"""Create Fuel environments, using the broker pattern."""
self.context.setdefault("fuel", {})
self.context["fuel"].setdefault("environments", [])
threads = self.config["resource_management_workers"]
LOG.debug("Creating %(envs)d environments using %(threads)s threads" %
{"envs": self.config["environments"],
"threads": threads})
self.fscenario = fuel_utils.FuelScenario(self.context)
self.context["fuel"]["environments"] = self._create_envs()
if len(self.context[
"fuel"]["environments"]) != self.config["environments"]:
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(),
msg=_("failed to create the requested"
" number of environments."))
def setup(self):
client = client_heketi.HeketiClient(
host=self.config.get("server"),
user=self.config.get("username"),
key=self.config.get("secret"),
)
try:
if client.hello():
self.context["heketi_client"] = client
LOG.debug("Successfully connected to the Heketi server.")
return
msg = "Failed to connect to the Heketi server."
except Exception as e:
msg = "Can't connect to the Heketi server: %s" % e.message
if logging.is_debug():
LOG.exception(msg)
else:
LOG.warning(msg)
raise exceptions.ContextSetupFailure(ctx_name=CONTEXT_NAME, msg=msg)
def setup(self):
# FIXME(andreykurilin): move all checks to validate method.
# use admin only when `service_name` is presented
admin_clients = osclients.Clients(
self.context.get("admin", {}).get("credential"))
clients = osclients.Clients(random.choice(
self.context["users"])["credential"])
services = clients.keystone.service_catalog.get_endpoints()
services_from_admin = None
for client_name, conf in self.config.items():
if "service_type" in conf and conf["service_type"] not in services:
raise exceptions.ValidationError(
"There is no service with '%s' type in your environment."
% conf["service_type"])
elif "service_name" in conf:
if not self.context.get("admin", {}).get("credential"):
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(),
msg="Setting 'service_name' is admin only operation.")
if not services_from_admin:
services_from_admin = dict(
[(s.name, s.type)
for s in admin_clients.keystone().services.list()])
if conf["service_name"] not in services_from_admin:
raise exceptions.ValidationError(
"There is no '%s' service in your environment"
% conf["service_name"])
# TODO(boris-42): Use separate key ["openstack"]["versions"]
self.context["config"]["api_versions@openstack"][client_name][
"service_type"] = services_from_admin[conf["service_name"]]
admin_cred = self.context.get("admin", {}).get("credential")
if admin_cred:
admin_cred["api_info"].update(
self.context["config"]["api_versions@openstack"]
)
for user in self.context["users"]:
user["credential"]["api_info"].update(
self.context["config"]["api_versions@openstack"]
)
def _setup_neutron_floating_ip_pool(self, name_or_id):
if name_or_id:
if uuidutils.is_uuid_like(name_or_id):
# Looks like an id is provided Return as is.
return name_or_id
else:
# It's a name. Changing to id.
for net in self.clients("neutron").list_networks()["networks"]:
if net["name"] == name_or_id:
return net["id"]
# If the name is not found in the list. Exit with error.
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(),
msg="Could not resolve Floating IP Pool name %s to id" %
name_or_id)
else:
# Pool is not provided. Using the one set as GW for current router.
net = self.context["tenant"]["networks"][0]
router_id = net["router_id"]
router = self.clients("neutron").show_router(router_id)["router"]
net_id = router["external_gateway_info"]["network_id"]
return net_id
