def setup(self):
utils.init_sahara_context(self)
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
clients = osclients.Clients(user["endpoint"])
sahara = clients.sahara()
if self.config["output_type"] == "swift":
swift = swift_utils.SwiftScenario(clients=clients,
context=self.context)
container_name = rutils.generate_random_name(
prefix=self.config["output_url_prefix"])
self.context["tenants"][tenant_id]["sahara"]["container"] = {
"name": swift._create_container(
container_name=container_name),
"output_swift_objects": []
}
self.setup_outputs_swift(swift, sahara, tenant_id,
container_name,
user["endpoint"].username,
user["endpoint"].password)
else:
self.setup_outputs_hdfs(sahara, tenant_id,
self.config["output_url_prefix"])
def setup(self):
utils.init_sahara_context(self)
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
clients = osclients.Clients(user["credential"])
sahara = clients.sahara()
self.context["tenants"][tenant_id]["sahara"]["mains"] = []
self.context["tenants"][tenant_id]["sahara"]["libs"] = []
for main in self.config.get("mains", []):
self.download_and_save_lib(
sahara=sahara,
lib_type="mains",
name=main["name"],
download_url=main["download_url"],
tenant_id=tenant_id)
for lib in self.config.get("libs", []):
self.download_and_save_lib(
sahara=sahara,
lib_type="libs",
name=lib["name"],
download_url=lib["download_url"],
tenant_id=tenant_id)
def setup(self):
utils.init_sahara_context(self)
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
clients = osclients.Clients(user["credential"])
sahara = clients.sahara()
self.context["tenants"][tenant_id]["sahara"]["mains"] = []
self.context["tenants"][tenant_id]["sahara"]["libs"] = []
for main in self.config.get("mains", []):
self.download_and_save_lib(
sahara=sahara,
lib_type="mains",
name=main["name"],
download_url=main["download_url"],
tenant_id=tenant_id)
for lib in self.config.get("libs", []):
self.download_and_save_lib(
sahara=sahara,
lib_type="libs",
name=lib["name"],
download_url=lib["download_url"],
tenant_id=tenant_id)
def setup(self):
utils.init_sahara_context(self)
self.context["sahara"]["clusters"] = {}
wait_dict = {}
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
image_id = self.context["tenants"][tenant_id]["sahara"]["image"]
floating_ip_pool = self.config.get("floating_ip_pool")
temporary_context = {
"user": user,
"tenant": self.context["tenants"][tenant_id],
"task": self.context["task"],
"owner_id": self.context["owner_id"]
}
scenario = utils.SaharaScenario(context=temporary_context)
cluster = scenario._launch_cluster(
plugin_name=self.config["plugin_name"],
hadoop_version=self.config["hadoop_version"],
flavor_id=self.config.get("flavor_id"),
master_flavor_id=self.config["master_flavor_id"],
worker_flavor_id=self.config["worker_flavor_id"],
workers_count=self.config["workers_count"],
image_id=image_id,
floating_ip_pool=floating_ip_pool,
volumes_per_node=self.config.get("volumes_per_node"),
volumes_size=self.config.get("volumes_size", 1),
auto_security_group=self.config.get("auto_security_group",
True),
security_groups=self.config.get("security_groups"),
node_configs=self.config.get("node_configs"),
cluster_configs=self.config.get("cluster_configs"),
enable_anti_affinity=self.config.get("enable_anti_affinity",
False),
enable_proxy=self.config.get("enable_proxy", False),
wait_active=False,
use_autoconfig=self.config.get("use_autoconfig", True)
)
self.context["tenants"][tenant_id]["sahara"]["cluster"] = (
cluster.id)
# Need to save the client instance to poll for active status
wait_dict[cluster] = scenario.clients("sahara")
bench_utils.wait_for(
resource=wait_dict,
update_resource=self.update_clusters_dict,
is_ready=self.all_clusters_active,
timeout=CONF.benchmark.sahara_cluster_create_timeout,
check_interval=CONF.benchmark.sahara_cluster_check_interval)
def setup(self):
utils.init_sahara_context(self)
self.context["sahara"]["clusters"] = {}
wait_dict = {}
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
image_id = self.context["tenants"][tenant_id]["sahara"]["image"]
floating_ip_pool = self.config.get("floating_ip_pool")
temporary_context = {
"user": user,
"tenant": self.context["tenants"][tenant_id],
"task": self.context["task"],
"owner_id": self.context["owner_id"]
}
scenario = utils.SaharaScenario(context=temporary_context)
cluster = scenario._launch_cluster(
plugin_name=self.config["plugin_name"],
hadoop_version=self.config["hadoop_version"],
flavor_id=self.config.get("flavor_id"),
master_flavor_id=self.config["master_flavor_id"],
worker_flavor_id=self.config["worker_flavor_id"],
workers_count=self.config["workers_count"],
image_id=image_id,
floating_ip_pool=floating_ip_pool,
volumes_per_node=self.config.get("volumes_per_node"),
volumes_size=self.config.get("volumes_size", 1),
auto_security_group=self.config.get("auto_security_group",
True),
security_groups=self.config.get("security_groups"),
node_configs=self.config.get("node_configs"),
cluster_configs=self.config.get("cluster_configs"),
enable_anti_affinity=self.config.get("enable_anti_affinity",
False),
enable_proxy=self.config.get("enable_proxy", False),
wait_active=False,
use_autoconfig=self.config.get("use_autoconfig", True)
)
self.context["tenants"][tenant_id]["sahara"]["cluster"] = (
cluster.id)
# Need to save the client instance to poll for active status
wait_dict[cluster] = scenario.clients("sahara")
bench_utils.wait_for(
resource=wait_dict,
update_resource=self.update_clusters_dict,
is_ready=self.all_clusters_active,
timeout=CONF.openstack.sahara_cluster_create_timeout,
check_interval=CONF.openstack.sahara_cluster_check_interval)
def setup(self):
utils.init_sahara_context(self)
self.context["sahara"]["images"] = {}
# The user may want to use the existing image. In this case he should
# make sure that the image is public and has all required metadata.
image_uuid = self.config.get("image_uuid")
self.context["sahara"]["need_image_cleanup"] = not image_uuid
if image_uuid:
# Using the first user to check the existing image.
user = self.context["users"][0]
clients = osclients.Clients(user["credential"])
image = clients.glance().images.get(image_uuid)
visibility = None
if hasattr(image, "is_public"):
visibility = "public" if image.is_public else "private"
else:
visibility = image["visibility"]
if visibility != "public":
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(),
msg=_("Image provided in the Sahara context"
" should be public.")
)
image_id = image_uuid
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
self.context["tenants"][tenant_id]["sahara"]["image"] = (
image_id)
else:
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
image_id = self._create_image(
hadoop_version=self.config["hadoop_version"],
image_url=self.config["image_url"],
plugin_name=self.config["plugin_name"],
user=user,
user_name=self.config["username"])
self.context["tenants"][tenant_id]["sahara"]["image"] = (
image_id)
def setup(self):
utils.init_sahara_context(self)
self.context["sahara"]["images"] = {}
# The user may want to use the existing image. In this case he should
# make sure that the image is public and has all required metadata.
image_uuid = self.config.get("image_uuid")
self.context["sahara"]["need_image_cleanup"] = not image_uuid
if image_uuid:
# Using the first user to check the existing image.
user = self.context["users"][0]
clients = osclients.Clients(user["credential"])
image = clients.glance().images.get(image_uuid)
visibility = None
if hasattr(image, "is_public"):
visibility = "public" if image.is_public else "private"
else:
visibility = image["visibility"]
if visibility != "public":
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(),
msg="Use only public image for sahara_image context"
)
image_id = image_uuid
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
self.context["tenants"][tenant_id]["sahara"]["image"] = (
image_id)
else:
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
image_id = self._create_image(
hadoop_version=self.config["hadoop_version"],
image_url=self.config["image_url"],
plugin_name=self.config["plugin_name"],
user=user,
user_name=self.config["username"])
self.context["tenants"][tenant_id]["sahara"]["image"] = (
image_id)
def setup(self):
utils.init_sahara_context(self)
self.context["sahara"]["swift_objects"] = []
self.context["sahara"]["container_name"] = None
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
clients = osclients.Clients(user["credential"])
if self.config["input_type"] == "swift":
self.setup_inputs_swift(clients, tenant_id,
self.config["input_url"],
self.config["swift_files"],
user["credential"].username,
user["credential"].password)
else:
self.setup_inputs(clients, tenant_id,
self.config["input_type"],
self.config["input_url"])
def setup(self):
utils.init_sahara_context(self)
self.context["sahara"]["swift_objects"] = []
self.context["sahara"]["container_name"] = None
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
clients = osclients.Clients(user["credential"])
if self.config["input_type"] == "swift":
self.setup_inputs_swift(clients, tenant_id,
self.config["input_url"],
self.config["swift_files"],
user["credential"].username,
user["credential"].password)
else:
self.setup_inputs(clients, tenant_id,
self.config["input_type"],
self.config["input_url"])
def setup(self):
utils.init_sahara_context(self)
self.context["sahara"]["images"] = {}
# The user may want to use the existing image. In this case he should
# make sure that the image is public and has all required metadata.
image_uuid = self.config.get("image_uuid")
self.context["sahara"]["need_image_cleanup"] = not image_uuid
if image_uuid:
# Using the first user to check the existing image.
user = self.context["users"][0]
clients = osclients.Clients(user["credential"])
image = clients.glance().images.get(image_uuid)
if not image.is_public:
raise exceptions.BenchmarkSetupFailure(
"Image provided in the Sahara context should be public.")
image_id = image_uuid
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
self.context["tenants"][tenant_id]["sahara"]["image"] = (
image_id)
else:
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
image_id = self._create_image(
hadoop_version=self.config["hadoop_version"],
image_url=self.config["image_url"],
plugin_name=self.config["plugin_name"],
user=user,
user_name=self.config["username"])
self.context["tenants"][tenant_id]["sahara"]["image"] = (
image_id)
def setup(self):
utils.init_sahara_context(self)
self.context["sahara"]["images"] = {}
# The user may want to use the existing image. In this case he should
# make sure that the image is public and has all required metadata.
image_uuid = self.config.get("image_uuid")
self.context["sahara"]["need_image_cleanup"] = not image_uuid
if image_uuid:
# Using the first user to check the existing image.
user = self.context["users"][0]
clients = osclients.Clients(user["endpoint"])
image = clients.glance().images.get(image_uuid)
if not image.is_public:
raise exceptions.BenchmarkSetupFailure(
"Image provided in the Sahara context should be public.")
image_id = image_uuid
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
self.context["tenants"][tenant_id]["sahara"]["image"] = (
image_id)
else:
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
image_id = self._create_image(
hadoop_version=self.config["hadoop_version"],
image_url=self.config["image_url"],
plugin_name=self.config["plugin_name"],
user=user,
user_name=self.config["username"])
self.context["tenants"][tenant_id]["sahara"]["image"] = (
image_id)
def setup(self):
utils.init_sahara_context(self)
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
clients = osclients.Clients(user["credential"])
sahara = clients.sahara()
if self.config["output_type"] == "swift":
swift = swift_utils.SwiftScenario(clients=clients,
context=self.context)
container_name = self.generate_random_name()
self.context["tenants"][tenant_id]["sahara"]["container"] = {
"name":
swift._create_container(container_name=container_name),
"output_swift_objects": []
}
self.setup_outputs_swift(swift, sahara, tenant_id,
container_name,
user["credential"].username,
user["credential"].password)
else:
self.setup_outputs_hdfs(sahara, tenant_id,
self.config["output_url_prefix"])
