Beispiel #1
0
    def setup(self):
        # The user may want to use the existing image. In this case he should
        # make sure that the image is public and has all required metadata.

        image_uuid = self.config.get("image_uuid")

        self.context["need_sahara_image_cleanup"] = not image_uuid

        if image_uuid:
            # Using the first user to check the existing image.
            user = self.context["users"][0]
            clients = osclients.Clients(user["endpoint"])

            image = clients.glance().images.get(image_uuid)

            if not image.is_public:
                raise exceptions.BenchmarkSetupFailure(
                    "Image provided in the Sahara context should be public.")
            image_id = image_uuid

            for user, tenant_id in rutils.iterate_per_tenants(
                    self.context["users"]):
                self.context["tenants"][tenant_id]["sahara_image"] = image_id
        else:
            for user, tenant_id in rutils.iterate_per_tenants(
                    self.context["users"]):

                image_id = self._create_image(
                    hadoop_version=self.config["hadoop_version"],
                    image_url=self.config["image_url"],
                    plugin_name=self.config["plugin_name"],
                    user=user,
                    user_name=self.config["username"])

                self.context["tenants"][tenant_id]["sahara_image"] = image_id
Beispiel #2
0
    def setup(self):
        image = self.config["image"]
        flavor = self.config["flavor"]

        clients = osclients.Clients(self.context["users"][0]["credential"])
        image_id = types.EC2ImageResourceType.transform(clients=clients,
                                                        resource_config=image)

        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):
            LOG.debug("Booting servers for tenant %s "
                      % (user["tenant_id"]))
            user_clients = osclients.Clients(user["credential"])
            ec2_scenario = ec2_utils.EC2Scenario(clients=user_clients)

            LOG.debug(
                "Calling _boot_servers with "
                "image_id={image_id} flavor_name={flavor_name} "
                "servers_per_tenant={servers_per_tenant}".format(
                    image_id=image_id, flavor_name=flavor["name"],
                    servers_per_tenant=self.config["servers_per_tenant"]))

            servers = ec2_scenario._boot_servers(
                image_id, flavor["name"], self.config["servers_per_tenant"])

            current_servers = [server.id for server in servers]

            self.context["tenants"][tenant_id]["ec2_servers"] = current_servers
Beispiel #3
0
    def setup(self):
        image = self.config["image"]
        flavor = self.config["flavor"]

        clients = osclients.Clients(self.context["users"][0]["credential"])
        image_id = types.EC2Image.transform(clients=clients,
                                            resource_config=image)

        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):
            LOG.debug("Booting servers for tenant %s " % user["tenant_id"])
            ec2_scenario = ec2_utils.EC2Scenario({
                "user": user,
                "task": self.context["task"],
                "owner_id": self.context["owner_id"]})

            LOG.debug(
                "Calling _boot_servers with "
                "image_id=%(image_id)s flavor_name=%(flavor_name)s "
                "servers_per_tenant=%(servers_per_tenant)s"
                % {"image_id": image_id,
                   "flavor_name": flavor["name"],
                   "servers_per_tenant": self.config["servers_per_tenant"]})

            servers = ec2_scenario._boot_servers(
                image_id, flavor["name"], self.config["servers_per_tenant"])

            current_servers = [server.id for server in servers]

            self.context["tenants"][tenant_id]["ec2_servers"] = current_servers
 def cleanup(self):
     for user, tenant_id in rutils.iterate_per_tenants(
             self.context["users"]):
         args = user['ec2args']
         client = botocoreclient.get_ec2_client(
             args['url'], args['region'], args['access'], args['secret'])
         self.cleanup_networks(tenant_id, client)
    def setup(self):
        image = self.config["image"]
        image_id = None

        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):
            LOG.info("Booting servers for user tenant %s "
                     % (user["tenant_id"]))

            args = user['ec2args']
            client = botocoreclient.APIClientEC2(
                args['url'], args['region'], args['access'], args['secret'])

            if image_id is None:
                resp, data = client.DescribeImages(
                    Filters=[{'Name': 'name', 'Values': [image]},
                             {'Name': 'image-type', 'Values': ['machine']}])
                if resp.status_code != 200:
                    LOG.error(ec2_tests_base.EC2ErrorConverter(data))
                    assert 200 == resp.status_code
                image_id = data['Images'][0]['ImageId']

            self.context["tenants"][tenant_id]["servers"] = list()
            self.context["tenants"][tenant_id]["networks"] = list()
            self.run_instances(tenant_id, client, image_id)
            self.wait_for_instances(tenant_id, client)
            self.assign_floating_ips(tenant_id, client)
Beispiel #6
0
    def setup(self):
        image_url = self.config["image_url"]
        image_type = self.config["image_type"]
        image_container = self.config["image_container"]
        images_per_tenant = self.config["images_per_tenant"]
        image_name = self.config.get("image_name")

        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):
            current_images = []
            clients = osclients.Clients(user["endpoint"])
            glance_scenario = glance_utils.GlanceScenario(
                clients=clients)
            for i in range(images_per_tenant):
                if image_name and i > 0:
                    cur_name = image_name + str(i)
                elif image_name:
                    cur_name = image_name
                else:
                    cur_name = glance_scenario._generate_random_name(
                        prefix="rally_ctx_image_")

                image = glance_scenario._create_image(
                    cur_name, image_container, image_url, image_type,
                    min_ram=self.config.get("min_ram", 0),
                    min_disk=self.config.get("min_disk", 0))
                current_images.append(image.id)

            self.context["tenants"][tenant_id]["images"] = current_images
Beispiel #7
0
 def setup(self):
     # NOTE(rkiran): Some clients are not thread-safe. Thus during
     #               multithreading/multiprocessing, it is likely the
     #               sockets are left open. This problem is eliminated by
     #               creating a connection in setup and cleanup separately.
     net_wrapper = network_wrapper.wrap(
         osclients.Clients(self.context["admin"]["credential"]),
         self, config=self.config)
     kwargs = {}
     if self.config["dns_nameservers"] is not None:
         kwargs["dns_nameservers"] = self.config["dns_nameservers"]
     for user, tenant_id in (utils.iterate_per_tenants(
             self.context.get("users", []))):
         self.context["tenants"][tenant_id]["networks"] = []
         for i in range(self.config["networks_per_tenant"]):
             # NOTE(amaretskiy): add_router and subnets_num take effect
             #                   for Neutron only.
             network_create_args = self.config["network_create_args"].copy()
             network = net_wrapper.create_network(
                 tenant_id,
                 add_router=True,
                 subnets_num=self.config["subnets_per_network"],
                 network_create_args=network_create_args,
                 **kwargs)
             self.context["tenants"][tenant_id]["networks"].append(network)
Beispiel #8
0
    def setup(self):
        is_config_app_dir = False
        pckg_path = os.path.expanduser(self.config["app_package"])
        if zipfile.is_zipfile(pckg_path):
            zip_name = pckg_path
        elif os.path.isdir(pckg_path):
            is_config_app_dir = True
            zip_name = fileutils.pack_dir(pckg_path)
        else:
            msg = "There is no zip archive or directory by this path: %s"
            raise exceptions.ContextSetupFailure(msg=msg % pckg_path,
                                                 ctx_name=self.get_name())

        for user, tenant_id in utils.iterate_per_tenants(
                self.context["users"]):
            clients = osclients.Clients(user["credential"])
            self.context["tenants"][tenant_id]["packages"] = []
            if is_config_app_dir:
                self.context["tenants"][tenant_id]["murano_ctx"] = zip_name
            # TODO(astudenov): use self.generate_random_name()
            package = clients.murano().packages.create(
                {"categories": ["Web"], "tags": ["tag"]},
                {"file": open(zip_name)})

            self.context["tenants"][tenant_id]["packages"].append(package)
    def setup(self):
        utils.init_sahara_context(self)
        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):

            clients = osclients.Clients(user["endpoint"])
            sahara = clients.sahara()

            if self.config["output_type"] == "swift":
                swift = swift_utils.SwiftScenario(clients=clients,
                                                  context=self.context)
                container_name = rutils.generate_random_name(
                    prefix=self.config["output_url_prefix"])
                self.context["tenants"][tenant_id]["sahara"]["container"] = {
                    "name": swift._create_container(
                        container_name=container_name),
                    "output_swift_objects": []
                }
                self.setup_outputs_swift(swift, sahara, tenant_id,
                                         container_name,
                                         user["endpoint"].username,
                                         user["endpoint"].password)
            else:
                self.setup_outputs_hdfs(sahara, tenant_id,
                                        self.config["output_url_prefix"])
    def setup(self):
        """This method is called before the task start."""
        try:
            for user in self.context['users']:
                osclient = osclients.Clients(user['credential'])
                keystone = osclient.keystone()
                creds = keystone.ec2.list(user['id'])
                if not creds:
                    creds = keystone.ec2.create(user['id'], user['tenant_id'])
                else:
                    creds = creds[0]
                url = keystone.service_catalog.url_for(service_type='ec2')
                user['ec2args'] = {
                    'region': 'RegionOne',
                    'url': url,
                    'access': creds.access,
                    'secret': creds.secret
                }

            if self.net_wrapper.SERVICE_IMPL == consts.Service.NEUTRON:
                for user, tenant_id in rutils.iterate_per_tenants(
                        self.context["users"]):
                    body = {"quota": {"router": -1, "floatingip": -1}}
                    self.net_wrapper.client.update_quota(tenant_id, body)
                    network = self.net_wrapper.create_network(
                        tenant_id, add_router=True, subnets_num=1)
                    self.context["tenants"][tenant_id]["network"] = network

        except Exception as e:
            msg = "Can't prepare ec2 client: %s" % e.message
            if logging.is_debug():
                LOG.exception(msg)
            else:
                LOG.warning(msg)
Beispiel #11
0
 def setup(self):
     template = get_data(self.config["template"])
     files = {}
     for key, filename in self.config.get("files", {}).items():
         files[key] = get_data(filename)
     parameters = self.config.get("parameters", rutils.LockedDict())
     with parameters.unlocked():
         if "network_id" not in parameters:
             parameters["network_id"] = self._get_public_network_id()
         for user, tenant_id in rutils.iterate_per_tenants(
                 self.context["users"]):
             for name, path in self.config.get("context_parameters",
                                               {}).items():
                 parameters[name] = self._get_context_parameter(user,
                                                                tenant_id,
                                                                path)
             if "router_id" not in parameters:
                 networks = self.context["tenants"][tenant_id]["networks"]
                 parameters["router_id"] = networks[0]["router_id"]
             if "key_name" not in parameters:
                 parameters["key_name"] = user["keypair"]["name"]
             heat_scenario = heat_utils.HeatScenario(
                 {"user": user, "task": self.context["task"],
                  "owner_id": self.context["owner_id"]})
             self.context["tenants"][tenant_id]["stack_dataplane"] = []
             for i in range(self.config["stacks_per_tenant"]):
                 stack = heat_scenario._create_stack(template, files=files,
                                                     parameters=parameters)
                 tenant_data = self.context["tenants"][tenant_id]
                 tenant_data["stack_dataplane"].append([stack.id, template,
                                                        files, parameters])
Beispiel #12
0
 def setup(self):
     new_sample = {
         "counter_name": self.config["counter_name"],
         "counter_type": self.config["counter_type"],
         "counter_unit": self.config["counter_unit"],
         "counter_volume": self.config["counter_volume"],
     }
     for user, tenant_id in rutils.iterate_per_tenants(
             self.context["users"]):
         self.context["tenants"][tenant_id]["samples"] = []
         self.context["tenants"][tenant_id]["resources"] = []
         scenario = ceilo_utils.CeilometerScenario(
             context={"user": user, "task": self.context["task"]}
         )
         for i in moves.xrange(self.config["resources_per_tenant"]):
             samples_to_create = scenario._make_samples(
                 count=self.config["samples_per_resource"],
                 interval=self.config["timestamp_interval"],
                 metadata_list=self.config.get("metadata_list"),
                 batch_size=self.config.get("batch_size"),
                 **new_sample)
             samples = self._store_batch_samples(
                 scenario, samples_to_create,
                 self.config.get("batches_allow_lose")
             )
             for sample in samples:
                 self.context["tenants"][tenant_id]["samples"].append(
                     sample.to_dict())
             self.context["tenants"][tenant_id]["resources"].append(
                 samples[0].resource_id)
Beispiel #13
0
 def setup(self):
     for user, tenant_id in utils.iterate_per_tenants(
             self.context.get("users", [])):
         net_wrapper = network_wrapper.wrap(
             osclients.Clients(user["endpoint"]), self, config=self.config)
         self.context["tenants"][tenant_id]["networks"] = (
             net_wrapper.list_networks())
Beispiel #14
0
    def setup(self):
        net_wrapper = network_wrapper.wrap(
            osclients.Clients(self.context["admin"]["endpoint"]),
            self.context["task"],
            config=self.config)

        use_lb, msg = net_wrapper.supports_extension("lbaas")
        if not use_lb:
            LOG.info(msg)
            return

        # Creates a lb-pool for every subnet created in network context.
        for user, tenant_id in (utils.iterate_per_tenants(
                self.context.get("users", []))):
            for network in self.context["tenants"][tenant_id]["networks"]:
                for subnet in network.get("subnets", []):
                    if self.config["lbaas_version"] == 1:
                        network.setdefault("lb_pools", []).append(
                            net_wrapper.create_v1_pool(
                                tenant_id,
                                subnet,
                                **self.config["pool"]))
                    else:
                        raise NotImplementedError(
                            "Context for LBaaS version %s not implemented."
                            % self.config["lbaas_version"])
Beispiel #15
0
    def setup(self):

        input_type = self.config["input_type"]
        input_url = self.config["input_url"]
        mains = self.config.get("mains", [])
        libs = self.config.get("libs", [])

        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):

            clients = osclients.Clients(user["endpoint"])
            sahara = clients.sahara()

            self.setup_inputs(sahara, tenant_id, input_type, input_url)

            self.context["tenants"][tenant_id]["sahara_mains"] = []
            self.context["tenants"][tenant_id]["sahara_libs"] = []

            for main in mains:
                self.download_and_save_lib(
                    sahara=sahara,
                    lib_type="sahara_mains",
                    name=main["name"],
                    download_url=main["download_url"],
                    tenant_id=tenant_id)

            for lib in libs:
                self.download_and_save_lib(
                    sahara=sahara,
                    lib_type="sahara_libs",
                    name=lib["name"],
                    download_url=lib["download_url"],
                    tenant_id=tenant_id)
Beispiel #16
0
    def setup(self):
        image_url = self.config["image_url"]
        plugin_name = self.config["plugin_name"]
        hadoop_version = self.config["hadoop_version"]
        user_name = self.config["username"]

        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):

            clients = osclients.Clients(user["endpoint"])
            glance_util_class = glance_utils.GlanceScenario(
                clients=clients)

            image_name = scenarios_base.Scenario._generate_random_name(
                prefix="sahara_image_", length=15)
            image = glance_util_class._create_image(image_name,
                                                    "bare",
                                                    image_url,
                                                    "qcow2")

            clients.sahara().images.update_image(image_id=image.id,
                                                 user_name=user_name,
                                                 desc="")

            clients.sahara().images.update_tags(image_id=image.id,
                                                new_tags=[plugin_name,
                                                          hadoop_version])

            self.context["tenants"][tenant_id]["sahara_image"] = image.id
Beispiel #17
0
def init_sahara_context(context_instance):
    context_instance.context["sahara"] = context_instance.context.get("sahara",
                                                                      {})
    for user, tenant_id in rutils.iterate_per_tenants(
            context_instance.context["users"]):
        context_instance.context["tenants"][tenant_id]["sahara"] = (
            context_instance.context["tenants"][tenant_id].get("sahara", {}))
Beispiel #18
0
 def publish(queue):
     for user, tenant_id in (rutils.iterate_per_tenants(
             context.get("users", []))):
         context["tenants"][tenant_id]["containers"] = []
         for i in range(containers_per_tenant):
             args = (user, context["tenants"][tenant_id]["containers"])
             queue.append(args)
Beispiel #19
0
    def cleanup(self):
        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):

            magnum_scenario = magnum_utils.MagnumScenario({
                "user": user,
                "task": self.context["task"],
                "config": {"api_versions": self.context["config"].get(
                    "api_versions", [])}
            })

            # get the cluster and cluster_template
            cluster_uuid = str(self.context["tenants"][tenant_id]["cluster"])
            cluster = magnum_scenario._get_cluster(cluster_uuid)
            cluster_template = magnum_scenario._get_cluster_template(
                cluster.cluster_template_id)

            if not cluster_template.tls_disabled:
                dir = self.context["ca_certs_directory"]
                fname = os.path.join(dir, cluster_uuid + ".key")
                os.remove(fname)
                fname = os.path.join(dir, cluster_uuid + "_ca.crt")
                os.remove(fname)
                fname = os.path.join(dir, cluster_uuid + ".crt")
                os.remove(fname)
Beispiel #20
0
    def setup(self):
        image_url = self.config["image_url"]
        image_type = self.config["image_type"]
        image_container = self.config["image_container"]
        images_per_tenant = self.config["images_per_tenant"]
        image_name = self.config.get("image_name")

        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):
            current_images = []
            glance_scenario = glance_utils.GlanceScenario(
                {"user": user, "task": self.context["task"]})
            for i in range(images_per_tenant):
                if image_name and i > 0:
                    cur_name = image_name + str(i)
                elif image_name:
                    cur_name = image_name
                else:
                    cur_name = None

                image = glance_scenario._create_image(
                    image_container, image_url, image_type,
                    name=cur_name, min_ram=self.config.get("min_ram", 0),
                    min_disk=self.config.get("min_disk", 0))
                current_images.append(image.id)

            self.context["tenants"][tenant_id]["images"] = current_images
Beispiel #21
0
    def setup(self):
        image = self.config["image"]
        flavor = self.config["flavor"]
        servers_per_tenant = self.config["servers_per_tenant"]

        clients = osclients.Clients(self.context["users"][0]["endpoint"])
        image_id = types.ImageResourceType.transform(clients=clients, resource_config=image)
        flavor_id = types.FlavorResourceType.transform(clients=clients, resource_config=flavor)

        for user, tenant_id in rutils.iterate_per_tenants(self.context["users"]):
            LOG.debug("Booting servers for user tenant %s " % (user["tenant_id"]))
            nova_scenario = nova_utils.NovaScenario({"user": user})

            LOG.debug(
                "Calling _boot_servers with image_id=%(image_id)s "
                "flavor_id=%(flavor_id)s "
                "servers_per_tenant=%(servers_per_tenant)s"
                % {"image_id": image_id, "flavor_id": flavor_id, "servers_per_tenant": servers_per_tenant}
            )

            servers = nova_scenario._boot_servers(image_id, flavor_id, servers_per_tenant)

            current_servers = [server.id for server in servers]

            LOG.debug("Adding booted servers %s to context" % current_servers)

            self.context["tenants"][tenant_id]["servers"] = current_servers
    def setup(self):
        disk_format = self.config["disk_format"]
        container_format = self.config["container_format"]
        images_per_tenant = self.config["images_per_tenant"]

        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):
            glance = osclients.Clients(user["credential"]).glance().images
            current_images = []
            for i in range(images_per_tenant):
                kw = {
                    "name": "image-" + tenant_id[0:8] + "-" + str(i),
                    "container_format": container_format,
                    "disk_format": disk_format,
                    "size": 1000000,
                }
                image = glance.create(**kw)
                current_images.append(image.id)

            self.context["tenants"][tenant_id]["images"] = current_images

            # NOTE(andrey-mp): call ec2 api to initialize it
            args = user['ec2args']
            client = botocoreclient.get_ec2_client(
                args['url'], args['region'], args['access'], args['secret'])
            data = client.describe_images()
Beispiel #23
0
    def setup(self):
        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):

            nova_scenario = nova_utils.NovaScenario({
                "user": user,
                "task": self.context["task"],
                "config": {"api_versions": self.context["config"].get(
                    "api_versions", [])}
            })
            keypair = nova_scenario._create_keypair()

            magnum_scenario = magnum_utils.MagnumScenario({
                "user": user,
                "task": self.context["task"],
                "owner_id": self.context["owner_id"],
                "config": {"api_versions": self.context["config"].get(
                    "api_versions", [])}
            })

            # create a cluster
            ct_uuid = self.config.get("cluster_template_uuid", None)
            if ct_uuid is None:
                ctx = self.context["tenants"][tenant_id]
                ct_uuid = ctx.get("cluster_template")
            cluster = magnum_scenario._create_cluster(
                cluster_template=ct_uuid,
                node_count=self.config.get("node_count"), keypair=keypair)
            self.context["tenants"][tenant_id]["cluster"] = cluster.uuid
Beispiel #24
0
 def setup(self):
     counter_name = self.config["counter_name"]
     counter_type = self.config["counter_type"]
     counter_unit = self.config["counter_unit"]
     counter_volume = self.config["counter_volume"]
     resources_per_tenant = self.config["resources_per_tenant"]
     samples_per_resource = self.config["samples_per_resource"]
     for user, tenant_id in rutils.iterate_per_tenants(
             self.context["users"]):
         self.context["tenants"][tenant_id]["samples"] = []
         self.context["tenants"][tenant_id]["resources"] = []
         clients = osclients.Clients(user["endpoint"])
         scenario = ceilo_utils.CeilometerScenario(
             clients=clients)
         for i in range(resources_per_tenant):
             for j in range(samples_per_resource):
                 try:
                     sample = scenario._create_sample(counter_name,
                                                      counter_type,
                                                      counter_unit,
                                                      counter_volume)
                     self.context["tenants"][tenant_id]["samples"].append(
                         sample[0].to_dict())
                 except Exception as e:
                     LOG.error("Creating a sample failed: %(error)s"
                               % {"error": e})
             self.context["tenants"][tenant_id]["resources"].append(
                 sample[0].resource_id)
Beispiel #25
0
    def _cleanup_tenant_resources(self, resources_plural_name,
                                  resources_singular_name):
        """Cleans up tenant resources.

        :param resources_plural_name: plural name for resources
        :param resources_singular_name: singular name for resource. Expected
            to be part of resource deletion method name (obj._delete_%s)
        """
        for user, tenant_id in (utils.iterate_per_tenants(
                self.context.get("users", []))):
            manila_scenario = manila_utils.ManilaScenario({
                "user": user,
                "config": {
                    "api_versions": self.context["config"].get(
                        "api_versions", [])}
            })
            resources = self.context["tenants"][tenant_id][CONTEXT_NAME].get(
                resources_plural_name, [])
            for resource in resources:
                logger = logging.ExceptionLogger(
                    LOG,
                    _("Failed to delete %(name)s %(id)s for tenant %(t)s.") % {
                        "id": resource, "t": tenant_id,
                        "name": resources_singular_name})
                with logger:
                    delete_func = getattr(
                        manila_scenario,
                        "_delete_%s" % resources_singular_name)
                    delete_func(resource)
    def setup(self):
        utils.init_sahara_context(self)
        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):

            clients = osclients.Clients(user["credential"])
            sahara = clients.sahara()

            self.context["tenants"][tenant_id]["sahara"]["mains"] = []
            self.context["tenants"][tenant_id]["sahara"]["libs"] = []

            for main in self.config.get("mains", []):
                self.download_and_save_lib(
                    sahara=sahara,
                    lib_type="mains",
                    name=main["name"],
                    download_url=main["download_url"],
                    tenant_id=tenant_id)

            for lib in self.config.get("libs", []):
                self.download_and_save_lib(
                    sahara=sahara,
                    lib_type="libs",
                    name=lib["name"],
                    download_url=lib["download_url"],
                    tenant_id=tenant_id)
Beispiel #27
0
    def _cleanup_tenant_resources(self, resources_plural_name,
                                  resources_singular_name):
        """Cleans up tenant resources.

        :param resources_plural_name: plural name for resources, should be
            one of "shares", "share_networks" or "security_services".
        :param resources_singular_name: singular name for resource. Expected
            to be part of resource deletion method name (obj._delete_%s)
        """
        for user, tenant_id in (utils.iterate_per_tenants(
                self.context.get("users", []))):
            clients = osclients.Clients(user["endpoint"])
            manila_scenario = manila_utils.ManilaScenario(clients=clients)
            resources = self.context["tenants"][tenant_id].get(
                resources_plural_name, [])
            for resource in resources:
                logger = log.ExceptionLogger(
                    LOG,
                    _("Failed to delete %(name)s %(id)s for tenant %(t)s.") % {
                        "id": resource, "t": tenant_id,
                        "name": resources_singular_name})
                with logger:
                    delete_func = getattr(
                        manila_scenario,
                        "_delete_%s" % resources_singular_name)
                    delete_func(resource)
Beispiel #28
0
    def setup(self):
        utils.init_sahara_context(self)
        self.context["sahara"]["images"] = {}

        # The user may want to use the existing image. In this case he should
        # make sure that the image is public and has all required metadata.
        image_uuid = self.config.get("image_uuid")

        self.context["sahara"]["need_image_cleanup"] = not image_uuid

        if image_uuid:
            # Using the first user to check the existing image.
            user = self.context["users"][0]
            clients = osclients.Clients(user["credential"])

            image = clients.glance().images.get(image_uuid)

            visibility = None
            if hasattr(image, "is_public"):
                visibility = "public" if image.is_public else "private"
            else:
                visibility = image["visibility"]

            if visibility != "public":
                raise exceptions.ContextSetupFailure(
                    ctx_name=self.get_name(),
                    msg="Use only public image for sahara_image context"
                )
            image_id = image_uuid

            for user, tenant_id in rutils.iterate_per_tenants(
                    self.context["users"]):
                self.context["tenants"][tenant_id]["sahara"]["image"] = (
                    image_id)
        else:
            for user, tenant_id in rutils.iterate_per_tenants(
                    self.context["users"]):

                image_id = self._create_image(
                    hadoop_version=self.config["hadoop_version"],
                    image_url=self.config["image_url"],
                    plugin_name=self.config["plugin_name"],
                    user=user,
                    user_name=self.config["username"])

                self.context["tenants"][tenant_id]["sahara"]["image"] = (
                    image_id)
Beispiel #29
0
 def setup(self):
     template = self._prepare_stack_template(self.config["resources_per_stack"])
     for user, tenant_id in rutils.iterate_per_tenants(self.context["users"]):
         heat_scenario = heat_utils.HeatScenario(clients=osclients.Clients(user["endpoint"]))
         self.context["tenants"][tenant_id]["stacks"] = []
         for i in range(self.config["stacks_per_tenant"]):
             stack = heat_scenario._create_stack(template)
             self.context["tenants"][tenant_id]["stacks"].append(stack.id)
Beispiel #30
0
 def cleanup(self):
     for user, tenant_id in utils.iterate_per_tenants(
             self.context["users"]):
         with logging.ExceptionLogger(
                 LOG, _("Unable to delete secgroup: %s.") %
                 user["secgroup"]["name"]):
             clients = osclients.Clients(user["endpoint"])
             clients.nova().security_groups.get(
                 user["secgroup"]["id"]).delete()
Beispiel #31
0
    def setup(self):
        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):

            nova_scenario = nova_utils.NovaScenario({
                "user":
                user,
                "task":
                self.context["task"],
                "config": {
                    "api_versions":
                    self.context["config"].get("api_versions", [])
                }
            })
            keypair = nova_scenario._create_keypair()

            magnum_scenario = magnum_utils.MagnumScenario({
                "user":
                user,
                "task":
                self.context["task"],
                "owner_id":
                self.context["owner_id"],
                "config": {
                    "api_versions":
                    self.context["config"].get("api_versions", [])
                }
            })

            # create a cluster
            ct_uuid = self.config.get("cluster_template_uuid", None)
            if ct_uuid is None:
                ctx = self.context["tenants"][tenant_id]
                ct_uuid = ctx.get("cluster_template")
            cluster = magnum_scenario._create_cluster(
                cluster_template=ct_uuid,
                node_count=self.config.get("node_count"),
                keypair=keypair)
            self.context["tenants"][tenant_id]["cluster"] = cluster.uuid
    def setup(self):
        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):

            magnum_scenario = magnum_utils.MagnumScenario({
                "user":
                user,
                "task":
                self.context["task"],
                "owner_id":
                self.context["owner_id"],
                "config": {
                    "api_versions":
                    self.context["config"].get("api_versions", [])
                }
            })

            cluster_template = magnum_scenario._create_cluster_template(
                **self.config)

            ct_uuid = cluster_template.uuid
            self.context["tenants"][tenant_id]["cluster_template"] = ct_uuid
    def setup(self):
        """This method is called before the task start."""
        try:
            for user in self.context['users']:
                osclient = osclients.Clients(user['credential'])
                keystone = osclient.keystone()
                creds = keystone.ec2.list(user['id'])
                if not creds:
                    creds = keystone.ec2.create(user['id'], user['tenant_id'])
                else:
                    creds = creds[0]
                url = keystone.service_catalog.url_for(service_type='ec2')
                url_parts = url.rpartition(':')
                nova_url = (url_parts[0] + ':8773/' +
                            url_parts[2].partition('/')[2])
                self.context['users'][0]['ec2args'] = {
                    'region': 'RegionOne',
                    'url': url,
                    'nova_url': nova_url,
                    'access': creds.access,
                    'secret': creds.secret
                }

            if self.net_wrapper.SERVICE_IMPL == consts.Service.NEUTRON:
                for user, tenant_id in rutils.iterate_per_tenants(
                        self.context["users"]):
                    body = {"quota": {"router": -1, "floatingip": -1}}
                    self.net_wrapper.client.update_quota(tenant_id, body)
                    network = self.net_wrapper.create_network(tenant_id,
                                                              add_router=True,
                                                              subnets_num=1)
                    self.context["tenants"][tenant_id]["network"] = network

        except Exception as e:
            msg = "Can't prepare ec2 client: %s" % e.message
            if logging.is_debug():
                LOG.exception(msg)
            else:
                LOG.warning(msg)
Beispiel #34
0
    def setup(self):
        image = self.config["image"]
        flavor = self.config["flavor"]
        auto_nic = self.config["auto_assign_nic"]
        servers_per_tenant = self.config["servers_per_tenant"]

        clients = osclients.Clients(self.context["users"][0]["endpoint"])
        image_id = types.ImageResourceType.transform(clients=clients,
                                                     resource_config=image)
        flavor_id = types.FlavorResourceType.transform(clients=clients,
                                                       resource_config=flavor)

        for iter_, (user, tenant_id) in enumerate(rutils.iterate_per_tenants(
                self.context["users"])):
            LOG.debug("Booting servers for user tenant %s "
                      % (user["tenant_id"]))
            tenant = self.context["tenants"][tenant_id]
            nova_scenario = nova_utils.NovaScenario({"user": user,
                                                     "tenant": tenant,
                                                     "iteration": iter_})

            LOG.debug("Calling _boot_servers with image_id=%(image_id)s "
                      "flavor_id=%(flavor_id)s "
                      "servers_per_tenant=%(servers_per_tenant)s"
                      % {"image_id": image_id,
                         "flavor_id": flavor_id,
                         "servers_per_tenant": servers_per_tenant})

            servers = nova_scenario._boot_servers(image_id, flavor_id,
                                                  requests=servers_per_tenant,
                                                  auto_assign_nic=auto_nic)

            current_servers = [server.id for server in servers]

            LOG.debug("Adding booted servers %s to context"
                      % current_servers)

            self.context["tenants"][tenant_id][
                "servers"] = current_servers
Beispiel #35
0
 def setup(self):
     for user, tenant_id in (utils.iterate_per_tenants(
             self.context.get("users", []))):
         manila_scenario = manila_utils.ManilaScenario({
             "task":
             self.task,
             "owner_id":
             self.context["owner_id"],
             "user":
             user,
             "config": {
                 "api_versions":
                 self.context["config"].get("api_versions", [])
             }
         })
         self._create_shares(
             manila_scenario,
             tenant_id,
             self.config["share_proto"],
             self.config["size"],
             self.config["share_type"],
         )
Beispiel #36
0
    def setup(self):
        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):

            magnum_scenario = magnum_utils.MagnumScenario({
                "user": user,
                "task": self.context["task"],
                "config": {"api_versions": self.context["config"].get(
                    "api_versions", [])}
            })

            # get the cluster and cluster_template
            cluster_uuid = str(self.context["tenants"][tenant_id]["cluster"])
            cluster = magnum_scenario._get_cluster(cluster_uuid)
            cluster_template = magnum_scenario._get_cluster_template(
                cluster.cluster_template_id)

            if not cluster_template.tls_disabled:
                tls = self._generate_csr_and_key()
                dir = ""
                if self.config.get("directory") is not None:
                    dir = self.config.get("directory")
                self.context["ca_certs_directory"] = dir
                fname = os.path.join(dir, cluster_uuid + ".key")
                with open(fname, "w") as key_file:
                    key_file.write(tls["key"])
                # get CA certificate for this cluster
                ca_cert = magnum_scenario._get_ca_certificate(cluster_uuid)
                fname = os.path.join(dir, cluster_uuid + "_ca.crt")
                with open(fname, "w") as ca_cert_file:
                    ca_cert_file.write(ca_cert.pem)
                # send csr to Magnum to have it signed
                csr_req = {"cluster_uuid": cluster_uuid,
                           "csr": tls["csr"]}
                cert = magnum_scenario._create_ca_certificate(csr_req)
                fname = os.path.join(dir, cluster_uuid + ".crt")
                with open(fname, "w") as cert_file:
                    cert_file.write(cert.pem)
Beispiel #37
0
    def _remove_default_security_group(self):
        """Delete default security group for tenants."""
        clients = osclients.Clients(self.credential)

        if consts.Service.NEUTRON not in clients.services().values():
            return

        use_sg, msg = network.wrap(clients, self).supports_extension(
            "security-group")
        if not use_sg:
            LOG.debug("Security group context is disabled: %s" % msg)
            return

        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):
            with logging.ExceptionLogger(
                    LOG, "Unable to delete default security group"):
                uclients = osclients.Clients(user["credential"])
                security_groups = uclients.neutron().list_security_groups()
                default = [sg for sg in security_groups["security_groups"]
                           if sg["name"] == "default"]
                if default:
                    clients.neutron().delete_security_group(default[0]["id"])
    def setup(self):
        self.context["sahara_output_conf"] = {
            "output_type": self.config["output_type"],
            "output_url_prefix": self.config["output_url_prefix"]
        }
        self.context["sahara_mains"] = {}
        self.context["sahara_libs"] = {}

        input_type = self.config["input_type"]
        input_url = self.config["input_url"]
        mains = self.config.get("mains", [])
        libs = self.config.get("libs", [])

        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):

            clients = osclients.Clients(user["endpoint"])
            sahara = clients.sahara()

            self.setup_inputs(sahara, tenant_id, input_type, input_url)

            self.context["tenants"][tenant_id]["sahara_mains"] = []
            self.context["tenants"][tenant_id]["sahara_libs"] = []

            for main in mains:
                self.download_and_save_lib(sahara=sahara,
                                           lib_type="sahara_mains",
                                           name=main["name"],
                                           download_url=main["download_url"],
                                           tenant_id=tenant_id)

            for lib in libs:
                self.download_and_save_lib(sahara=sahara,
                                           lib_type="sahara_libs",
                                           name=lib["name"],
                                           download_url=lib["download_url"],
                                           tenant_id=tenant_id)
    def setup(self):
        image = self.config["image"]
        image_id = None

        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):
            LOG.info("Booting servers for user tenant %s "
                     % (user["tenant_id"]))

            args = user['ec2args']
            client = botocoreclient.get_ec2_client(
                args['url'], args['region'], args['access'], args['secret'])

            if image_id is None:
                data = client.describe_images(
                    Filters=[{'Name': 'name', 'Values': [image]},
                             {'Name': 'image-type', 'Values': ['machine']}])
                image_id = data['Images'][0]['ImageId']

            self.context["tenants"][tenant_id]["servers"] = list()
            self.context["tenants"][tenant_id]["networks"] = list()
            self.run_instances(tenant_id, client, image_id)
            self.wait_for_instances(tenant_id, client)
            self.assign_floating_ips(tenant_id, client)
    def _setup_for_autocreated_users(self):
        # Create share network for each network of tenant
        for user, tenant_id in (utils.iterate_per_tenants(
                self.context.get("users", []))):
            networks = self.context["tenants"][tenant_id].get("networks")
            manila_scenario = manila_utils.ManilaScenario({
                "task": self.task,
                "user": user,
                "config": {
                    "api_versions": self.context["config"].get(
                        "api_versions", [])}
            })
            self.context["tenants"][tenant_id][CONTEXT_NAME] = {
                "share_networks": []}
            data = {}

            def _setup_share_network(tenant_id, data):
                share_network = manila_scenario._create_share_network(
                    **data).to_dict()
                self.context["tenants"][tenant_id][CONTEXT_NAME][
                    "share_networks"].append(share_network)

            if networks:
                for network in networks:
                    if network.get("cidr"):
                        data["nova_net_id"] = network["id"]
                    elif network.get("subnets"):
                        data["neutron_net_id"] = network["id"]
                        data["neutron_subnet_id"] = network["subnets"][0]
                    else:
                        LOG.warning(_(
                            "Can not determine network service provider. "
                            "Share network will have no data."))
                    _setup_share_network(tenant_id, data)
            else:
                _setup_share_network(tenant_id, data)
Beispiel #41
0
    def setup(self):
        utils.init_sahara_context(self)
        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):

            clients = osclients.Clients(user["credential"])
            sahara = clients.sahara()

            self.context["tenants"][tenant_id]["sahara"]["mains"] = []
            self.context["tenants"][tenant_id]["sahara"]["libs"] = []

            for main in self.config.get("mains", []):
                self.download_and_save_lib(sahara=sahara,
                                           lib_type="mains",
                                           name=main["name"],
                                           download_url=main["download_url"],
                                           tenant_id=tenant_id)

            for lib in self.config.get("libs", []):
                self.download_and_save_lib(sahara=sahara,
                                           lib_type="libs",
                                           name=lib["name"],
                                           download_url=lib["download_url"],
                                           tenant_id=tenant_id)
Beispiel #42
0
    def setup(self):
        utils.init_sahara_context(self)
        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):

            clients = osclients.Clients(user["credential"])
            sahara = clients.sahara()

            if self.config["output_type"] == "swift":
                swift = swift_utils.SwiftScenario(clients=clients,
                                                  context=self.context)
                container_name = self.generate_random_name()
                self.context["tenants"][tenant_id]["sahara"]["container"] = {
                    "name":
                    swift._create_container(container_name=container_name),
                    "output_swift_objects": []
                }
                self.setup_outputs_swift(swift, sahara, tenant_id,
                                         container_name,
                                         user["credential"].username,
                                         user["credential"].password)
            else:
                self.setup_outputs_hdfs(sahara, tenant_id,
                                        self.config["output_url_prefix"])
Beispiel #43
0
    def setup(self):
        image = self.config["image"]
        flavor = self.config["flavor"]
        servers_per_tenant = self.config["servers_per_tenant"]

        clients = osclients.Clients(self.context["users"][0]["endpoint"])
        image_id = types.ImageResourceType.transform(clients=clients,
                                                     resource_config=image)
        flavor_id = types.FlavorResourceType.transform(clients=clients,
                                                       resource_config=flavor)

        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):
            LOG.debug("Booting servers for user tenant %s "
                      % (user["tenant_id"]))
            clients = osclients.Clients(user["endpoint"])
            nova_scenario = nova_utils.NovaScenario(clients=clients)
            server_name_prefix = nova_scenario._generate_random_name(
                                                prefix="ctx_rally_server_")

            LOG.debug("Calling _boot_servers with server_name_prefix=%s "
                      "image_id=%s flavor_id=%s servers_per_tenant=%s"
                      % (server_name_prefix, image_id,
                         flavor_id, flavor_id))

            servers = nova_scenario._boot_servers(
                server_name_prefix, image_id, flavor_id,
                servers_per_tenant)

            current_servers = [server.id for server in servers]

            LOG.debug("Adding booted servers %s to context"
                      % current_servers)

            self.context["tenants"][tenant_id][
                "servers"] = current_servers
 def setup(self):
     for user, tenant_id in (utils.iterate_per_tenants(
             self.context.get("users", []))):
         self.context["tenants"][tenant_id][CONTEXT_NAME] = {
             "security_services": [],
         }
         if self.config["security_services"]:
             manila_scenario = manila_utils.ManilaScenario({
                 "task":
                 self.task,
                 "owner_id":
                 self.context["owner_id"],
                 "user":
                 user,
                 "config": {
                     "api_versions":
                     self.context["config"].get("api_versions", [])
                 }
             })
             for ss in self.config["security_services"]:
                 inst = manila_scenario._create_security_service(
                     **ss).to_dict()
                 self.context["tenants"][tenant_id][CONTEXT_NAME][
                     "security_services"].append(inst)
Beispiel #45
0
 def setup(self):
     template = get_data(self.config["template"])
     files = {}
     for key, filename in self.config.get("files", {}).items():
         files[key] = get_data(filename)
     parameters = self.config.get("parameters", rutils.LockedDict())
     with parameters.unlocked():
         if "network_id" not in parameters:
             parameters["network_id"] = self._get_public_network_id()
         for user, tenant_id in rutils.iterate_per_tenants(
                 self.context["users"]):
             for name, path in self.config.get("context_parameters",
                                               {}).items():
                 parameters[name] = self._get_context_parameter(
                     user, tenant_id, path)
             if "router_id" not in parameters:
                 networks = self.context["tenants"][tenant_id]["networks"]
                 parameters["router_id"] = networks[0]["router_id"]
             if "key_name" not in parameters:
                 parameters["key_name"] = user["keypair"]["name"]
             heat_scenario = heat_utils.HeatScenario({
                 "user":
                 user,
                 "task":
                 self.context["task"],
                 "owner_id":
                 self.context["owner_id"]
             })
             self.context["tenants"][tenant_id]["stack_dataplane"] = []
             for i in range(self.config["stacks_per_tenant"]):
                 stack = heat_scenario._create_stack(template,
                                                     files=files,
                                                     parameters=parameters)
                 tenant_data = self.context["tenants"][tenant_id]
                 tenant_data["stack_dataplane"].append(
                     [stack.id, template, files, parameters])
Beispiel #46
0
    def setup(self):
        net_wrapper = network_wrapper.wrap(osclients.Clients(
            self.context["admin"]["credential"]),
                                           self,
                                           config=self.config)

        use_lb, msg = net_wrapper.supports_extension("lbaas")
        if not use_lb:
            LOG.info(msg)
            return

        # Creates a lb-pool for every subnet created in network context.
        for user, tenant_id in (utils.iterate_per_tenants(
                self.context.get("users", []))):
            for network in self.context["tenants"][tenant_id]["networks"]:
                for subnet in network.get("subnets", []):
                    if self.config["lbaas_version"] == 1:
                        network.setdefault("lb_pools", []).append(
                            net_wrapper.create_v1_pool(tenant_id, subnet,
                                                       **self.config["pool"]))
                    else:
                        raise NotImplementedError(
                            "Context for LBaaS version %s not implemented." %
                            self.config["lbaas_version"])
Beispiel #47
0
    def setup(self):
        image_url = self.config.get("image_url")
        image_type = self.config.get("image_type")
        disk_format = self.config.get("disk_format")
        image_container = self.config.get("image_container")
        container_format = self.config.get("container_format")
        images_per_tenant = self.config.get("images_per_tenant")
        image_name = self.config.get("image_name")
        visibility = self.config.get("visibility", "private")
        min_disk = self.config.get("min_disk", 0)
        min_ram = self.config.get("min_ram", 0)
        image_args = self.config.get("image_args", {})
        is_public = image_args.get("is_public")

        if is_public:
            LOG.warning(
                _("The 'is_public' argument is deprecated "
                  "since Rally 0.10.0; specify visibility "
                  "arguments instead"))
            if "visibility" not in self.config:
                visibility = "public" if is_public else "private"

        if image_type:
            LOG.warning(
                _("The 'image_type' argument is deprecated "
                  "since Rally 0.10.0; specify disk_format "
                  "arguments instead"))
            disk_format = image_type

        if image_container:
            LOG.warning(
                _("The 'image_container' argument is deprecated "
                  "since Rally 0.10.0; specify container_format "
                  "arguments instead"))
            container_format = image_container

        if image_args:
            LOG.warning(
                _("The 'kwargs' argument is deprecated since "
                  "Rally 0.10.0; specify exact arguments instead"))

        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):
            current_images = []
            clients = osclients.Clients(
                user["credential"],
                api_info=self.context["config"].get("api_versions"))
            image_service = image.Image(
                clients, name_generator=self.generate_random_name)

            for i in range(images_per_tenant):
                if image_name and i > 0:
                    cur_name = image_name + str(i)
                elif image_name:
                    cur_name = image_name
                else:
                    cur_name = self.generate_random_name()

                image_obj = image_service.create_image(
                    image_name=cur_name,
                    container_format=container_format,
                    image_location=image_url,
                    disk_format=disk_format,
                    visibility=visibility,
                    min_disk=min_disk,
                    min_ram=min_ram)
                current_images.append(image_obj.id)

            self.context["tenants"][tenant_id]["images"] = current_images
 def publish(queue):
     users = self.context.get("users", [])
     for user, tenant_id in utils.iterate_per_tenants(users):
         queue.append((user, tenant_id))
Beispiel #49
0
    def setup(self):
        image_url = self.config.get("image_url")
        disk_format = self.config.get("disk_format")
        container_format = self.config.get("container_format")
        images_per_tenant = self.config.get("images_per_tenant")
        visibility = self.config.get("visibility", "private")
        min_disk = self.config.get("min_disk", 0)
        min_ram = self.config.get("min_ram", 0)
        image_args = self.config.get("image_args", {})

        if "image_type" in self.config:
            LOG.warning("The 'image_type' argument is deprecated since "
                        "Rally 0.10.0, use disk_format argument instead")
            if not disk_format:
                disk_format = self.config["image_type"]

        if "image_container" in self.config:
            LOG.warning("The 'image_container' argument is deprecated since "
                        "Rally 0.10.0; use container_format argument instead")
            if not container_format:
                container_format = self.config["image_container"]

        if image_args:
            LOG.warning(
                "The 'image_args' argument is deprecated since Rally 0.10.0; "
                "specify arguments in a root section of context instead")

            if "is_public" in image_args:
                if "visibility" not in self.config:
                    visibility = ("public" if image_args["is_public"]
                                  else "private")
            if "min_ram" in image_args:
                if "min_ram" not in self.config:
                    min_ram = image_args["min_ram"]

            if "min_disk" in image_args:
                if "min_disk" not in self.config:
                    min_disk = image_args["min_disk"]

        # None image_name means that image.Image will generate a random name
        image_name = None
        if "image_name" in self.config and images_per_tenant == 1:
            image_name = self.config["image_name"]

        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):
            current_images = []
            clients = osclients.Clients(
                user["credential"],
                api_info=self.context["config"].get("api_versions"))
            image_service = image.Image(
                clients, name_generator=self.generate_random_name)

            for i in range(images_per_tenant):
                image_obj = image_service.create_image(
                    image_name=image_name,
                    container_format=container_format,
                    image_location=image_url,
                    disk_format=disk_format,
                    visibility=visibility,
                    min_disk=min_disk,
                    min_ram=min_ram)
                current_images.append(image_obj.id)

            self.context["tenants"][tenant_id]["images"] = current_images