def test_update_endpoints(self, mock_deployment_update):
mock_deployment_update.return_value = self.deployment
deploy = objects.Deployment(deployment=self.deployment)
endpoints = {
"admin":
objects.Endpoint("url", "user", "pwd", "tenant",
consts.EndpointPermission.ADMIN),
"users": [
objects.Endpoint("url1", "user1", "pwd1", "tenant1",
consts.EndpointPermission.USER),
objects.Endpoint("url2", "user2", "pwd2", "tenant2",
consts.EndpointPermission.USER),
]
}
expected_users = [
u.to_dict(include_permission=True) for u in endpoints["users"]
]
deploy.update_endpoints(endpoints)
mock_deployment_update.assert_called_once_with(
self.deployment["uuid"], {
"admin": endpoints["admin"].to_dict(include_permission=True),
"users": expected_users
})
def __init__(self,
config,
task,
admin=None,
users=None,
abort_on_sla_failure=False):
"""BenchmarkEngine constructor.
:param config: Dict with configuration of specified benchmark scenarios
:param task: Instance of Task,
the current task which is being performed
:param admin: Dict with admin credentials
:param users: List of dicts with user credentials
:param abort_on_sla_failure: True if the execution should be stopped
when some SLA check fails
"""
try:
self.config = TaskConfig(config)
except Exception as e:
log = [str(type(e)), str(e), json.dumps(traceback.format_exc())]
task.set_failed(log=log)
raise exceptions.InvalidTaskException(str(e))
self.task = task
self.admin = admin and objects.Endpoint(**admin) or None
self.existing_users = users or []
self.abort_on_sla_failure = abort_on_sla_failure
def create_from_env(cls):
return cls(
objects.Endpoint(os.environ["OS_AUTH_URL"],
os.environ["OS_USERNAME"],
os.environ["OS_PASSWORD"],
os.environ.get("OS_TENANT_NAME"),
region_name=os.environ.get("OS_REGION_NAME")))
def images(self, deployment=None):
"""Display available images.
:param deployment: UUID or name of a deployment
"""
headers = ["UUID", "Name", "Size (B)"]
mixed_case_fields = ["UUID", "Name"]
float_cols = ["Size (B)"]
formatters = dict(
zip(float_cols,
[cliutils.pretty_float_formatter(col) for col in float_cols]))
for endpoint_dict in self._get_endpoints(deployment):
self._print_header("Images", endpoint_dict)
table_rows = []
clients = osclients.Clients(objects.Endpoint(**endpoint_dict))
glance_client = clients.glance()
for image in glance_client.images.list():
data = [image.id, image.name, image.size]
table_rows.append(utils.Struct(**dict(zip(headers, data))))
cliutils.print_list(table_rows,
fields=headers,
formatters=formatters,
mixed_case_fields=mixed_case_fields)
def flavors(self, deployment=None):
"""Display available flavors.
:param deployment: UUID or name of a deployment
"""
headers = ["ID", "Name", "vCPUs", "RAM (MB)", "Swap (MB)", "Disk (GB)"]
mixed_case_fields = ["ID", "Name", "vCPUs"]
float_cols = ["RAM (MB)", "Swap (MB)", "Disk (GB)"]
formatters = dict(
zip(float_cols,
[cliutils.pretty_float_formatter(col) for col in float_cols]))
for endpoint_dict in self._get_endpoints(deployment):
self._print_header("Flavors", endpoint_dict)
table_rows = []
clients = osclients.Clients(objects.Endpoint(**endpoint_dict))
nova_client = clients.nova()
for flavor in nova_client.flavors.list():
data = [
flavor.id, flavor.name, flavor.vcpus, flavor.ram,
flavor.swap, flavor.disk
]
table_rows.append(utils.Struct(**dict(zip(headers, data))))
cliutils.print_list(table_rows,
fields=headers,
formatters=formatters,
mixed_case_fields=mixed_case_fields)
def consume(cache, args):
username, password, project_dom, user_dom, tenant_id = args
if "client" not in cache:
clients = osclients.Clients(self.endpoint)
cache["client"] = keystone.wrap(clients.keystone())
client = cache["client"]
user = client.create_user(username, password,
"*****@*****.**" % username, tenant_id,
user_dom)
user_endpoint = objects.Endpoint(
client.auth_url,
user.name,
password,
self.context["tenants"][tenant_id]["name"],
consts.EndpointPermission.USER,
client.region_name,
project_domain_name=project_dom,
user_domain_name=user_dom,
endpoint_type=self.endpoint.endpoint_type,
https_insecure=self.endpoint.insecure,
https_cacert=self.endpoint.cacert)
users.append({
"id": user.id,
"endpoint": user_endpoint,
"tenant_id": tenant_id
})
def test_users_contains_correct_endpoint_type(self, mock_keystone):
endpoint = objects.Endpoint("foo_url",
"foo",
"foo_pass",
endpoint_type=consts.EndpointType.INTERNAL)
config = {
"config": {
"users": {
"tenants": 1,
"users_per_tenant": 2,
"resource_management_workers": 1
}
},
"admin": {
"endpoint": endpoint
},
"task": {
"uuid": "task_id"
}
}
user_generator = users.UserGenerator(config)
users_ = user_generator._create_users()
for user in users_:
self.assertEqual("internal", user["endpoint"].endpoint_type)
def __init__(self, deployment, conf_path):
endpoint = db.deployment_get(deployment)["admin"]
self.clients = osclients.Clients(objects.Endpoint(**endpoint))
self.keystone = self.clients.verified_keystone()
self.conf_path = conf_path
self.conf = configparser.ConfigParser()
self.conf.read(conf_path)
def service_list(cls, deployment):
"""Get the services list.
:param deployment: Deployment object
:returns: Service list
"""
# TODO(kun): put this work into objects.Deployment
clients = osclients.Clients(objects.Endpoint(**deployment["admin"]))
return clients.services()
def test_deployment_check_raise(self, mock_deployment_get,
mock_clients_verified_keystone):
deployment_id = "e87e4dca-b515-4477-888d-5f6103f13b42"
sample_endpoint = objects.Endpoint("http://192.168.1.1:5000/v2.0/",
"admin", "adminpass").to_dict()
sample_endpoint["not-exist-key"] = "error"
mock_deployment_get.return_value = {"admin": sample_endpoint}
mock_clients_verified_keystone.services.list.return_value = []
self.assertRaises(TypeError, self.deployment.check, deployment_id)
def test_deployment_check_connect_failed(self, mock_deployment_get,
mock_clients_services):
deployment_id = "e87e4dca-b515-4477-888d-5f6103f13b42"
sample_endpoint = objects.Endpoint("http://192.168.1.1:5000/v2.0/",
"admin", "adminpass").to_dict()
mock_deployment_get.return_value = {"admin": sample_endpoint}
refused = keystone_exceptions.ConnectionRefused()
mock_clients_services.side_effect = refused
self.assertEqual(self.deployment.check(deployment_id), 1)
def test_users_and_tenants_in_context(self, mock_keystone):
wrapped_keystone = mock.MagicMock()
mock_keystone.wrap.return_value = wrapped_keystone
endpoint = objects.Endpoint("foo_url",
"foo",
"foo_pass",
https_insecure=True,
https_cacert="cacert")
tmp_context = dict(self.context)
tmp_context["config"]["users"] = {
"tenants": 1,
"users_per_tenant": 2,
"resource_management_workers": 1
}
tmp_context["admin"]["endpoint"] = endpoint
endpoint_dict = endpoint.to_dict(False)
user_list = [
mock.MagicMock(id="id_%d" % i) for i in range(self.users_num)
]
wrapped_keystone.create_user.side_effect = user_list
with users.UserGenerator(tmp_context) as ctx:
ctx.generate_random_name = mock.Mock()
ctx.setup()
create_tenant_calls = []
for i, t in enumerate(ctx.context["tenants"]):
create_tenant_calls.append(
mock.call(ctx.generate_random_name.return_value,
ctx.config["project_domain"]))
for user in ctx.context["users"]:
self.assertEqual(set(["id", "endpoint", "tenant_id"]),
set(user.keys()))
user_endpoint_dict = user["endpoint"].to_dict(False)
excluded_keys = [
"auth_url", "username", "password", "tenant_name",
"region_name", "project_domain_name", "admin_domain_name",
"user_domain_name"
]
for key in (set(endpoint_dict.keys()) - set(excluded_keys)):
self.assertEqual(endpoint_dict[key],
user_endpoint_dict[key])
tenants_ids = []
for t in ctx.context["tenants"].keys():
tenants_ids.append(t)
for (user, tenant_id, orig_user) in zip(ctx.context["users"],
tenants_ids, user_list):
self.assertEqual(user["id"], orig_user.id)
self.assertEqual(user["tenant_id"], tenant_id)
class FakeUserContext(FakeContext):
admin = {
"id": "adminuuid",
"endpoint": objects.Endpoint("aurl", "aname", "apwd", "atenant")
}
user = {
"id": "uuid",
"endpoint": objects.Endpoint("url", "name", "pwd", "tenant"),
"tenant_id": "uuid"
}
tenants = {"uuid": {"name": "tenant"}}
def __init__(self, ctx):
super(FakeUserContext, self).__init__(ctx)
self.context.setdefault("admin", FakeUserContext.admin)
self.context.setdefault("users", [FakeUserContext.user])
self.context.setdefault("tenants", FakeUserContext.tenants)
self.context.setdefault(
"scenario_name", "NovaServers.boot_server_from_volume_and_delete")
def test_deployment_check(self, mock_deployment_get, mock_clients_keystone,
mock_clients_verified_keystone):
deployment_id = "e87e4dca-b515-4477-888d-5f6103f13b42"
sample_endpoint = objects.Endpoint("http://192.168.1.1:5000/v2.0/",
"admin", "adminpass").to_dict()
mock_deployment_get.return_value = {
"admin": sample_endpoint,
"users": [sample_endpoint]
}
self.deployment.check(deployment_id)
mock_deployment_get.assert_called_once_with(deployment_id)
def setUp(self):
super(SaharaInputDataSourcesTestCase, self).setUp()
credential = objects.Endpoint("foo_url", "user", "passwd")
self.tenants_num = 2
self.users_per_tenant = 2
self.users = self.tenants_num * self.users_per_tenant
self.task = mock.MagicMock()
self.tenants = {}
self.users_key = []
for i in range(self.tenants_num):
self.tenants[str(i)] = {
"id": str(i),
"name": str(i),
"sahara": {
"image": "42"
}
}
for j in range(self.users_per_tenant):
self.users_key.append({
"id": "%s_%s" % (str(i), str(j)),
"tenant_id": str(i),
"credential": credential
})
self.user_key = [{
"id": i,
"tenant_id": j,
"credential": "credential"
} for j in range(self.tenants_num)
for i in range(self.users_per_tenant)]
self.context.update({
"config": {
"users": {
"tenants": self.tenants_num,
"users_per_tenant": self.users_per_tenant,
},
"sahara_input_data_sources": {
"input_type": "hdfs",
"input_url": "hdfs://test_host/",
},
},
"admin": {
"credential": mock.MagicMock()
},
"users": self.users_key,
"tenants": self.tenants
})
def setUp(self):
super(OSClientsTestCase, self).setUp()
self.endpoint = objects.Endpoint("http://auth_url", "use", "pass",
"tenant")
self.clients = osclients.Clients(self.endpoint)
self.fake_keystone = fakes.FakeKeystoneClient()
self.fake_keystone.auth_token = mock.MagicMock()
self.service_catalog = self.fake_keystone.service_catalog
self.service_catalog.url_for = mock.MagicMock()
keystone_patcher = mock.patch("rally.osclients.create_keystone_client")
self.mock_create_keystone_client = keystone_patcher.start()
self.addCleanup(keystone_patcher.stop)
self.mock_create_keystone_client.return_value = self.fake_keystone
def __init__(self, deployment):
self.deployment = deployment
self.endpoint = db.deployment_get(deployment)["admin"]
self.clients = osclients.Clients(objects.Endpoint(**self.endpoint))
self.keystone = self.clients.verified_keystone()
self.available_services = self.clients.services().values()
self.data_dir = _create_or_get_data_dir()
self.conf = configparser.ConfigParser()
self.conf.read(os.path.join(os.path.dirname(__file__), "config.ini"))
self._download_cirros_image()
def __init__(self, deployment, config):
super(OpenStackProvider, self).__init__(deployment, config)
user_endpoint = objects.Endpoint(config["auth_url"],
config["user"],
config["password"],
config["tenant"],
region_name=config.get("region"))
clients = osclients.Clients(user_endpoint)
self.nova = clients.nova()
try:
self.glance = clients.glance()
except KeyError:
self.glance = None
LOG.warning(
_("Glance endpoint not available in service catalog"
", only existing images can be used"))
def secgroups(self, deployment=None):
"""Display security groups."""
headers = ["ID", "Name", "Description"]
mixed_case_fields = ["ID", "Name", "Description"]
for endpoint_dict in self._get_endpoints(deployment):
self._print_header("Security groups", endpoint_dict)
table_rows = []
clients = osclients.Clients(objects.Endpoint(**endpoint_dict))
nova_client = clients.nova()
for secgroup in nova_client.security_groups.list():
data = [secgroup.id, secgroup.name, secgroup.description]
table_rows.append(utils.Struct(**dict(zip(headers, data))))
cliutils.print_list(table_rows,
fields=headers,
mixed_case_fields=mixed_case_fields)
def _create_endpoint(self, common, user, permission):
return objects.Endpoint(
common["auth_url"],
user["username"],
user["password"],
tenant_name=user.get("project_name", user.get("tenant_name")),
permission=permission,
region_name=common.get("region_name"),
endpoint_type=common.get("endpoint_type",
consts.EndpointType.PUBLIC),
endpoint=common.get("endpoint"),
domain_name=user.get("domain_name"),
user_domain_name=user.get("user_domain_name", "Default"),
admin_domain_name=user.get("admin_domain_name", "Default"),
project_domain_name=user.get("project_domain_name", "Default"),
https_insecure=common.get("https_insecure", False),
https_cacert=common.get("https_cacert"))
def keypairs(self, deployment=None):
"""Display available ssh keypairs."""
headers = ["Name", "Fingerprint"]
mixed_case_fields = ["Name", "Fingerprint"]
for endpoint_dict in self._get_endpoints(deployment):
self._print_header("Keypairs", endpoint_dict)
table_rows = []
clients = osclients.Clients(objects.Endpoint(**endpoint_dict))
nova_client = clients.nova()
for keypair in nova_client.keypairs.list():
data = [keypair.name, keypair.fingerprint]
table_rows.append(utils.Struct(**dict(zip(headers, data))))
cliutils.print_list(table_rows,
fields=headers,
mixed_case_fields=mixed_case_fields)
def networks(self, deployment=None):
"""Display configured networks."""
headers = ["ID", "Label", "CIDR"]
mixed_case_fields = ["ID", "Label", "CIDR"]
for endpoint_dict in self._get_endpoints(deployment):
self._print_header("Networks", endpoint_dict)
table_rows = []
clients = osclients.Clients(objects.Endpoint(**endpoint_dict))
nova_client = clients.nova()
for network in nova_client.networks.list():
data = [network.id, network.label, network.cidr]
table_rows.append(utils.Struct(**dict(zip(headers, data))))
cliutils.print_list(table_rows,
fields=headers,
mixed_case_fields=mixed_case_fields)
def required_clients(config, clients, deployment, *components, **kwargs):
"""Validator checks if specified OpenStack clients are available.
:param *components: list of client components names
:param **kwargs: optional parameters:
admin - bool, whether to use admin clients
"""
if kwargs.get("admin", False):
clients = osclients.Clients(objects.Endpoint(**deployment["admin"]))
for client_component in components:
try:
getattr(clients, client_component)()
except ImportError:
return ValidationResult(
False,
_("Client for %s is not installed. To install it run "
"`pip install -r"
" optional-requirements.txt`") % client_component)
def required_cinder_services(config, clients, deployment, service_name):
"""Validator checks that specified Cinder service is available.
It uses Cinder client with admin permissions to call 'cinder service-list'
call
:param service_name: Cinder service name
"""
admin_client = osclients.Clients(
objects.Endpoint(**deployment["admin"])).cinder()
for service in admin_client.services.list():
if (service.binary == six.text_type(service_name)
and service.state == six.text_type("up")):
return ValidationResult(True)
msg = _("%s service is not available") % service_name
return ValidationResult(False, msg)
def __init__(self, endpoint_=None):
self._nova = None
self._glance = None
self._keystone = None
self._cinder = None
self._neutron = None
self._sahara = None
self._heat = None
self._designate = None
self._ceilometer = None
self._zaqar = None
self._trove = None
self._mistral = None
self._swift = None
self._murano = None
self._ec2 = None
self._endpoint = endpoint_ or objects.Endpoint(
"http://fake.example.org:5000/v2.0/", "fake_username",
"fake_password", "fake_tenant_name")
def __oops(self):
self.context["users"] = []
self.context["tenants"] = {}
for user in self.config:
user_endpoint = objects.Endpoint(**user)
user_kclient = osclients.Clients(user_endpoint).keystone()
if user_kclient.tenant_id not in self.context["tenants"]:
self.context["tenants"][user_kclient.tenant_id] = {
"id": user_kclient.tenant_id,
"name": user_kclient.tenant_name
}
self.context["users"].append({
"endpoint": user_endpoint,
"id": user_kclient.user_id,
"tenant_id": user_kclient.tenant_id
})
def __init__(self,
config,
task,
admin=None,
users=None,
abort_on_sla_failure=False):
"""BenchmarkEngine constructor.
:param config: The configuration with specified benchmark scenarios
:param task: The current task which is being performed
:param admin: Dict with admin credentials
:param users: List of dicts with user credentials
:param abort_on_sla_failure: True if the execution should be stopped
when some SLA check fails
"""
self.config = config
self.task = task
self.admin = admin and objects.Endpoint(**admin) or None
self.existing_users = users or []
self.abort_on_sla_failure = abort_on_sla_failure
def deploy(self):
name = self.config["container_name"]
start_script = self.config.get("start_script",
get_script_path(START_SCRIPT))
distribution = self.config["distribution"]
release = self.config.get("release", None)
network = self.config.get("start_lxc_network")
if network:
network = netaddr.IPNetwork(network)
else:
ip = "0"
self.provider = self._get_provider()
for server in self.provider.create_servers():
config = {"tunnel_to": self.config.get("tunnel_to", [])}
if network:
config["network"] = str(network)
ip = str(network.ip).replace(".", "-")
else:
ip = "0"
name_prefix = "%s-%s" % (name, ip)
first_name = name_prefix + "-000"
lxc_host = lxc.LxcHost(server, config)
self._deploy_first(lxc_host, first_name, distribution, release)
for i in range(1, self.config["containers_per_host"]):
clone_name = "%s-%03d" % (name_prefix, i)
lxc_host.create_clone(clone_name, first_name)
lxc_host.start_containers()
info = {
"host": server.get_credentials(),
"containers": lxc_host.containers,
"forwarded_ports": lxc_host._port_cache.items(),
"config": config
}
self.deployment.add_resource(provider_name="LxcEngine", info=info)
for container in lxc_host.get_server_objects():
container.ssh.run("/bin/sh -e", stdin=open(start_script, "rb"))
if network:
network += 1
return {"admin": objects.Endpoint("", "", "", "")}
def deploy(self):
self.client = fuelclient.FuelClient(self.config["api_url"])
self.nodes = self.client.get_nodes()
controllers = self._get_nodes("controller")
computes = self._get_nodes("compute")
cinders = self._get_nodes("cinder")
computes_cinders = self._get_nodes("cinder+compute")
cluster = fuelclient.FuelCluster(
self.client,
name=self.config["deploy_name"],
release=self._get_release_id(),
mode=self.config["mode"],
net_provider=self.config["net_provider"],
net_segment_type=self.config.get("net_segment_type", "gre"),
)
cluster.set_nodes(controllers, ["controller"])
cluster.set_nodes(computes, ["compute"])
cluster.set_nodes(cinders, ["cinder"])
cluster.set_nodes(computes_cinders, ["compute", "cinder"])
cluster.configure_network(self.config["networks"])
cluster.deploy()
self.deployment.add_resource("FuelEngine",
type="cloud",
info={"id": cluster.cluster["id"]})
ip = cluster.get_endpoint_ip()
attrs = cluster.get_attributes()["editable"]["access"]
admin_endpoint = objects.Endpoint("http://%s:5000/v2.0/" % ip,
attrs["user"]["value"],
attrs["password"]["value"],
attrs["tenant"]["value"],
consts.EndpointPermission.ADMIN)
return {"admin": admin_endpoint}
def required_services(config, clients, deployment, *required_services):
"""Validator checks if specified OpenStack services are available.
:param *required_services: list of services names
"""
available_services = list(clients.services().values())
if consts.Service.NOVA_NET in required_services:
nova = osclients.Clients(
objects.Endpoint(**deployment["admin"])).nova()
for service in nova.services.list():
if (service.binary == consts.Service.NOVA_NET
and service.status == "enabled"):
available_services.append(consts.Service.NOVA_NET)
for service in required_services:
if service not in consts.Service:
return ValidationResult(False, _("Unknown service: %s") % service)
if service not in available_services:
return ValidationResult(
False,
_("Service is not available: %s") % service)