def generate_key_pair(key_length=2048):
"""Create RSA key pair with specified number of bits in key.
Returns tuple of private and public keys.
"""
with tempfiles.tempdir() as tmpdir:
keyfile = os.path.join(tmpdir, 'tempkey')
# The key is generated in the old PEM format, instead of the native
# format of OpenSSH >=6.5, because paramiko does not support it:
# https://github.com/paramiko/paramiko/issues/602
args = [
'ssh-keygen',
'-q', # quiet
'-N', '', # w/o passphrase
'-m', 'PEM', # old PEM format
'-t', 'rsa', # create key of rsa type
'-f', keyfile, # filename of the key file
'-C', 'Generated-by-Sahara' # key comment
]
if key_length is not None:
args.extend(['-b', key_length])
processutils.execute(*args)
if not os.path.exists(keyfile):
raise ex.TempestException("Private key file hasn't been created")
with open(keyfile) as keyfile_fd:
private_key = keyfile_fd.read()
public_key_path = keyfile + '.pub'
if not os.path.exists(public_key_path):
raise ex.TempestException("Public key file hasn't been created")
with open(public_key_path) as public_key_path_fd:
public_key = public_key_path_fd.read()
return private_key, public_key
def generate_key_pair(key_length=2048):
"""Create RSA key pair with specified number of bits in key.
Returns tuple of private and public keys.
"""
with tempfiles.tempdir() as tmpdir:
keyfile = os.path.join(tmpdir, 'tempkey')
args = [
'ssh-keygen',
'-q', # quiet
'-N', '', # w/o passphrase
'-t', 'rsa', # create key of rsa type
'-f', keyfile, # filename of the key file
'-C', 'Generated-by-Sahara' # key comment
]
if key_length is not None:
args.extend(['-b', key_length])
processutils.execute(*args)
if not os.path.exists(keyfile):
raise ex.TempestException("Private key file hasn't been created")
with open(keyfile) as keyfile_fd:
private_key = keyfile_fd.read()
public_key_path = keyfile + '.pub'
if not os.path.exists(public_key_path):
raise ex.TempestException("Public key file hasn't been created")
with open(public_key_path) as public_key_path_fd:
public_key = public_key_path_fd.read()
return private_key, public_key
def create_shares(cls, share_data_list):
"""Creates several shares in parallel with retries.
Use this method when you want to create more than one share at same
time. Especially if config option 'share.share_creation_retry_number'
has value more than zero (0).
All shares will be expected to have 'available' status with or without
recreation else error will be raised.
:param share_data_list: list -- list of dictionaries with 'args' and
'kwargs' for '_create_share' method of this base class.
example of data:
share_data_list=[{'args': ['quuz'], 'kwargs': {'foo': 'bar'}}}]
:returns: list -- list of shares created using provided data.
"""
data = [copy.deepcopy(d) for d in share_data_list]
for d in data:
if not isinstance(d, dict):
raise exceptions.TempestException("Expected 'dict', got '%s'" %
type(d))
if "args" not in d:
d["args"] = []
if "kwargs" not in d:
d["kwargs"] = {}
if len(d) > 2:
raise exceptions.TempestException(
"Expected only 'args' and 'kwargs' keys. "
"Provided %s" % list(d))
d["kwargs"]["client"] = d["kwargs"].get("client",
cls.shares_v2_client)
d["share"] = cls._create_share(*d["args"], **d["kwargs"])
d["cnt"] = 0
d["available"] = False
while not all(d["available"] for d in data):
for d in data:
if d["available"]:
continue
try:
d["kwargs"]["client"].wait_for_share_status(
d["share"]["id"], "available")
d["available"] = True
except (share_exceptions.ShareBuildErrorException,
exceptions.TimeoutException) as e:
if CONF.share.share_creation_retry_number > d["cnt"]:
d["cnt"] += 1
msg = ("Share '%s' failed to be built. "
"Trying create another." % d["share"]["id"])
LOG.error(msg)
LOG.error(e)
d["share"] = cls._create_share(*d["args"],
**d["kwargs"])
else:
raise e
return [d["share"] for d in data]
def wait_deployment_result(self, env_id, timeout=180):
start_time = time.time()
env = self.listing('environment-show', params=env_id)
env_status = self.get_property_value(env, 'status')
expected_statuses = ['ready', 'deploying']
while env_status != 'ready':
if time.time() - start_time > timeout:
msg = ("Environment exceeds timeout {0} to change state "
"to Ready. Environment: {1}".format(timeout, env))
raise exceptions.TimeoutException(msg)
env = self.listing('environment-show', params=env_id)
env_status = self.get_property_value(env, 'status')
if env_status not in expected_statuses:
msg = ("Environment status %s is not in expected "
"statuses: %s" % (env_status, expected_statuses))
raise exceptions.TempestException(msg)
time.sleep(2)
return True
def wait_execution(self,
ex_body,
timeout=180,
url='executions',
target_state='SUCCESS'):
start_time = time.time()
expected_states = [target_state, 'RUNNING']
while ex_body['state'] != target_state:
if time.time() - start_time > timeout:
msg = ("Execution exceeds timeout {0} "
"to change state to {1}. "
"Execution: {2}".format(timeout, target_state, ex_body))
raise exceptions.TimeoutException(msg)
_, ex_body = self.get_object(url, ex_body['id'])
if ex_body['state'] not in expected_states:
msg = ("Execution state %s is not in expected "
"states: %s" % (ex_body['state'], expected_states))
raise exceptions.TempestException(msg)
time.sleep(1)
return ex_body
def get_instance_ip(cls, instance=None):
if not instance:
instance = cls.client.get_resource(
"instances", cls.instance_id)['instance']
# TODO(lxkong): IPv6 needs to be tested.
v4_ip = None
if 'addresses' in instance:
for addr_info in instance['addresses']:
if addr_info['type'] == 'private':
v4_ip = addr_info['address']
if addr_info['type'] == 'public':
v4_ip = addr_info['address']
break
else:
ips = instance.get('ip', [])
for ip in ips:
if netutils.is_valid_ipv4(ip):
v4_ip = ip
if not v4_ip:
message = ('Failed to get instance IP address.')
raise exceptions.TempestException(message)
return v4_ip
def wait_execution_success(self, exec_id, timeout=180):
start_time = time.time()
ex = self.mistral_admin('execution-get', params=exec_id)
exec_state = self.get_field_value(ex, 'State')
expected_states = ['SUCCESS', 'RUNNING']
while exec_state != 'SUCCESS':
if time.time() - start_time > timeout:
msg = ("Execution exceeds timeout {0} to change state "
"to SUCCESS. Execution: {1}".format(timeout, ex))
raise exceptions.TimeoutException(msg)
ex = self.mistral_admin('execution-get', params=exec_id)
exec_state = self.get_field_value(ex, 'State')
if exec_state not in expected_states:
msg = ("Execution state %s is not in expected "
"states: %s" % (exec_state, expected_states))
raise exceptions.TempestException(msg)
time.sleep(2)
return True
def mysql_execute(self, cmds, **kwargs):
try:
with self.engine.begin() as conn:
return self.conn_execute(conn, cmds)
except Exception as e:
raise exceptions.TempestException(
'Failed to execute database command %s, error: %s' %
(cmds, str(e)))
def resource_setup(cls):
super(AutoAllocateNetworkTest, cls).resource_setup()
# Sanity check that there are no networks available to the tenant.
# This is essentially what Nova does for getting available networks.
tenant_id = cls.networks_client.tenant_id
# (1) Retrieve non-public network list owned by the tenant.
search_opts = {'tenant_id': tenant_id, 'shared': False}
nets = cls.networks_client.list_networks(**search_opts).get(
'networks', [])
if nets:
raise lib_excs.TempestException('Found tenant networks: %s' % nets)
# (2) Retrieve shared network list.
search_opts = {'shared': True}
nets = cls.networks_client.list_networks(**search_opts).get(
'networks', [])
if nets:
raise lib_excs.TempestException('Found shared networks: %s' % nets)
def check_negative_scenarios(self, error_message, cmd, name):
msg_exist = None
try:
self.openstack('dataprocessing %s' % cmd, params=name)
except exc.CommandFailed as e:
# lower() is required because "result" string could
# have the first letter capitalized.
if error_message.lower() in str(e).lower():
msg_exist = True
if not msg_exist:
raise exc.TempestException('"%s" is not a part of output of '
'executed command "%s" (%s)'
% (error_message, cmd, output_msg))
else:
raise exc.TempestException('"%s %s" in negative scenarios has '
'been executed without any errors'
% (cmd, name))
def pgsql_execute(self, cmds, **kwargs):
try:
with self.engine.connect() as conn:
conn.connection.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
return self.conn_execute(conn, cmds)
except Exception as e:
raise exceptions.TempestException(
'Failed to execute database command %s, error: %s' %
(cmds, str(e)))
def _get_availability(network, net_availability):
if 'network_ip_availabilities' in net_availability:
for availability in net_availability['network_ip_availabilities']:
if availability['network_id'] == network['id']:
return availability
raise exceptions.TempestException('Network IP Availability not '
'found')
else:
return net_availability['network_ip_availability']
def setUp(self):
try:
# We expect here RuntimeError exception because 'setUpClass'
# has not called 'super'.
super(TestSetUpClass, self).setUp()
except RuntimeError:
pass
else:
raise exceptions.TempestException(
"If you see this, then expected exception was not raised.")
def _poll_cluster_status(self, cluster_name):
with fixtures.Timeout(TEMPEST_CONF.data_processing.cluster_timeout,
gentle=True):
while True:
status = self._get_cluster_status(cluster_name)
if status == 'Active':
break
if status == 'Error':
raise exc.TempestException("Cluster in %s state" % status)
time.sleep(3)
def check():
result[0] = client.show_allocation(allocation_ident)
allocation = result[0][1]
if allocation['state'] == 'error' and not expect_error:
raise lib_exc.TempestException(
"Allocation %(ident)s failed: %(error)s" %
{'ident': allocation_ident,
'error': allocation.get('last_error')})
else:
return allocation['state'] != 'allocating'
def check_negative_scenarios(self, error_message, cmd, name):
msg_exist = None
try:
self.openstack('dataprocessing %s' % cmd, params=name)
except exc.CommandFailed as e:
# lower() is required because "result" string could
# have the first letter capitalized.
output_msg = str(e).splitlines()
for msg in output_msg:
if msg.lower() == error_message.lower():
self.assertEqual(error_message.lower(), msg.lower())
msg_exist = True
if not msg_exist:
raise exc.TempestException('%s is not a part of output of '
'executed command %s' %
(error_message, cmd))
else:
raise exc.TempestException('%s %s in negative scenarios have been '
'executed without any errors' %
(cmd, name))
def resize_server(cls, server_id, new_flavor_id, **kwargs):
"""resize and confirm_resize an server, waits for it to be ACTIVE."""
cls.servers_client.resize_server(server_id, new_flavor_id, **kwargs)
waiters.wait_for_server_status(cls.servers_client, server_id,
'VERIFY_RESIZE')
cls.servers_client.confirm_resize_server(server_id)
waiters.wait_for_server_status(cls.servers_client, server_id, 'ACTIVE')
server = cls.servers_client.show_server(server_id)['server']
# Nova API > 2.46 no longer includes flavor.id
if server['flavor'].get('id'):
if new_flavor_id != server['flavor']['id']:
msg = ('Flavor id of %s is not equal to new_flavor_id.' %
server_id)
raise lib_exc.TempestException(msg)
def is_attr_in_status():
node = utils.get_node(client, node_id=node_id)
if node[attr] in status:
return True
elif (abort_on_error_state
and node['provision_state'].endswith(' failed')):
msg = ('Node %(node)s reached failure state %(state)s while '
'waiting for %(attr)s=%(expected)s. '
'Error: %(error)s' %
{'node': node_id, 'state': node['provision_state'],
'attr': attr, 'expected': status,
'error': node.get('last_error')})
LOG.debug(msg)
raise lib_exc.TempestException(msg)
return False
def tempdir(**kwargs):
argdict = kwargs.copy()
if 'dir' not in argdict:
argdict['dir'] = '/tmp/'
tmpdir = tempfile.mkdtemp(**argdict)
try:
yield tmpdir
finally:
try:
shutil.rmtree(tmpdir)
except OSError as e:
raise ex.TempestException(
_("Failed to delete temp dir %(dir)s (reason: %(reason)s)") % {
'dir': tmpdir,
'reason': e
})
def _poll_cluster_status(self, cluster_id):
with fixtures.Timeout(
timeouts.Defaults.instance.timeout_poll_cluster_status,
gentle=True):
while True:
status = self.sahara.get_cluster_status(cluster_id)
if status == CLUSTER_STATUS_ACTIVE:
break
if status == CLUSTER_STATUS_ERROR:
cluster = self.sahara.get_cluster(cluster_id)
failure_desc = cluster.status_description
message = ("Cluster in %s state with"
" a message below:\n%s") % (status,
failure_desc)
raise exc.TempestException(message)
time.sleep(3)
def _check_event_logs(self, cluster):
invalid_steps = []
if cluster.is_transient:
# skip event log testing
return
for step in cluster.provision_progress:
if not step['successful']:
invalid_steps.append(step)
if len(invalid_steps) > 0:
invalid_steps_info = "\n".join(
six.text_type(e) for e in invalid_steps)
steps_info = "\n".join(
six.text_type(e) for e in cluster.provision_progress)
raise exc.TempestException(
"Issues with event log work: "
"\n Incomplete steps: \n\n {invalid_steps}"
"\n All steps: \n\n {steps}".format(
steps=steps_info, invalid_steps=invalid_steps_info))
def wait_for_volume_migration(client, volume_id, new_host):
"""Waits for a Volume to move to a new host."""
body = client.show_volume(volume_id)['volume']
host = body['os-vol-host-attr:host']
migration_status = body['migration_status']
start = int(time.time())
# new_host is hostname@backend while current_host is hostname@backend#type
while migration_status != 'success' or new_host not in host:
time.sleep(client.build_interval)
body = client.show_volume(volume_id)['volume']
host = body['os-vol-host-attr:host']
migration_status = body['migration_status']
if migration_status == 'error':
message = ('volume %s failed to migrate.' % (volume_id))
raise lib_exc.TempestException(message)
if int(time.time()) - start >= client.build_timeout:
message = ('Volume %s failed to migrate to %s (current %s) '
'within the required time (%s s).' %
(volume_id, new_host, host, client.build_timeout))
raise lib_exc.TimeoutException(message)
def create_instance(cls, name=None, datastore_version=None,
database=constants.DB_NAME, username=constants.DB_USER,
password=constants.DB_PASS, backup_id=None,
replica_of=None, create_user=True):
"""Create database instance.
Creating database instance is time-consuming, so we define this method
as a class method, which means the instance is shared in a single
TestCase. According to
https://docs.openstack.org/tempest/latest/write_tests.html#adding-a-new-testcase,
all test methods within a TestCase are assumed to be executed serially.
"""
name = name or cls.get_resource_name("instance")
# Flavor, volume, datastore are not needed for creating replica.
if replica_of:
body = {
"instance": {
"name": name,
"nics": [{"net-id": cls.private_network}],
"access": {"is_public": True},
"replica_of": replica_of,
}
}
# Get datastore version. Get from API if the default ds version is not
# configured.
elif not datastore_version:
default_versions = CONF.database.default_datastore_versions
datastore_version = default_versions.get(cls.datastore)
if not datastore_version:
res = cls.client.list_resources("datastores")
for d in res['datastores']:
if d['name'] == cls.datastore:
if d.get('default_version'):
datastore_version = d['default_version']
else:
datastore_version = d['versions'][0]['name']
break
if not datastore_version:
message = ('Failed to get available datastore version.')
raise exceptions.TempestException(message)
if not replica_of:
body = {
"instance": {
"name": name,
"datastore": {
"type": cls.datastore,
"version": datastore_version
},
"flavorRef": CONF.database.flavor_id,
"volume": {
"size": 1,
"type": CONF.database.volume_type
},
"nics": [{"net-id": cls.private_network}],
"access": {"is_public": True}
}
}
if backup_id:
body['instance'].update(
{'restorePoint': {'backupRef': backup_id}})
if create_user:
body['instance'].update({
'databases': [{"name": database}],
"users": [
{
"name": username,
"password": password,
"databases": [{"name": database}]
}
]
})
res = cls.client.create_resource("instances", body)
cls.addClassResourceCleanup(cls.wait_for_instance_status,
res["instance"]["id"],
need_delete=True,
expected_status="DELETED")
return res["instance"]
def create_shares(cls, share_data_list):
"""Creates several shares in parallel with retries.
Use this method when you want to create more than one share at same
time. Especially if config option 'share.share_creation_retry_number'
has value more than zero (0).
All shares will be expected to have 'available' status with or without
recreation else error will be raised.
:param share_data_list: list -- list of dictionaries with 'args' and
'kwargs' for '_create_share' method of this base class.
example of data:
share_data_list=[{'args': ['quuz'], 'kwargs': {'foo': 'bar'}}}]
:returns: list -- list of shares created using provided data.
"""
for d in share_data_list:
if not isinstance(d, dict):
raise exceptions.TempestException("Expected 'dict', got '%s'" %
type(d))
if "args" not in d:
d["args"] = []
if "kwargs" not in d:
d["kwargs"] = {}
if len(d) > 2:
raise exceptions.TempestException(
"Expected only 'args' and 'kwargs' keys. "
"Provided %s" % list(d))
data = []
for d in share_data_list:
client = d["kwargs"].pop("client", cls.shares_v2_client)
local_d = {
"args": d["args"],
"kwargs": copy.deepcopy(d["kwargs"]),
}
local_d["kwargs"]["client"] = client
local_d["share"] = cls._create_share(*local_d["args"],
**local_d["kwargs"])
local_d["cnt"] = 0
local_d["available"] = False
data.append(local_d)
while not all(d["available"] for d in data):
for d in data:
if d["available"]:
continue
client = d["kwargs"]["client"]
share_id = d["share"]["id"]
try:
client.wait_for_share_status(share_id, "available")
d["available"] = True
except (share_exceptions.ShareBuildErrorException,
exceptions.TimeoutException) as e:
if CONF.share.share_creation_retry_number > d["cnt"]:
d["cnt"] += 1
msg = ("Share '%s' failed to be built. "
"Trying create another." % share_id)
LOG.error(msg)
LOG.error(e)
cg_id = d["kwargs"].get("consistency_group_id")
if cg_id:
# NOTE(vponomaryov): delete errored share
# immediately in case share is part of CG.
client.delete_share(
share_id,
params={"consistency_group_id": cg_id})
client.wait_for_resource_deletion(
share_id=share_id)
d["share"] = cls._create_share(*d["args"],
**d["kwargs"])
else:
raise e
return [d["share"] for d in data]
def _get_net_id(cls):
for net in cls.networks_client.list_networks()['networks']:
if net['name'] == CONF.compute.fixed_network_name:
return net['id']
else:
raise lib_exc.TempestException('Could not find fixed network!')