def fill_memory():
"""
Use `stress-ng` tool to stress swap memory for 4 minutes to given
percentage
"""
run_time = 240
SSH = usmssh.get_ssh()
host = CONF.config["usmqe"]["cluster_member"]
# get total and swap memory of machine via /proc/meminfo file
meminfo_cmd = """awk '{if ($1=="MemTotal:" || $1=="SwapTotal:") print $2}' /proc/meminfo"""
_, stdout, _ = SSH[host].run(meminfo_cmd)
mem_total, swap_total, _ = stdout.decode("utf-8").split("\n")
# how much memory is going to be consumed considered both normal memory
# and swap
memory_percent = 100 + (int(swap_total) / int(mem_total) *
int(request.param))
stress_cmd = "stress-ng --vm-method flip --vm {} --vm-bytes {}%".format(
1, int(memory_percent))
stress_cmd += " --timeout {}s --vm-hang 0 --vm-keep --verify".format(
run_time)
stress_cmd += " --syslog"
retcode, stdout, stderr = SSH[host].run(stress_cmd)
if retcode != 0:
raise OSError(stderr)
teardown_cmd = "sleep 3; swapoff -a && swapon -a; sleep 5"
SSH[host].run(teardown_cmd)
return request.param
def workload_memory_utilization(request, stress_tools):
"""
Returns:
dict: contains information about `start` and `stop` time of stress-ng
command and its `result`
"""
def fill_memory():
"""
Use `stress` tool to stress memory for 4 minutes to given percentage
"""
# stress memory for for 240 seconds
run_time = 240
SSH = usmssh.get_ssh()
host = CONF.config["usmqe"]["cluster_member"]
stress_cmd = "stress --vm-bytes $(awk '/MemAvailable/{{printf "\
"\"%d\\n\" , $2 * ({0}/100);}}' < /proc/meminfo)k "\
"--vm-keep -m {1}".format(request.param, 1)
stress_cmd += " --timeout {}s".format(run_time)
retcode, stdout, stderr = SSH[host].run(stress_cmd)
if retcode != 0:
raise OSError(stderr)
return request.param
SSH = usmssh.get_ssh()
host = CONF.config["usmqe"]["cluster_member"]
meminfo_cmd = "free -b | awk '{if (NR==2) print $2}'"
retcode, stdout, stderr = SSH[host].run(meminfo_cmd)
if retcode != 0:
raise OSError(stderr)
mem_total = stdout.decode("utf-8")
return measure_operation(fill_memory, metadata={'total_memory': mem_total})
def __init__(self, nodes):
"""
Args:
nodes (list): list of nodes used for command scheduling
"""
self.nodes = nodes
self.ssh = usmssh.get_ssh()
def os_info():
"""
Return information from /etc/os-release file about current os distribution.
"""
SSH = usmssh.get_ssh()
os_release = 'cat /etc/os-release'
node_connection = SSH[pytest.config.getini("usm_cluster_member")]
f_content = node_connection.run(os_release)
f_content = f_content[1].decode("utf-8").replace('"', '')
config = configparser.ConfigParser()
config.read_string('[os_info]\n' + f_content)
LOGGER.debug(config['os_info'])
return dict(config['os_info'])
def test_scheduler_workflow():
conf = UsmConfig()
nodes = conf.inventory.get_groups_dict()["gluster_servers"]
scheduler = Scheduler(nodes)
scheduler.run_at("echo '1' > /tmp/test_task")
jobs = scheduler.jobs()
LOGGER.info("Jobs: {}".format(jobs))
assert len(jobs) == len(nodes)
time.sleep(70)
for node in nodes:
SSH = usmssh.get_ssh()
_, message, _ = SSH[node].run("cat /tmp/test_task")
assert message.decode("utf-8").rstrip("\n") == "1"
# teardown
SSH[node].run("rm -f /tmp/test_task")
def fill_memory():
"""
Use `stress` tool to stress memory for 4 minutes to given percentage
"""
# stress memory for for 240 seconds
run_time = 240
SSH = usmssh.get_ssh()
host = CONF.config["usmqe"]["cluster_member"]
stress_cmd = "stress --vm-bytes $(awk '/MemAvailable/{{printf "\
"\"%d\\n\" , $2 * ({0}/100);}}' < /proc/meminfo)k "\
"--vm-keep -m {1}".format(request.param, 1)
stress_cmd += " --timeout {}s".format(run_time)
retcode, stdout, stderr = SSH[host].run(stress_cmd)
if retcode != 0:
raise OSError(stderr)
return request.param
def fill_cpu():
"""
Use `stress-ng` tool to stress cpu for 3 minutes to given percentage
"""
# stress cpu for for 180 seconds
run_time = 180
SSH = usmssh.get_ssh()
host = CONF.config["usmqe"]["cluster_member"]
processors_cmd = "grep -c ^processor /proc/cpuinfo"
retcode, processors_count, _ = SSH[host].run(processors_cmd)
stress_cmd = "stress-ng --cpu {} -l {} --timeout {}s".format(
int(processors_count), request.param, run_time)
retcode, stdout, stderr = SSH[host].run(stress_cmd)
if retcode != 0:
raise OSError(stderr)
return request.param
def delete_new_user(user_data):
"""
Delete user with given user_data.
"""
auth = login(pytest.config.getini("usm_username"),
pytest.config.getini("usm_password"))
admin = tendrlapi_user.ApiUser(auth=auth)
if user_data['email'].endswith(
usmqe.inventory.role2hosts("usm_client")[0]):
SSH = usmssh.get_ssh()
node_connection = SSH[usmqe.inventory.role2hosts("usm_client")[0]]
userdel = 'userdel {}'.format(user_data['username'])
userdel_response = node_connection.run(userdel)
# userdel command returned 0 return code
assert userdel_response[0] == 0
admin.del_user(user_data["username"])
logout(auth=auth)
def volume_mount_points():
"""
Returns dictionary where keys are volume names in gluster and values
are directory paths to volume mount points.
"""
SSH = usmssh.get_ssh()
host = CONF.inventory.get_groups_dict()["usm_client"][0]
gluster_volume = GlusterVolume()
volumes = gluster_volume.list()
mount_points = {}
for volume in volumes:
mount_point_cmd = "mount | awk '/{}/ {{print $3}}'".format(volume)
retcode, mount_point, stderr = SSH[host].run(mount_point_cmd)
if retcode != 0:
raise OSError(stderr.decode("utf-8"))
mount_points[volume] = mount_point.decode("utf-8")
return mount_points
def delete_new_user(user_data):
"""
Delete user with given user_data.
"""
auth = login(CONF.config["usmqe"]["username"],
CONF.config["usmqe"]["password"])
admin = tendrlapi_user.ApiUser(auth=auth)
if user_data['email'].endswith(
CONF.inventory.get_groups_dict()["usm_client"][0]):
SSH = usmssh.get_ssh()
node_connection = SSH[CONF.inventory.get_groups_dict()["usm_client"]
[0]]
userdel = 'userdel {}'.format(user_data['username'])
userdel_response = node_connection.run(userdel)
# userdel command returned 0 return code
assert userdel_response[0] == 0
admin.del_user(user_data["username"])
logout(auth=auth)
def create_new_user(user_data):
"""
Create user from given user_data.
"""
auth = login(pytest.config.getini("usm_username"),
pytest.config.getini("usm_password"))
admin = tendrlapi_user.ApiUser(auth=auth)
admin.add_user(user_data)
if user_data['email'].endswith(
usmqe.inventory.role2hosts("usm_client")[0]):
SSH = usmssh.get_ssh()
useradd = 'useradd {}'.format(user_data['username'])
node_connection = SSH[usmqe.inventory.role2hosts("usm_client")[0]]
node_connection.run(useradd)
passwd = 'echo "{}" | passwd --stdin {}'.format(
user_data['password'], user_data['username'])
passwd_response = node_connection.run(passwd)
# passwd command returned 0 return code
assert passwd_response[0] == 0
def create_new_user(user_data):
"""
Create user from given user_data.
"""
auth = login(CONF.config["usmqe"]["username"],
CONF.config["usmqe"]["password"])
admin = tendrlapi_user.ApiUser(auth=auth)
admin.add_user(user_data)
if user_data['email'].endswith(
CONF.inventory.get_groups_dict()["usm_client"][0]):
SSH = usmssh.get_ssh()
useradd = 'useradd {}'.format(user_data['username'])
node_connection = SSH[CONF.inventory.get_groups_dict()["usm_client"]
[0]]
node_connection.run(useradd)
passwd = 'echo "{}" | passwd --stdin {}'.format(
user_data['password'], user_data['username'])
passwd_response = node_connection.run(passwd)
# passwd command returned 0 return code
assert passwd_response[0] == 0
def test_alerting_settings(application, receive_alerts,
valid_normal_user_data):
"""
Create normal user with email notifications switched on or off.
Check that alerts appear in the mailbox according to notification settings.
"""
"""
:step:
Change admin's email to avoid collision.
:result:
Admin's email is changed
"""
# create a user with the email address configured in ansible playbook
new_data = {
"email": "alerting_test" + str(receive_alerts) + "@example.com",
"password": CONF.config["usmqe"]["password"],
"confirm_password": CONF.config["usmqe"]["password"]
}
application.collections.users.edit_logged_in_user(new_data)
"""
:step:
Create normal user with notifications switched on or off
and email configured in ansible playbook
:result:
Normal user is created
"""
user = application.collections.users.create(
user_id=valid_normal_user_data["username"],
name=valid_normal_user_data["name"],
email="root@" + CONF.inventory.get_groups_dict()["usm_client"][0],
notifications_on=receive_alerts,
password=valid_normal_user_data["password"],
role=valid_normal_user_data["role"])
"""
:step:
Stop glusterd on one of the cluster nodes.
:result:
After some time alerts are generated
"""
start_time = datetime.datetime.now().timestamp()
SSH = usmssh.get_ssh()
host = CONF.config["usmqe"]["cluster_member"]
stop_cmd = "systemctl stop glusterd"
time.sleep(90)
retcode, stdout, stderr = SSH[host].run(stop_cmd)
if retcode != 0:
raise OSError(stderr)
"""
:step:
Restart glusterd.
:result:
After some time alerts disappear
"""
restart_cmd = "systemctl restart glusterd"
time.sleep(90)
retcode, stdout, stderr = SSH[host].run(restart_cmd)
if retcode != 0:
raise OSError(stderr)
stop_time = datetime.datetime.now().timestamp()
"""
:step:
Gather e-mails that came after glusterd was stopped and check if there were any.
:result:
There should be some or no e-mails according to user settings.
"""
messages = usmmail.get_msgs_by_time(start_timestamp=start_time,
end_timestamp=stop_time)
LOGGER.debug("Selected messages count: {}".format(len(messages)))
alert_received = False
for message in messages:
LOGGER.debug("Message date: {}".format(message['Date']))
LOGGER.debug("Message subject: {}".format(message['Subject']))
LOGGER.debug("Message body: {}".format(
message.get_payload(decode=True)))
if message['Subject'].count("]"):
alert_received = True
if receive_alerts:
pytest.check(alert_received, "Check that alert has been received")
else:
pytest.check(not alert_received,
"Check that alert hasn't been recieved")
user.delete()
def test_ui_alerts(application, managed_cluster):
"""
Test UI alert appearance and disapearance.
"""
"""
:step:
Stop glusterd on one of the cluster nodes.
:result:
After some time alerts appear in UI
"""
SSH = usmssh.get_ssh()
host = CONF.config["usmqe"]["cluster_member"]
stop_cmd = "systemctl stop glusterd"
time.sleep(10)
retcode, stdout, stderr = SSH[host].run(stop_cmd)
if retcode != 0:
raise OSError(stderr)
time.sleep(40)
alert_found = False
alerts = application.collections.alerts.get_alerts()
for alert in alerts:
LOGGER.debug("Alert description: {}".format(alert.description))
LOGGER.debug("Alert date: {}".format(alert.date))
LOGGER.debug("Alert severity: {}".format(alert.severity))
if alert.description.find(
"is Disconnected") > 0 and alert.description.find(host) > 0:
alert_found = True
LOGGER.debug("Alert found: {}".format(alert_found))
pytest.check(
alert.severity == "warning",
"Check that severity of alert about disconnection is ``warning``"
)
pytest.check(
int(alert.date.split(" ")[2]) > 2018,
"Check that the year in the alert date is integer, 2019 or greater"
)
pytest.check(
alert_found,
"Check that the alert about disconnection exists in the list of UI alerts"
)
"""
:step:
Restart glusterd.
:result:
After some time alerts disappear
"""
restart_cmd = "systemctl restart glusterd"
time.sleep(10)
retcode, stdout, stderr = SSH[host].run(restart_cmd)
time.sleep(50)
if retcode != 0:
raise OSError(stderr)
alerts = application.collections.alerts.get_alerts()
alert_found = False
for alert in alerts:
LOGGER.debug("Alert description: {}".format(alert.description))
LOGGER.debug("Alert date: {}".format(alert.date))
LOGGER.debug("Alert severity: {}".format(alert.severity))
if alert.description.find(
"is Connected") > 0 and alert.description.find(host) > 0:
alert_found = True
LOGGER.debug("Alert found: {}".format(alert_found))
pytest.check(
alert.severity == "info",
"Check that severity of the alert about re-connection is ``info``"
)
pytest.check(
int(alert.date.split(" ")[2]) > 2018,
"Check that the year in the alert date is integer, 2019 or greater"
)
pytest.check(
alert_found,
"Check that the alert about re-connection exists in the list of UI alerts"
)
def workload_capacity_utilization(request, volume_mount_points):
"""
Returns:
dict: contains information about `start` and `stop` time of dd
command and `result` as number presenting percentage of disk
utilization.
"""
volume_name = list(volume_mount_points.keys())[0]
mount_point = volume_mount_points[volume_name].strip()
SSH = usmssh.get_ssh()
host = CONF.inventory.get_groups_dict()["usm_client"][0]
def fill_volume():
"""
Use `dd` command to utilize mounted volume.
"""
disk_space_cmd = "df {0} | awk '/{1}/ " \
"{{print $3 \" \" $4}}'".format(
mount_point,
mount_point.split("/")[-1])
retcode, disk_space, stderr = SSH[host].run(disk_space_cmd)
if retcode != 0:
raise OSError(stderr.decode("utf-8"))
# disk values in M
disk_used, disk_available = [
int(size.rstrip("\n")) / 1024
for size in disk_space.decode("utf-8").split(" ")
]
# block size = 100M
block_size = 100
# compute disk space that is going to be used and number of files
# to create with regard to already utilized space
file_count = int(
((int(disk_available) / 100 * request.param) - int(disk_used)) /
block_size)
stress_cmd = "for x in {{1..{}}}; do dd if=/dev/zero" \
" of={}/test_file$x count=1 bs={}M; done".format(
file_count,
mount_point[:-1] if mount_point.endswith("/") else mount_point,
block_size)
retcode, _, stderr = SSH[host].run(stress_cmd)
if retcode != 0:
raise OSError(stderr.decode("utf-8"))
return request.param
disk_space_cmd = "df {0} | awk '/{1}/ {{print $2}}'".format(
mount_point,
mount_point.split("/")[-1])
retcode, disk_total, stderr = SSH[host].run(disk_space_cmd)
if retcode != 0:
raise OSError(stderr.decode("utf-8"))
time_to_measure = 180
yield measure_operation(fill_volume,
minimal_time=time_to_measure,
metadata={
"volume_name":
volume_name,
"total_capacity":
int(disk_total.decode("utf-8").rstrip("\n"))
},
measure_after=True)
cleanup_cmd = "rm -f {}/test_file*".format(
mount_point[:-1] if mount_point.endswith("/") else mount_point)
retcode, _, stderr = SSH[host].run(cleanup_cmd)
if retcode != 0:
raise OSError(stderr.decode("utf-8"))
def test_create_brick_valid(valid_cluster_id, valid_brick_path, valid_devices,
valid_session_credentials):
"""@pylatest api/gluster.create_brick_valid
API-gluster: create_brick
******************************
.. test_metadata:: author [email protected]
Description
===========
Create ``GlusterCreateBrick`` job with a ``brick_name`` on specified
``valid_device`` with nodes from cluster with ``valid_cluster_id``
.. test_step:: 1
Connect to Tendrl API via POST request to ``APIURL/:cluster_id/GlusterCreateBrick``
Where cluster_id is set to predefined value.
.. test_result:: 1
Server should return response in JSON format:
Return code should be **202** with data ``{"message": "Accepted"}``.
job should finish.
"""
api = glusterapi.TendrlApiGluster(auth=valid_session_credentials)
brick_name = re.search("(.*)_mount",
os.path.split(valid_brick_path)[1]).group(1)
cluster_info = [
x for x in api.get_cluster_list()
if x["integration_id"] == valid_cluster_id
]
nodes = cluster_info[0]["nodes"]
job_id = api.create_bricks(valid_cluster_id, nodes, valid_devices,
brick_name)["job_id"]
api.wait_for_job_status(job_id)
"""@pylatest api/gluster.create_brick_valid
API-gluster: create_brick
******************************
.. test_metadata:: author [email protected]
Description
===========
Check if the bricks were created on hosts of cluster with ``valid_cluster_id``.
.. test_step:: 2
Via ssh check on cluster nodes that there exists directory with called
``brick_name`` in `/tendrl_gluster_bricks/brick_mount`:
[ -d /tendrl_gluster_bricks/brick_mount/``brick_name`` ] && echo "exists"
.. test_result:: 2
There should be string ``exists`` in output of ssh.
"""
SSH = usmssh.get_ssh()
pytest.check(
len(nodes) > 0,
"In cluster have to be at least one node. There are {}".format(
len(nodes)))
cmd_exists = "[ -d {} ] && echo 'exists'".format(valid_brick_path)
cmd_fs = 'mount | grep $(df --output=source {} | tail -1)'.format(
valid_brick_path)
expected_output = '/dev/mapper/tendrl{0}_vg-tendrl{0}_lv on {1} type xfs'\
.format(brick_name, valid_brick_path)
for node in nodes:
_, output, _ = SSH[nodes[node]["fqdn"]].run(cmd_exists)
output = str(output).strip("'b\\n")
pytest.check(
output == "exists",
"Output of command `{}` should be `exists`. Output is: `{}`".
format(cmd_exists, output))
"""@pylatest api/gluster.create_brick_valid
API-gluster: create_brick
******************************
.. test_metadata:: author [email protected]
Description
===========
Check if the bricks have ``xfs`` filesystem and set correct device.
.. test_step:: 3
Via ssh check filesystem and deviceof directory with
``valid_brick_path``:
mount | grep $(df --output=source ``valid_brick_path`` | tail -1)
.. test_result:: 3
Output of the command should be:
``/dev/mapper/tendrl`brick_name`_vg-tendrl`brick_name`_lv on `brick_path` type xfs``
"""
_, output, _ = SSH[nodes[node]["fqdn"]].run(cmd_fs)
output = str(output).strip("'b\\n")
output = re.sub("\s*\(.*\)$", "", output)
pytest.check(
output == expected_output,
"Output of command {} should be `{}`. Output is: `{}`".format(
cmd_fs, expected_output, output))
