def setup_rgw_conf(node: CephNode) -> None:
"""
Execute the pre-setup workflow on the provided node.
Below are the steps executed
- Stop the RadosGW service
- Add `rgw_lc_debug_interval` with interval as 10 seconds
- Start the RadosGW service
Args:
node: The node object has the rgw role
Returns:
None
Raises:
CommandFailed: when any remote command execution has returned a non-zero
"""
commands = [
"sudo systemctl stop ceph-radosgw.target",
"sudo sed -i -e '$argw_lc_debug_interval = 10' /etc/ceph/ceph.conf",
"sudo systemctl start ceph-radosgw.target",
]
for cmd in commands:
node.exec_command(cmd=cmd)
def teardown_rgw_conf(node: CephNode) -> None:
"""
Execute the pre-setup cleanup workflow on the given node.
Below are the steps executed as part of the cleanup activity
- Stop the RadosGW service
- Remove the conf changes
- Start the RadosGW service
Args:
node: The node that has the rgw role
Returns:
None
Raises:
CommandFailed: when any of the
"""
commands = [
"sudo systemctl stop ceph-radosgw.target",
"sudo sed -i '/rgw_lc_debug_interval/d' /etc/ceph/ceph.conf",
"sudo systemctl start ceph-radosgw.target",
]
for cmd in commands:
node.exec_command(cmd=cmd)
def enable_ports(node: CephNode, port: int = 18088) -> None:
"""
Opens the required firewall ports on the COSBench role type nodes.
Args:
node (CephNode): The list of nodes for which the port has to be opened.
port (int): The network port that needs to be opened
Returns:
None
Raises:
CommandFailed
"""
LOG.debug("Opening the required network ports if firewall is configured.")
try:
out, err = node.exec_command(sudo=True, cmd="firewall-cmd --state")
if out.lower() != "running":
return
except CommandFailed:
LOG.debug(f"{node.shortname} has no firewall configuration.")
return
node.exec_command(
sudo=True,
cmd=f"firewall-cmd --zone public --permanent --port {port}/tcp")
def _create_agent_systemd(node: CephNode) -> None:
"""
Configures and runs the vault-agent as a system daemon.
This method creates two files i.e. a launcher file and a system service unit. It
also enables the service to start.
Args:
node The node for which the vault agent needs to be set.
Returns:
None
Raises:
CommandFailed
"""
_write_remote_file(
node=node,
file_name="/usr/bin/vault-agent",
content=AGENT_LAUNCHER,
)
_write_remote_file(
node=node,
file_name="/usr/lib/systemd/system/vault-agent.service",
content=AGENT_SYSTEMD,
)
commands = [
"chmod +x /usr/bin/vault-agent",
"systemctl start vault-agent.service",
"systemctl enable vault-agent.service",
]
for command in commands:
node.exec_command(sudo=True, cmd=command)
def restart_service(node: CephNode) -> None:
"""
restart HAproxy service.
Args:
node : The node on which service need to restart.
Returns:
None
Raises:
CommandFailed
"""
node.exec_command(sudo=True, cmd="systemctl restart haproxy")
LOG.info("HAproxy Service restarted!!!")
def create_s3_user(node: CephNode, user_prefix: str, data: Dict) -> None:
"""
Create a S3 user with the given display_name.
The other required information for creating an user is auto generated.
Args:
node: node in the cluster to create the user on
user_prefix: Prefix to be added to the new user
data: a reference to the payload that needs to be updated.
Returns:
user_info dict
"""
uid = binascii.hexlify(os.urandom(32)).decode()
display_name = f"{user_prefix}-user"
log.info("Creating user: {display_name}".format(display_name=display_name))
cmd = f"radosgw-admin user create --uid={uid} --display_name={display_name}"
cmd += " --email={email}@foo.bar".format(email=uid)
out, err = node.exec_command(sudo=True, cmd=cmd)
user_info = json.loads(out)
data[user_prefix] = {
"id": user_info["keys"][0]["user"],
"access_key": user_info["keys"][0]["access_key"],
"secret_key": user_info["keys"][0]["secret_key"],
"name": user_info["display_name"],
"email": user_info["email"],
}
def install(node: CephNode) -> None:
"""
Installs HAproxy.
Args:
nodes (list): The list of nodes on which the packages are installed.
Returns:
None
Raises:
CommandFailed
"""
node.exec_command(sudo=True, cmd="yum install -y haproxy")
node.exec_command(sudo=True, cmd="chmod 666 /etc/haproxy/haproxy.cfg")
LOG.info("Successfully installed HAproxy!!!")
def execute_s3_tests(node: CephNode, build: str) -> int:
"""
Return the result of S3 test run.
Args:
node: The node from which the test execution is triggered.
build: the RH build version
Returns:
0 - Success
1 - Failure
"""
log.info("Executing s3-tests")
try:
base_cmd = "cd s3-tests; S3TEST_CONF=config.yaml virtualenv/bin/nosetests -v"
extra_args = "-a '!fails_on_rgw,!fails_strict_rfc2616,!encryption'"
tests = "s3tests"
if build.startswith("5"):
extra_args = "-a '!fails_on_rgw,!fails_strict_rfc2616,!encryption"
extra_args += ",!test_of_sts,!lifecycle,!s3select,!user-policy"
extra_args += ",!webidentity_test'"
tests = "s3tests_boto3"
cmd = f"{base_cmd} {extra_args} {tests}"
out, err = node.exec_command(cmd=cmd, timeout=3600)
log.info(out.read().decode())
log.error(err.read().decode())
except CommandFailed as e:
log.warning("Received CommandFailed")
log.warning(e)
return 1
return 0
def execute_s3_tests(node: CephNode,
build: str,
encryption: bool = False) -> int:
"""
Return the result of S3 test run.
Args:
node The node from which the test execution is triggered.
build The RH build version
encryption include encryption test or not
Returns:
0 - Success
1 - Failure
"""
log.debug("Executing s3-tests")
try:
base_cmd = "cd s3-tests; S3TEST_CONF=config.yaml virtualenv/bin/nosetests -v"
extra_args = "-a '!fails_on_rgw,!fails_strict_rfc2616,!encryption'"
tests = "s3tests"
if build.startswith("5"):
extra_args = "-a '!fails_on_rgw,!fails_strict_rfc2616"
if not encryption:
extra_args += ",!encryption"
extra_args += ",!test_of_sts,!s3select,!user-policy,!webidentity_test'"
tests = "s3tests_boto3"
cmd = f"{base_cmd} {extra_args} {tests}"
return node.exec_command(cmd=cmd, long_running=True)
except CommandFailed as e:
log.warning("Received CommandFailed")
log.warning(e)
return 1
def _install_vault_packages(node: CephNode) -> None:
"""
Installs the required packages for vault
Args:
node The system on which the package needs to be installed
Returns:
None
Raises:
CommandFailed
"""
vault_repo = "https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo"
commands = [f"yum-config-manager --add-repo {vault_repo}", "yum install -y vault"]
for command in commands:
node.exec_command(sudo=True, cmd=command, check_ec=False)
def get_or_create_user(node: CephNode) -> Dict:
"""Creates or retrieves a RADOS user.
Returns:
Dictionary holding the keys user, access_key & secret_key
"""
LOG.debug("Get or Create cosbench01 user using radosgw-admin.")
user = "******"
try:
out, err = node.exec_command(
cmd=f"radosgw-admin user info --uid {user}")
out = loads(out)
return out["keys"][0]
except CommandFailed:
out, err = node.exec_command(
cmd=f"radosgw-admin user create --uid {user} --display-name {user}"
f" --email {user}@noreply.com")
out = loads(out)
return out["keys"][0]
def exec_command(
node: CephNode,
command: str,
sudo: Optional[bool] = False,
check_ec: Optional[bool] = True,
) -> Tuple:
"""Executes the given command on the provided node."""
out, err = node.exec_command(sudo=sudo, cmd=command, check_ec=check_ec)
LOG.debug(f"Output: {out}")
LOG.debug(f"Error: {err}")
return out, err
def _s3tests_req_install(node: CephNode, os_ver: str) -> None:
"""Install S3 prerequisites via pip."""
packages = [
"python2-virtualenv",
"python2-devel",
"libevent-devel",
"libffi-devel",
"libxml2-devel",
"libxslt-devel",
"zlib-devel",
]
node.exec_command(sudo=True,
cmd="yum groupinstall -y 'Development Tools'",
check_ec=False)
node.exec_command(sudo=True,
cmd=f"yum install -y --nogpgcheck {' '.join(packages)}")
venv_cmd = "virtualenv" if os_ver == "7" else "virtualenv-2"
commands = [
f"{venv_cmd} -p python2 --no-site-packages --distribute s3-tests/virtualenv",
"s3-tests/virtualenv/bin/pip install --upgrade pip setuptools",
"s3-tests/virtualenv/bin/pip install -r s3-tests/requirements.txt",
"s3-tests/virtualenv/bin/python s3-tests/setup.py develop",
]
for cmd in commands:
node.exec_command(cmd=cmd)
def create_nodes(conf, inventory, osp_cred, run_id, report_portal_session=None, instances_name=None):
if report_portal_session:
name = create_unique_test_name("ceph node creation", test_names)
test_names.append(name)
desc = "Ceph cluster preparation"
report_portal_session.start_test_item(name=name,
description=desc,
start_time=timestamp(),
item_type="STEP")
log.info("Destroying existing osp instances..")
cleanup_ceph_nodes(osp_cred, instances_name)
ceph_cluster_dict = {}
log.info('Creating osp instances')
for cluster in conf.get('globals'):
ceph_vmnodes = create_ceph_nodes(cluster, inventory, osp_cred, run_id, instances_name)
ceph_nodes = []
clients = []
for node in ceph_vmnodes.values():
if node.role == 'win-iscsi-clients':
clients.append(WinNode(ip_address=node.ip_address,
private_ip=node.get_private_ip()))
else:
ceph = CephNode(username='******',
password='******',
root_password='******',
root_login=node.root_login,
role=node.role,
no_of_volumes=node.no_of_volumes,
ip_address=node.ip_address,
subnet=node.subnet,
private_ip=node.get_private_ip(),
hostname=node.hostname,
ceph_vmnode=node)
ceph_nodes.append(ceph)
cluster_name = cluster.get('ceph-cluster').get('name', 'ceph')
ceph_cluster_dict[cluster_name] = Ceph(cluster_name, ceph_nodes)
# TODO: refactor cluster dict to cluster list
log.info('Done creating osp instances')
log.info("Waiting for Floating IPs to be available")
log.info("Sleeping 15 Seconds")
time.sleep(15)
for cluster_name, cluster in ceph_cluster_dict.items():
for instance in cluster:
try:
instance.connect()
except BaseException:
if report_portal_session:
report_portal_session.finish_test_item(end_time=timestamp(), status="FAILED")
raise
if report_portal_session:
report_portal_session.finish_test_item(end_time=timestamp(), status="PASSED")
return ceph_cluster_dict, clients
def _s3tests_req_install(node: CephNode) -> None:
"""Install S3 prerequisites via pip."""
packages = [
"python2-virtualenv",
"python2-devel",
"libevent-devel",
"libffi-devel",
"libxml2-devel",
"libxslt-devel",
"zlib-devel",
]
node.exec_command(sudo=True,
cmd=f"yum install -y --nogpgcheck {' '.join(packages)}")
commands = [
"virtualenv -p python2 --no-site-packages --distribute s3-tests/virtualenv",
"s3-tests/virtualenv/bin/pip install --upgrade pip setuptools",
"s3-tests/virtualenv/bin/pip install -r s3-tests/requirements.txt",
"s3-tests/virtualenv/bin/python s3-tests/setup.py develop",
]
for cmd in commands:
node.exec_command(cmd=cmd)
def _create_agent_config(node: CephNode, config: Dict) -> None:
"""
Writes the required configuration file to the provided node.
The following files are created .app-role-id, .app-secret-id and agent.hcl
Args:
node The system on which files have to be copied
config Dictionary holding the tokens
Returns:
None
Raises:
CommandFailed
"""
node.exec_command(sudo=True, cmd="mkdir -p /usr/local/etc/vault/")
_write_remote_file(
node=node,
file_name="/usr/local/etc/vault/.app-role-id",
content=config["agent"]["role-id"],
)
_write_remote_file(
node=node,
file_name="/usr/local/etc/vault/.app-secret-id",
content=config["agent"]["secret-id"],
)
# hcl file
agent_conf = {"url": config["url"], "auth": config["agent"]["auth"]}
tmpl = Template(AGENT_HCL)
data = tmpl.render(data=agent_conf)
_write_remote_file(
node=node,
file_name="/usr/local/etc/vault/agent.hcl",
content=data,
)
def config(node: CephNode, data: List) -> None:
"""
Writes the haproxy.cfg configuration file.
Args:
node: Haproxy client node
data: A list having rgw endpoints(ip_address & port)
Returns:
None
Raises:
CommandFailed
"""
LOG.info("Generating the HAproxy.cfg file.")
templ = Template(HAPRX_CONF)
conf = templ.render(data=data)
conf_file = node.remote_file(file_name="/etc/haproxy/haproxy.cfg",
file_mode="w")
conf_file.write(conf)
conf_file.flush()
def _write_remote_file(node: CephNode, file_name: str, content: str) -> None:
"""
Copies the provide content to the specified file on the given node.
Args:
node The target system
file_name The name of the remote file to which the content needs to be written
content The content of the file to be written
Returns:
None
Raises:
CommandFailed
"""
LOG.debug(f"Writing to remote file {file_name}")
file_handle = node.remote_file(sudo=True, file_mode="w", file_name=file_name)
file_handle.write(data=content)
file_handle.flush()
file_handle.close()
def config(node: CephNode, data: List) -> None:
"""
Writes the COS Bench controller configuration file.
Args:
node: The node that is designated to be a COS controller
data: A list of dictionaries having driver details (name & ip_address)
Returns:
None
Raises:
CommandFailed
"""
LOG.info("Generating the COS Bench controller file.")
templ = Template(CTRL_CONF)
conf = templ.render(data=data)
conf_file = node.remote_file(
file_name="/opt/cosbench/conf/controller.conf", file_mode="w")
conf_file.write(conf)
conf_file.flush()
def install_s3test_requirements(node: CephNode, branch: str) -> None:
"""
Install the required packages required by S3tests.
The S3test requirements are installed via bootstrap however for RHCS it is a
simulation of the manual steps.
Args:
node: The node that is consider for running S3Tests.
branch: The branch to be installed
Raises:
CommandFailed: Whenever a command returns a non-zero value part of the method.
"""
rhel8, err = node.exec_command(
cmd="grep -i 'release 8' /etc/redhat-release", check_ec=False)
rhel8 = rhel8.read().decode()
if branch == "ceph-nautilus" and rhel8:
return _s3tests_req_install(node)
_s3tests_req_bootstrap(node)
def _configure_rgw_daemons(node: CephNode, config: Dict) -> None:
"""
Updates the RGW daemons with the provided configuration.
Args:
node Server that has privilege to perform ceph config set commands.
config Key/value pairs to be used for configuration
Returns:
None
Raises:
CommandFailed
"""
out, err = node.exec_command(
sudo=True, cmd="ceph orch ps --daemon_type rgw --format json")
rgw_daemons = [
f"client.rgw.{x['daemon_id']}" for x in loads(out.read().decode())
]
out, err = node.exec_command(
sudo=True, cmd="ceph orch ls --service_type rgw --format json")
rgw_services = [x["service_name"] for x in loads(out.read().decode())]
configs = [
("rgw_crypt_s3_kms_backend", "vault"),
("rgw_crypt_vault_secret_engine", config["agent"]["engine"]),
("rgw_crypt_vault_auth", config["agent"]["auth"]),
]
if config["agent"]["auth"] == "token":
configs += [
("rgw_crypt_vault_token_file", config["agent"]["token_file"]),
("rgw_crypt_vault_addr", config["url"]),
]
else:
configs += [
("rgw_crypt_vault_prefix", config["agent"]["prefix"]),
("rgw_crypt_vault_addr", "http://127.0.0.1:8100"),
]
for daemon in rgw_daemons:
for key, value in configs:
node.exec_command(sudo=True,
cmd=f"ceph config set {daemon} {key} {value}")
for service in rgw_services:
node.exec_command(sudo=True, cmd=f"ceph orch restart {service}")
def create_nodes(
conf,
inventory,
osp_cred,
run_id,
cloud_type="openstack",
report_portal_session=None,
instances_name=None,
enable_eus=False,
rp_logger: Optional[ReportPortal] = None,
):
"""Creates the system under test environment."""
if report_portal_session:
name = create_unique_test_name("ceph node creation", test_names)
test_names.append(name)
desc = "Ceph cluster preparation"
rp_logger.start_test_item(name=name,
description=desc,
item_type="STEP")
log.info("Destroying existing osp instances..")
if cloud_type == "openstack":
cleanup_ceph_nodes(osp_cred, instances_name)
elif cloud_type == "ibmc":
cleanup_ibmc_ceph_nodes(osp_cred, instances_name)
ceph_cluster_dict = {}
log.info("Creating osp instances")
clients = []
for cluster in conf.get("globals"):
if cloud_type == "openstack":
ceph_vmnodes = create_ceph_nodes(
cluster,
inventory,
osp_cred,
run_id,
instances_name,
enable_eus=enable_eus,
)
elif cloud_type == "ibmc":
ceph_vmnodes = create_ibmc_ceph_nodes(cluster, inventory, osp_cred,
run_id, instances_name)
elif cloud_type == "baremetal":
ceph_vmnodes = create_baremetal_ceph_nodes(cluster)
ceph_nodes = []
root_password = None
for node in ceph_vmnodes.values():
look_for_key = False
private_key_path = ""
if cloud_type == "openstack":
private_ip = node.get_private_ip()
elif cloud_type == "baremetal":
private_key_path = node.private_key if node.private_key else ""
private_ip = node.ip_address
look_for_key = True if node.private_key else False
root_password = node.root_password
elif cloud_type == "ibmc":
glbs = osp_cred.get("globals")
ibmc = glbs.get("ibm-credentials")
private_key_path = ibmc.get("private_key_path")
private_ip = node.ip_address
look_for_key = True
if node.role == "win-iscsi-clients":
clients.append(
WinNode(ip_address=node.ip_address, private_ip=private_ip))
else:
ceph = CephNode(
username="******",
password="******",
root_password="******"
if not root_password else root_password,
look_for_key=look_for_key,
private_key_path=private_key_path,
root_login=node.root_login,
role=node.role,
no_of_volumes=node.no_of_volumes,
ip_address=node.ip_address,
subnet=node.subnet,
private_ip=private_ip,
hostname=node.hostname,
ceph_vmnode=node,
)
ceph_nodes.append(ceph)
cluster_name = cluster.get("ceph-cluster").get("name", "ceph")
ceph_cluster_dict[cluster_name] = Ceph(cluster_name, ceph_nodes)
# TODO: refactor cluster dict to cluster list
log.info("Done creating osp instances")
log.info("Waiting for Floating IPs to be available")
log.info("Sleeping 15 Seconds")
time.sleep(15)
for cluster_name, cluster in ceph_cluster_dict.items():
for instance in cluster:
try:
instance.connect()
except BaseException:
rp_logger.finish_test_item(status="FAILED")
raise
rp_logger.finish_test_item(status="PASSED")
return ceph_cluster_dict, clients
def _s3tests_req_bootstrap(node: CephNode) -> None:
"""Install the S3tests using bootstrap script."""
node.exec_command(cmd="cd s3-tests; ./bootstrap")
def clone_s3_tests(node: CephNode, branch="ceph-luminous") -> None:
"""Clone the S3 repository on the given node."""
repo_url = "https://github.com/ceph/s3-tests.git"
node.exec_command(cmd="if test -d s3-tests; then sudo rm -r s3-tests; fi")
node.exec_command(cmd=f"git clone -b {branch} {repo_url}")
