def _remove_hosts(hosts):
"""
Remove a list of compute nodes from the cluster.
If one of the steps fails then the procedure for the failing host continues. This is done
to clean up as much as possible the scheduler configuration in case a node is terminated
but some of the steps are failing.
All operations should be idempotent in order to allow retries in case the procedure
fails at any of the steps.
:return: the list of hostnames that were removed correctly from the cluster
"""
if not hosts:
return []
succeeded_hosts = set(hosts)
succeeded_hosts = succeeded_hosts.intersection(set(remove_hosts_from_queue(hosts)))
succeeded_hosts = succeeded_hosts.intersection(set(remove_hosts_from_group(hosts)))
succeeded_hosts = succeeded_hosts.intersection(
set(exec_qconf_command(hosts, QCONF_COMMANDS["REMOVE_ADMINISTRATIVE_HOST"]))
)
succeeded_hosts = succeeded_hosts.intersection(set(exec_qconf_command(hosts, QCONF_COMMANDS["REMOVE_SUBMIT_HOST"])))
succeeded_hosts = succeeded_hosts.intersection(
set(exec_qconf_command(hosts, QCONF_COMMANDS["REMOVE_EXECUTION_HOST"]))
)
return [host.hostname for host in succeeded_hosts]
def _add_hosts(hosts, cluster_user):
"""
Add a list of compute nodes to the cluster.
If one of the steps fails then the procedure for the failing host is stopped.
All operations should be idempotent in order to allow retries in case the procedure
fails at any of the steps.
:return: the list of hostnames that were added correctly to the cluster
"""
if not hosts:
return []
succeeded_hosts = exec_qconf_command(hosts, QCONF_COMMANDS["ADD_ADMINISTRATIVE_HOST"])
succeeded_hosts = exec_qconf_command(succeeded_hosts, QCONF_COMMANDS["ADD_SUBMIT_HOST"])
succeeded_hosts = install_sge_on_compute_nodes(succeeded_hosts, cluster_user)
succeeded_hosts = add_hosts_to_group(succeeded_hosts)
succeeded_hosts = add_host_slots(succeeded_hosts)
return [host.hostname for host in succeeded_hosts]
def test_qconf_commands(qconf_output, command, expected_succeeded_hosts, mocker):
mock = mocker.patch(
"common.schedulers.sge_commands.check_sge_command_output", return_value=qconf_output, autospec=True
)
hosts = [Host("id", "ip-10-0-0-157", 1, 0), Host("id", "ip-10-0-0-155", 1, 0), Host("id", "ip-10-0-0", 1, 0)]
succeeded_hosts = exec_qconf_command(hosts, QCONF_COMMANDS[command])
mock.assert_called_with(
"qconf {0} ip-10-0-0-157,ip-10-0-0-155,ip-10-0-0".format(QCONF_COMMANDS[command].command_flags),
raise_on_error=False,
)
assert_that([host.hostname for host in succeeded_hosts]).contains_only(*expected_succeeded_hosts)