def delete_workers(worker_ids, signal_to_pass=signal.SIGINT):
"""
Expect worker ID without RQ REDIS WORKER NAMESPACE PREFIX of rq:worker:
By default performs warm shutdown
:param worker_id: list of worker id's to delete
:param signal_to_pass:
:return:
"""
# find worker instance by key, refreshes worker implicitly
def attach_rq_worker_prefix(worker_id):
return Worker.redis_worker_namespace_prefix + worker_id
for worker_instance in [Worker.find_by_key(attach_rq_worker_prefix(worker_id))
for worker_id in worker_ids]:
requested_hostname = worker_instance.hostname
requested_hostname = requested_hostname.decode('utf-8')
# kill if on same instance
if socket.gethostname() == requested_hostname:
os.kill(worker_instance.pid, signal_to_pass)
else:
required_host_ip = socket.gethostbyname(requested_hostname)
fabric_config_wrapper = Config()
# loads from user level ssh config (~/.ssh/config) and system level
# config /etc/ssh/ssh_config
fabric_config_wrapper.load_ssh_config()
# to use its ssh_config parser abilities
paramiko_ssh_config = fabric_config_wrapper.base_ssh_config
for hostname in paramiko_ssh_config.get_hostnames():
ssh_info = paramiko_ssh_config.lookup(hostname)
available_host_ip = ssh_info.get('hostname')
if available_host_ip == required_host_ip:
process_owner = None
# make connection via fabric and send SIGINT for now
ssh_connection = Connection(hostname)
try:
#find owner of process https://unix.stackexchange.com/questions/284934/return-owner-of-process-given-pid
process_owner = ssh_connection.run('ps -o user= -p {0}'.format(worker_instance.pid))
# have permission to kill so this works without sudo
# need to plan for other cases
process_owner = process_owner.stdout.strip(' \n\t')
result_kill = ssh_connection.run('kill -{0} {1}'.format(2, worker_instance.pid), hide=True)
if result_kill.failed:
raise RQMonitorException("Some issue occured on running command {0.command!r} "
"on {0.connection.host}, we got stdout:\n{0.stdout}"
"and stderr:\n{0.stderr}".format(result_kill))
except UnexpectedExit as e:
stdout, stderr = e.streams_for_display()
# plan to accept password from user and proceed with sudo in future
if "Operation not permitted" in stderr.strip(' \n\t'):
raise RQMonitorException('Logged in user {0} does not have permission to kill worker'
' process with pid {1} on {2} because it is owned '
' by user {3}'.format(ssh_info.get('user'), worker_instance.pid,
required_host_ip, process_owner))
raise RQMonitorException('Invoke\'s UnexpectedExit occurred with'
'stdout: {0}\nstderr: {1}\nresult: {2}\nreason {3}'.format(stdout.strip(' \n\t'),
stderr.strip(' \n\t'),
e.result, e.reason))
return
def may_use_lazy_plus_explicit_methods_to_control_flow(self, method):
c = Config(lazy=True)
assert not method.called
c.set_runtime_ssh_path(self._runtime_path)
c.load_ssh_config()
method.assert_called_once_with(self._runtime_path)
def may_use_lazy_plus_explicit_methods_to_control_flow(self, method):
c = Config(lazy=True)
assert not method.called
c.set_runtime_ssh_path(self._runtime_path)
c.load_ssh_config()
method.assert_called_once_with(self._runtime_path)
