def delete_workers(worker_ids, signal_to_pass=signal.SIGINT): """ Expect worker ID without RQ REDIS WORKER NAMESPACE PREFIX of rq:worker: By default performs warm shutdown :param worker_id: list of worker id's to delete :param signal_to_pass: :return: """ # find worker instance by key, refreshes worker implicitly def attach_rq_worker_prefix(worker_id): return Worker.redis_worker_namespace_prefix + worker_id for worker_instance in [Worker.find_by_key(attach_rq_worker_prefix(worker_id)) for worker_id in worker_ids]: requested_hostname = worker_instance.hostname requested_hostname = requested_hostname.decode('utf-8') # kill if on same instance if socket.gethostname() == requested_hostname: os.kill(worker_instance.pid, signal_to_pass) else: required_host_ip = socket.gethostbyname(requested_hostname) fabric_config_wrapper = Config() # loads from user level ssh config (~/.ssh/config) and system level # config /etc/ssh/ssh_config fabric_config_wrapper.load_ssh_config() # to use its ssh_config parser abilities paramiko_ssh_config = fabric_config_wrapper.base_ssh_config for hostname in paramiko_ssh_config.get_hostnames(): ssh_info = paramiko_ssh_config.lookup(hostname) available_host_ip = ssh_info.get('hostname') if available_host_ip == required_host_ip: process_owner = None # make connection via fabric and send SIGINT for now ssh_connection = Connection(hostname) try: #find owner of process https://unix.stackexchange.com/questions/284934/return-owner-of-process-given-pid process_owner = ssh_connection.run('ps -o user= -p {0}'.format(worker_instance.pid)) # have permission to kill so this works without sudo # need to plan for other cases process_owner = process_owner.stdout.strip(' \n\t') result_kill = ssh_connection.run('kill -{0} {1}'.format(2, worker_instance.pid), hide=True) if result_kill.failed: raise RQMonitorException("Some issue occured on running command {0.command!r} " "on {0.connection.host}, we got stdout:\n{0.stdout}" "and stderr:\n{0.stderr}".format(result_kill)) except UnexpectedExit as e: stdout, stderr = e.streams_for_display() # plan to accept password from user and proceed with sudo in future if "Operation not permitted" in stderr.strip(' \n\t'): raise RQMonitorException('Logged in user {0} does not have permission to kill worker' ' process with pid {1} on {2} because it is owned ' ' by user {3}'.format(ssh_info.get('user'), worker_instance.pid, required_host_ip, process_owner)) raise RQMonitorException('Invoke\'s UnexpectedExit occurred with' 'stdout: {0}\nstderr: {1}\nresult: {2}\nreason {3}'.format(stdout.strip(' \n\t'), stderr.strip(' \n\t'), e.result, e.reason)) return
def may_use_lazy_plus_explicit_methods_to_control_flow(self, method): c = Config(lazy=True) assert not method.called c.set_runtime_ssh_path(self._runtime_path) c.load_ssh_config() method.assert_called_once_with(self._runtime_path)