def delete_workers(worker_ids, signal_to_pass=signal.SIGINT):
"""
Expect worker ID without RQ REDIS WORKER NAMESPACE PREFIX of rq:worker:
By default performs warm shutdown
:param worker_id: list of worker id's to delete
:param signal_to_pass:
:return:
"""
# find worker instance by key, refreshes worker implicitly
def attach_rq_worker_prefix(worker_id):
return Worker.redis_worker_namespace_prefix + worker_id
for worker_instance in [Worker.find_by_key(attach_rq_worker_prefix(worker_id))
for worker_id in worker_ids]:
requested_hostname = worker_instance.hostname
requested_hostname = requested_hostname.decode('utf-8')
# kill if on same instance
if socket.gethostname() == requested_hostname:
os.kill(worker_instance.pid, signal_to_pass)
else:
required_host_ip = socket.gethostbyname(requested_hostname)
fabric_config_wrapper = Config()
# loads from user level ssh config (~/.ssh/config) and system level
# config /etc/ssh/ssh_config
fabric_config_wrapper.load_ssh_config()
# to use its ssh_config parser abilities
paramiko_ssh_config = fabric_config_wrapper.base_ssh_config
for hostname in paramiko_ssh_config.get_hostnames():
ssh_info = paramiko_ssh_config.lookup(hostname)
available_host_ip = ssh_info.get('hostname')
if available_host_ip == required_host_ip:
process_owner = None
# make connection via fabric and send SIGINT for now
ssh_connection = Connection(hostname)
try:
#find owner of process https://unix.stackexchange.com/questions/284934/return-owner-of-process-given-pid
process_owner = ssh_connection.run('ps -o user= -p {0}'.format(worker_instance.pid))
# have permission to kill so this works without sudo
# need to plan for other cases
process_owner = process_owner.stdout.strip(' \n\t')
result_kill = ssh_connection.run('kill -{0} {1}'.format(2, worker_instance.pid), hide=True)
if result_kill.failed:
raise RQMonitorException("Some issue occured on running command {0.command!r} "
"on {0.connection.host}, we got stdout:\n{0.stdout}"
"and stderr:\n{0.stderr}".format(result_kill))
except UnexpectedExit as e:
stdout, stderr = e.streams_for_display()
# plan to accept password from user and proceed with sudo in future
if "Operation not permitted" in stderr.strip(' \n\t'):
raise RQMonitorException('Logged in user {0} does not have permission to kill worker'
' process with pid {1} on {2} because it is owned '
' by user {3}'.format(ssh_info.get('user'), worker_instance.pid,
required_host_ip, process_owner))
raise RQMonitorException('Invoke\'s UnexpectedExit occurred with'
'stdout: {0}\nstderr: {1}\nresult: {2}\nreason {3}'.format(stdout.strip(' \n\t'),
stderr.strip(' \n\t'),
e.result, e.reason))
return
def resume_workers_api():
if request.method == 'POST':
try:
resume(connection=get_current_connection())
except ActionFailed:
raise RQMonitorException('Unable to resume worker/s',
status_code=500)
return {'message': 'Successfully resumed all workers'}
raise RQMonitorException('Invalid HTTP Request type', status_code=400)
def empty_queue_api():
if request.method == 'POST':
queue_id = request.form.get('queue_id', None)
if queue_id is None:
raise RQMonitorException('Queue Name not received',
status_code=400)
try:
empty_queue(queue_id)
except ActionFailed as e:
raise RQMonitorException(
'Unable to empty Queue {0}'.format(queue_id), status_code=500)
return {'message': 'Successfully emptied {0}'.format(queue_id)}
raise RQMonitorException('Invalid HTTP Request type', status_code=400)
def cancel_queued_jobs_api():
if request.method == 'POST':
requested_queues = request.form.getlist('queues[]')
if requested_queues is None:
raise RQMonitorException('No queue/s selected', status_code=400)
try:
fail_count = cancel_all_queued_jobs(requested_queues)
except ActionFailed:
raise RQMonitorException('Unable to cancel all jobs',
status_code=500)
return {'message': 'Successfully requeued all jobs'}
raise RQMonitorException('Invalid HTTP Request type', status_code=400)
def cancel_job_api():
if request.method == 'POST':
job_id = request.form.get('job_id')
if job_id is None:
raise RQMonitorException('Job ID not received', status_code=400)
try:
cancel_job(job_id)
except ActionFailed:
raise RQMonitorException('Unable to cancel {0}'.format(job_id),
status_code=500)
return {
'message': 'Successfully cancelled job with ID {0}'.format(job_id)
}
raise RQMonitorException('Invalid HTTP Request type', status_code=400)
def requeue_failed_jobs_api():
if request.method == 'POST':
requested_queues = request.form.getlist('queues[]')
if requested_queues is None:
raise RQMonitorException('No queue/s selected', status_code=400)
fail_count = 0
try:
fail_count = requeue_all_jobs_in_failed_registry(requested_queues)
except ActionFailed:
raise RQMonitorException('Unable to requeue all', status_code=500)
return {
'message':
'Successfully requeued all jobs on queues {0}'.format(
", ".join(requested_queues))
}
raise RQMonitorException('Invalid HTTP Request type', status_code=400)
def delete_queue_api():
if request.method == 'POST':
queue_id = request.form.get('queue_id', None)
if queue_id is None:
raise RQMonitorException('Queue Name not received',
status_code=400)
delete_queue(queue_id)
return {'message': 'Successfully deleted {0}'.format(queue_id)}
def cancel_queued_jobs_api():
if request.method == 'POST':
requested_queues = request.form.getlist('queues[]')
if requested_queues is None:
raise RQMonitorException('No queue/s selected', status_code=400)
fail_count = cancel_all_queued_jobs(requested_queues)
return {'message': 'Successfully requeued all jobs'}
def delete_job_api():
if request.method == 'POST':
job_id = request.form.get('job_id')
if job_id is None:
raise RQMonitorException('Job ID not received', status_code=400)
delete_job(job_id)
return {
'message': 'Successfully deleted job with ID {0}'.format(job_id)
}
def empty_all_queues_api():
if request.method == 'POST':
queue_names = [queue.name for queue in list_all_queues()]
for queue in list_all_queues():
queue.empty()
return {
'message':
'Successfully emptied queues {0}'.format(", ".join(queue_names))
}
else:
raise RQMonitorException('Invalid HTTP Request type', status_code=400)
def worker_info_api():
if request.method == 'GET':
worker_id = request.args.get('worker_id', None)
if worker_id is None:
raise RQMonitorException('Worker ID not received !',
status_code=400)
worker_instance = Worker.find_by_key(
Worker.redis_worker_namespace_prefix + worker_id)
return {
'worker_host_name':
worker_instance.hostname.decode('utf-8'),
'worker_ttl':
worker_instance.default_worker_ttl,
'worker_result_ttl':
worker_instance.default_result_ttl,
'worker_name':
worker_instance.name,
'worker_birth_date':
worker_instance.birth_date.strftime('%d-%m-%Y %H:%M:%S')
if worker_instance.birth_date is not None else "Not Available",
'worker_death_date':
worker_instance.death_date.strftime('%d-%m-%Y %H:%M:%S')
if worker_instance.death_date is not None else "Is Alive",
'worker_last_cleaned_at':
worker_instance.last_cleaned_at.strftime('%d-%m-%Y %H:%M:%S') if
worker_instance.last_cleaned_at is not None else "Not Yet Cleaned",
'worker_failed_job_count':
worker_instance.failed_job_count,
'worker_successful_job_count':
worker_instance.successful_job_count,
'worker_job_monitoring_interval':
worker_instance.job_monitoring_interval,
'worker_last_heartbeat':
worker_instance.last_heartbeat.strftime('%d-%m-%Y %H:%M:%S')
if worker_instance.last_heartbeat is not None else "Not Available",
'worker_current_job_id':
worker_instance.get_current_job_id(),
}
raise RQMonitorException('Invalid HTTP Request type', status_code=400)
def requeue_failed_jobs_api():
if request.method == 'POST':
requested_queues = request.form.getlist('queues[]')
if requested_queues is None:
raise RQMonitorException('No queue/s selected', status_code=400)
fail_count = requeue_all_jobs_in_failed_registry(requested_queues)
return {
'message':
'Successfully requeued all jobs on queues {0}'.format(
", ".join(requested_queues))
}
def delete_all_jobs_api():
if request.method == 'POST':
requested_queues = request.form.getlist('queues[]')
requested_job_status = request.form.getlist('jobstatus[]')
if requested_queues is None or requested_job_status is None:
raise RQMonitorException('No queue/status selected',
status_code=400)
try:
delete_all_jobs_in_queues_registries(requested_queues,
requested_job_status)
except ActionFailed:
raise RQMonitorException('Unable to delete all jobs',
status_code=500)
return {
'message':
'Successfully deleted all jobs with status as {0} on queues {1}'.
format(", ".join(requested_job_status),
", ".join(requested_queues))
}
raise RQMonitorException('Invalid HTTP Request type', status_code=400)
def _wrapper(*args, **kwargs):
try:
inner_response = func(*args, **kwargs)
except Exception as e:
tb = sys.exc_info()[2]
error_message = getattr(e, 'message', None)
status_code = getattr(e, 'status_code', None)
kwargs = {}
if error_message is not None:
kwargs.update({'message': error_message})
if status_code is not None:
kwargs.update({'status_code': status_code})
raise RQMonitorException(**kwargs).with_traceback(tb)
return inner_response
def delete_workers_api():
worker_names = []
if request.method == 'POST':
worker_id = request.form.get('worker_id', None)
delete_all = request.form.get('delete_all')
if worker_id is None and (delete_all is "false" or delete_all is None):
raise RQMonitorException('Worker ID not received', status_code=400)
try:
if delete_all == "true":
for worker_instance in Worker.all():
worker_names.append(worker_instance.name)
delete_workers(worker_names)
else:
worker_names.append(worker_id)
delete_workers([worker_id])
except ActionFailed:
raise RQMonitorException('Unable to delete worker/s',
status_code=500)
return {
'message':
'Successfully deleted worker/s {0}'.format(", ".join(worker_names))
}
raise RQMonitorException('Invalid HTTP Request type', status_code=400)
def delete_workers_api():
worker_names = []
if request.method == 'POST':
worker_id = request.form.get('worker_id', None)
delete_all = request.form.get('delete_all')
if worker_id == None and (delete_all == "false" or delete_all == None):
raise RQMonitorException('Worker ID not received', status_code=400)
if delete_all == "true":
for worker_instance in Worker.all():
worker_names.append(worker_instance.name)
delete_workers(worker_names)
else:
worker_names.append(worker_id)
delete_workers([worker_id])
return {
'message':
'Successfully deleted worker/s {0}'.format(", ".join(worker_names))
}
def push_rq_connection():
new_instance_index = None
if request.method == 'GET':
new_instance_index = request.args.get('redis_instance_index')
if request.method == 'POST':
new_instance_index = request.form.get('redis_instance_index')
if new_instance_index is None:
new_instance_index = request.view_args.get('redis_instance_index')
#print(new_instance_index, type(new_instance_index))
if new_instance_index is not None:
redis_url = current_app.config.get("RQ_MONITOR_REDIS_URL")
new_instance_index = int(new_instance_index)
if int(new_instance_index) < len(redis_url):
new_instance = create_redis_connection(
redis_url[new_instance_index])
else:
raise RQMonitorException("Invalid redis instance index!",
status_code=400)
else:
new_instance = current_app.redis_connection
push_connection(new_instance)
current_app.redis_connection = new_instance