class Queue(object):
def __init__(self):
self._qm = QueueManager()
self._qm.subscriber('job_failure', handler='marteau.queue:failure')
self._qm.subscriber('job_postrun', handler='marteau.queue:success')
self._qm.subscriber('job_prerun', handler='marteau.queue:starting')
self._conn = self._qm.redis
def pid_to_jobid(self, pid):
return self._conn.get('retools:jobpid:%s' % str(pid))
def get_console(self, job_id):
return self._conn.get('retools:jobconsole:%s' % job_id)
def append_console(self, job_id, data):
key = 'retools:jobconsole:%s' % job_id
current = self._conn.get(key)
if current is not None:
data = current + data
self._conn.set(key, data)
def get_node(self, name):
data = self._conn.get('retools:node:%s' % name)
return Node(**json.loads(data))
def delete_node(self, name):
if not self._conn.sismember('retools:nodes', name):
return
self._conn.srem('retools:nodes', name)
self._conn.delete('retools:node:%s' % name)
def get_nodes(self):
names = self._conn.smembers('retools:nodes')
for name in sorted(names):
node = self._conn.get('retools:node:%s' % name)
yield Node(**json.loads(node))
def reset_nodes(self):
for node in self.get_nodes():
node.status = 'idle'
self.save_node(node)
def save_node(self, node):
names = self._conn.smembers('retools:nodes')
if node.name not in names:
self._conn.sadd('retools:nodes', node.name)
self._conn.set('retools:node:%s' % node.name, node.to_json())
def purge_console(self, job_id):
key = 'retools:jobconsole:%s' % job_id
try:
return self._conn.get(key)
finally:
self._conn.delete(key)
def get_result(self, job_id):
res = self._conn.lindex('retools:result:%s' % job_id, 0)
console = self.get_console(job_id)
if res is None:
return None, console
return json.loads(res), console
def sorter(self, field):
def _sort_jobs(job1, job2):
return -cmp(job1.metadata[field], job2.metadata[field])
return _sort_jobs
def get_failures(self):
jobs = [self.get_job(job_id)
for job_id in self._conn.smembers('retools:queue:failures')]
jobs.sort(self.sorter('ended'))
return jobs
def get_successes(self):
jobs = [self.get_job(job_id)
for job_id in self._conn.smembers('retools:queue:successes')]
jobs.sort(self.sorter('ended'))
return jobs
def delete_job(self, job_id):
self._conn.delete('retools:started', job_id)
self._conn.delete('retools:job:%s' % job_id)
self._conn.delete('retools:jobpid:%s' % job_id)
self._conn.delete('retools:jobconsole:%s' % job_id)
if self._conn.sismember('retools:consoles', job_id):
self._conn.srem('retools:consoles', job_id)
self._conn.srem('retools:queue:failures', job_id)
self._conn.srem('retools:queue:successes', job_id)
def purge(self):
for queue in self._conn.smembers('retools:queues'):
self._conn.delete('retools:queue:%s' % queue)
for job_id in self._conn.smembers('retools:queue:started'):
self._conn.delete('retools:job:%s' % job_id)
self._conn.delete('retools:jobpid:%s' % job_id)
self._conn.delete('retools:queue:failures')
self._conn.delete('retools:queue:successes')
self._conn.delete('retools:queue:starting')
for job_id in self._conn.smembers('retools:consoles'):
self._conn.delete('retools:jobconsole:%s' % job_id)
self._conn.delete('retools:consoles')
self._conn.delete('retools:started')
for node in self.get_nodes():
node.status = 'idle'
self.save_node(node)
def cancel_job(self, job_id):
# first, find out which worker is working on this
for worker_id in self._conn.smembers('retools:workers'):
status_key = "retools:worker:%s" % worker_id
status = self._conn.get(status_key)
if status is None:
continue
status = json.loads(status)
job_payload = status['payload']
if job_payload['job_id'] != job_id:
continue
# that's the worker !
# get its pid and ask it to stop
pid = int(worker_id.split(':')[1])
os.kill(pid, signal.SIGUSR1)
break
# XXX we make the assumption all went well...
def replay(self, job_id):
job = self.get_job(job_id)
data = job.to_dict()
job_name = data['job']
kwargs = data['kwargs']
metadata = {'created': time.time(),
'repo': data['metadata']['repo']}
kwargs['metadata'] = metadata
self.enqueue(job_name, **kwargs)
def enqueue(self, funcname, **kwargs):
return self._qm.enqueue(funcname, **kwargs)
def _get_job(self, job_id, queue_names, redis):
for queue_name in queue_names:
current_len = self._conn.llen(queue_name)
# that's O(n), we should do better
for i in range(current_len):
# the list can change while doing this
# so we need to catch any index error
job = self._conn.lindex(queue_name, i)
job_data = json.loads(job)
if job_data['job_id'] == job_id:
return Job(queue_name, job, redis)
raise IndexError(job_id)
def get_job(self, job_id):
try:
return self._qm.get_job(job_id)
except IndexError:
job = self._conn.get('retools:job:%s' % job_id)
if job is None:
raise
return Job(self._qm.default_queue_name, job, self._conn)
def get_jobs(self):
return list(self._qm.get_jobs())
def get_running_jobs(self):
return [self.get_job(job_id)
for job_id in self._conn.smembers('retools:started')]
def get_workers(self):
ids = list(Worker.get_worker_ids(redis=self._conn))
return [wid.split(':')[1] for wid in ids]
def delete_pids(self, job_id):
self._conn.delete('retools:%s:pids' % job_id)
def add_pid(self, job_id, pid):
self._conn.sadd('retools:%s:pids' % job_id, str(pid))
def remove_pid(self, job_id, pid):
self._conn.srem('retools:%s:pids' % job_id, str(pid))
def get_pids(self, job_id):
return [int(pid) for pid in
self._conn.smembers('retools:%s:pids' % job_id)]
def cleanup_job(self, job_id):
for pid in self.get_pids(job_id):
os.kill(pid, signal.SIGTERM)
self.delete_pids(job_id)
class Queue(object):
def __init__(self):
if not redis_available():
raise IOError('Marteau needs Redis to run.')
self._qm = QueueManager()
self._qm.subscriber('job_failure', handler='marteau.queue:failure')
self._qm.subscriber('job_postrun', handler='marteau.queue:success')
self._qm.subscriber('job_prerun', handler='marteau.queue:starting')
self._conn = self._qm.redis
def pid_to_jobid(self, pid):
return self._conn.get('retools:jobpid:%s' % str(pid))
def get_console(self, job_id):
return self._conn.get('retools:jobconsole:%s' % job_id)
def append_console(self, job_id, data):
key = 'retools:jobconsole:%s' % job_id
current = self._conn.get(key)
if current is not None:
data = current + data
self._conn.set(key, data)
def get_node(self, name):
data = self._conn.get('retools:node:%s' % name)
return Node(**json.loads(data))
def delete_node(self, name):
if not self._conn.sismember('retools:nodes', name):
return
self._conn.srem('retools:nodes', name)
self._conn.delete('retools:node:%s' % name)
def get_nodes(self, check_available=False):
names = self._conn.smembers('retools:nodes')
nodes = []
for name in sorted(names):
node = self._conn.get('retools:node:%s' % name)
node = Node(**json.loads(node))
if check_available and (node.status != 'idle' or not node.enabled):
continue
nodes.append(node)
return nodes
def reset_nodes(self):
for node in self.get_nodes():
node.status = 'idle'
self.save_node(node)
def save_node(self, node):
names = self._conn.smembers('retools:nodes')
if node.name not in names:
self._conn.sadd('retools:nodes', node.name)
self._conn.set('retools:node:%s' % node.name, node.to_json())
def purge_console(self, job_id):
key = 'retools:jobconsole:%s' % job_id
try:
return self._conn.get(key)
finally:
self._conn.delete(key)
def get_result(self, job_id):
res = self._conn.lindex('retools:result:%s' % job_id, 0)
console = self.get_console(job_id)
if res is None:
return None, console
return json.loads(res), console
def sorter(self, field):
def _sort_jobs(job1, job2):
return -cmp(job1.metadata[field], job2.metadata[field])
return _sort_jobs
def get_failures(self):
jobs = [
self.get_job(job_id)
for job_id in self._conn.smembers('retools:queue:failures')
]
jobs.sort(self.sorter('ended'))
return jobs
def get_successes(self):
jobs = [
self.get_job(job_id)
for job_id in self._conn.smembers('retools:queue:successes')
]
jobs.sort(self.sorter('ended'))
return jobs
def delete_job(self, job_id):
self._conn.delete('retools:started', job_id)
self._conn.delete('retools:job:%s' % job_id)
self._conn.delete('retools:jobpid:%s' % job_id)
self._conn.delete('retools:jobconsole:%s' % job_id)
if self._conn.sismember('retools:consoles', job_id):
self._conn.srem('retools:consoles', job_id)
self._conn.srem('retools:queue:failures', job_id)
self._conn.srem('retools:queue:successes', job_id)
def purge(self):
for queue in self._conn.smembers('retools:queues'):
self._conn.delete('retools:queue:%s' % queue)
for job_id in self._conn.smembers('retools:queue:started'):
self._conn.delete('retools:job:%s' % job_id)
self._conn.delete('retools:jobpid:%s' % job_id)
self._conn.delete('retools:queue:failures')
self._conn.delete('retools:queue:successes')
self._conn.delete('retools:queue:starting')
for job_id in self._conn.smembers('retools:consoles'):
self._conn.delete('retools:jobconsole:%s' % job_id)
self._conn.delete('retools:consoles')
self._conn.delete('retools:started')
for node in self.get_nodes():
node.status = 'idle'
self.save_node(node)
def cancel_job(self, job_id):
# first, find out which worker is working on this
for worker_id in self._conn.smembers('retools:workers'):
status_key = "retools:worker:%s" % worker_id
status = self._conn.get(status_key)
if status is None:
continue
status = json.loads(status)
job_payload = status['payload']
if job_payload['job_id'] != job_id:
continue
# that's the worker !
# get its pid and ask it to stop
pid = int(worker_id.split(':')[1])
os.kill(pid, signal.SIGUSR1)
break
# XXX we make the assumption all went well...
def replay(self, job_id):
job = self.get_job(job_id)
data = job.to_dict()
job_name = data['job']
kwargs = data['kwargs']
metadata = {'created': time.time(), 'repo': data['metadata']['repo']}
kwargs['metadata'] = metadata
return self.enqueue(job_name, **kwargs)
def enqueue(self, funcname, **kwargs):
return self._qm.enqueue(funcname, **kwargs)
def _get_job(self, job_id, queue_names, redis):
for queue_name in queue_names:
current_len = self._conn.llen(queue_name)
# that's O(n), we should do better
for i in range(current_len):
# the list can change while doing this
# so we need to catch any index error
job = self._conn.lindex(queue_name, i)
job_data = json.loads(job)
if job_data['job_id'] == job_id:
return Job(queue_name, job, redis)
raise IndexError(job_id)
def get_job(self, job_id):
try:
return self._qm.get_job(job_id)
except IndexError:
job = self._conn.get('retools:job:%s' % job_id)
if job is None:
raise
return Job(self._qm.default_queue_name, job, self._conn)
def get_jobs(self):
return list(self._qm.get_jobs())
def get_running_jobs(self):
return [
self.get_job(job_id)
for job_id in self._conn.smembers('retools:started')
]
def get_workers(self):
ids = list(Worker.get_worker_ids(redis=self._conn))
return [wid.split(':')[1] for wid in ids]
def delete_pids(self, job_id):
self._conn.delete('retools:%s:pids' % job_id)
def add_pid(self, job_id, pid):
self._conn.sadd('retools:%s:pids' % job_id, str(pid))
def remove_pid(self, job_id, pid):
self._conn.srem('retools:%s:pids' % job_id, str(pid))
def get_pids(self, job_id):
return [
int(pid) for pid in self._conn.smembers('retools:%s:pids' % job_id)
]
def cleanup_job(self, job_id):
for pid in self.get_pids(job_id):
os.kill(pid, signal.SIGTERM)
self.delete_pids(job_id)
def get_key(self, user):
return self._conn.get('retools:apikey:%s' % user)
def set_key(self, user, key):
return self._conn.set('retools:apikey:%s' % user, key)
