def run(self):
'''Work on jobs'''
# Register signal handlers
self.signals()
# Start listening
with self.listener():
try:
generator = self.jobs()
while not self.shutdown:
self.pool.wait_available()
job = next(generator)
if job:
# For whatever reason, doing imports within a greenlet
# (there's one implicitly invoked in job.process), was
# throwing exceptions. The hacky way to get around this
# is to force the import to happen before the greenlet
# is spawned.
job.klass
greenlet = gevent.Greenlet(self.process, job)
self.greenlets[job.jid] = greenlet
self.pool.start(greenlet)
else:
logger.debug('Sleeping for %fs' % self.interval)
gevent.sleep(self.interval)
except StopIteration:
logger.info('Exhausted jobs')
finally:
logger.info('Waiting for greenlets to finish')
self.pool.join()
def title(cls, message=None):
'''Set the title of the process'''
if message == None:
return getproctitle()
else:
setproctitle('qless-py-worker %s' % message)
logger.info(message)
def title(cls, message=None):
'''Set the title of the process'''
if message == None:
return getproctitle()
else:
setproctitle('qless-py-worker %s' % message)
logger.info(message)
def process(self):
"""Load the module containing your class, and run the appropriate
method. For example, if this job was popped from the queue
``testing``, then this would invoke the ``testing`` staticmethod of
your class."""
try:
method = getattr(self.klass, self.queue_name, getattr(self.klass, "process", None))
except Exception as exc:
# We failed to import the module containing this class
logger.exception("Failed to import %s" % self.klass_name)
return self.fail(self.queue_name + "-" + exc.__class__.__name__, "Failed to import %s" % self.klass_name)
if method:
if isinstance(method, types.FunctionType):
try:
logger.info("Processing %s in %s" % (self.jid, self.queue_name))
method(self)
logger.info("Completed %s in %s" % (self.jid, self.queue_name))
except Exception as exc:
# Make error type based on exception type
logger.exception("Failed %s in %s: %s" % (self.jid, self.queue_name, repr(method)))
self.fail(self.queue_name + "-" + exc.__class__.__name__, traceback.format_exc())
else:
# Or fail with a message to that effect
logger.error("Failed %s in %s : %s is not static" % (self.jid, self.queue_name, repr(method)))
self.fail(self.queue_name + "-method-type", repr(method) + " is not static")
else:
# Or fail with a message to that effect
logger.error(
'Failed %s : %s is missing a method "%s" or "process"' % (self.jid, self.klass_name, self.queue_name)
)
self.fail(
self.queue_name + "-method-missing",
self.klass_name + ' is missing a method "' + self.queue_name + '" or "process"',
)
def complete(self, nextq=None, delay=None, depends=None):
"""Turn this job in as complete, optionally advancing it to another
queue. Like ``Queue.put`` and ``move``, it accepts a delay, and
dependencies"""
if nextq:
logger.info("Advancing %s to %s from %s" % (self.jid, nextq, self.queue_name))
return (
self.client(
"complete",
self.jid,
self.client.worker_name,
self.queue_name,
json.dumps(self.data),
"next",
nextq,
"delay",
delay or 0,
"depends",
json.dumps(depends or []),
)
or False
)
else:
logger.info("Completing %s" % self.jid)
return (
self.client("complete", self.jid, self.client.worker_name, self.queue_name, json.dumps(self.data))
or False
)
def run(self):
'''Work on jobs'''
# Register signal handlers
self.signals()
# And monkey-patch before doing any imports
self.patch()
# Start listening
with self.listener():
try:
generator = self.jobs()
while not self.shutdown:
self.pool.wait_available()
job = generator.next()
if job:
# For whatever reason, doing imports within a greenlet
# (there's one implicitly invoked in job.process), was
# throwing exceptions. The hacky way to get around this
# is to force the import to happen before the greenlet
# is spawned.
job.klass
greenlet = gevent.Greenlet(self.process, job)
self.greenlets[job.jid] = greenlet
self.pool.start(greenlet)
else:
logger.debug('Sleeping for %fs' % self.interval)
gevent.sleep(self.interval)
except StopIteration:
logger.info('Exhausted jobs')
finally:
logger.info('Waiting for greenlets to finish')
self.pool.join()
def move(self, queue, delay=0, depends=None):
"""Move this job out of its existing state and into another queue. If
a worker has been given this job, then that worker's attempts to
heartbeat that job will fail. Like ``Queue.put``, this accepts a
delay, and dependencies"""
logger.info("Moving %s to %s from %s" % (self.jid, queue, self.queue_name))
return self.client(
"put", queue, self.jid, self.klass_name, json.dumps(self.data), delay, "depends", json.dumps(depends or [])
)
def title(cls, message=None, level='INFO'):
'''Set the title of the process'''
if message == None:
return getproctitle()
else:
setproctitle('qless-py-worker %s' % message)
if level == 'DEBUG':
logger.debug(message)
elif level == 'INFO':
logger.info(message)
def move(self, queue, delay=0, depends=None):
'''Move this job out of its existing state and into another queue. If
a worker has been given this job, then that worker's attempts to
heartbeat that job will fail. Like ``Queue.put``, this accepts a
delay, and dependencies'''
logger.info('Moving %s to %s from %s',
self.jid, queue, self.queue_name)
return self.client('put', queue, self.jid, self.klass_name,
json.dumps(self.data), delay, 'depends', json.dumps(depends or [])
)
def clean(self):
# This cleans the sandbox -- changing the working directory to it,
# as well as clearing out any files that might be in there
# Make sure we're running in our sandbox
os.chdir(self.sandbox)
# And that it's clear of any files
for p in os.listdir(self.sandbox):
p = os.path.join(self.sandbox, p)
if os.path.isdir(p):
logger.info('Removing tree %s...' % p)
shutil.rmtree(p)
else:
logger.info('Removing file %s...' % p)
os.remove(p)
def move(self, queue, delay=0, depends=None):
'''Move this job out of its existing state and into another queue. If
a worker has been given this job, then that worker's attempts to
heartbeat that job will fail. Like ``Queue.put``, this accepts a
delay, and dependencies'''
logger.info('Moving %s to %s from %s' % (
self.jid, queue, self.queue_name))
return self.client._put([queue], [
self.jid,
self.klass_name,
json.dumps(self.data),
repr(time.time()),
delay,
'depends', json.dumps(depends or [])
])
def complete(self, nextq=None, delay=None, depends=None):
'''Turn this job in as complete, optionally advancing it to another
queue. Like ``Queue.put`` and ``move``, it accepts a delay, and
dependencies'''
if nextq:
logger.info('Advancing %s to %s from %s',
self.jid, nextq, self.queue_name)
return self.client('complete', self.jid, self.client.worker_name,
self.queue_name, json.dumps(self.data), 'next', nextq,
'delay', delay or 0, 'depends',
json.dumps(depends or [])) or False
else:
logger.info('Completing %s', self.jid)
return self.client('complete', self.jid, self.client.worker_name,
self.queue_name, json.dumps(self.data)) or False
def complete(self, nextq=None, delay=None, depends=None):
'''Turn this job in as complete, optionally advancing it to another
queue. Like ``Queue.put`` and ``move``, it accepts a delay, and
dependencies'''
if nextq:
logger.info('Advancing %s to %s from %s',
self.jid, nextq, self.queue_name)
return self.client('complete', self.jid, self.client.worker_name,
self.queue_name, json.dumps(self.data), 'next', nextq,
'delay', delay or 0, 'depends', json.dumps(depends or [])
) or False
else:
logger.info('Completing %s', self.jid)
return self.client('complete', self.jid, self.client.worker_name,
self.queue_name, json.dumps(self.data)) or False
def work(self):
# We should probably open up our own redis client
self.client = qless.client(url=self.host)
self.queues = [self.client.queues[q] for q in self.queues]
if not os.path.isdir(self.sandbox):
os.makedirs(self.sandbox)
self.clean()
# First things first, we should clear out any jobs that
# we're responsible for off-hand
while len(self.jids):
try:
job = self.client.jobs[self.jids.pop(0)]
# If we still have access to it, then we should process it
if job.heartbeat():
logger.info('Resuming %s' % job.jid)
self.setproctitle('Working %s (%s)' % (job.jid, job.klass_name))
job.process()
self.clean()
else:
logger.warn('Lost heart on would-be resumed job %s' % job.jid)
except KeyboardInterrupt:
return
sleep_cycles = 0
while True:
try:
for queue in self.queues:
job = queue.pop()
if job:
sleep_cycles = -1
self.setproctitle('Working %s (%s)' % (job.jid, job.klass_name))
job.process()
self.clean()
if self.stop_on_idle and sleep_cycles >= 2:
logger.info("Idle for too long, quiting")
import sys
sys.exit(self.IDLE_EXIT_STATUS)
if sleep_cycles >= 0:
self.setproctitle('sleeping...')
logger.debug('Sleeping for %fs' % self.interval)
time.sleep(self.interval)
sleep_cycles += 1
else:
sleep_cycles = 0
except KeyboardInterrupt:
return
def stop(self, sig=signal.SIGINT):
'''Stop all the workers, and then wait for them'''
for cpid in self.sandboxes.keys():
logger.warn('Stopping %i...' % cpid)
os.kill(cpid, sig)
# While we still have children running, wait for them
for cpid in self.sandboxes.keys():
try:
logger.info('Waiting for %i...' % cpid)
pid, status = os.waitpid(cpid, 0)
logger.warn('%i stopped with status %i' % (pid, status >> 8))
except OSError: # pragma: no cover
logger.exception('Error waiting for %i...' % cpid)
finally:
self.sandboxes.pop(pid, None)
def run(self):
'''Run this worker'''
self.signals(('TERM', 'INT', 'QUIT'))
# Divide up the jobs that we have to divy up between the workers. This
# produces evenly-sized groups of jobs
resume = self.divide(self.resume, self.count)
for index in range(self.count):
# The sandbox for the child worker
sandbox = os.path.join(
os.getcwd(), 'qless-py-workers', 'sandbox-%s' % index)
cpid = os.fork()
if cpid:
logger.info('Spawned worker %i' % cpid)
self.sandboxes[cpid] = sandbox
else: # pragma: no cover
# Move to the sandbox as the current working directory
with Worker.sandbox(sandbox):
os.chdir(sandbox)
try:
self.spawn(resume=resume[index], sandbox=sandbox).run()
except:
logger.exception('Exception in spawned worker')
finally:
os._exit(0)
try:
while not self.shutdown:
pid, status = os.wait()
logger.warn('Worker %i died with status %i from signal %i' % (
pid, status >> 8, status & 0xff))
sandbox = self.sandboxes.pop(pid)
cpid = os.fork()
if cpid:
logger.info('Spawned replacement worker %i' % cpid)
self.sandboxes[cpid] = sandbox
else: # pragma: no cover
with Worker.sandbox(sandbox):
os.chdir(sandbox)
try:
self.spawn(sandbox=sandbox).run()
except:
logger.exception('Exception in spawned worker')
finally:
os._exit(0)
finally:
self.stop(signal.SIGKILL)
def run(self):
'''Run this worker'''
self.signals(('TERM', 'INT', 'QUIT'))
# Divide up the jobs that we have to divy up between the workers. This
# produces evenly-sized groups of jobs
resume = self.divide(self.resume, self.count)
for index in range(self.count):
# The sandbox for the child worker
sandbox = os.path.join(os.getcwd(), 'qless-py-workers',
'sandbox-%s' % index)
cpid = os.fork()
if cpid:
logger.info('Spawned worker %i' % cpid)
self.sandboxes[cpid] = sandbox
else: # pragma: no cover
# Move to the sandbox as the current working directory
with Worker.sandbox(sandbox):
os.chdir(sandbox)
try:
self.spawn(resume=resume[index], sandbox=sandbox).run()
except:
logger.exception('Exception in spawned worker')
finally:
os._exit(0)
try:
while not self.shutdown:
pid, status = os.wait()
logger.warn('Worker %i died with status %i from signal %i' %
(pid, status >> 8, status & 0xff))
sandbox = self.sandboxes.pop(pid)
cpid = os.fork()
if cpid:
logger.info('Spawned replacement worker %i' % cpid)
self.sandboxes[cpid] = sandbox
else: # pragma: no cover
with Worker.sandbox(sandbox):
os.chdir(sandbox)
try:
self.spawn(sandbox=sandbox).run()
except:
logger.exception('Exception in spawned worker')
finally:
os._exit(0)
finally:
self.stop(signal.SIGKILL)
def work(self):
# We should probably open up our own redis client
self.client = qless.client(self.host, self.port)
self.queues = [self.client.queues[q] for q in self.queues]
if not os.path.isdir(self.sandbox):
os.makedirs(self.sandbox)
self.clean()
# First things first, we should clear out any jobs that
# we're responsible for off-hand
while len(self.jids):
try:
job = self.client.jobs[self.jids.pop(0)]
# If we still have access to it, then we should process it
if job.heartbeat():
logger.info('Resuming %s' % job.jid)
self.setproctitle('Working %s (%s)' %
(job.jid, job.klass_name))
job.process()
self.clean()
else:
logger.warn('Lost heart on would-be resumed job %s' %
job.jid)
except KeyboardInterrupt:
return
while True:
try:
seen = False
for queue in self.queues:
job = queue.pop()
if job:
seen = True
self.setproctitle('Working %s (%s)' %
(job.jid, job.klass_name))
job.process()
self.clean()
if not seen:
self.setproctitle('sleeping...')
logger.debug('Sleeping for %fs' % self.interval)
time.sleep(self.interval)
except KeyboardInterrupt:
return
def stop(self, sig=signal.SIGINT):
'''Stop all the workers, and then wait for them'''
for cpid in self.sandboxes.keys():
logger.warn('Stopping %i...' % cpid)
try:
os.kill(cpid, sig)
except OSError: # pragma: no cover
logger.exception('Error stopping %s...' % cpid)
# While we still have children running, wait for them
for cpid in self.sandboxes.keys():
try:
logger.info('Waiting for %i...' % cpid)
pid, status = os.waitpid(cpid, 0)
logger.warn('%i stopped with status %i' % (pid, status >> 8))
except OSError: # pragma: no cover
logger.exception('Error waiting for %i...' % cpid)
finally:
self.sandboxes.pop(cpid, None)
def process(self):
'''Load the module containing your class, and run the appropriate
method. For example, if this job was popped from the queue
``testing``, then this would invoke the ``testing`` staticmethod of
your class.'''
try:
method = getattr(self.klass, self.queue_name,
getattr(self.klass, 'process', None))
except Exception as exc:
# We failed to import the module containing this class
logger.exception('Failed to import %s' % self.klass_name)
self.fail(self.queue_name + '-' + exc.__class__.__name__,
'Failed to import %s' % self.klass_name)
if method:
if isinstance(method, types.FunctionType):
try:
logger.info('Processing %s in %s' %
(self.jid, self.queue_name))
method(self)
logger.info('Completed %s in %s' %
(self.jid, self.queue_name))
except Exception as e:
# Make error type based on exception type
logger.exception('Failed %s in %s: %s' %
(self.jid, self.queue_name, repr(method)))
self.fail(self.queue_name + '-' + e.__class__.__name__,
traceback.format_exc())
else:
# Or fail with a message to that effect
logger.error('Failed %s in %s : %s is not static' %
(self.jid, self.queue_name, repr(method)))
self.fail(self.queue_name + '-method-type',
repr(method) + ' is not static')
else:
# Or fail with a message to that effect
logger.error(
'Failed %s : %s is missing a method "%s" or "process"' %
(self.jid, self.klass_name, self.queue_name))
self.fail(
self.queue_name + '-method-missing', self.klass_name +
' is missing a method "' + self.queue_name + '" or "process"')
def stop(self, sig=signal.SIGINT):
'''Stop all the workers, and then wait for them'''
for cpid in self.sandboxes:
logger.warn('Stopping %i...' % cpid)
try:
os.kill(cpid, sig)
except OSError: # pragma: no cover
logger.exception('Error stopping %s...' % cpid)
# While we still have children running, wait for them
# We edit the dictionary during the loop, so we need to copy its keys
for cpid in list(self.sandboxes):
try:
logger.info('Waiting for %i...' % cpid)
pid, status = os.waitpid(cpid, 0)
logger.warn('%i stopped with status %i' % (pid, status >> 8))
except OSError: # pragma: no cover
logger.exception('Error waiting for %i...' % cpid)
finally:
self.sandboxes.pop(cpid, None)
def work(self):
# We should probably open up our own redis client
self.client = qless.client(self.host, self.port, password=self.password)
self.queues = [self.client.queues[q] for q in self.queues]
if not os.path.isdir(self.sandbox):
os.makedirs(self.sandbox)
self.clean()
# First things first, we should clear out any jobs that
# we're responsible for off-hand
while len(self.jids):
try:
job = self.client.jobs[self.jids.pop(0)]
# If we still have access to it, then we should process it
if job.heartbeat():
logger.info('Resuming %s' % job.jid)
self.setproctitle('Working %s (%s)' % (job.jid, job.klass_name))
job.process()
self.clean()
else:
logger.warn('Lost heart on would-be resumed job %s' % job.jid)
except KeyboardInterrupt:
return
while True:
try:
seen = False
for queue in self.queues:
job = queue.pop()
if job:
seen = True
self.setproctitle('Working %s (%s)' % (job.jid, job.klass_name))
job.process()
self.clean()
if not seen:
self.setproctitle('sleeping...')
logger.debug('Sleeping for %fs' % self.interval)
time.sleep(self.interval)
except KeyboardInterrupt:
return
def multitau(job):
global proc
logger.info("Launching multitau.sh process in dir %s"%(os.environ['EXE_DIR']))
command = "%s/%s"%(os.environ['EXE_DIR'], 'multitau.sh')
args = []
# if 'input' in job:
# args.append('-i')
# args.append(job['input'])
# else:
# job.fail("Input argument", "Input file path is missing from job definition")
# if 'endpoint' in job:
# args.append('-e')
# args.append(job['endpoint'])
args.append('-i')
args.append(job['input'])
proc = Process(command, args)
proc.start()
time_last_heartbeat = 0
try:
while proc.isAlive():
time.sleep(10)
job.heartbeat()
if (proc.retcode != 0):
#TODO better error reporting
job.fail("Child process failed", "error")
else:
job.complete()
except:
cleanUp()
exit(1)
def run(self):
# If this worker is meant to be resumable, then we should find out
# what jobs this worker was working on beforehand.
if self.resume:
jids_to_resume = self.client.workers[self.client.worker_name]['jobs']
else:
jids_to_resume = []
pids = []
for i in range(self.count):
slot = {
'worker_id': i,
'sandbox' : os.path.join(self.workdir, 'qless-py-workers', 'sandbox-%i' % i)
}
cpid = os.fork()
if cpid:
logger.info('Spawned worker %i' % cpid)
self.sandboxes[cpid] = slot
pids.append(str(cpid))
else:
# Set the value of the metadata so that jobs can detect
# what worker they're running on
import qless.worker
qless.worker.meta = slot
# Make note that we're not the master, and then save our
# sandbox and worker id for reference
self.master = False
self.sandbox = slot['sandbox']
self.worker_id = slot['worker_id']
# Also, we should take our share of the jobs that we want
# to resume, if any.
start = (i * len(jids_to_resume)) / self.count
end = ((i+1) * len(jids_to_resume)) / self.count
self.jids = jids_to_resume[start:end]
return self.work()
f=open(os.path.join(self.workdir, 'workers-pid.txt'),'w')
f.write(str(os.getpid()))
f.write('\n')
for pid in pids:
f.write(pid)
f.write('\n')
f.close()
while self.master:
try:
pid, status = os.wait()
logger.warn('Worker %i died with status %i from signal %i' % (pid, status >> 8, status & 0xff))
slot = self.sandboxes.pop(pid)
cpid = os.fork()
if cpid:
logger.info('Spawned replacement worker %i' % cpid)
self.sandboxes[cpid] = slot
else:
# Set the value of the metadata so that jobs can detect
# what worker they're running on
import qless.worker
qless.worker.meta = slot
# Make note that we're not the master, and then save our
# sandbox and worker id for reference
self.master = False
self.sandbox = slot['sandbox']
self.worker_id = slot['worker_id']
# NOTE: In the case that the worker died, we're going to
# assume that something about the job(s) it was working
# made the worker exit, and so we're going to ignore any
# jobs that we might have been working on. It's also
# significantly more difficult than the above problem of
# simply distributing work to /new/ workers, rather than
# a respawned worker.
return self.work()
except KeyboardInterrupt:
break
if self.master:
self.stop()
