def run(self): '''Work on jobs''' # Register signal handlers self.signals() # Start listening with self.listener(): try: generator = self.jobs() while not self.shutdown: self.pool.wait_available() job = next(generator) if job: # For whatever reason, doing imports within a greenlet # (there's one implicitly invoked in job.process), was # throwing exceptions. The hacky way to get around this # is to force the import to happen before the greenlet # is spawned. job.klass greenlet = gevent.Greenlet(self.process, job) self.greenlets[job.jid] = greenlet self.pool.start(greenlet) else: logger.debug('Sleeping for %fs' % self.interval) gevent.sleep(self.interval) except StopIteration: logger.info('Exhausted jobs') finally: logger.info('Waiting for greenlets to finish') self.pool.join()
def title(cls, message=None): '''Set the title of the process''' if message == None: return getproctitle() else: setproctitle('qless-py-worker %s' % message) logger.info(message)
def process(self): """Load the module containing your class, and run the appropriate method. For example, if this job was popped from the queue ``testing``, then this would invoke the ``testing`` staticmethod of your class.""" try: method = getattr(self.klass, self.queue_name, getattr(self.klass, "process", None)) except Exception as exc: # We failed to import the module containing this class logger.exception("Failed to import %s" % self.klass_name) return self.fail(self.queue_name + "-" + exc.__class__.__name__, "Failed to import %s" % self.klass_name) if method: if isinstance(method, types.FunctionType): try: logger.info("Processing %s in %s" % (self.jid, self.queue_name)) method(self) logger.info("Completed %s in %s" % (self.jid, self.queue_name)) except Exception as exc: # Make error type based on exception type logger.exception("Failed %s in %s: %s" % (self.jid, self.queue_name, repr(method))) self.fail(self.queue_name + "-" + exc.__class__.__name__, traceback.format_exc()) else: # Or fail with a message to that effect logger.error("Failed %s in %s : %s is not static" % (self.jid, self.queue_name, repr(method))) self.fail(self.queue_name + "-method-type", repr(method) + " is not static") else: # Or fail with a message to that effect logger.error( 'Failed %s : %s is missing a method "%s" or "process"' % (self.jid, self.klass_name, self.queue_name) ) self.fail( self.queue_name + "-method-missing", self.klass_name + ' is missing a method "' + self.queue_name + '" or "process"', )
def complete(self, nextq=None, delay=None, depends=None): """Turn this job in as complete, optionally advancing it to another queue. Like ``Queue.put`` and ``move``, it accepts a delay, and dependencies""" if nextq: logger.info("Advancing %s to %s from %s" % (self.jid, nextq, self.queue_name)) return ( self.client( "complete", self.jid, self.client.worker_name, self.queue_name, json.dumps(self.data), "next", nextq, "delay", delay or 0, "depends", json.dumps(depends or []), ) or False ) else: logger.info("Completing %s" % self.jid) return ( self.client("complete", self.jid, self.client.worker_name, self.queue_name, json.dumps(self.data)) or False )
def run(self): '''Work on jobs''' # Register signal handlers self.signals() # And monkey-patch before doing any imports self.patch() # Start listening with self.listener(): try: generator = self.jobs() while not self.shutdown: self.pool.wait_available() job = generator.next() if job: # For whatever reason, doing imports within a greenlet # (there's one implicitly invoked in job.process), was # throwing exceptions. The hacky way to get around this # is to force the import to happen before the greenlet # is spawned. job.klass greenlet = gevent.Greenlet(self.process, job) self.greenlets[job.jid] = greenlet self.pool.start(greenlet) else: logger.debug('Sleeping for %fs' % self.interval) gevent.sleep(self.interval) except StopIteration: logger.info('Exhausted jobs') finally: logger.info('Waiting for greenlets to finish') self.pool.join()
def move(self, queue, delay=0, depends=None): """Move this job out of its existing state and into another queue. If a worker has been given this job, then that worker's attempts to heartbeat that job will fail. Like ``Queue.put``, this accepts a delay, and dependencies""" logger.info("Moving %s to %s from %s" % (self.jid, queue, self.queue_name)) return self.client( "put", queue, self.jid, self.klass_name, json.dumps(self.data), delay, "depends", json.dumps(depends or []) )
def title(cls, message=None, level='INFO'): '''Set the title of the process''' if message == None: return getproctitle() else: setproctitle('qless-py-worker %s' % message) if level == 'DEBUG': logger.debug(message) elif level == 'INFO': logger.info(message)
def move(self, queue, delay=0, depends=None): '''Move this job out of its existing state and into another queue. If a worker has been given this job, then that worker's attempts to heartbeat that job will fail. Like ``Queue.put``, this accepts a delay, and dependencies''' logger.info('Moving %s to %s from %s', self.jid, queue, self.queue_name) return self.client('put', queue, self.jid, self.klass_name, json.dumps(self.data), delay, 'depends', json.dumps(depends or []) )
def clean(self): # This cleans the sandbox -- changing the working directory to it, # as well as clearing out any files that might be in there # Make sure we're running in our sandbox os.chdir(self.sandbox) # And that it's clear of any files for p in os.listdir(self.sandbox): p = os.path.join(self.sandbox, p) if os.path.isdir(p): logger.info('Removing tree %s...' % p) shutil.rmtree(p) else: logger.info('Removing file %s...' % p) os.remove(p)
def move(self, queue, delay=0, depends=None): '''Move this job out of its existing state and into another queue. If a worker has been given this job, then that worker's attempts to heartbeat that job will fail. Like ``Queue.put``, this accepts a delay, and dependencies''' logger.info('Moving %s to %s from %s' % ( self.jid, queue, self.queue_name)) return self.client._put([queue], [ self.jid, self.klass_name, json.dumps(self.data), repr(time.time()), delay, 'depends', json.dumps(depends or []) ])
def complete(self, nextq=None, delay=None, depends=None): '''Turn this job in as complete, optionally advancing it to another queue. Like ``Queue.put`` and ``move``, it accepts a delay, and dependencies''' if nextq: logger.info('Advancing %s to %s from %s', self.jid, nextq, self.queue_name) return self.client('complete', self.jid, self.client.worker_name, self.queue_name, json.dumps(self.data), 'next', nextq, 'delay', delay or 0, 'depends', json.dumps(depends or [])) or False else: logger.info('Completing %s', self.jid) return self.client('complete', self.jid, self.client.worker_name, self.queue_name, json.dumps(self.data)) or False
def complete(self, nextq=None, delay=None, depends=None): '''Turn this job in as complete, optionally advancing it to another queue. Like ``Queue.put`` and ``move``, it accepts a delay, and dependencies''' if nextq: logger.info('Advancing %s to %s from %s', self.jid, nextq, self.queue_name) return self.client('complete', self.jid, self.client.worker_name, self.queue_name, json.dumps(self.data), 'next', nextq, 'delay', delay or 0, 'depends', json.dumps(depends or []) ) or False else: logger.info('Completing %s', self.jid) return self.client('complete', self.jid, self.client.worker_name, self.queue_name, json.dumps(self.data)) or False
def work(self): # We should probably open up our own redis client self.client = qless.client(url=self.host) self.queues = [self.client.queues[q] for q in self.queues] if not os.path.isdir(self.sandbox): os.makedirs(self.sandbox) self.clean() # First things first, we should clear out any jobs that # we're responsible for off-hand while len(self.jids): try: job = self.client.jobs[self.jids.pop(0)] # If we still have access to it, then we should process it if job.heartbeat(): logger.info('Resuming %s' % job.jid) self.setproctitle('Working %s (%s)' % (job.jid, job.klass_name)) job.process() self.clean() else: logger.warn('Lost heart on would-be resumed job %s' % job.jid) except KeyboardInterrupt: return sleep_cycles = 0 while True: try: for queue in self.queues: job = queue.pop() if job: sleep_cycles = -1 self.setproctitle('Working %s (%s)' % (job.jid, job.klass_name)) job.process() self.clean() if self.stop_on_idle and sleep_cycles >= 2: logger.info("Idle for too long, quiting") import sys sys.exit(self.IDLE_EXIT_STATUS) if sleep_cycles >= 0: self.setproctitle('sleeping...') logger.debug('Sleeping for %fs' % self.interval) time.sleep(self.interval) sleep_cycles += 1 else: sleep_cycles = 0 except KeyboardInterrupt: return
def stop(self, sig=signal.SIGINT): '''Stop all the workers, and then wait for them''' for cpid in self.sandboxes.keys(): logger.warn('Stopping %i...' % cpid) os.kill(cpid, sig) # While we still have children running, wait for them for cpid in self.sandboxes.keys(): try: logger.info('Waiting for %i...' % cpid) pid, status = os.waitpid(cpid, 0) logger.warn('%i stopped with status %i' % (pid, status >> 8)) except OSError: # pragma: no cover logger.exception('Error waiting for %i...' % cpid) finally: self.sandboxes.pop(pid, None)
def run(self): '''Run this worker''' self.signals(('TERM', 'INT', 'QUIT')) # Divide up the jobs that we have to divy up between the workers. This # produces evenly-sized groups of jobs resume = self.divide(self.resume, self.count) for index in range(self.count): # The sandbox for the child worker sandbox = os.path.join( os.getcwd(), 'qless-py-workers', 'sandbox-%s' % index) cpid = os.fork() if cpid: logger.info('Spawned worker %i' % cpid) self.sandboxes[cpid] = sandbox else: # pragma: no cover # Move to the sandbox as the current working directory with Worker.sandbox(sandbox): os.chdir(sandbox) try: self.spawn(resume=resume[index], sandbox=sandbox).run() except: logger.exception('Exception in spawned worker') finally: os._exit(0) try: while not self.shutdown: pid, status = os.wait() logger.warn('Worker %i died with status %i from signal %i' % ( pid, status >> 8, status & 0xff)) sandbox = self.sandboxes.pop(pid) cpid = os.fork() if cpid: logger.info('Spawned replacement worker %i' % cpid) self.sandboxes[cpid] = sandbox else: # pragma: no cover with Worker.sandbox(sandbox): os.chdir(sandbox) try: self.spawn(sandbox=sandbox).run() except: logger.exception('Exception in spawned worker') finally: os._exit(0) finally: self.stop(signal.SIGKILL)
def run(self): '''Run this worker''' self.signals(('TERM', 'INT', 'QUIT')) # Divide up the jobs that we have to divy up between the workers. This # produces evenly-sized groups of jobs resume = self.divide(self.resume, self.count) for index in range(self.count): # The sandbox for the child worker sandbox = os.path.join(os.getcwd(), 'qless-py-workers', 'sandbox-%s' % index) cpid = os.fork() if cpid: logger.info('Spawned worker %i' % cpid) self.sandboxes[cpid] = sandbox else: # pragma: no cover # Move to the sandbox as the current working directory with Worker.sandbox(sandbox): os.chdir(sandbox) try: self.spawn(resume=resume[index], sandbox=sandbox).run() except: logger.exception('Exception in spawned worker') finally: os._exit(0) try: while not self.shutdown: pid, status = os.wait() logger.warn('Worker %i died with status %i from signal %i' % (pid, status >> 8, status & 0xff)) sandbox = self.sandboxes.pop(pid) cpid = os.fork() if cpid: logger.info('Spawned replacement worker %i' % cpid) self.sandboxes[cpid] = sandbox else: # pragma: no cover with Worker.sandbox(sandbox): os.chdir(sandbox) try: self.spawn(sandbox=sandbox).run() except: logger.exception('Exception in spawned worker') finally: os._exit(0) finally: self.stop(signal.SIGKILL)
def work(self): # We should probably open up our own redis client self.client = qless.client(self.host, self.port) self.queues = [self.client.queues[q] for q in self.queues] if not os.path.isdir(self.sandbox): os.makedirs(self.sandbox) self.clean() # First things first, we should clear out any jobs that # we're responsible for off-hand while len(self.jids): try: job = self.client.jobs[self.jids.pop(0)] # If we still have access to it, then we should process it if job.heartbeat(): logger.info('Resuming %s' % job.jid) self.setproctitle('Working %s (%s)' % (job.jid, job.klass_name)) job.process() self.clean() else: logger.warn('Lost heart on would-be resumed job %s' % job.jid) except KeyboardInterrupt: return while True: try: seen = False for queue in self.queues: job = queue.pop() if job: seen = True self.setproctitle('Working %s (%s)' % (job.jid, job.klass_name)) job.process() self.clean() if not seen: self.setproctitle('sleeping...') logger.debug('Sleeping for %fs' % self.interval) time.sleep(self.interval) except KeyboardInterrupt: return
def stop(self, sig=signal.SIGINT): '''Stop all the workers, and then wait for them''' for cpid in self.sandboxes.keys(): logger.warn('Stopping %i...' % cpid) try: os.kill(cpid, sig) except OSError: # pragma: no cover logger.exception('Error stopping %s...' % cpid) # While we still have children running, wait for them for cpid in self.sandboxes.keys(): try: logger.info('Waiting for %i...' % cpid) pid, status = os.waitpid(cpid, 0) logger.warn('%i stopped with status %i' % (pid, status >> 8)) except OSError: # pragma: no cover logger.exception('Error waiting for %i...' % cpid) finally: self.sandboxes.pop(cpid, None)
def process(self): '''Load the module containing your class, and run the appropriate method. For example, if this job was popped from the queue ``testing``, then this would invoke the ``testing`` staticmethod of your class.''' try: method = getattr(self.klass, self.queue_name, getattr(self.klass, 'process', None)) except Exception as exc: # We failed to import the module containing this class logger.exception('Failed to import %s' % self.klass_name) self.fail(self.queue_name + '-' + exc.__class__.__name__, 'Failed to import %s' % self.klass_name) if method: if isinstance(method, types.FunctionType): try: logger.info('Processing %s in %s' % (self.jid, self.queue_name)) method(self) logger.info('Completed %s in %s' % (self.jid, self.queue_name)) except Exception as e: # Make error type based on exception type logger.exception('Failed %s in %s: %s' % (self.jid, self.queue_name, repr(method))) self.fail(self.queue_name + '-' + e.__class__.__name__, traceback.format_exc()) else: # Or fail with a message to that effect logger.error('Failed %s in %s : %s is not static' % (self.jid, self.queue_name, repr(method))) self.fail(self.queue_name + '-method-type', repr(method) + ' is not static') else: # Or fail with a message to that effect logger.error( 'Failed %s : %s is missing a method "%s" or "process"' % (self.jid, self.klass_name, self.queue_name)) self.fail( self.queue_name + '-method-missing', self.klass_name + ' is missing a method "' + self.queue_name + '" or "process"')
def stop(self, sig=signal.SIGINT): '''Stop all the workers, and then wait for them''' for cpid in self.sandboxes: logger.warn('Stopping %i...' % cpid) try: os.kill(cpid, sig) except OSError: # pragma: no cover logger.exception('Error stopping %s...' % cpid) # While we still have children running, wait for them # We edit the dictionary during the loop, so we need to copy its keys for cpid in list(self.sandboxes): try: logger.info('Waiting for %i...' % cpid) pid, status = os.waitpid(cpid, 0) logger.warn('%i stopped with status %i' % (pid, status >> 8)) except OSError: # pragma: no cover logger.exception('Error waiting for %i...' % cpid) finally: self.sandboxes.pop(cpid, None)
def work(self): # We should probably open up our own redis client self.client = qless.client(self.host, self.port, password=self.password) self.queues = [self.client.queues[q] for q in self.queues] if not os.path.isdir(self.sandbox): os.makedirs(self.sandbox) self.clean() # First things first, we should clear out any jobs that # we're responsible for off-hand while len(self.jids): try: job = self.client.jobs[self.jids.pop(0)] # If we still have access to it, then we should process it if job.heartbeat(): logger.info('Resuming %s' % job.jid) self.setproctitle('Working %s (%s)' % (job.jid, job.klass_name)) job.process() self.clean() else: logger.warn('Lost heart on would-be resumed job %s' % job.jid) except KeyboardInterrupt: return while True: try: seen = False for queue in self.queues: job = queue.pop() if job: seen = True self.setproctitle('Working %s (%s)' % (job.jid, job.klass_name)) job.process() self.clean() if not seen: self.setproctitle('sleeping...') logger.debug('Sleeping for %fs' % self.interval) time.sleep(self.interval) except KeyboardInterrupt: return
def multitau(job): global proc logger.info("Launching multitau.sh process in dir %s"%(os.environ['EXE_DIR'])) command = "%s/%s"%(os.environ['EXE_DIR'], 'multitau.sh') args = [] # if 'input' in job: # args.append('-i') # args.append(job['input']) # else: # job.fail("Input argument", "Input file path is missing from job definition") # if 'endpoint' in job: # args.append('-e') # args.append(job['endpoint']) args.append('-i') args.append(job['input']) proc = Process(command, args) proc.start() time_last_heartbeat = 0 try: while proc.isAlive(): time.sleep(10) job.heartbeat() if (proc.retcode != 0): #TODO better error reporting job.fail("Child process failed", "error") else: job.complete() except: cleanUp() exit(1)
def run(self): # If this worker is meant to be resumable, then we should find out # what jobs this worker was working on beforehand. if self.resume: jids_to_resume = self.client.workers[self.client.worker_name]['jobs'] else: jids_to_resume = [] pids = [] for i in range(self.count): slot = { 'worker_id': i, 'sandbox' : os.path.join(self.workdir, 'qless-py-workers', 'sandbox-%i' % i) } cpid = os.fork() if cpid: logger.info('Spawned worker %i' % cpid) self.sandboxes[cpid] = slot pids.append(str(cpid)) else: # Set the value of the metadata so that jobs can detect # what worker they're running on import qless.worker qless.worker.meta = slot # Make note that we're not the master, and then save our # sandbox and worker id for reference self.master = False self.sandbox = slot['sandbox'] self.worker_id = slot['worker_id'] # Also, we should take our share of the jobs that we want # to resume, if any. start = (i * len(jids_to_resume)) / self.count end = ((i+1) * len(jids_to_resume)) / self.count self.jids = jids_to_resume[start:end] return self.work() f=open(os.path.join(self.workdir, 'workers-pid.txt'),'w') f.write(str(os.getpid())) f.write('\n') for pid in pids: f.write(pid) f.write('\n') f.close() while self.master: try: pid, status = os.wait() logger.warn('Worker %i died with status %i from signal %i' % (pid, status >> 8, status & 0xff)) slot = self.sandboxes.pop(pid) cpid = os.fork() if cpid: logger.info('Spawned replacement worker %i' % cpid) self.sandboxes[cpid] = slot else: # Set the value of the metadata so that jobs can detect # what worker they're running on import qless.worker qless.worker.meta = slot # Make note that we're not the master, and then save our # sandbox and worker id for reference self.master = False self.sandbox = slot['sandbox'] self.worker_id = slot['worker_id'] # NOTE: In the case that the worker died, we're going to # assume that something about the job(s) it was working # made the worker exit, and so we're going to ignore any # jobs that we might have been working on. It's also # significantly more difficult than the above problem of # simply distributing work to /new/ workers, rather than # a respawned worker. return self.work() except KeyboardInterrupt: break if self.master: self.stop()