def __init__(self, fork_worker=None, wait_interval=None, connection=None,
q_name=None, q_names=None, top_bound=None):
if fork_worker is None:
fork_worker = bool(os.environ.get('QC_FORK_WORKER', ''))
if wait_interval is None:
wait_interval = int(os.environ.get('QC_LISTEN_TIME', '5'))
self.fork_worker = fork_worker
self.wait_interval = wait_interval
self.conn_adapter = ConnAdapter(connection)
if q_name is None:
q_name = os.environ.get('QUEUE', 'default')
if q_names is None:
q_names = os.environ.get('QUEUES', '')
if not q_names:
q_names = []
else:
q_names = q_names.split(',')
self.queues = self.__setup_queues(
self.conn_adapter, q_name, q_names, top_bound)
self.running = True
log(at="worker_initialized")
def drop(connection=None):
conn_adapter = ConnAdapter(connection)
conn_adapter.execute(open(DropSqlFunctions).read())
if connection is None:
conn_adapter.disconnect()
class Worker(object):
# Creates a new worker but does not start the worker. See Worker#start.
# This method takes a single hash argument. The following keys are read:
# fork_worker:: Worker forks each job execution.
# wait_interval:: Time to wait between failed lock attempts
# connection:: PGConn object.
# q_name:: Name of a single queue to process.
# q_names:: Names of queues to process. Will process left to right.
# top_bound:: Offset to the head of the queue. 1 == strict FIFO.
def __init__(self, fork_worker=None, wait_interval=None, connection=None,
q_name=None, q_names=None, top_bound=None):
if fork_worker is None:
fork_worker = bool(os.environ.get('QC_FORK_WORKER', ''))
if wait_interval is None:
wait_interval = int(os.environ.get('QC_LISTEN_TIME', '5'))
self.fork_worker = fork_worker
self.wait_interval = wait_interval
self.conn_adapter = ConnAdapter(connection)
if q_name is None:
q_name = os.environ.get('QUEUE', 'default')
if q_names is None:
q_names = os.environ.get('QUEUES', '')
if not q_names:
q_names = []
else:
q_names = q_names.split(',')
self.queues = self.__setup_queues(
self.conn_adapter, q_name, q_names, top_bound)
self.running = True
log(at="worker_initialized")
# Commences the working of jobs.
# start() spins on @running -which is initialized as true.
# This method is the primary entry point to starting the worker.
# The canonical example of starting a worker is as follows:
# QC::Worker.new.start
def start(self):
while self.running:
if self.fork_worker:
self.fork_and_work()
else:
self.work()
# Signals the worker to stop taking new work.
# This method has no immediate effect. However, there are
# two loops in the worker (one in #start and another in #lock_job)
# which check the @running variable to determine if further progress
# is desirable. In the case that @running is false, the aforementioned
# methods will short circuit and cause the blocking call to #start
# to unblock.
def stop(self):
self.running = False
# Calls Worker#work but after the current process is forked.
# The parent process will wait on the child process to exit.
def fork_and_work(self):
cpid = os.fork()
if cpid == 0:
try:
self.setup_child()
self.work()
except:
# prevent going up in the stack
os._exit(1)
else:
# prevent going up in the stack
os._exit(0)
else:
log(at="fork", pid=str(cpid))
os.waitpid(cpid, 0)
# Blocks on locking a job, and once a job is locked,
# it will process the job.
def work(self):
queue, job = self.lock_job()
if queue and job:
log(at="work", job=str(job['id']))
self.process(queue, job)
# Attempt to lock a job in the queue's table.
# If a job can be locked, this method returns an array with
# 2 elements. The first element is the queue from which the job was locked
# and the second is a hash representation of the job.
# If a job is returned, its locked_at column has been set in the
# job's row. It is the caller's responsibility to delete the job row
# from the table when the job is complete.
def lock_job(self):
log(at="lock_job")
job = None
while self.running:
for queue in self.queues:
job = queue.lock()
if job:
return (queue, job)
self.conn_adapter.wait(self.wait_interval,
*[queue.name for queue in self.queues])
# A job is processed by evaluating the target code.
# if the job is evaluated with no exceptions
# then it is deleted from the queue.
# If the job has raised an exception the responsibility of what
# to do with the job is delegated to Worker#handle_failure.
# If the job is not finished and an INT signal is trapped,
# this method will unlock the job in the queue.
def process(self, queue, job):
start = datetime.datetime.now()
finished = False
try:
self.call(job)
queue.delete(job['id'])
finished = True
except Exception, e:
self.handle_failure(job, e)
finished = True
finally:
def create(connection=None):
conn_adapter = ConnAdapter(connection)
conn_adapter.execute(open(CreateTable).read())
conn_adapter.execute(open(SqlFunctions).read())
if connection is None:
conn_adapter.disconnect()