def configure(self):
# Configure the worker based on the configuration settings.
qdo_section = self.settings.getsection('qdo-worker')
self.name = '%s-%s' % (socket.getfqdn(), os.getpid())
identifier = qdo_section['name']
if identifier:
self.name += '-' + identifier
self.wait_interval = qdo_section['wait_interval']
resolve(self, qdo_section, 'job')
resolve(self, qdo_section, 'job_context')
resolve(self, qdo_section, 'job_failure')
queuey_section = self.settings.getsection('queuey')
self.queuey_conn = Client(queuey_section['app_key'],
connection=queuey_section['connection'])
zk_section = self.settings.getsection('zookeeper')
self.zk_hosts = zk_section['connection']
self.zk_party_wait = zk_section['party_wait']
def configure(self):
# Configure the worker based on the configuration settings.
qdo_section = self.settings.getsection('qdo-worker')
self.name = '%s-%s' % (socket.getfqdn(), os.getpid())
identifier = qdo_section['name']
if identifier:
self.name += '-' + identifier
self.wait_interval = qdo_section['wait_interval']
resolve(self, qdo_section, 'job')
resolve(self, qdo_section, 'job_context')
resolve(self, qdo_section, 'job_failure')
queuey_section = self.settings.getsection('queuey')
self.queuey_conn = Client(
queuey_section['app_key'],
connection=queuey_section['connection'])
zk_section = self.settings.getsection('zookeeper')
self.zk_hosts = zk_section['connection']
self.zk_party_wait = zk_section['party_wait']
class Worker(object):
"""A Worker works on jobs.
:param settings: Configuration settings
:type settings: dict
"""
def __init__(self, settings):
self.settings = settings
self.shutdown = False
self.job = None
self.job_context = dict_context
self.job_failure = log_failure
self.partition_policy = 'manual'
self.queuey_conn = None
self.zk = None
self.partitioner = None
self.partition_cache = PartitionCache(self)
self.configure()
def configure(self):
# Configure the worker based on the configuration settings.
qdo_section = self.settings.getsection('qdo-worker')
self.name = '%s-%s' % (socket.getfqdn(), os.getpid())
identifier = qdo_section['name']
if identifier:
self.name += '-' + identifier
self.wait_interval = qdo_section['wait_interval']
resolve(self, qdo_section, 'job')
resolve(self, qdo_section, 'job_context')
resolve(self, qdo_section, 'job_failure')
queuey_section = self.settings.getsection('queuey')
self.queuey_conn = Client(
queuey_section['app_key'],
connection=queuey_section['connection'])
zk_section = self.settings.getsection('zookeeper')
self.zk_hosts = zk_section['connection']
self.zk_party_wait = zk_section['party_wait']
def setup_zookeeper(self):
self.zk = KazooClient(hosts=self.zk_hosts, max_retries=1)
self.zk.start()
def all_partitions(self):
# List all partitions
queuey_conn = self.queuey_conn
response = queuey_conn.get(params={'details': True})
queues = ujson_decode(response.text)['queues']
partitions = []
for q in queues:
name = q['queue_name']
part = q['partitions']
for i in xrange(1, part + 1):
partitions.append('%s-%s' % (name, i))
return partitions
def configure_partitions(self):
section = self.settings.getsection('partitions')
self.partition_policy = policy = section['policy']
queuey_conn = self.queuey_conn
all_partitions = self.all_partitions()
partition_ids = section.get('ids')
partitioner_class = StaticPartitioner
if not partition_ids:
partition_ids = all_partitions
if policy == 'automatic':
self.setup_zookeeper()
partitioner_class = self.zk.SetPartitioner
partition_ids = [p for p in partition_ids if not
p.startswith((ERROR_QUEUE, STATUS_QUEUE))]
self.partitioner = partitioner_class(
'/worker', set=tuple(partition_ids), identifier=self.name,
time_boundary=self.zk_party_wait)
def cond_create(queue_name):
if queue_name + '-1' not in all_partitions:
queuey_conn.create_queue(
queue_name=queue_name, partitions=STATUS_PARTITIONS)
cond_create(ERROR_QUEUE)
cond_create(STATUS_QUEUE)
self.status = self.status_partitions()
def status_partitions(self):
status = {}
# get all status messages, starting with the newest ones
status_messages = self.queuey_conn.messages(
STATUS_QUEUE, limit=1000, order='descending')
if len(status_messages) >= 1000: # pragma: no cover
# TODO deal with more than 1000 status messages / partitions
raise RuntimeError('More than 1000 status messages detected!')
for message in status_messages:
body = ujson_decode(message['body'])
partition = body['partition']
if partition not in status:
# don't overwrite newer messages with older status
status[partition] = message['message_id']
return status
def work(self):
"""Work on jobs."""
if not self.job:
return
# Try Queuey heartbeat connection
self.queuey_conn.connect()
self.configure_partitions()
atexit.register(self.stop)
timer = get_logger().timer
partitioner = self.partitioner
with self.job_context() as context:
if partitioner.allocating:
partitioner.wait_for_acquire(self.zk_party_wait)
waited = 0
while 1:
if self.shutdown or partitioner.failed:
break
if partitioner.release:
partitioner.release_set()
elif partitioner.allocating:
partitioner.wait_for_acquire(self.zk_party_wait)
elif partitioner.acquired:
no_messages = 0
partitions = list(self.partitioner)
for name in partitions:
partition = self.partition_cache[name]
messages = partition.messages(limit=2)
if not messages:
no_messages += 1
continue
message = messages[0]
message_id = message['message_id']
try:
with timer('worker.job_time'):
self.job(message, context)
except StopWorker:
self.shutdown = True
break
except Exception as exc:
with timer('worker.job_failure_time'):
self.job_failure(message, context,
name, exc, self.queuey_conn)
# record successful message processing
partition.last_message = message_id
if no_messages == len(partitions):
# if none of the partitions had a message, wait
self.wait(waited)
waited += 1
else:
waited = 0
# give up the partitions and leave party
self.partitioner.finish()
def wait(self, waited=1):
get_logger().incr('worker.wait_for_jobs')
jitter = random.uniform(0.8, 1.2)
time.sleep(self.wait_interval * jitter * 2 ** min(waited, 10))
def stop(self):
"""Stop the worker loop. Used in an `atexit` hook."""
self.shutdown = True
if self.zk is not None:
self.partitioner.finish()
self.zk.stop()
def _make_one(self, connection=u'https://127.0.0.1:5001/v1/queuey/'):
return Client(self.queuey_app_key, connection=connection)
class Worker(object):
"""A Worker works on jobs.
:param settings: Configuration settings
:type settings: dict
"""
def __init__(self, settings):
self.settings = settings
self.shutdown = False
self.job = None
self.job_context = dict_context
self.job_failure = log_failure
self.partition_policy = 'manual'
self.queuey_conn = None
self.zk = None
self.partitioner = None
self.partition_cache = PartitionCache(self)
self.configure()
def configure(self):
# Configure the worker based on the configuration settings.
qdo_section = self.settings.getsection('qdo-worker')
self.name = '%s-%s' % (socket.getfqdn(), os.getpid())
identifier = qdo_section['name']
if identifier:
self.name += '-' + identifier
self.wait_interval = qdo_section['wait_interval']
resolve(self, qdo_section, 'job')
resolve(self, qdo_section, 'job_context')
resolve(self, qdo_section, 'job_failure')
queuey_section = self.settings.getsection('queuey')
self.queuey_conn = Client(queuey_section['app_key'],
connection=queuey_section['connection'])
zk_section = self.settings.getsection('zookeeper')
self.zk_hosts = zk_section['connection']
self.zk_party_wait = zk_section['party_wait']
def setup_zookeeper(self):
self.zk = KazooClient(hosts=self.zk_hosts, max_retries=1)
self.zk.start()
def all_partitions(self):
# List all partitions
queuey_conn = self.queuey_conn
response = queuey_conn.get(params={'details': True})
queues = ujson_decode(response.text)['queues']
partitions = []
for q in queues:
name = q['queue_name']
part = q['partitions']
for i in xrange(1, part + 1):
partitions.append('%s-%s' % (name, i))
return partitions
def configure_partitions(self):
section = self.settings.getsection('partitions')
self.partition_policy = policy = section['policy']
queuey_conn = self.queuey_conn
all_partitions = self.all_partitions()
partition_ids = section.get('ids')
partitioner_class = StaticPartitioner
if not partition_ids:
partition_ids = all_partitions
if policy == 'automatic':
self.setup_zookeeper()
partitioner_class = self.zk.SetPartitioner
partition_ids = [
p for p in partition_ids
if not p.startswith((ERROR_QUEUE, STATUS_QUEUE))
]
self.partitioner = partitioner_class('/worker',
set=tuple(partition_ids),
identifier=self.name,
time_boundary=self.zk_party_wait)
def cond_create(queue_name):
if queue_name + '-1' not in all_partitions:
queuey_conn.create_queue(queue_name=queue_name,
partitions=STATUS_PARTITIONS)
cond_create(ERROR_QUEUE)
cond_create(STATUS_QUEUE)
self.status = self.status_partitions()
def status_partitions(self):
status = {}
# get all status messages, starting with the newest ones
status_messages = self.queuey_conn.messages(STATUS_QUEUE,
limit=1000,
order='descending')
if len(status_messages) >= 1000: # pragma: no cover
# TODO deal with more than 1000 status messages / partitions
raise RuntimeError('More than 1000 status messages detected!')
for message in status_messages:
body = ujson_decode(message['body'])
partition = body['partition']
if partition not in status:
# don't overwrite newer messages with older status
status[partition] = message['message_id']
return status
def work(self):
"""Work on jobs."""
if not self.job:
return
# Try Queuey heartbeat connection
self.queuey_conn.connect()
self.configure_partitions()
atexit.register(self.stop)
timer = get_logger().timer
partitioner = self.partitioner
with self.job_context() as context:
if partitioner.allocating:
partitioner.wait_for_acquire(self.zk_party_wait)
waited = 0
while 1:
if self.shutdown or partitioner.failed:
break
if partitioner.release:
partitioner.release_set()
elif partitioner.allocating:
partitioner.wait_for_acquire(self.zk_party_wait)
elif partitioner.acquired:
no_messages = 0
partitions = list(self.partitioner)
for name in partitions:
partition = self.partition_cache[name]
messages = partition.messages(limit=2)
if not messages:
no_messages += 1
continue
message = messages[0]
message_id = message['message_id']
try:
with timer('worker.job_time'):
self.job(message, context)
except StopWorker:
self.shutdown = True
break
except Exception as exc:
with timer('worker.job_failure_time'):
self.job_failure(message, context, name, exc,
self.queuey_conn)
# record successful message processing
partition.last_message = message_id
if no_messages == len(partitions):
# if none of the partitions had a message, wait
self.wait(waited)
waited += 1
else:
waited = 0
# give up the partitions and leave party
self.partitioner.finish()
def wait(self, waited=1):
get_logger().incr('worker.wait_for_jobs')
jitter = random.uniform(0.8, 1.2)
time.sleep(self.wait_interval * jitter * 2**min(waited, 10))
def stop(self):
"""Stop the worker loop. Used in an `atexit` hook."""
self.shutdown = True
if self.zk is not None:
self.partitioner.finish()
self.zk.stop()
def _make_queuey_conn(cls, connection='http://127.0.0.1:5000/v1/queuey/'):
return Client(cls.queuey_app_key, connection=connection)
