def configure(self): # Configure the worker based on the configuration settings. qdo_section = self.settings.getsection('qdo-worker') self.name = '%s-%s' % (socket.getfqdn(), os.getpid()) identifier = qdo_section['name'] if identifier: self.name += '-' + identifier self.wait_interval = qdo_section['wait_interval'] resolve(self, qdo_section, 'job') resolve(self, qdo_section, 'job_context') resolve(self, qdo_section, 'job_failure') queuey_section = self.settings.getsection('queuey') self.queuey_conn = Client(queuey_section['app_key'], connection=queuey_section['connection']) zk_section = self.settings.getsection('zookeeper') self.zk_hosts = zk_section['connection'] self.zk_party_wait = zk_section['party_wait']
def configure(self): # Configure the worker based on the configuration settings. qdo_section = self.settings.getsection('qdo-worker') self.name = '%s-%s' % (socket.getfqdn(), os.getpid()) identifier = qdo_section['name'] if identifier: self.name += '-' + identifier self.wait_interval = qdo_section['wait_interval'] resolve(self, qdo_section, 'job') resolve(self, qdo_section, 'job_context') resolve(self, qdo_section, 'job_failure') queuey_section = self.settings.getsection('queuey') self.queuey_conn = Client( queuey_section['app_key'], connection=queuey_section['connection']) zk_section = self.settings.getsection('zookeeper') self.zk_hosts = zk_section['connection'] self.zk_party_wait = zk_section['party_wait']
class Worker(object): """A Worker works on jobs. :param settings: Configuration settings :type settings: dict """ def __init__(self, settings): self.settings = settings self.shutdown = False self.job = None self.job_context = dict_context self.job_failure = log_failure self.partition_policy = 'manual' self.queuey_conn = None self.zk = None self.partitioner = None self.partition_cache = PartitionCache(self) self.configure() def configure(self): # Configure the worker based on the configuration settings. qdo_section = self.settings.getsection('qdo-worker') self.name = '%s-%s' % (socket.getfqdn(), os.getpid()) identifier = qdo_section['name'] if identifier: self.name += '-' + identifier self.wait_interval = qdo_section['wait_interval'] resolve(self, qdo_section, 'job') resolve(self, qdo_section, 'job_context') resolve(self, qdo_section, 'job_failure') queuey_section = self.settings.getsection('queuey') self.queuey_conn = Client( queuey_section['app_key'], connection=queuey_section['connection']) zk_section = self.settings.getsection('zookeeper') self.zk_hosts = zk_section['connection'] self.zk_party_wait = zk_section['party_wait'] def setup_zookeeper(self): self.zk = KazooClient(hosts=self.zk_hosts, max_retries=1) self.zk.start() def all_partitions(self): # List all partitions queuey_conn = self.queuey_conn response = queuey_conn.get(params={'details': True}) queues = ujson_decode(response.text)['queues'] partitions = [] for q in queues: name = q['queue_name'] part = q['partitions'] for i in xrange(1, part + 1): partitions.append('%s-%s' % (name, i)) return partitions def configure_partitions(self): section = self.settings.getsection('partitions') self.partition_policy = policy = section['policy'] queuey_conn = self.queuey_conn all_partitions = self.all_partitions() partition_ids = section.get('ids') partitioner_class = StaticPartitioner if not partition_ids: partition_ids = all_partitions if policy == 'automatic': self.setup_zookeeper() partitioner_class = self.zk.SetPartitioner partition_ids = [p for p in partition_ids if not p.startswith((ERROR_QUEUE, STATUS_QUEUE))] self.partitioner = partitioner_class( '/worker', set=tuple(partition_ids), identifier=self.name, time_boundary=self.zk_party_wait) def cond_create(queue_name): if queue_name + '-1' not in all_partitions: queuey_conn.create_queue( queue_name=queue_name, partitions=STATUS_PARTITIONS) cond_create(ERROR_QUEUE) cond_create(STATUS_QUEUE) self.status = self.status_partitions() def status_partitions(self): status = {} # get all status messages, starting with the newest ones status_messages = self.queuey_conn.messages( STATUS_QUEUE, limit=1000, order='descending') if len(status_messages) >= 1000: # pragma: no cover # TODO deal with more than 1000 status messages / partitions raise RuntimeError('More than 1000 status messages detected!') for message in status_messages: body = ujson_decode(message['body']) partition = body['partition'] if partition not in status: # don't overwrite newer messages with older status status[partition] = message['message_id'] return status def work(self): """Work on jobs.""" if not self.job: return # Try Queuey heartbeat connection self.queuey_conn.connect() self.configure_partitions() atexit.register(self.stop) timer = get_logger().timer partitioner = self.partitioner with self.job_context() as context: if partitioner.allocating: partitioner.wait_for_acquire(self.zk_party_wait) waited = 0 while 1: if self.shutdown or partitioner.failed: break if partitioner.release: partitioner.release_set() elif partitioner.allocating: partitioner.wait_for_acquire(self.zk_party_wait) elif partitioner.acquired: no_messages = 0 partitions = list(self.partitioner) for name in partitions: partition = self.partition_cache[name] messages = partition.messages(limit=2) if not messages: no_messages += 1 continue message = messages[0] message_id = message['message_id'] try: with timer('worker.job_time'): self.job(message, context) except StopWorker: self.shutdown = True break except Exception as exc: with timer('worker.job_failure_time'): self.job_failure(message, context, name, exc, self.queuey_conn) # record successful message processing partition.last_message = message_id if no_messages == len(partitions): # if none of the partitions had a message, wait self.wait(waited) waited += 1 else: waited = 0 # give up the partitions and leave party self.partitioner.finish() def wait(self, waited=1): get_logger().incr('worker.wait_for_jobs') jitter = random.uniform(0.8, 1.2) time.sleep(self.wait_interval * jitter * 2 ** min(waited, 10)) def stop(self): """Stop the worker loop. Used in an `atexit` hook.""" self.shutdown = True if self.zk is not None: self.partitioner.finish() self.zk.stop()
def _make_one(self, connection=u'https://127.0.0.1:5001/v1/queuey/'): return Client(self.queuey_app_key, connection=connection)
class Worker(object): """A Worker works on jobs. :param settings: Configuration settings :type settings: dict """ def __init__(self, settings): self.settings = settings self.shutdown = False self.job = None self.job_context = dict_context self.job_failure = log_failure self.partition_policy = 'manual' self.queuey_conn = None self.zk = None self.partitioner = None self.partition_cache = PartitionCache(self) self.configure() def configure(self): # Configure the worker based on the configuration settings. qdo_section = self.settings.getsection('qdo-worker') self.name = '%s-%s' % (socket.getfqdn(), os.getpid()) identifier = qdo_section['name'] if identifier: self.name += '-' + identifier self.wait_interval = qdo_section['wait_interval'] resolve(self, qdo_section, 'job') resolve(self, qdo_section, 'job_context') resolve(self, qdo_section, 'job_failure') queuey_section = self.settings.getsection('queuey') self.queuey_conn = Client(queuey_section['app_key'], connection=queuey_section['connection']) zk_section = self.settings.getsection('zookeeper') self.zk_hosts = zk_section['connection'] self.zk_party_wait = zk_section['party_wait'] def setup_zookeeper(self): self.zk = KazooClient(hosts=self.zk_hosts, max_retries=1) self.zk.start() def all_partitions(self): # List all partitions queuey_conn = self.queuey_conn response = queuey_conn.get(params={'details': True}) queues = ujson_decode(response.text)['queues'] partitions = [] for q in queues: name = q['queue_name'] part = q['partitions'] for i in xrange(1, part + 1): partitions.append('%s-%s' % (name, i)) return partitions def configure_partitions(self): section = self.settings.getsection('partitions') self.partition_policy = policy = section['policy'] queuey_conn = self.queuey_conn all_partitions = self.all_partitions() partition_ids = section.get('ids') partitioner_class = StaticPartitioner if not partition_ids: partition_ids = all_partitions if policy == 'automatic': self.setup_zookeeper() partitioner_class = self.zk.SetPartitioner partition_ids = [ p for p in partition_ids if not p.startswith((ERROR_QUEUE, STATUS_QUEUE)) ] self.partitioner = partitioner_class('/worker', set=tuple(partition_ids), identifier=self.name, time_boundary=self.zk_party_wait) def cond_create(queue_name): if queue_name + '-1' not in all_partitions: queuey_conn.create_queue(queue_name=queue_name, partitions=STATUS_PARTITIONS) cond_create(ERROR_QUEUE) cond_create(STATUS_QUEUE) self.status = self.status_partitions() def status_partitions(self): status = {} # get all status messages, starting with the newest ones status_messages = self.queuey_conn.messages(STATUS_QUEUE, limit=1000, order='descending') if len(status_messages) >= 1000: # pragma: no cover # TODO deal with more than 1000 status messages / partitions raise RuntimeError('More than 1000 status messages detected!') for message in status_messages: body = ujson_decode(message['body']) partition = body['partition'] if partition not in status: # don't overwrite newer messages with older status status[partition] = message['message_id'] return status def work(self): """Work on jobs.""" if not self.job: return # Try Queuey heartbeat connection self.queuey_conn.connect() self.configure_partitions() atexit.register(self.stop) timer = get_logger().timer partitioner = self.partitioner with self.job_context() as context: if partitioner.allocating: partitioner.wait_for_acquire(self.zk_party_wait) waited = 0 while 1: if self.shutdown or partitioner.failed: break if partitioner.release: partitioner.release_set() elif partitioner.allocating: partitioner.wait_for_acquire(self.zk_party_wait) elif partitioner.acquired: no_messages = 0 partitions = list(self.partitioner) for name in partitions: partition = self.partition_cache[name] messages = partition.messages(limit=2) if not messages: no_messages += 1 continue message = messages[0] message_id = message['message_id'] try: with timer('worker.job_time'): self.job(message, context) except StopWorker: self.shutdown = True break except Exception as exc: with timer('worker.job_failure_time'): self.job_failure(message, context, name, exc, self.queuey_conn) # record successful message processing partition.last_message = message_id if no_messages == len(partitions): # if none of the partitions had a message, wait self.wait(waited) waited += 1 else: waited = 0 # give up the partitions and leave party self.partitioner.finish() def wait(self, waited=1): get_logger().incr('worker.wait_for_jobs') jitter = random.uniform(0.8, 1.2) time.sleep(self.wait_interval * jitter * 2**min(waited, 10)) def stop(self): """Stop the worker loop. Used in an `atexit` hook.""" self.shutdown = True if self.zk is not None: self.partitioner.finish() self.zk.stop()
def _make_queuey_conn(cls, connection='http://127.0.0.1:5000/v1/queuey/'): return Client(cls.queuey_app_key, connection=connection)