def __init__(self, connection_manager=None): super(GearmanWorkerCommandHandler, self).__init__(connection_manager=connection_manager) self._handler_abilities = [] self._client_id = None self._worker_grab = WorkerGrab()
def __init__(self, host_list=None): super(GearmanWorker, self).__init__(host_list=host_list) self.randomized_connections = None self.worker_abilities = {} self.worker_client_id = None self.command_handler_holding_job_lock = None self._update_initial_state() self.worker_grab = WorkerGrab()
class GearmanWorkerCommandHandler(GearmanCommandHandler): """GearmanWorker state machine on a per connection basis A worker can be in the following distinct states: SLEEP -> Doing nothing, can be awoken AWAKE -> Transitional state (for NOOP) AWAITING_JOB -> Holding worker level job lock and awaiting a server response EXECUTING_JOB -> Transitional state (for ASSIGN_JOB) """ def __init__(self, connection_manager=None): super(GearmanWorkerCommandHandler, self).__init__(connection_manager=connection_manager) self._handler_abilities = [] self._client_id = None self._worker_grab = WorkerGrab() def initial_state(self, abilities=None, client_id=None): self.set_client_id(client_id) self.set_abilities(abilities) self._sleep() ################################################################## ##### Public interface methods to be called by GearmanWorker ##### ################################################################## def set_abilities(self, connection_abilities_list): assert type(connection_abilities_list) in (list, tuple) self._handler_abilities = connection_abilities_list self.send_command(GEARMAN_COMMAND_RESET_ABILITIES) for task in self._handler_abilities: self.send_command(GEARMAN_COMMAND_CAN_DO, task=task) def set_client_id(self, client_id): self._client_id = client_id if self._client_id is not None: self.send_command(GEARMAN_COMMAND_SET_CLIENT_ID, client_id=self._client_id) ############################################################### #### Convenience methods for typical gearman jobs to call ##### ############################################################### def send_job_status(self, current_job, numerator, denominator): assert type(numerator) in (int, float), 'Numerator must be a numeric value' assert type(denominator) in ( int, float), 'Denominator must be a numeric value' self.send_command(GEARMAN_COMMAND_WORK_STATUS, job_handle=current_job.handle, numerator=str(numerator), denominator=str(denominator)) def send_job_complete(self, current_job, data): """Removes a job from the queue if its backgrounded""" self.send_command(GEARMAN_COMMAND_WORK_COMPLETE, job_handle=current_job.handle, data=self.encode_data(data)) def send_job_failure(self, current_job): """Removes a job from the queue if its backgrounded""" self.send_command(GEARMAN_COMMAND_WORK_FAIL, job_handle=current_job.handle) def send_job_exception(self, current_job, data): # Using GEARMAND_COMMAND_WORK_EXCEPTION is not recommended at time of this writing [2010-02-24] # http://groups.google.com/group/gearman/browse_thread/thread/5c91acc31bd10688/529e586405ed37fe # self.send_command(GEARMAN_COMMAND_WORK_EXCEPTION, job_handle=current_job.handle, data=self.encode_data(data)) def send_job_data(self, current_job, data): self.send_command(GEARMAN_COMMAND_WORK_DATA, job_handle=current_job.handle, data=self.encode_data(data)) def send_job_warning(self, current_job, data): self.send_command(GEARMAN_COMMAND_WORK_WARNING, job_handle=current_job.handle, data=self.encode_data(data)) ########################################################### ### Callbacks when we receive a command from the server ### ########################################################### def _grab_job(self): self.send_command(GEARMAN_COMMAND_GRAB_JOB_UNIQ) def _sleep(self): self.send_command(GEARMAN_COMMAND_PRE_SLEEP) def _check_job_lock(self): return self.connection_manager.check_job_lock(self) def _acquire_job_lock(self): return self.connection_manager.set_job_lock(self, lock=True) def _release_job_lock(self): if not self.connection_manager.set_job_lock(self, lock=False): raise InvalidWorkerState("Unable to release job lock for %r" % self) return True def recv_noop(self, dummy_noop=False): """Transition from being SLEEP --> AWAITING_JOB / SLEEP AWAITING_JOB -> AWAITING_JOB :: Noop transition, we're already awaiting a job SLEEP -> AWAKE -> AWAITING_JOB :: Transition if we can acquire the worker job lock SLEEP -> AWAKE -> SLEEP :: Transition if we can NOT acquire a worker job lock """ if self._check_job_lock(): pass elif self._acquire_job_lock(): if WorkerGrab.DO_GRAB and dummy_noop: self._worker_grab.set_dummy_noop_flag_by_handler(self, False) self._grab_job() else: self._sleep() return True def recv_no_job(self): """Transition from being AWAITING_JOB --> SLEEP AWAITING_JOB -> SLEEP :: Always transition to sleep if we have nothing to do """ self._release_job_lock() self._sleep() if WorkerGrab.DO_GRAB: self._send_dummy_noop() return True def _send_dummy_noop(self): if not WorkerGrab.DO_GRAB: return handler = self._worker_grab.get_dummy_noop_handler() if handler != None: handler.recv_command(GEARMAN_COMMAND_NOOP, dummy_noop=True) return def recv_job_assign_uniq(self, job_handle, task, unique, data): """Transition from being AWAITING_JOB --> EXECUTE_JOB --> SLEEP AWAITING_JOB -> EXECUTE_JOB -> SLEEP :: Always transition once we're given a job """ assert task in self._handler_abilities, '%s not found in %r' % ( task, self._handler_abilities) # After this point, we know this connection handler is holding onto the job lock so we don't need to acquire it again if not self.connection_manager.check_job_lock(self): raise InvalidWorkerState( "Received a job when we weren't expecting one") gearman_job = self.connection_manager.create_job( self, job_handle, task, unique, self.decode_data(data)) # Create a new job self.connection_manager.on_job_execute(gearman_job) # Release the job lock once we're doing and go back to sleep self._release_job_lock() self._sleep() if WorkerGrab.DO_GRAB: self._worker_grab.set_all_dummy_noop_flag_true() self._send_dummy_noop() return True def recv_job_assign(self, job_handle, task, data): """JOB_ASSIGN and JOB_ASSIGN_UNIQ are essentially the same""" return self.recv_job_assign_uniq(job_handle=job_handle, task=task, unique=None, data=data)
class GearmanWorkerCommandHandler(GearmanCommandHandler): """GearmanWorker state machine on a per connection basis A worker can be in the following distinct states: SLEEP -> Doing nothing, can be awoken AWAKE -> Transitional state (for NOOP) AWAITING_JOB -> Holding worker level job lock and awaiting a server response EXECUTING_JOB -> Transitional state (for ASSIGN_JOB) """ def __init__(self, connection_manager=None): super(GearmanWorkerCommandHandler, self).__init__(connection_manager=connection_manager) self._handler_abilities = [] self._client_id = None self._worker_grab = WorkerGrab() def initial_state(self, abilities=None, client_id=None): self.set_client_id(client_id) self.set_abilities(abilities) self._sleep() ################################################################## ##### Public interface methods to be called by GearmanWorker ##### ################################################################## def set_abilities(self, connection_abilities_list): assert type(connection_abilities_list) in (list, tuple) self._handler_abilities = connection_abilities_list self.send_command(GEARMAN_COMMAND_RESET_ABILITIES) for task in self._handler_abilities: self.send_command(GEARMAN_COMMAND_CAN_DO, task=task) def set_client_id(self, client_id): self._client_id = client_id if self._client_id is not None: self.send_command(GEARMAN_COMMAND_SET_CLIENT_ID, client_id=self._client_id) ############################################################### #### Convenience methods for typical gearman jobs to call ##### ############################################################### def send_job_status(self, current_job, numerator, denominator): assert type(numerator) in (int, float), 'Numerator must be a numeric value' assert type(denominator) in (int, float), 'Denominator must be a numeric value' self.send_command(GEARMAN_COMMAND_WORK_STATUS, job_handle=current_job.handle, numerator=str(numerator), denominator=str(denominator)) def send_job_complete(self, current_job, data): """Removes a job from the queue if its backgrounded""" self.send_command(GEARMAN_COMMAND_WORK_COMPLETE, job_handle=current_job.handle, data=self.encode_data(data)) def send_job_failure(self, current_job): """Removes a job from the queue if its backgrounded""" self.send_command(GEARMAN_COMMAND_WORK_FAIL, job_handle=current_job.handle) def send_job_exception(self, current_job, data): # Using GEARMAND_COMMAND_WORK_EXCEPTION is not recommended at time of this writing [2010-02-24] # http://groups.google.com/group/gearman/browse_thread/thread/5c91acc31bd10688/529e586405ed37fe # self.send_command(GEARMAN_COMMAND_WORK_EXCEPTION, job_handle=current_job.handle, data=self.encode_data(data)) def send_job_data(self, current_job, data): self.send_command(GEARMAN_COMMAND_WORK_DATA, job_handle=current_job.handle, data=self.encode_data(data)) def send_job_warning(self, current_job, data): self.send_command(GEARMAN_COMMAND_WORK_WARNING, job_handle=current_job.handle, data=self.encode_data(data)) ########################################################### ### Callbacks when we receive a command from the server ### ########################################################### def _grab_job(self): self.send_command(GEARMAN_COMMAND_GRAB_JOB_UNIQ) def _sleep(self): self.send_command(GEARMAN_COMMAND_PRE_SLEEP) def _check_job_lock(self): return self.connection_manager.check_job_lock(self) def _acquire_job_lock(self): return self.connection_manager.set_job_lock(self, lock=True) def _release_job_lock(self): if not self.connection_manager.set_job_lock(self, lock=False): raise InvalidWorkerState("Unable to release job lock for %r" % self) return True def recv_noop(self, dummy_noop=False): """Transition from being SLEEP --> AWAITING_JOB / SLEEP AWAITING_JOB -> AWAITING_JOB :: Noop transition, we're already awaiting a job SLEEP -> AWAKE -> AWAITING_JOB :: Transition if we can acquire the worker job lock SLEEP -> AWAKE -> SLEEP :: Transition if we can NOT acquire a worker job lock """ if self._check_job_lock(): pass elif self._acquire_job_lock(): if WorkerGrab.DO_GRAB and dummy_noop: self._worker_grab.set_dummy_noop_flag_by_handler(self, False) self._grab_job() else: self._sleep() return True def recv_no_job(self): """Transition from being AWAITING_JOB --> SLEEP AWAITING_JOB -> SLEEP :: Always transition to sleep if we have nothing to do """ self._release_job_lock() self._sleep() if WorkerGrab.DO_GRAB: self._send_dummy_noop() return True def _send_dummy_noop(self): if not WorkerGrab.DO_GRAB: return handler = self._worker_grab.get_dummy_noop_handler() if handler != None: handler.recv_command(GEARMAN_COMMAND_NOOP, dummy_noop=True) return def recv_job_assign_uniq(self, job_handle, task, unique, data): """Transition from being AWAITING_JOB --> EXECUTE_JOB --> SLEEP AWAITING_JOB -> EXECUTE_JOB -> SLEEP :: Always transition once we're given a job """ assert task in self._handler_abilities, '%s not found in %r' % (task, self._handler_abilities) # After this point, we know this connection handler is holding onto the job lock so we don't need to acquire it again if not self.connection_manager.check_job_lock(self): raise InvalidWorkerState("Received a job when we weren't expecting one") gearman_job = self.connection_manager.create_job(self, job_handle, task, unique, self.decode_data(data)) # Create a new job self.connection_manager.on_job_execute(gearman_job) # Release the job lock once we're doing and go back to sleep self._release_job_lock() self._sleep() if WorkerGrab.DO_GRAB: self._worker_grab.set_all_dummy_noop_flag_true() self._send_dummy_noop() return True def recv_job_assign(self, job_handle, task, data): """JOB_ASSIGN and JOB_ASSIGN_UNIQ are essentially the same""" return self.recv_job_assign_uniq(job_handle=job_handle, task=task, unique=None, data=data)
class GearmanWorker(GearmanConnectionManager): """ GearmanWorker :: Interface to accept jobs from a Gearman server """ command_handler_class = GearmanWorkerCommandHandler def __init__(self, host_list=None): super(GearmanWorker, self).__init__(host_list=host_list) self.randomized_connections = None self.worker_abilities = {} self.worker_client_id = None self.command_handler_holding_job_lock = None self._update_initial_state() self.worker_grab = WorkerGrab() def _update_initial_state(self): self.handler_initial_state['abilities'] = self.worker_abilities.keys() self.handler_initial_state['client_id'] = self.worker_client_id ######################################################## ##### Public methods for general GearmanWorker use ##### ######################################################## def register_task(self, task, callback_function): """Register a function with this worker def function_callback(calling_gearman_worker, current_job): return current_job.data """ self.worker_abilities[task] = callback_function self._update_initial_state() for command_handler in self.handler_to_connection_map.iterkeys(): command_handler.set_abilities(self.handler_initial_state['abilities']) return task def unregister_task(self, task): """Unregister a function with worker""" self.worker_abilities.pop(task, None) self._update_initial_state() for command_handler in self.handler_to_connection_map.iterkeys(): command_handler.set_abilities(self.handler_initial_state['abilities']) return task def set_client_id(self, client_id): """Notify the server that we should be identified as this client ID""" self.worker_client_id = client_id self._update_initial_state() for command_handler in self.handler_to_connection_map.iterkeys(): command_handler.set_client_id(self.handler_initial_state['client_id']) return client_id def work(self, do_grab=False, poll_timeout=POLL_TIMEOUT_IN_SECONDS): """Loop indefinitely, complete tasks from all connections.""" continue_working = True worker_connections = [] first_run = True # We're going to track whether a previous call to our closure indicated # we were processing a job. This is just a list of possibly a single # element indicating we had a job. It's a list so that through the # magic of closures we can reference and write to it each call. # This is all so that we can determine when we've finished processing a job # correctly. had_job = [] WorkerGrab.DO_GRAB = do_grab def continue_while_connections_alive(any_activity): if had_job and not self.has_job_lock(): return self.after_poll(any_activity) and self.after_job() del had_job[:] if self.has_job_lock(): had_job.append(True) return self.after_poll(any_activity) def before_handling_activity(read_connections, write_connections, dead_connections): # Called before actually handling activity... if not read_connections and not self.has_job_lock(): if not WorkerGrab.DO_GRAB: return self.worker_grab.set_all_dummy_noop_flag_true() # If nothing has data to read and we are not processing a job... for connection in set(worker_connections) - set(dead_connections): # Have all of the handlers (that are still active)... handler = self.connection_to_handler_map[connection] # Act as if they have received a NOOP. handler.recv_command(GEARMAN_COMMAND_NOOP, dummy_noop=True) return # Shuffle our connections after the poll timeout while continue_working: worker_connections = self.establish_worker_connections() if do_grab: if first_run: self.worker_grab.init_connection_handler_list(self.connection_to_handler_map) first_run = False else: self.worker_grab.update_connection_handler_list(self.connection_to_handler_map) continue_working = self.poll_connections_until_stopped(worker_connections, continue_while_connections_alive, timeout=poll_timeout, prehandle=before_handling_activity) # If we were kicked out of the worker loop, we should shutdown all our connections for current_connection in worker_connections: current_connection.close() def shutdown(self): self.command_handler_holding_job_lock = None super(GearmanWorker, self).shutdown() ############################################################### ## Methods to override when dealing with connection polling ## ############################################################## def establish_worker_connections(self): """Return a shuffled list of connections that are alive, and try to reconnect to dead connections if necessary.""" self.randomized_connections = list(self.connection_list) random.shuffle(self.randomized_connections) output_connections = [] for current_connection in self.randomized_connections: try: valid_connection = self.establish_connection(current_connection) output_connections.append(valid_connection) except ConnectionError: pass return output_connections def after_poll(self, any_activity): """Polling callback to notify any outside listeners whats going on with the GearmanWorker. Return True to continue polling, False to exit the work loop""" return True def after_job(self): """Callback to notify any outside listeners that a GearmanWorker has completed the current job. This is useful for accomplishing work or stopping the GearmanWorker in between jobs. Return True to continue polling, False to exit the work loop """ return True def handle_error(self, current_connection): """If we discover that a connection has a problem, we better release the job lock""" current_handler = self.connection_to_handler_map.get(current_connection) if current_handler: self.set_job_lock(current_handler, lock=False) super(GearmanWorker, self).handle_error(current_connection) ############################################################# ## Public methods so Gearman jobs can send Gearman updates ## ############################################################# def _get_handler_for_job(self, current_job): return self.connection_to_handler_map[current_job.connection] def wait_until_updates_sent(self, multiple_gearman_jobs, poll_timeout=None): connection_set = set([current_job.connection for current_job in multiple_gearman_jobs]) def continue_while_updates_pending(any_activity): return compat.any(current_connection.writable() for current_connection in connection_set) self.poll_connections_until_stopped(connection_set, continue_while_updates_pending, timeout=poll_timeout) def send_job_status(self, current_job, numerator, denominator, poll_timeout=None): """Send a Gearman JOB_STATUS update for an inflight job""" current_handler = self._get_handler_for_job(current_job) current_handler.send_job_status(current_job, numerator=numerator, denominator=denominator) self.wait_until_updates_sent([current_job], poll_timeout=poll_timeout) def send_job_complete(self, current_job, data, poll_timeout=None): current_handler = self._get_handler_for_job(current_job) current_handler.send_job_complete(current_job, data=data) self.wait_until_updates_sent([current_job], poll_timeout=poll_timeout) def send_job_failure(self, current_job, poll_timeout=None): """Removes a job from the queue if its backgrounded""" current_handler = self._get_handler_for_job(current_job) current_handler.send_job_failure(current_job) self.wait_until_updates_sent([current_job], poll_timeout=poll_timeout) def send_job_exception(self, current_job, data, poll_timeout=None): """Removes a job from the queue if its backgrounded""" # Using GEARMAND_COMMAND_WORK_EXCEPTION is not recommended at time of this writing [2010-02-24] # http://groups.google.com/group/gearman/browse_thread/thread/5c91acc31bd10688/529e586405ed37fe # current_handler = self._get_handler_for_job(current_job) current_handler.send_job_exception(current_job, data=data) current_handler.send_job_failure(current_job) self.wait_until_updates_sent([current_job], poll_timeout=poll_timeout) def send_job_data(self, current_job, data, poll_timeout=None): """Send a Gearman JOB_DATA update for an inflight job""" current_handler = self._get_handler_for_job(current_job) current_handler.send_job_data(current_job, data=data) self.wait_until_updates_sent([current_job], poll_timeout=poll_timeout) def send_job_warning(self, current_job, data, poll_timeout=None): """Send a Gearman JOB_WARNING update for an inflight job""" current_handler = self._get_handler_for_job(current_job) current_handler.send_job_warning(current_job, data=data) self.wait_until_updates_sent([current_job], poll_timeout=poll_timeout) ##################################################### ##### Callback methods for GearmanWorkerHandler ##### ##################################################### def create_job(self, command_handler, job_handle, task, unique, data): """Create a new job using our self.job_class""" current_connection = self.handler_to_connection_map[command_handler] return self.job_class(current_connection, job_handle, task, unique, data) def on_job_execute(self, current_job): try: function_callback = self.worker_abilities[current_job.task] job_result = function_callback(self, current_job) except Exception: return self.on_job_exception(current_job, sys.exc_info()) return self.on_job_complete(current_job, job_result) def on_job_exception(self, current_job, exc_info): self.send_job_failure(current_job) return False def on_job_complete(self, current_job, job_result): self.send_job_complete(current_job, job_result) return True def set_job_lock(self, command_handler, lock): """Set a worker level job lock so we don't try to hold onto 2 jobs at anytime""" if command_handler not in self.handler_to_connection_map: return False failed_lock = bool(lock and self.command_handler_holding_job_lock is not None) failed_unlock = bool(not lock and self.command_handler_holding_job_lock != command_handler) # If we've already been locked, we should say the lock failed # If we're attempting to unlock something when we don't have a lock, we're in a bad state if failed_lock or failed_unlock: return False if lock: self.command_handler_holding_job_lock = command_handler else: self.command_handler_holding_job_lock = None return True def has_job_lock(self): return bool(self.command_handler_holding_job_lock is not None) def check_job_lock(self, command_handler): """Check to see if we hold the job lock""" return bool(self.command_handler_holding_job_lock == command_handler)