def get_announcement(self): '''Connect to the announcing socket and get an analysis announcement returns an analysis_id / worker_request address pair raises a CancelledException if we detect cancellation. ''' poller = zmq.Poller() poller.register(self.notify_socket, zmq.POLLIN) announce_socket = the_zmq_context.socket(zmq.SUB) announce_socket.setsockopt(zmq.SUBSCRIBE, "") announce_socket.connect(self.work_announce_address) try: poller.register(announce_socket, zmq.POLLIN) while True: for socket, state in poller.poll(): if socket == self.notify_socket and state == zmq.POLLIN: msg = self.notify_socket.recv() if msg == NOTIFY_STOP: from cellprofiler.pipeline import CancelledException self.cancelled = True raise CancelledException() elif socket == announce_socket and state == zmq.POLLIN: announcement = dict(announce_socket.recv_json()) if len(announcement) == 0: threading._sleep(0.25) continue if self.current_analysis_id in announcement: analysis_id = self.current_analysis_id else: analysis_id = random.choice(announcement.keys()) return analysis_id, announcement[analysis_id] finally: announce_socket.close()
def send(self, req, work_socket=None): '''Send a request and receive a reply req - request to send socket - socket to use for send. Default is current work socket returns a reply on success. If cancelled, throws a CancelledException ''' if self.current_analysis_id is None: from cellprofiler.pipeline import CancelledException raise CancelledException("Can't send after cancelling") if work_socket is None: work_socket = self.work_socket poller = zmq.Poller() poller.register(self.notify_socket, zmq.POLLIN) poller.register(work_socket, zmq.POLLIN) req.send_only(work_socket) response = None while response is None: for socket, state in poller.poll(): if socket == self.notify_socket and state == zmq.POLLIN: notify_msg = self.notify_socket.recv() if notify_msg == NOTIFY_STOP: self.cancelled = True self.raise_cancel( "Received stop notification while waiting for " "response from %s" % str(req)) if socket == work_socket and state == zmq.POLLIN: response = req.recv(work_socket) if isinstance(response, UpstreamExit): self.raise_cancel( "Received UpstreamExit for analysis %s during request %s" % (self.current_analysis_id, str(req))) return response
def raise_cancel(self, msg="Cancelling analysis"): '''Handle the cleanup after some proximate cause of cancellation msg - reason for cancellation This should only be called upon detection of a server-driven cancellation of analysis: either UpstreamExit or a stop notification from the deadman thread. ''' logger.debug(msg) self.cancelled = True if self.current_analysis_id in self.initial_measurements: self.initial_measurements[self.current_analysis_id].close() del self.initial_measurements[self.current_analysis_id] if self.current_analysis_id in self.pipelines_and_preferences: del self.pipelines_and_preferences[self.current_analysis_id] self.current_analysis_id = None raise CancelledException(msg)
def do_job(self, job): '''Handle a work request to its completion job - WorkRequest ''' job_measurements = [] try: send_dictionary = job.wants_dictionary logger.info("Starting job") # Fetch the pipeline and preferences for this analysis if we don't have it current_pipeline, current_preferences = \ self.pipelines_and_preferences.get( self.current_analysis_id, (None, None)) if not current_pipeline: logger.debug("Fetching pipeline and preferences") rep = self.send( PipelinePreferencesRequest(self.current_analysis_id)) logger.debug("Received pipeline and preferences response") preferences_dict = rep.preferences # update preferences to match remote values cpprefs.set_preferences_from_dict(preferences_dict) logger.debug("Loading pipeline") pipeline_blob = rep.pipeline_blob.tostring() current_pipeline = cpp.Pipeline() current_pipeline.loadtxt(StringIO.StringIO(pipeline_blob), raise_on_error=True) logger.debug("Pipeline loaded") current_pipeline.add_listener( self.pipeline_listener.handle_event) current_preferences = rep.preferences self.pipelines_and_preferences[self.current_analysis_id] = ( current_pipeline, current_preferences) else: # update preferences to match remote values cpprefs.set_preferences_from_dict(current_preferences) # Reset the listener's state self.pipeline_listener.reset() logger.debug("Getting initial measurements") # Fetch the path to the intial measurements if needed. current_measurements = self.initial_measurements.get( self.current_analysis_id) if current_measurements is None: logger.debug("Sending initial measurements request") rep = self.send( InitialMeasurementsRequest(self.current_analysis_id)) logger.debug("Got initial measurements") current_measurements = \ self.initial_measurements[self.current_analysis_id] = \ cpmeas.load_measurements_from_buffer(rep.buf) else: logger.debug("Has initial measurements") # Make a copy of the measurements for writing during this job current_measurements = cpmeas.Measurements( copy=current_measurements) all_measurements.add(current_measurements) job_measurements.append(current_measurements) successful_image_set_numbers = [] image_set_numbers = job.image_set_numbers worker_runs_post_group = job.worker_runs_post_group logger.info("Doing job: " + ",".join(map(str, image_set_numbers))) self.pipeline_listener.image_set_number = image_set_numbers[0] if not worker_runs_post_group: # Get the shared state from the first imageset in this run. shared_dicts = self.send( SharedDictionaryRequest( self.current_analysis_id)).dictionaries assert len(shared_dicts) == len(current_pipeline.modules()) for module, new_dict in zip(current_pipeline.modules(), shared_dicts): module.set_dictionary_for_worker(new_dict) # Run prepare group if this is the first image in the group. We do # this here (even if there's no grouping in the pipeline) to ensure # that any changes to the modules' shared state dictionaries get # propagated correctly. should_process = True if current_measurements[cpmeas.IMAGE, cpmeas.GROUP_INDEX, image_set_numbers[0]] == 1: workspace = cpw.Workspace(current_pipeline, None, None, None, current_measurements, None, None) if not current_pipeline.prepare_group( workspace, current_measurements.get_grouping_keys(), image_set_numbers): # exception handled elsewhere, possibly cancelling this run. should_process = False # process the images if should_process: abort = False for image_set_number in image_set_numbers: gc.collect() try: self.pipeline_listener.image_set_number = image_set_number current_pipeline.run_image_set( current_measurements, image_set_number, self.interaction_handler, self.display_handler) if self.pipeline_listener.should_abort: abort = True break elif self.pipeline_listener.should_skip: # Report skipped image sets as successful so that # analysis can complete. # Report their measurements because some modules # may have provided measurements before skipping. pass successful_image_set_numbers.append(image_set_number) # Send an indication that the image set finished successfully. if send_dictionary: # The jobserver would like a copy of our modules' # run_state dictionaries. ws = cpw.Workspace(current_pipeline, None, None, None, current_measurements, None, None) dicts = [ m.get_dictionary_for_worker() for m in current_pipeline.modules() ] req = ImageSetSuccessWithDictionary( self.current_analysis_id, image_set_number=image_set_number, shared_dicts=dicts) else: req = ImageSetSuccess( self.current_analysis_id, image_set_number=image_set_number) rep = self.send(req) except CancelledException: logging.info("Aborting job after cancellation") abort = True except Exception: try: logging.error("Error in pipeline", exc_info=True) if self.handle_exception( image_set_number=image_set_number ) == ED_STOP: abort = True break except: logging.error( "Error in handling of pipeline exception", exc_info=True) # this is bad. We can't handle nested exceptions # remotely so we just fail on this run. abort = True if abort: current_measurements.close() job_measurements.remove(current_measurements) return if worker_runs_post_group: workspace = cpw.Workspace(current_pipeline, None, current_measurements, None, current_measurements, None, None) workspace.post_group_display_handler = \ self.post_group_display_handler # There might be an exception in this call, but it will be # handled elsewhere, and there's nothing we can do for it # here. current_pipeline.post_group( workspace, current_measurements.get_grouping_keys()) # send measurements back to server req = MeasurementsReport(self.current_analysis_id, buf=current_measurements.file_contents(), image_set_numbers=image_set_numbers) rep = self.send(req) except CancelledException: # Main thread received shutdown signal raise except Exception: logging.error("Error in worker", exc_info=True) if self.handle_exception() == ED_STOP: raise CancelledException( "Cancelling after user-requested stop") finally: # Clean up any measurements owned by us for m in job_measurements: m.close()