def get_announcement(self):
'''Connect to the announcing socket and get an analysis announcement
returns an analysis_id / worker_request address pair
raises a CancelledException if we detect cancellation.
'''
poller = zmq.Poller()
poller.register(self.notify_socket, zmq.POLLIN)
announce_socket = the_zmq_context.socket(zmq.SUB)
announce_socket.setsockopt(zmq.SUBSCRIBE, "")
announce_socket.connect(self.work_announce_address)
try:
poller.register(announce_socket, zmq.POLLIN)
while True:
for socket, state in poller.poll():
if socket == self.notify_socket and state == zmq.POLLIN:
msg = self.notify_socket.recv()
if msg == NOTIFY_STOP:
from cellprofiler.pipeline import CancelledException
self.cancelled = True
raise CancelledException()
elif socket == announce_socket and state == zmq.POLLIN:
announcement = dict(announce_socket.recv_json())
if len(announcement) == 0:
threading._sleep(0.25)
continue
if self.current_analysis_id in announcement:
analysis_id = self.current_analysis_id
else:
analysis_id = random.choice(announcement.keys())
return analysis_id, announcement[analysis_id]
finally:
announce_socket.close()
def send(self, req, work_socket=None):
'''Send a request and receive a reply
req - request to send
socket - socket to use for send. Default is current work socket
returns a reply on success. If cancelled, throws a CancelledException
'''
if self.current_analysis_id is None:
from cellprofiler.pipeline import CancelledException
raise CancelledException("Can't send after cancelling")
if work_socket is None:
work_socket = self.work_socket
poller = zmq.Poller()
poller.register(self.notify_socket, zmq.POLLIN)
poller.register(work_socket, zmq.POLLIN)
req.send_only(work_socket)
response = None
while response is None:
for socket, state in poller.poll():
if socket == self.notify_socket and state == zmq.POLLIN:
notify_msg = self.notify_socket.recv()
if notify_msg == NOTIFY_STOP:
self.cancelled = True
self.raise_cancel(
"Received stop notification while waiting for "
"response from %s" % str(req))
if socket == work_socket and state == zmq.POLLIN:
response = req.recv(work_socket)
if isinstance(response, UpstreamExit):
self.raise_cancel(
"Received UpstreamExit for analysis %s during request %s" %
(self.current_analysis_id, str(req)))
return response
def raise_cancel(self, msg="Cancelling analysis"):
'''Handle the cleanup after some proximate cause of cancellation
msg - reason for cancellation
This should only be called upon detection of a server-driven
cancellation of analysis: either UpstreamExit or a stop notification
from the deadman thread.
'''
logger.debug(msg)
self.cancelled = True
if self.current_analysis_id in self.initial_measurements:
self.initial_measurements[self.current_analysis_id].close()
del self.initial_measurements[self.current_analysis_id]
if self.current_analysis_id in self.pipelines_and_preferences:
del self.pipelines_and_preferences[self.current_analysis_id]
self.current_analysis_id = None
raise CancelledException(msg)
def do_job(self, job):
'''Handle a work request to its completion
job - WorkRequest
'''
job_measurements = []
try:
send_dictionary = job.wants_dictionary
logger.info("Starting job")
# Fetch the pipeline and preferences for this analysis if we don't have it
current_pipeline, current_preferences = \
self.pipelines_and_preferences.get(
self.current_analysis_id, (None, None))
if not current_pipeline:
logger.debug("Fetching pipeline and preferences")
rep = self.send(
PipelinePreferencesRequest(self.current_analysis_id))
logger.debug("Received pipeline and preferences response")
preferences_dict = rep.preferences
# update preferences to match remote values
cpprefs.set_preferences_from_dict(preferences_dict)
logger.debug("Loading pipeline")
pipeline_blob = rep.pipeline_blob.tostring()
current_pipeline = cpp.Pipeline()
current_pipeline.loadtxt(StringIO.StringIO(pipeline_blob),
raise_on_error=True)
logger.debug("Pipeline loaded")
current_pipeline.add_listener(
self.pipeline_listener.handle_event)
current_preferences = rep.preferences
self.pipelines_and_preferences[self.current_analysis_id] = (
current_pipeline, current_preferences)
else:
# update preferences to match remote values
cpprefs.set_preferences_from_dict(current_preferences)
# Reset the listener's state
self.pipeline_listener.reset()
logger.debug("Getting initial measurements")
# Fetch the path to the intial measurements if needed.
current_measurements = self.initial_measurements.get(
self.current_analysis_id)
if current_measurements is None:
logger.debug("Sending initial measurements request")
rep = self.send(
InitialMeasurementsRequest(self.current_analysis_id))
logger.debug("Got initial measurements")
current_measurements = \
self.initial_measurements[self.current_analysis_id] = \
cpmeas.load_measurements_from_buffer(rep.buf)
else:
logger.debug("Has initial measurements")
# Make a copy of the measurements for writing during this job
current_measurements = cpmeas.Measurements(
copy=current_measurements)
all_measurements.add(current_measurements)
job_measurements.append(current_measurements)
successful_image_set_numbers = []
image_set_numbers = job.image_set_numbers
worker_runs_post_group = job.worker_runs_post_group
logger.info("Doing job: " + ",".join(map(str, image_set_numbers)))
self.pipeline_listener.image_set_number = image_set_numbers[0]
if not worker_runs_post_group:
# Get the shared state from the first imageset in this run.
shared_dicts = self.send(
SharedDictionaryRequest(
self.current_analysis_id)).dictionaries
assert len(shared_dicts) == len(current_pipeline.modules())
for module, new_dict in zip(current_pipeline.modules(),
shared_dicts):
module.set_dictionary_for_worker(new_dict)
# Run prepare group if this is the first image in the group. We do
# this here (even if there's no grouping in the pipeline) to ensure
# that any changes to the modules' shared state dictionaries get
# propagated correctly.
should_process = True
if current_measurements[cpmeas.IMAGE, cpmeas.GROUP_INDEX,
image_set_numbers[0]] == 1:
workspace = cpw.Workspace(current_pipeline, None, None, None,
current_measurements, None, None)
if not current_pipeline.prepare_group(
workspace, current_measurements.get_grouping_keys(),
image_set_numbers):
# exception handled elsewhere, possibly cancelling this run.
should_process = False
# process the images
if should_process:
abort = False
for image_set_number in image_set_numbers:
gc.collect()
try:
self.pipeline_listener.image_set_number = image_set_number
current_pipeline.run_image_set(
current_measurements, image_set_number,
self.interaction_handler, self.display_handler)
if self.pipeline_listener.should_abort:
abort = True
break
elif self.pipeline_listener.should_skip:
# Report skipped image sets as successful so that
# analysis can complete.
# Report their measurements because some modules
# may have provided measurements before skipping.
pass
successful_image_set_numbers.append(image_set_number)
# Send an indication that the image set finished successfully.
if send_dictionary:
# The jobserver would like a copy of our modules'
# run_state dictionaries.
ws = cpw.Workspace(current_pipeline, None, None,
None, current_measurements,
None, None)
dicts = [
m.get_dictionary_for_worker()
for m in current_pipeline.modules()
]
req = ImageSetSuccessWithDictionary(
self.current_analysis_id,
image_set_number=image_set_number,
shared_dicts=dicts)
else:
req = ImageSetSuccess(
self.current_analysis_id,
image_set_number=image_set_number)
rep = self.send(req)
except CancelledException:
logging.info("Aborting job after cancellation")
abort = True
except Exception:
try:
logging.error("Error in pipeline", exc_info=True)
if self.handle_exception(
image_set_number=image_set_number
) == ED_STOP:
abort = True
break
except:
logging.error(
"Error in handling of pipeline exception",
exc_info=True)
# this is bad. We can't handle nested exceptions
# remotely so we just fail on this run.
abort = True
if abort:
current_measurements.close()
job_measurements.remove(current_measurements)
return
if worker_runs_post_group:
workspace = cpw.Workspace(current_pipeline, None,
current_measurements, None,
current_measurements, None, None)
workspace.post_group_display_handler = \
self.post_group_display_handler
# There might be an exception in this call, but it will be
# handled elsewhere, and there's nothing we can do for it
# here.
current_pipeline.post_group(
workspace, current_measurements.get_grouping_keys())
# send measurements back to server
req = MeasurementsReport(self.current_analysis_id,
buf=current_measurements.file_contents(),
image_set_numbers=image_set_numbers)
rep = self.send(req)
except CancelledException:
# Main thread received shutdown signal
raise
except Exception:
logging.error("Error in worker", exc_info=True)
if self.handle_exception() == ED_STOP:
raise CancelledException(
"Cancelling after user-requested stop")
finally:
# Clean up any measurements owned by us
for m in job_measurements:
m.close()