def test_02_01_send(self):
self.awthread = self.AWThread(self.announce_addr)
self.awthread.start()
self.set_work_socket()
def send_something():
reply = self.awthread.aw.send(anarequest.Work("foo"))
return reply
self.awthread.ex(send_something)
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.Work)
self.assertEqual(req.analysis_id, "foo")
req.reply(anareply.Work(foo="bar"))
reply = self.awthread.ecute()
self.assertIsInstance(reply, anareply.Work)
self.assertEqual(reply.foo, "bar")
def test_03_08_a_sad_moment(self):
#
# Run using the good pipeline, but change one of the URLs so
# an exception is thrown.
#
self.awthread = self.AWThread(self.announce_addr)
self.awthread.start()
self.set_work_socket()
self.awthread.ex(
self.awthread.aw.do_job,
anareply.Work(
image_set_numbers=[2, 3],
worker_runs_post_group=False,
wants_dictionary=False,
),
)
#
# The worker should ask for the pipeline and preferences next.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.PipelinePreferences)
self.assertEqual(req.analysis_id, self.analysis_id)
input_dir = os.path.join(
tests.modules.example_images_directory(), "ExampleSBSImages"
)
cellprofiler_core.preferences.set_default_image_directory(input_dir)
preferences = {
cellprofiler_core.preferences.DEFAULT_IMAGE_DIRECTORY: cellprofiler_core.preferences.config_read(
cellprofiler_core.preferences.DEFAULT_IMAGE_DIRECTORY
)
}
rep = cellprofiler_core.utilities.zmq.Reply(
pipeline_blob=numpy.array(GOOD_PIPELINE), preferences=preferences
)
req.reply(rep)
#
# The worker asks for the initial measurements.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.InitialMeasurements)
self.assertEqual(req.analysis_id, self.analysis_id)
m = get_measurements_for_good_pipeline(nimages=3)
m[
cellprofiler_core.constants.measurement.IMAGE,
cellprofiler_core.modules.namesandtypes.M_IMAGE_SET,
2,
] = numpy.zeros(100, numpy.uint8)
try:
req.reply(cellprofiler_core.utilities.zmq.Reply(buf=m.file_contents()))
finally:
m.close()
#
# Next, the worker asks for the shared dictionary
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.SharedDictionary)
shared_dictionaries = [{("foo%d" % i): "bar%d" % i} for i in range(1, 8)]
rep = anareply.SharedDictionary(dictionaries=shared_dictionaries)
req.reply(rep)
#
# The worker should choke somewhere in NamesAndTypes, but we
# tell the worker to skip the rest of the imageset.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.ExceptionReport)
req.reply(anareply.ExceptionPleaseDebug(disposition="Skip"))
#
# The worker should send ImageSetSuccess for image set 2 anyway.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anareply.ImageSetSuccess)
self.assertEqual(req.image_set_number, 2)
req.reply(anareply.Ack())
#
# And then it tells us about image set 3
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anareply.ImageSetSuccess)
self.assertEqual(req.image_set_number, 3)
req.reply(anareply.Ack())
#
# The worker should then report the measurements for both 2 and 3
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.MeasurementsReport)
self.assertSequenceEqual(req.image_set_numbers, [2, 3])
m = cellprofiler_core.utilities.measurement.load_measurements_from_buffer(
req.buf
)
#
# Spot check for some expected stuff
#
self.assertTrue(
m.has_feature(
cellprofiler_core.constants.measurement.IMAGE,
cellprofiler_core.constants.measurement.C_COUNT + "_Nuclei",
)
)
self.assertTrue(
m.has_feature(
"Nuclei", cellprofiler_core.constants.measurement.M_LOCATION_CENTER_X
)
)
self.assertTrue(m.has_feature("Nuclei", "AreaShape_Area"))
#
# The count for the skipped image should be None
#
count = m[
cellprofiler_core.constants.measurement.IMAGE,
cellprofiler_core.constants.measurement.C_COUNT + "_Nuclei",
2,
]
self.assertIsNone(count)
count = m[
cellprofiler_core.constants.measurement.IMAGE,
cellprofiler_core.constants.measurement.C_COUNT + "_Nuclei",
3,
]
center_x = m[
"Nuclei", cellprofiler_core.constants.measurement.M_LOCATION_CENTER_X, 3
]
self.assertEqual(count, len(center_x))
req.reply(anareply.Ack())
self.awthread.ecute()
def test_03_06_the_happy_path_chapter_2(self):
#
# Give the worker image sets # 2 and 3 and tell it to run post_group
#
self.awthread = self.AWThread(self.announce_addr)
self.awthread.start()
self.set_work_socket()
self.awthread.ex(
self.awthread.aw.do_job,
anareply.Work(
image_set_numbers=[2, 3],
worker_runs_post_group=True,
wants_dictionary=False,
),
)
#
# The worker should ask for the pipeline and preferences next.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.PipelinePreferences)
self.assertEqual(req.analysis_id, self.analysis_id)
input_dir = os.path.join(
tests.modules.example_images_directory(), "ExampleSBSImages"
)
cellprofiler_core.preferences.set_default_image_directory(input_dir)
preferences = {
cellprofiler_core.preferences.DEFAULT_IMAGE_DIRECTORY: cellprofiler_core.preferences.config_read(
cellprofiler_core.preferences.DEFAULT_IMAGE_DIRECTORY
)
}
rep = cellprofiler_core.utilities.zmq.Reply(
pipeline_blob=numpy.array(DISPLAY_PIPELINE), preferences=preferences
)
req.reply(rep)
#
# The worker asks for the initial measurements.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.InitialMeasurements)
self.assertEqual(req.analysis_id, self.analysis_id)
m = get_measurements_for_good_pipeline(nimages=3)
try:
req.reply(cellprofiler_core.utilities.zmq.Reply(buf=m.file_contents()))
finally:
m.close()
#
# In group mode, the worker issues a display request and constructs
# its own dictonaries
#
for image_number in (2, 3):
#
# The worker sends a display request for FlipAndRotate
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.Display)
req.reply(anareply.Ack())
#
# The worker sends ImageSetSuccess.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anareply.ImageSetSuccess)
self.assertEqual(req.image_set_number, image_number)
req.reply(anareply.Ack())
#
# The worker sends a DisplayPostGroup request for FlipAndRotate
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.DisplayPostGroup)
self.assertEqual(req.image_set_number, 3)
req.reply(anareply.Ack())
#
# The worker sends a measurement report for image sets 2 and 3
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.MeasurementsReport)
self.assertSequenceEqual(req.image_set_numbers, [2, 3])
m = cellprofiler_core.utilities.measurement.load_measurements_from_buffer(
req.buf
)
#
# Spot check for some expected stuff
#
self.assertTrue(
m.has_feature(
cellprofiler_core.constants.measurement.IMAGE,
cellprofiler_core.constants.measurement.C_COUNT + "_Nuclei",
)
)
self.assertTrue(
m.has_feature(
"Nuclei", cellprofiler_core.constants.measurement.M_LOCATION_CENTER_X
)
)
self.assertTrue(m.has_feature("Nuclei", "AreaShape_Area"))
req.reply(anareply.Ack())
self.awthread.ecute()
def test_03_05_the_happy_path_chapter_1(self):
#
# Run the worker clear through to the end
# for the first imageset
#
self.awthread = self.AWThread(self.announce_addr)
self.awthread.start()
self.set_work_socket()
self.awthread.ex(
self.awthread.aw.do_job,
anareply.Work(
image_set_numbers=[1],
worker_runs_post_group=False,
wants_dictionary=True,
),
)
#
# The worker should ask for the pipeline and preferences next.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.PipelinePreferences)
self.assertEqual(req.analysis_id, self.analysis_id)
input_dir = os.path.join(
tests.modules.example_images_directory(), "ExampleSBSImages"
)
cellprofiler_core.preferences.set_default_image_directory(input_dir)
preferences = {
cellprofiler_core.preferences.DEFAULT_IMAGE_DIRECTORY: cellprofiler_core.preferences.config_read(
cellprofiler_core.preferences.DEFAULT_IMAGE_DIRECTORY
)
}
rep = cellprofiler_core.utilities.zmq.Reply(
pipeline_blob=numpy.array(DISPLAY_PIPELINE), preferences=preferences
)
req.reply(rep)
#
# The worker asks for the initial measurements.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.InitialMeasurements)
self.assertEqual(req.analysis_id, self.analysis_id)
m = get_measurements_for_good_pipeline()
try:
req.reply(cellprofiler_core.utilities.zmq.Reply(buf=m.file_contents()))
finally:
m.close()
#
# Next, the worker asks for the shared dictionary
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.SharedDictionary)
shared_dictionaries = [{("foo%d" % i): "bar%d" % i} for i in range(1, 8)]
rep = anareply.SharedDictionary(dictionaries=shared_dictionaries)
req.reply(rep)
#
# The worker sends a display request for FlipAndRotate
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.Display)
self.assertEqual(req.image_set_number, 1)
d = req.display_data_dict
# Possibly, this will break if someone edits FlipAndRotate. Sorry.
testkeys = ["vmax", "output_image_pixel_data", "image_pixel_data", "vmin"]
self.assertCountEqual(testkeys, list(d.keys()))
for item in testkeys:
self.assertIn(item, list(d.keys()))
self.assertIsInstance(d["output_image_pixel_data"], numpy.ndarray)
req.reply(anareply.Ack())
#
# The worker sends ImageSetSuccessWithDictionary.
#
req = self.awthread.recv(self.work_socket)
print(req)
self.assertIsInstance(req, anareply.ImageSetSuccessWithDictionary)
self.assertEqual(req.image_set_number, 1)
for expected, actual in zip(shared_dictionaries, req.shared_dicts):
self.assertDictEqual(expected, actual)
req.reply(anareply.Ack())
#
# The worker sends the measurement report
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.MeasurementsReport)
self.assertSequenceEqual(req.image_set_numbers, [1])
m = cellprofiler_core.utilities.measurement.load_measurements_from_buffer(
req.buf
)
#
# Spot check for some expected stuff
#
self.assertTrue(
m.has_feature(
cellprofiler_core.constants.measurement.IMAGE,
cellprofiler_core.constants.measurement.C_COUNT + "_Nuclei",
)
)
self.assertTrue(
m.has_feature(
"Nuclei", cellprofiler_core.constants.measurement.M_LOCATION_CENTER_X
)
)
self.assertTrue(m.has_feature("Nuclei", "AreaShape_Area"))
req.reply(anareply.Ack())
self.awthread.ecute()
def test_03_04_shared_dictionary_request(self):
#
# The request.SharedDictionary
#
self.awthread = self.AWThread(self.announce_addr)
self.awthread.start()
self.set_work_socket()
self.awthread.ex(
self.awthread.aw.do_job,
anareply.Work(
image_set_numbers=[1],
worker_runs_post_group=False,
wants_dictionary=True,
),
)
#
# The worker should ask for the pipeline and preferences next.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.PipelinePreferences)
self.assertEqual(req.analysis_id, self.analysis_id)
input_dir = os.path.join(
tests.modules.example_images_directory(), "ExampleSBSImages"
)
cellprofiler_core.preferences.set_default_image_directory(input_dir)
preferences = {
cellprofiler_core.preferences.DEFAULT_IMAGE_DIRECTORY: cellprofiler_core.preferences.config_read(
cellprofiler_core.preferences.DEFAULT_IMAGE_DIRECTORY
)
}
rep = cellprofiler_core.utilities.zmq.Reply(
pipeline_blob=numpy.array(DISPLAY_PIPELINE), preferences=preferences
)
req.reply(rep)
#
# The worker asks for the initial measurements.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.InitialMeasurements)
self.assertEqual(req.analysis_id, self.analysis_id)
m = get_measurements_for_good_pipeline()
try:
req.reply(cellprofiler_core.utilities.zmq.Reply(buf=m.file_contents()))
finally:
m.close()
#
# Next, the worker asks for the shared dictionary
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.SharedDictionary)
rep = anareply.SharedDictionary(
dictionaries=[{("foo%d" % i): "bar%d" % i} for i in range(1, 8)]
)
req.reply(rep)
#
# Sneaky way to get pipeline. First, synchronize with the next message
#
req = self.awthread.recv(self.work_socket)
pipe, prefs = self.awthread.aw.pipelines_and_preferences[self.analysis_id]
for d, module in zip(rep.dictionaries, pipe.modules()):
self.assertDictEqual(module.get_dictionary(), d)
#
# Might as well torpedo the app. It should be stalled waiting
# for the FlipAndRotate display.
#
self.cancel()
self.awthread.ecute()
def test_03_03_initial_measurements(self):
#
# Walk to the initial measurements
#
self.awthread = self.AWThread(self.announce_addr)
self.awthread.start()
self.set_work_socket()
work_reply = anareply.Work(
image_set_numbers=[1], worker_runs_post_group=False, wants_dictionary=True,
)
self.awthread.ex(
self.awthread.aw.do_job, work_reply,
)
#
# The worker should ask for the pipeline and preferences next.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.PipelinePreferences)
self.assertEqual(req.analysis_id, self.analysis_id)
input_dir = os.path.abspath(
os.path.join(
os.path.dirname(cellprofiler_core.__file__),
"..",
"tests/data/ExampleSBSImages",
)
)
cellprofiler_core.preferences.set_default_image_directory(input_dir)
preferences = {
cellprofiler_core.preferences.DEFAULT_IMAGE_DIRECTORY: cellprofiler_core.preferences.config_read(
cellprofiler_core.preferences.DEFAULT_IMAGE_DIRECTORY
)
}
rep = cellprofiler_core.utilities.zmq.Reply(
pipeline_blob=numpy.array(GOOD_PIPELINE), preferences=preferences
)
req.reply(rep)
#
# The worker asks for the initial measurements.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.InitialMeasurements)
self.assertEqual(req.analysis_id, self.analysis_id)
m = get_measurements_for_good_pipeline()
try:
req.reply(cellprofiler_core.utilities.zmq.Reply(buf=m.file_contents()))
req = self.awthread.recv(self.work_socket)
#
# Check that they were installed
#
self.assertIn(self.analysis_id, self.awthread.aw.initial_measurements)
cm = self.awthread.aw.initial_measurements[self.analysis_id]
for object_name in m.get_object_names():
for feature_name in m.get_feature_names(object_name):
self.assertTrue(cm.has_feature(object_name, feature_name))
if (
feature_name
== cellprofiler_core.modules.namesandtypes.M_IMAGE_SET
):
numpy.testing.assert_array_equal(
cm[object_name, feature_name, 1],
m[object_name, feature_name, 1],
)
else:
self.assertEqual(
cm[object_name, feature_name, 1],
m[object_name, feature_name, 1],
)
#
# Cancel and check for exit
#
req.reply(anareply.ServerExited())
self.assertRaises(
cellprofiler_core.pipeline.event.CancelledException, self.awthread.ecute
)
finally:
m.close()
def test_03_02_pipeline_preferences(self):
#
# Walk the worker up through pipelines and preferences.
#
self.awthread = self.AWThread(self.announce_addr)
self.awthread.start()
self.set_work_socket()
self.awthread.ex(
self.awthread.aw.do_job,
anareply.Work(
image_set_numbers=[1],
worker_runs_post_group=False,
wants_dictionary=True,
),
)
#
# The worker should ask for the pipeline and preferences next.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, anarequest.PipelinePreferences)
self.assertEqual(req.analysis_id, self.analysis_id)
tests.modules.maybe_download_example_image(
["ExampleSBSImages"], "Channel1-01-A-01.tif"
)
tests.modules.maybe_download_example_image(
["ExampleHT29"], "AS_09125_050116030001_D03f00d0.tif"
)
input_dir = os.path.normcase(
os.path.join(tests.modules.example_images_directory(), "ExampleSBSImages")
)
output_dir = os.path.normcase(
os.path.join(tests.modules.example_images_directory(), "ExampleHT29")
)
cellprofiler_core.preferences.set_default_image_directory(input_dir)
input_dir = cellprofiler_core.preferences.get_default_image_directory()
cellprofiler_core.preferences.set_default_output_directory(output_dir)
output_dir = cellprofiler_core.preferences.get_default_output_directory()
preferences = {
cellprofiler_core.preferences.DEFAULT_IMAGE_DIRECTORY: cellprofiler_core.preferences.config_read(
cellprofiler_core.preferences.DEFAULT_IMAGE_DIRECTORY
),
cellprofiler_core.preferences.DEFAULT_OUTPUT_DIRECTORY: cellprofiler_core.preferences.config_read(
cellprofiler_core.preferences.DEFAULT_OUTPUT_DIRECTORY
),
}
cellprofiler_core.preferences.set_default_image_directory(
tests.modules.example_images_directory()
)
cellprofiler_core.preferences.set_default_output_directory(
tests.modules.example_images_directory()
)
rep = cellprofiler_core.utilities.zmq.Reply(
pipeline_blob=numpy.array(GOOD_PIPELINE), preferences=preferences
)
req.reply(rep)
#
# Get the next request so that we know the worker has
# processed the preferences.
#
req = self.awthread.recv(self.work_socket)
self.assertEqual(
cellprofiler_core.preferences.get_default_image_directory(), input_dir
)
self.assertEqual(
cellprofiler_core.preferences.get_default_output_directory(), output_dir
)
self.assertIn(self.analysis_id, self.awthread.aw.pipelines_and_preferences)
pipe, prefs = self.awthread.aw.pipelines_and_preferences[self.analysis_id]
self.assertEqual(len(pipe.modules()), 7)
#
# Cancel and check for exit
#
req.reply(anareply.ServerExited())
self.assertRaises(
cellprofiler_core.pipeline.event.CancelledException, self.awthread.ecute
)