def test_load_objects(): r = numpy.random.RandomState() r.seed(1101) labels = r.randint(0, 10, size=(30, 20)).astype(numpy.uint8) handle, name = tempfile.mkstemp(".png") bioformats.write_image(name, labels, bioformats.PT_UINT8) os.close(handle) png_path, png_file = os.path.split(name) sbs_dir = os.path.join(tests.modules.example_images_directory(), "ExampleSBSImages") csv_text = """%s_%s,%s_%s,%s_DNA,%s_DNA %s,%s,Channel2-01-A-01.tif,%s """ % ( cellprofiler.measurement.C_OBJECTS_FILE_NAME, OBJECTS_NAME, cellprofiler.measurement.C_OBJECTS_PATH_NAME, OBJECTS_NAME, cellprofiler.measurement.C_FILE_NAME, cellprofiler.measurement.C_PATH_NAME, png_file, png_path, sbs_dir, ) pipeline, module, csv_name = make_pipeline(csv_text) assert isinstance(pipeline, cellprofiler.pipeline.Pipeline) assert isinstance(module, cellprofiler.modules.loaddata.LoadData) module.wants_images.value = True try: image_set_list = cellprofiler.image.ImageSetList() measurements = cellprofiler.measurement.Measurements() workspace = cellprofiler.workspace.Workspace(pipeline, module, None, None, measurements, image_set_list) pipeline.prepare_run(workspace) key_names, g = pipeline.get_groupings(workspace) assert len(g) == 1 module.prepare_group(workspace, g[0][0], g[0][1]) image_set = image_set_list.get_image_set(g[0][1][0] - 1) object_set = cellprofiler.object.ObjectSet() workspace = cellprofiler.workspace.Workspace(pipeline, module, image_set, object_set, measurements, image_set_list) module.run(workspace) objects = object_set.get_objects(OBJECTS_NAME) assert numpy.all(objects.segmented == labels) assert (measurements.get_current_image_measurement( cellprofiler.measurement.FF_COUNT % OBJECTS_NAME) == 9) for feature in ( cellprofiler.measurement.M_LOCATION_CENTER_X, cellprofiler.measurement.M_LOCATION_CENTER_Y, cellprofiler.measurement.M_NUMBER_OBJECT_NUMBER, ): value = measurements.get_current_measurement(OBJECTS_NAME, feature) assert len(value) == 9 finally: bioformats.formatreader.clear_image_reader_cache() os.remove(name) os.remove(csv_name)
def test_subtract(): """Test correction by subtraction""" numpy.random.seed(0) image = numpy.random.uniform(size=(10, 10)).astype(numpy.float32) illum = numpy.random.uniform(size=(10, 10)).astype(numpy.float32) expected = image - illum expected[expected < 0] = 0 pipeline = cellprofiler.pipeline.Pipeline() pipeline.add_listener(error_callback) input_module = cellprofiler.modules.injectimage.InjectImage( "InputImage", image) input_module.set_module_num(1) pipeline.add_module(input_module) illum_module = cellprofiler.modules.injectimage.InjectImage( "IllumImage", illum) illum_module.set_module_num(2) pipeline.add_module(illum_module) module = cellprofiler.modules.correctilluminationapply.CorrectIlluminationApply( ) module.set_module_num(3) pipeline.add_module(module) image = module.images[0] image.image_name.value = "InputImage" image.illum_correct_function_image_name.value = "IllumImage" image.corrected_image_name.value = "OutputImage" image.divide_or_subtract.value = ( cellprofiler.modules.correctilluminationapply.DOS_SUBTRACT) image.rescale_option = cellprofiler.modules.correctilluminationapply.RE_NONE measurements = cellprofiler.measurement.Measurements() image_set_list = cellprofiler.image.ImageSetList() measurements = cellprofiler.measurement.Measurements() workspace = cellprofiler.workspace.Workspace(pipeline, None, None, None, measurements, image_set_list) pipeline.prepare_run(workspace) input_module.prepare_group(workspace, {}, [1]) illum_module.prepare_group(workspace, {}, [1]) module.prepare_group(workspace, {}, [1]) image_set = image_set_list.get_image_set(0) object_set = cellprofiler.object.ObjectSet() workspace = cellprofiler.workspace.Workspace(pipeline, input_module, image_set, object_set, measurements, image_set_list) input_module.run(workspace) illum_module.run(workspace) module.run(workspace) output_image = workspace.image_set.get_image("OutputImage") assert numpy.all(output_image.pixel_data == expected)
def test_11_01_load_objects(self): r = numpy.random.RandomState() r.seed(1101) labels = r.randint(0, 10, size=(30, 20)).astype(numpy.uint8) handle, name = tempfile.mkstemp(".png") bioformats.write_image(name, labels, bioformats.PT_UINT8) os.close(handle) png_path, png_file = os.path.split(name) sbs_dir = os.path.join(tests.modules.example_images_directory(), "ExampleSBSImages") csv_text = """%s_%s,%s_%s,%s_DNA,%s_DNA %s,%s,Channel2-01-A-01.tif,%s """ % (cellprofiler.measurement.C_OBJECTS_FILE_NAME, OBJECTS_NAME, cellprofiler.measurement.C_OBJECTS_PATH_NAME, OBJECTS_NAME, cellprofiler.measurement.C_FILE_NAME, cellprofiler.measurement.C_PATH_NAME, png_file, png_path, sbs_dir) pipeline, module, csv_name = self.make_pipeline(csv_text) assert isinstance(pipeline, cellprofiler.pipeline.Pipeline) assert isinstance(module, cellprofiler.modules.loaddata.LoadData) module.wants_images.value = True try: image_set_list = cellprofiler.image.ImageSetList() measurements = cellprofiler.measurement.Measurements() workspace = cellprofiler.workspace.Workspace( pipeline, module, None, None, measurements, image_set_list) pipeline.prepare_run(workspace) key_names, g = pipeline.get_groupings(workspace) self.assertEqual(len(g), 1) module.prepare_group(workspace, g[0][0], g[0][1]) image_set = image_set_list.get_image_set(g[0][1][0] - 1) object_set = cellprofiler.object.ObjectSet() workspace = cellprofiler.workspace.Workspace(pipeline, module, image_set, object_set, measurements, image_set_list) module.run(workspace) objects = object_set.get_objects(OBJECTS_NAME) self.assertTrue(numpy.all(objects.segmented == labels)) self.assertEqual(measurements.get_current_image_measurement( cellprofiler.measurement.FF_COUNT % OBJECTS_NAME), 9) for feature in (cellprofiler.measurement.M_LOCATION_CENTER_X, cellprofiler.measurement.M_LOCATION_CENTER_Y, cellprofiler.measurement.M_NUMBER_OBJECT_NUMBER): value = measurements.get_current_measurement( OBJECTS_NAME, feature) self.assertEqual(len(value), 9) finally: bioformats.formatreader.clear_image_reader_cache() os.remove(name) os.remove(csv_name)
def test_prepare_run(): # regression test for issue #673 and #1161 # # If LoadSingleImage appears first, pathname data does not show # up in the measurements. # tests.modules.maybe_download_sbs() folder = "ExampleSBSImages" path = os.path.join(tests.modules.example_images_directory(), folder) filename = "Channel1-01-A-01.tif" pipeline = cellprofiler.pipeline.Pipeline() lsi = cellprofiler.modules.loadsingleimage.LoadSingleImage() lsi.set_module_num(1) lsi.directory.dir_choice = cellprofiler.setting.ABSOLUTE_FOLDER_NAME lsi.directory.custom_path = path lsi.file_settings[0].image_name.value = get_image_name(0) lsi.file_settings[0].file_name.value = filename pipeline.add_module(lsi) li = cellprofiler.modules.loadimages.LoadImages() li.set_module_num(2) pipeline.add_module(li) li.match_method.value = cellprofiler.modules.loadimages.MS_EXACT_MATCH li.location.dir_choice = cellprofiler.setting.ABSOLUTE_FOLDER_NAME li.location.custom_path = path li.images[0].common_text.value = "Channel2-" m = cellprofiler.measurement.Measurements() workspace = cellprofiler.workspace.Workspace( pipeline, lsi, m, cellprofiler.object.ObjectSet(), m, cellprofiler.image.ImageSetList(), ) assert pipeline.prepare_run(workspace) assert m.image_set_count > 1 pipeline.prepare_group(workspace, {}, m.get_image_numbers()) # # Skip to the second image set # m.next_image_set(2) lsi.run(workspace) # # Are the measurements populated? # m_file = "_".join( (cellprofiler.measurement.C_FILE_NAME, get_image_name(0))) assert m[cellprofiler.measurement.IMAGE, m_file, 2] == filename # # Can we retrieve the image? # pixel_data = m.get_image(get_image_name(0)).pixel_data assert not numpy.isscalar(pixel_data)
def test_02_03_prepare_run(self): # regression test for issue #673 and #1161 # # If LoadSingleImage appears first, pathname data does not show # up in the measurements. # tests.modules.maybe_download_sbs() folder = "ExampleSBSImages" path = os.path.join(tests.modules.example_images_directory(), folder) filename = "Channel1-01-A-01.tif" pipeline = cellprofiler.pipeline.Pipeline() lsi = cellprofiler.modules.loadsingleimage.LoadSingleImage() lsi.module_num = 1 lsi.directory.dir_choice = cellprofiler.setting.ABSOLUTE_FOLDER_NAME lsi.directory.custom_path = path lsi.file_settings[0].image_name.value = self.get_image_name(0) lsi.file_settings[0].file_name.value = filename pipeline.add_module(lsi) li = cellprofiler.modules.loadimages.LoadImages() li.module_num = 2 pipeline.add_module(li) li.match_method.value = cellprofiler.modules.loadimages.MS_EXACT_MATCH li.location.dir_choice = cellprofiler.setting.ABSOLUTE_FOLDER_NAME li.location.custom_path = path li.images[0].common_text.value = "Channel2-" m = cellprofiler.measurement.Measurements() workspace = cellprofiler.workspace.Workspace(pipeline, lsi, m, cellprofiler.object.ObjectSet(), m, cellprofiler.image.ImageSetList()) self.assertTrue(pipeline.prepare_run(workspace)) self.assertGreater(m.image_set_count, 1) pipeline.prepare_group(workspace, {}, m.get_image_numbers()) # # Skip to the second image set # m.next_image_set(2) lsi.run(workspace) # # Are the measurements populated? # m_file = "_".join((cellprofiler.measurement.C_FILE_NAME, self.get_image_name(0))) self.assertEqual(m[cellprofiler.measurement.IMAGE, m_file, 2], filename) # # Can we retrieve the image? # pixel_data = m.get_image(self.get_image_name(0)).pixel_data self.assertFalse(numpy.isscalar(pixel_data))
def make_jobs(args): # how to make this nicer? xvfb = args.xvfb software = args.software script = args.script script_arguments = args.script_arguments memory = args.memory queue = args.queue host_group = args.host_group num_jobs_max = 1000 print '' print 'make_cellprofiler_jobs_LSF:' print '' # # determine files to analyze # CELLPROFILERDIR = '/g/software/linux/pack/cellprofiler-2.0.11047/lib' sys.path.insert(0, CELLPROFILERDIR) # try importing cellprofiler modules global cellprofiler import cellprofiler import cellprofiler.pipeline import cellprofiler.workspace import cellprofiler.measurements import cellprofiler.cpimage # LOAD PIPELINE pipeline_file = script if not os.path.isfile(pipeline_file): raise Exception("-- ERROR: pipeline file not found") cp_plugin_directory = None if 'CP_PLUGIN_DIRECTORY' in os.environ: cp_plugin_directory = os.environ['CP_PLUGIN_DIRECTORY'] pipeline = cellprofiler.pipeline.Pipeline() def error_callback(event, caller): if (isinstance(event, cellprofiler.pipeline.LoadExceptionEvent) or isinstance(event, cellprofiler.pipeline.RunExceptionEvent)): sys.stdout.write("Handling exception: {}\n".format(event)) sys.stdout.write(traceback.format_exc()) sys.sydout.flush() pipeline.add_listener(error_callback) # pipeline.remove_listener(error_callback) # GET NUMBER OF IMAGES AND GROUPS # check wether we have a new or an old version of CellProfiler import inspect argspec = inspect.getargspec(pipeline.prepare_run) if argspec[0][1] == 'workspace' or len(argspec[0]) == 2: print 'New CellProfiler version' new_version = True else: print 'Old CellProfiler version' new_version = False if new_version: # this should work for plain pipeline files ... try: pipeline.load(pipeline_file) image_set_list = cellprofiler.cpimage.ImageSetList() measurements = cellprofiler.measurements.Measurements() workspace = cellprofiler.workspace.Workspace( pipeline, None, None, None, measurements, image_set_list ) grouping_argument = workspace result = pipeline.prepare_run(workspace) grouping_keys, groups = pipeline.get_groupings( grouping_argument ) pipeline.prepare_group( grouping_argument, groups[0][0], groups[0][1]) num_sets = image_set_list.count() except: import traceback traceback.print_exc() raise Exception('Unable to load pipeline file:', pipeline_file) # ... and this should work for files created with # the CreateBatchFile module measurements = cellprofiler.measurements.load_measurements( pipeline_file ) print 'Obtaining list of image sets...this can take a while...' image_set_list = measurements.get_image_numbers() grouping_keys = [] num_sets = len(image_set_list) else: try: pipeline.load(pipeline_file) except: import traceback traceback.print_exc() raise Exception('Unable to load pipeline file:', pipeline_file) workspace = None grouping_argument = workspace print 'Obtaining list of image sets...this can take a while...' result = pipeline.prepare_run(workspace) if not result: raise Exception("Failed to prepare running the pipeline") if not new_version: grouping_argument = result image_set_list = result grouping_keys, groups = pipeline.get_groupings(grouping_argument) if new_version: pipeline.prepare_group( grouping_argument, groups[0][0], groups[0][1]) num_sets = image_set_list.count() print("Image sets: {}".format(num_sets)) if num_sets == 0: print 'No image sets to process...finished' sys.exit(0) # GET IMAGE PATH input_dir = None # could be also an directory with image files if one does not use Batch_data.mat.... if input_dir is None: loadimage_module_name = 'LoadImages' cp_modules = pipeline.modules() loadimage_module = None for module in cp_modules: if module.module_name == loadimage_module_name: loadimage_module = module break if loadimage_module: input_dir = str(loadimage_module.location).partition('|')[2] print("Image path: {}".format(input_dir)) else: print '-- WARNING: The LoadImage module is not used in this' \ ' pipeline. Default input folder is undefined' #print('-- ERROR: Could not load the image module!') #sys.exit(1) # CREATE BATCHES jobStartImages = [] jobEndImages = [] jobLengths = [] if len(grouping_keys) > 0: print('Using groupings to assign the jobs to {} groups.'.format( len(groups))) for group in groups: #print 'group length',len(group[1]) #print group[1][1] jobStartImages.append(group[1][0]) jobEndImages.append(group[1][-1]) jobLengths.append(len(group[1])) #batch_size_max = max(jobLengths) print 'Starting images:' print jobStartImages else: print "No groupings assigned => " \ "images will be randomly assigned to the jobs." if int(args.batch_size) > 0: batch_size = int(args.batch_size) else: batch_size = max(4 , int(num_sets / float(num_jobs_max)) + 1) #batch_size = 4 #int(round(num_sets/num_jobs_max)+1) jobStartImages = range(1, num_sets + 1, batch_size) for x in jobStartImages: jobEndImages.append(x + batch_size - 1) jobEndImages[-1] = num_sets #batch_size_max = batch_size # # create directories # input_dir = input_dir.rstrip(os.path.sep) # remove trailing slash if exists output_dir = input_dir + '--cluster' print('Cluster directory: {}'.format(output_dir)) log_dir = os.path.join(output_dir, "log") # contains information about job status job_dir = os.path.join(output_dir, "jobs") # contains the actual job scripts # create directories ensure_empty_dir(output_dir) ensure_empty_dir(log_dir) ensure_empty_dir(job_dir) # # write the job files # for iJob in range(0, len(jobStartImages)): # chose image subset start = jobStartImages[iJob] end = jobEndImages[iJob] if end > num_sets: end = num_sets # write the jobs to files script_name = "job_{}.sh".format(iJob + 1) script_name = os.path.join(job_dir, script_name) script_file = file(script_name, "w") # information to LSF txt = ['#!/bin/bash', '#BSUB -oo "{}/job_{}--out.txt"'.format(log_dir,iJob+1), '#BSUB -eo "{}/job_{}--err.txt"'.format(log_dir,iJob+1), '#BSUB -M {}'.format(memory), '#BSUB -R select[mem>{}] -R rusage[mem={}]'.format(memory,memory), '#BSUB -R span[hosts=1]' ] txt = '\n'.join(txt) txt = txt + '\n' script_file.write(txt) if queue: script_file.write( '#BSUB -q {}\n'.format(queue) ) if host_group: script_file.write( '#BSUB -m {}\n'.format(host_group) ) script_file.write( 'echo "starting job"\n' ) # write a file to be able to check from the outside that the job has started script_file.write( 'touch "{}/job_{}--started.touch"\n'.format(log_dir,iJob+1) ) # do not produce core dumps script_file.write( 'ulimit -c 0\n' ) txt = ['echo "hostname:"', 'hostname', 'echo "number of cores:"', 'nproc', 'echo "total RAM:"', 'head -1 /proc/meminfo' ] txt = '\n'.join(txt) txt = txt + '\n' script_file.write(txt) # generate the actual command def make_command(software, script, script_arguments): cmd = [ software, "-c -b -r", "-p {}".format(script), script_arguments ] return ' '.join(cmd) script_arguments = "-f {} -l {}".format(start, end) # using software without quotation as it does not work with cmd = make_command(software, script, script_arguments) script_file.write(cmd + '\n') script_file.write( 'echo "job finished"\n' ) # this is the last line in the script, because this will be displayed as the job name by LSF ''' script_file.write( 'echo "# job {}"\n'.format(iJob) ) ''' script_file.close() # make script executable os.system('chmod a+x "{}"'.format(script_name)) return job_dir, len(jobStartImages)