def execute_merge_files(self, mm): input_files = [] output_fd, output_file = tempfile.mkstemp(".mat") pipeline = cpp.Pipeline() li = LoadImages() li.module_num = 1 pipeline.add_module(li) for m in mm: input_fd, input_file = tempfile.mkstemp(".mat") pipeline.save_measurements(input_file, m) input_files.append((input_fd, input_file)) M.MergeOutputFiles.merge_files(output_file, [x[1] for x in input_files]) m = cpmeas.load_measurements(output_file) os.close(output_fd) os.remove(output_file) for fd, filename in input_files: os.close(fd) os.remove(filename) return m
def run_pipeline_headless(options, args): '''Run a CellProfiler pipeline in headless mode''' if sys.platform == 'darwin': if options.start_awt: from javabridge import activate_awt activate_awt() if not options.first_image_set is None: if not options.first_image_set.isdigit(): raise ValueError( "The --first-image-set option takes a numeric argument") else: image_set_start = int(options.first_image_set) else: image_set_start = None image_set_numbers = None if not options.last_image_set is None: if not options.last_image_set.isdigit(): raise ValueError( "The --last-image-set option takes a numeric argument") else: image_set_end = int(options.last_image_set) if image_set_start is None: image_set_numbers = np.arange(1, image_set_end + 1) else: image_set_numbers = np.arange(image_set_start, image_set_end + 1) else: image_set_end = None if ((options.pipeline_filename is not None) and (not options.pipeline_filename.lower().startswith('http'))): options.pipeline_filename = os.path.expanduser( options.pipeline_filename) from cellprofiler.pipeline import Pipeline, EXIT_STATUS, M_PIPELINE import cellprofiler.measurement as cpmeas import cellprofiler.preferences as cpprefs pipeline = Pipeline() initial_measurements = None try: if h5py.is_hdf5(options.pipeline_filename): initial_measurements = cpmeas.load_measurements( options.pipeline_filename, image_numbers=image_set_numbers) except: logging.root.info("Failed to load measurements from pipeline") if initial_measurements is not None: pipeline_text = \ initial_measurements.get_experiment_measurement( M_PIPELINE) pipeline_text = pipeline_text.encode('us-ascii') pipeline.load(StringIO(pipeline_text)) if not pipeline.in_batch_mode(): # # Need file list in order to call prepare_run # from cellprofiler.utilities.hdf5_dict import HDF5FileList with h5py.File(options.pipeline_filename, "r") as src: if HDF5FileList.has_file_list(src): HDF5FileList.copy(src, initial_measurements.hdf5_dict.hdf5_file) else: pipeline.load(options.pipeline_filename) if options.groups is not None: kvs = [x.split('=') for x in options.groups.split(',')] groups = dict(kvs) else: groups = None file_list = cpprefs.get_image_set_file() if file_list is not None: pipeline.read_file_list(file_list) # # Fixup CreateBatchFiles with any command-line input or output directories # if pipeline.in_batch_mode(): create_batch_files = [ m for m in pipeline.modules() if m.is_create_batch_module() ] if len(create_batch_files) > 0: create_batch_files = create_batch_files[0] if options.output_directory is not None: create_batch_files.custom_output_directory.value = \ options.output_directory if options.image_directory is not None: create_batch_files.default_image_directory.value = \ options.image_directory use_hdf5 = len(args) > 0 and not args[0].lower().endswith(".mat") measurements = pipeline.run( image_set_start=image_set_start, image_set_end=image_set_end, grouping=groups, measurements_filename=None if not use_hdf5 else args[0], initial_measurements=initial_measurements) if len(args) > 0 and not use_hdf5: pipeline.save_measurements(args[0], measurements) if options.done_file is not None: if (measurements is not None and measurements.has_feature(cpmeas.EXPERIMENT, EXIT_STATUS)): done_text = measurements.get_experiment_measurement(EXIT_STATUS) exit_code = (0 if done_text == "Complete" else -1) else: done_text = "Failure" exit_code = -1 fd = open(options.done_file, "wt") fd.write("%s\n" % done_text) fd.close() if measurements is not None: measurements.close() return exit_code
def merge_files(destination, sources, force_headless=False): is_headless = force_headless or get_headless() if not is_headless: import wx if len(sources) == 0: return if not is_headless: progress = wx.ProgressDialog("Writing " + destination, "Loading " + sources[0], maximum=len(sources) * 4 + 1, style=wx.PD_CAN_ABORT | wx.PD_APP_MODAL | wx.PD_ELAPSED_TIME | wx.PD_REMAINING_TIME) count = 0 try: pipeline = cpp.Pipeline() has_error = [False] def callback(caller, event): if isinstance(event, cpp.LoadExceptionEvent): has_error = True wx.MessageBox( message="Could not load %s: %s" % ( sources[0], event.error), caption="Failed to load %s" % sources[0]) has_error[0] = True pipeline.add_listener(callback) pipeline.load(sources[0]) if has_error[0]: return if destination.lower().endswith(".h5"): mdest = cpmeas.Measurements(filename=destination, multithread=False) h5_dest = True else: mdest = cpmeas.Measurements(multithread=False) h5_dest = False for source in sources: if not is_headless: count += 1 keep_going, skip = progress.Update(count, "Loading " + source) if not keep_going: return if h5py.is_hdf5(source): msource = cpmeas.Measurements(filename=source, mode="r", multithread=False) else: msource = cpmeas.load_measurements(source) dest_image_numbers = mdest.get_image_numbers() source_image_numbers = msource.get_image_numbers() if (len(dest_image_numbers) == 0 or len(source_image_numbers) == 0): offset_source_image_numbers = source_image_numbers else: offset_source_image_numbers = ( np.max(dest_image_numbers) - np.min(source_image_numbers) + source_image_numbers + 1) for object_name in msource.get_object_names(): if object_name in mdest.get_object_names(): destfeatures = mdest.get_feature_names(object_name) else: destfeatures = [] for feature in msource.get_feature_names(object_name): if object_name == cpmeas.EXPERIMENT: if not mdest.has_feature(object_name, feature): src_value = msource.get_experiment_measurement( feature) mdest.add_experiment_measurement(feature, src_value) continue src_values = msource.get_measurement( object_name, feature, image_set_number=source_image_numbers) mdest[object_name, feature, offset_source_image_numbers] = src_values destset = set(destfeatures) if not is_headless: keep_going, skip = progress.Update(count + 1, "Saving to " + destination) if not keep_going: return if not h5_dest: pipeline.save_measurements(destination, mdest) finally: if not is_headless: progress.Destroy()
def run_pipeline_headless(options, args): '''Run a CellProfiler pipeline in headless mode''' if sys.platform == 'darwin': if options.start_awt: from javabridge import activate_awt activate_awt() if not options.first_image_set is None: if not options.first_image_set.isdigit(): raise ValueError("The --first-image-set option takes a numeric argument") else: image_set_start = int(options.first_image_set) else: image_set_start = None image_set_numbers = None if not options.last_image_set is None: if not options.last_image_set.isdigit(): raise ValueError("The --last-image-set option takes a numeric argument") else: image_set_end = int(options.last_image_set) if image_set_start is None: image_set_numbers = np.arange(1, image_set_end + 1) else: image_set_numbers = np.arange(image_set_start, image_set_end + 1) else: image_set_end = None if ((options.pipeline_filename is not None) and (not options.pipeline_filename.lower().startswith('http'))): options.pipeline_filename = os.path.expanduser(options.pipeline_filename) from cellprofiler.pipeline import Pipeline, EXIT_STATUS, M_PIPELINE import cellprofiler.measurement as cpmeas import cellprofiler.preferences as cpprefs pipeline = Pipeline() initial_measurements = None try: if h5py.is_hdf5(options.pipeline_filename): initial_measurements = cpmeas.load_measurements( options.pipeline_filename, image_numbers=image_set_numbers) except: logging.root.info("Failed to load measurements from pipeline") if initial_measurements is not None: pipeline_text = \ initial_measurements.get_experiment_measurement( M_PIPELINE) pipeline_text = pipeline_text.encode('us-ascii') pipeline.load(StringIO(pipeline_text)) if not pipeline.in_batch_mode(): # # Need file list in order to call prepare_run # from cellprofiler.utilities.hdf5_dict import HDF5FileList with h5py.File(options.pipeline_filename, "r") as src: if HDF5FileList.has_file_list(src): HDF5FileList.copy( src, initial_measurements.hdf5_dict.hdf5_file) else: pipeline.load(options.pipeline_filename) if options.groups is not None: kvs = [x.split('=') for x in options.groups.split(',')] groups = dict(kvs) else: groups = None file_list = cpprefs.get_image_set_file() if file_list is not None: pipeline.read_file_list(file_list) # # Fixup CreateBatchFiles with any command-line input or output directories # if pipeline.in_batch_mode(): create_batch_files = [ m for m in pipeline.modules() if m.is_create_batch_module()] if len(create_batch_files) > 0: create_batch_files = create_batch_files[0] if options.output_directory is not None: create_batch_files.custom_output_directory.value = \ options.output_directory if options.image_directory is not None: create_batch_files.default_image_directory.value = \ options.image_directory use_hdf5 = len(args) > 0 and not args[0].lower().endswith(".mat") measurements = pipeline.run( image_set_start=image_set_start, image_set_end=image_set_end, grouping=groups, measurements_filename=None if not use_hdf5 else args[0], initial_measurements=initial_measurements) if len(args) > 0 and not use_hdf5: pipeline.save_measurements(args[0], measurements) if options.done_file is not None: if (measurements is not None and measurements.has_feature(cpmeas.EXPERIMENT, EXIT_STATUS)): done_text = measurements.get_experiment_measurement(EXIT_STATUS) exit_code = (0 if done_text == "Complete" else -1) else: done_text = "Failure" exit_code = -1 fd = open(options.done_file, "wt") fd.write("%s\n" % done_text) fd.close() if measurements is not None: measurements.close() return exit_code