def __run_cp(self, img_list):
image_set_file = os.path.join(os.getcwd(), uuid.uuid4().hex)
with open(image_set_file, 'w') as fo:
for img in img_list:
fo.write('%s\n' % img)
cpprefs.set_image_set_file(image_set_file)
out_dir = os.path.join(os.getcwd(), uuid.uuid4().hex)
os.mkdir(out_dir)
cpprefs.set_default_output_directory(out_dir)
pipeline = Pipeline()
pipeline.load(self.pipeline_filename)
pipeline.read_file_list(image_set_file)
pipeline.run(image_set_start=None,
image_set_end=None,
grouping=None,
measurements_filename=None,
initial_measurements=None)
return self.__build_results_dict(out_dir)
def __run_cp(self, img_list):
image_set_file = os.path.join(os.getcwd(), uuid.uuid4().hex)
with open(image_set_file, 'w') as fo:
for img in img_list:
fo.write('%s\n' % img)
cpprefs.set_image_set_file(image_set_file)
out_dir = os.path.join(os.getcwd(), uuid.uuid4().hex)
os.mkdir(out_dir)
cpprefs.set_default_output_directory(out_dir)
pipeline = Pipeline()
pipeline.load(self.pipeline_filename)
pipeline.read_file_list(image_set_file)
pipeline.run(
image_set_start=None,
image_set_end=None,
grouping=None,
measurements_filename=None,
initial_measurements=None
)
return self.__build_results_dict(out_dir)
def run_pipeline_headless(options, args):
'''Run a CellProfiler pipeline in headless mode'''
if sys.platform == 'darwin':
if options.start_awt:
from javabridge import activate_awt
activate_awt()
if not options.first_image_set is None:
if not options.first_image_set.isdigit():
raise ValueError(
"The --first-image-set option takes a numeric argument")
else:
image_set_start = int(options.first_image_set)
else:
image_set_start = None
image_set_numbers = None
if not options.last_image_set is None:
if not options.last_image_set.isdigit():
raise ValueError(
"The --last-image-set option takes a numeric argument")
else:
image_set_end = int(options.last_image_set)
if image_set_start is None:
image_set_numbers = np.arange(1, image_set_end + 1)
else:
image_set_numbers = np.arange(image_set_start,
image_set_end + 1)
else:
image_set_end = None
if ((options.pipeline_filename is not None)
and (not options.pipeline_filename.lower().startswith('http'))):
options.pipeline_filename = os.path.expanduser(
options.pipeline_filename)
from cellprofiler.pipeline import Pipeline, EXIT_STATUS, M_PIPELINE
import cellprofiler.measurement as cpmeas
import cellprofiler.preferences as cpprefs
pipeline = Pipeline()
initial_measurements = None
try:
if h5py.is_hdf5(options.pipeline_filename):
initial_measurements = cpmeas.load_measurements(
options.pipeline_filename, image_numbers=image_set_numbers)
except:
logging.root.info("Failed to load measurements from pipeline")
if initial_measurements is not None:
pipeline_text = \
initial_measurements.get_experiment_measurement(
M_PIPELINE)
pipeline_text = pipeline_text.encode('us-ascii')
pipeline.load(StringIO(pipeline_text))
if not pipeline.in_batch_mode():
#
# Need file list in order to call prepare_run
#
from cellprofiler.utilities.hdf5_dict import HDF5FileList
with h5py.File(options.pipeline_filename, "r") as src:
if HDF5FileList.has_file_list(src):
HDF5FileList.copy(src,
initial_measurements.hdf5_dict.hdf5_file)
else:
pipeline.load(options.pipeline_filename)
if options.groups is not None:
kvs = [x.split('=') for x in options.groups.split(',')]
groups = dict(kvs)
else:
groups = None
file_list = cpprefs.get_image_set_file()
if file_list is not None:
pipeline.read_file_list(file_list)
#
# Fixup CreateBatchFiles with any command-line input or output directories
#
if pipeline.in_batch_mode():
create_batch_files = [
m for m in pipeline.modules() if m.is_create_batch_module()
]
if len(create_batch_files) > 0:
create_batch_files = create_batch_files[0]
if options.output_directory is not None:
create_batch_files.custom_output_directory.value = \
options.output_directory
if options.image_directory is not None:
create_batch_files.default_image_directory.value = \
options.image_directory
use_hdf5 = len(args) > 0 and not args[0].lower().endswith(".mat")
measurements = pipeline.run(
image_set_start=image_set_start,
image_set_end=image_set_end,
grouping=groups,
measurements_filename=None if not use_hdf5 else args[0],
initial_measurements=initial_measurements)
if len(args) > 0 and not use_hdf5:
pipeline.save_measurements(args[0], measurements)
if options.done_file is not None:
if (measurements is not None
and measurements.has_feature(cpmeas.EXPERIMENT, EXIT_STATUS)):
done_text = measurements.get_experiment_measurement(EXIT_STATUS)
exit_code = (0 if done_text == "Complete" else -1)
else:
done_text = "Failure"
exit_code = -1
fd = open(options.done_file, "wt")
fd.write("%s\n" % done_text)
fd.close()
if measurements is not None:
measurements.close()
return exit_code
def run_pipeline_headless(options, args):
'''Run a CellProfiler pipeline in headless mode'''
if sys.platform == 'darwin':
if options.start_awt:
import bioformats
from cellprofiler.utilities.jutil import activate_awt
activate_awt()
if not options.first_image_set is None:
if not options.first_image_set.isdigit():
raise ValueError("The --first-image-set option takes a numeric argument")
else:
image_set_start = int(options.first_image_set)
else:
image_set_start = None
image_set_numbers = None
if not options.last_image_set is None:
if not options.last_image_set.isdigit():
raise ValueError("The --last-image-set option takes a numeric argument")
else:
image_set_end = int(options.last_image_set)
if image_set_start is None:
image_set_numbers = np.arange(1, image_set_end+1)
else:
image_set_numbers = np.arange(image_set_start, image_set_end+1)
else:
image_set_end = None
if ((options.pipeline_filename is not None) and
(not options.pipeline_filename.lower().startswith('http'))):
options.pipeline_filename = os.path.expanduser(options.pipeline_filename)
from cellprofiler.pipeline import Pipeline, EXIT_STATUS, M_PIPELINE
import cellprofiler.measurements as cpmeas
pipeline = Pipeline()
initial_measurements = None
try:
if h5py.is_hdf5(options.pipeline_filename):
initial_measurements = cpmeas.load_measurements(
options.pipeline_filename,
image_numbers=image_set_numbers)
except:
logging.root.info("Failed to load measurements from pipeline")
if initial_measurements is not None:
pipeline_text = \
initial_measurements.get_experiment_measurement(
M_PIPELINE)
pipeline_text = pipeline_text.encode('us-ascii')
pipeline.load(StringIO(pipeline_text))
if not pipeline.in_batch_mode():
#
# Need file list in order to call prepare_run
#
from cellprofiler.utilities.hdf5_dict import HDF5FileList
with h5py.File(options.pipeline_filename, "r") as src:
if HDF5FileList.has_file_list(src):
HDF5FileList.copy(
src, initial_measurements.hdf5_dict.hdf5_file)
else:
pipeline.load(options.pipeline_filename)
if options.groups is not None:
kvs = [x.split('=') for x in options.groups.split(',')]
groups = dict(kvs)
else:
groups = None
use_hdf5 = len(args) > 0 and not args[0].lower().endswith(".mat")
measurements = pipeline.run(
image_set_start=image_set_start,
image_set_end=image_set_end,
grouping=groups,
measurements_filename = None if not use_hdf5 else args[0],
initial_measurements = initial_measurements)
if len(args) > 0 and not use_hdf5:
pipeline.save_measurements(args[0], measurements)
if options.done_file is not None:
if (measurements is not None and
measurements.has_feature(cpmeas.EXPERIMENT, EXIT_STATUS)):
done_text = measurements.get_experiment_measurement(EXIT_STATUS)
else:
done_text = "Failure"
fd = open(options.done_file, "wt")
fd.write("%s\n"%done_text)
fd.close()
if measurements is not None:
measurements.close()
def run_pipeline_headless(options, args):
'''Run a CellProfiler pipeline in headless mode'''
#
# Start Ilastik's workers
#
try:
from ilastik.core.jobMachine import GLOBAL_WM
GLOBAL_WM.set_thread_count(1)
except:
logging.root.warn("Failed to stop Ilastik")
if sys.platform == 'darwin':
if options.start_awt:
import bioformats
from javabridge import activate_awt
activate_awt()
if not options.first_image_set is None:
if not options.first_image_set.isdigit():
raise ValueError("The --first-image-set option takes a numeric argument")
else:
image_set_start = int(options.first_image_set)
else:
image_set_start = None
image_set_numbers = None
if not options.last_image_set is None:
if not options.last_image_set.isdigit():
raise ValueError("The --last-image-set option takes a numeric argument")
else:
image_set_end = int(options.last_image_set)
if image_set_start is None:
image_set_numbers = np.arange(1, image_set_end+1)
else:
image_set_numbers = np.arange(image_set_start, image_set_end+1)
else:
image_set_end = None
if ((options.pipeline_filename is not None) and
(not options.pipeline_filename.lower().startswith('http'))):
options.pipeline_filename = os.path.expanduser(options.pipeline_filename)
from cellprofiler.pipeline import Pipeline, EXIT_STATUS, M_PIPELINE
import cellprofiler.measurements as cpmeas
import cellprofiler.preferences as cpprefs
pipeline = Pipeline()
initial_measurements = None
try:
if h5py.is_hdf5(options.pipeline_filename):
initial_measurements = cpmeas.load_measurements(
options.pipeline_filename,
image_numbers=image_set_numbers)
except:
logging.root.info("Failed to load measurements from pipeline")
if initial_measurements is not None:
pipeline_text = \
initial_measurements.get_experiment_measurement(
M_PIPELINE)
pipeline_text = pipeline_text.encode('us-ascii')
pipeline.load(StringIO(pipeline_text))
if not pipeline.in_batch_mode():
#
# Need file list in order to call prepare_run
#
from cellprofiler.utilities.hdf5_dict import HDF5FileList
with h5py.File(options.pipeline_filename, "r") as src:
if HDF5FileList.has_file_list(src):
HDF5FileList.copy(
src, initial_measurements.hdf5_dict.hdf5_file)
else:
pipeline.load(options.pipeline_filename)
if options.groups is not None:
kvs = [x.split('=') for x in options.groups.split(',')]
groups = dict(kvs)
else:
groups = None
file_list = cpprefs.get_image_set_file()
if file_list is not None:
pipeline.read_file_list(file_list)
#
# Fixup CreateBatchFiles with any command-line input or output directories
#
if pipeline.in_batch_mode():
create_batch_files = [
m for m in pipeline.modules()
if m.is_create_batch_module()]
if len(create_batch_files) > 0:
create_batch_files = create_batch_files[0]
if options.output_directory is not None:
create_batch_files.custom_output_directory.value = \
options.output_directory
if options.image_directory is not None:
create_batch_files.default_image_directory.value = \
options.image_directory
use_hdf5 = len(args) > 0 and not args[0].lower().endswith(".mat")
measurements = pipeline.run(
image_set_start=image_set_start,
image_set_end=image_set_end,
grouping=groups,
measurements_filename = None if not use_hdf5 else args[0],
initial_measurements = initial_measurements)
if len(args) > 0 and not use_hdf5:
pipeline.save_measurements(args[0], measurements)
if options.done_file is not None:
if (measurements is not None and
measurements.has_feature(cpmeas.EXPERIMENT, EXIT_STATUS)):
done_text = measurements.get_experiment_measurement(EXIT_STATUS)
else:
done_text = "Failure"
fd = open(options.done_file, "wt")
fd.write("%s\n"%done_text)
fd.close()
if measurements is not None:
measurements.close()
def run_pipeline_headless(options, args):
'''Run a CellProfiler pipeline in headless mode'''
if not options.first_image_set is None:
if not options.first_image_set.isdigit():
raise ValueError(
"The --first-image-set option takes a numeric argument")
else:
image_set_start = int(options.first_image_set)
else:
image_set_start = None
image_set_numbers = None
if not options.last_image_set is None:
if not options.last_image_set.isdigit():
raise ValueError(
"The --last-image-set option takes a numeric argument")
else:
image_set_end = int(options.last_image_set)
if image_set_start is None:
image_set_numbers = np.arange(1, image_set_end + 1)
else:
image_set_numbers = np.arange(image_set_start,
image_set_end + 1)
else:
image_set_end = None
if ((options.pipeline_filename is not None)
and (not options.pipeline_filename.lower().startswith('http'))):
options.pipeline_filename = os.path.expanduser(
options.pipeline_filename)
from cellprofiler.pipeline import Pipeline, EXIT_STATUS, M_PIPELINE
import cellprofiler.measurements as cpmeas
pipeline = Pipeline()
initial_measurements = None
try:
if h5py.is_hdf5(options.pipeline_filename):
initial_measurements = cpmeas.load_measurements(
options.pipeline_filename, image_numbers=image_set_numbers)
except:
logging.root.info("Failed to load measurements from pipeline")
if initial_measurements is not None:
pipeline_text = \
initial_measurements.get_experiment_measurement(
M_PIPELINE)
pipeline_text = pipeline_text.encode('us-ascii')
pipeline.load(StringIO(pipeline_text))
if not pipeline.in_batch_mode():
#
# Need file list in order to call prepare_run
#
from cellprofiler.utilities.hdf5_dict import HDF5FileList
with h5py.File(options.pipeline_filename, "r") as src:
if HDF5FileList.has_file_list(src):
HDF5FileList.copy(src,
initial_measurements.hdf5_dict.hdf5_file)
else:
pipeline.load(options.pipeline_filename)
if options.groups is not None:
kvs = [x.split('=') for x in options.groups.split(',')]
groups = dict(kvs)
else:
groups = None
use_hdf5 = len(args) > 0 and not args[0].lower().endswith(".mat")
measurements = pipeline.run(
image_set_start=image_set_start,
image_set_end=image_set_end,
grouping=groups,
measurements_filename=None if not use_hdf5 else args[0],
initial_measurements=initial_measurements)
if len(args) > 0 and not use_hdf5:
pipeline.save_measurements(args[0], measurements)
if options.done_file is not None:
if (measurements is not None
and measurements.has_feature(cpmeas.EXPERIMENT, EXIT_STATUS)):
done_text = measurements.get_experiment_measurement(EXIT_STATUS)
else:
done_text = "Failure"
fd = open(options.done_file, "wt")
fd.write("%s\n" % done_text)
fd.close()
if measurements is not None:
measurements.close()
if batch_fd is not None:
os.close(batch_fd)
if batch_path is not None:
try:
os.unlink(batch_path)
except:
logging.root.warn("Failed to delete temporary file %s" % batch_path)
if options.groups is not None:
kvs = [x.split('=') for x in options.groups.split(',')]
groups = dict(kvs)
else:
groups = None
use_hdf5 = len(args) > 0 and not args[0].lower().endswith(".mat")
measurements = pipeline.run(
image_set_start=image_set_start,
image_set_end=image_set_end,
grouping=groups,
measurements_filename = None if not use_hdf5 else args[0],
initial_measurements = initial_measurements)
if options.worker_mode_URL is not None:
try:
assert measurements is not None
assert measurements.has_feature(cpmeas.EXPERIMENT, EXIT_STATUS)
assert measurements.get_experiment_measurement(EXIT_STATUS) != 'Failure'
jobinfo.report_measurements(pipeline, measurements)
last_success = time.time()
except:
logging.root.error("Couldn't return results to server", exc_info=True)
logging.root.info("Continuing...")
time.sleep(20 + random.randint(1, 10)) # avoid hammering server
continue
elif len(args) > 0 and not use_hdf5:
if batch_path is not None:
try:
os.unlink(batch_path)
except:
logging.root.warn(
"Failed to delete temporary file %s" %
batch_path)
if options.groups is not None:
kvs = [x.split('=') for x in options.groups.split(',')]
groups = dict(kvs)
else:
groups = None
use_hdf5 = len(args) > 0 and not args[0].lower().endswith(".mat")
measurements = pipeline.run(
image_set_start=image_set_start,
image_set_end=image_set_end,
grouping=groups,
measurements_filename=None if not use_hdf5 else args[0],
initial_measurements=initial_measurements)
if options.worker_mode_URL is not None:
try:
assert measurements is not None
assert measurements.has_feature(cpmeas.EXPERIMENT,
EXIT_STATUS)
assert measurements.get_experiment_measurement(
EXIT_STATUS) != 'Failure'
jobinfo.report_measurements(pipeline, measurements)
last_success = time.time()
except:
logging.root.error("Couldn't return results to server",
exc_info=True)
logging.root.info("Continuing...")