def test_03_02_pipeline_preferences(self):
#
# Walk the worker up through pipelines and preferences.
#
self.awthread = self.AWThread(self.announce_addr)
self.awthread.start()
self.set_work_socket()
self.awthread.ex(self.awthread.aw.do_job,
cpanalysis.WorkReply(
image_set_numbers = [1],
worker_runs_post_group = False,
wants_dictionary = True))
#
# The worker should ask for the pipeline and preferences next.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, cpanalysis.PipelinePreferencesRequest)
self.assertEqual(req.analysis_id, self.analysis_id)
maybe_download_example_image(["ExampleSBSImages"],
"Channel1-01-A-01.tif")
maybe_download_example_image(["ExampleHT29"],
"AS_09125_050116030001_D03f00d0.tif")
input_dir = os.path.normcase(
os.path.join(example_images_directory(), "ExampleSBSImages"))
output_dir = os.path.normcase(
os.path.join(example_images_directory(), "ExampleHT29"))
cpprefs.set_default_image_directory(input_dir)
input_dir = cpprefs.get_default_image_directory()
cpprefs.set_default_output_directory(output_dir)
output_dir = cpprefs.get_default_output_directory()
preferences = {cpprefs.DEFAULT_IMAGE_DIRECTORY:
cpprefs.config_read(cpprefs.DEFAULT_IMAGE_DIRECTORY),
cpprefs.DEFAULT_OUTPUT_DIRECTORY:
cpprefs.config_read(cpprefs.DEFAULT_OUTPUT_DIRECTORY)}
cpprefs.set_default_image_directory(example_images_directory())
cpprefs.set_default_output_directory(example_images_directory())
rep = cpanalysis.Reply(
pipeline_blob = np.array(GOOD_PIPELINE),
preferences = preferences)
req.reply(rep)
#
# Get the next request so that we know the worker has
# processed the preferences.
#
req = self.awthread.recv(self.work_socket)
self.assertEqual(cpprefs.get_default_image_directory(),
input_dir)
self.assertEqual(cpprefs.get_default_output_directory(),
output_dir)
self.assertIn(self.analysis_id,
self.awthread.aw.pipelines_and_preferences)
pipe, prefs = self.awthread.aw.pipelines_and_preferences[
self.analysis_id]
self.assertEqual(len(pipe.modules()), 7)
#
# Cancel and check for exit
#
req.reply(cpanalysis.ServerExited())
self.assertRaises(cpp.CancelledException, self.awthread.ecute)
def save_pipeline(self, workspace, outf=None):
'''Save the pipeline in Batch_data.mat
Save the pickled image_set_list state in a setting and put this
module in batch mode.
if outf is not None, it is used as a file object destination.
'''
from cellprofiler.utilities.get_revision import version
if self.wants_default_output_directory.value:
path = cpprefs.get_default_output_directory()
else:
path = cpprefs.get_absolute_path(self.custom_output_directory.value)
h5_path = os.path.join(path, F_BATCH_DATA_H5)
image_set_list = workspace.image_set_list
pipeline = workspace.pipeline
m = cpmeas.Measurements(copy = workspace.measurements,
filename = h5_path)
assert isinstance(image_set_list, cpi.ImageSetList)
assert isinstance(pipeline, cpp.Pipeline)
assert isinstance(m, cpmeas.Measurements)
pipeline = pipeline.copy()
target_workspace = cpw.Workspace(pipeline, None, None, None,
m, image_set_list,
workspace.frame)
pipeline.prepare_to_create_batch(target_workspace, self.alter_path)
bizarro_self = pipeline.module(self.module_num)
assert isinstance(bizarro_self, CreateBatchFiles)
state = image_set_list.save_state()
state = zlib.compress(state)
bizarro_self.revision.value = version
bizarro_self.batch_state = np.array(state)
if self.wants_default_output_directory:
bizarro_self.custom_output_directory.value = \
self.alter_path(cpprefs.get_default_output_directory())
bizarro_self.default_image_directory.value = \
self.alter_path(cpprefs.get_default_image_directory())
bizarro_self.batch_mode.value = True
pipeline.write_pipeline_measurement(m)
if outf is None:
mat_path = os.path.join(path, F_BATCH_DATA)
if os.path.exists(mat_path) and workspace.frame is not None:
import wx
if (wx.MessageBox("%s already exists. Do you want to overwrite it?"%
mat_path,
"Overwriting %s" % F_BATCH_DATA,
wx.YES_NO, workspace.frame) == wx.NO):
return
pipeline.save(mat_path, format=cpp.FMT_MATLAB) # Matlab... it's like the vestigial hole in the head of CP.
else:
pipeline.save(outf, format=cpp.FMT_NATIVE)
def save_pipeline(self, workspace, outf=None):
'''Save the pipeline in Batch_data.mat
Save the pickled image_set_list state in a setting and put this
module in batch mode.
if outf is not None, it is used as a file object destination.
'''
from cellprofiler.utilities.version import version_number
if outf is None:
if self.wants_default_output_directory.value:
path = cpprefs.get_default_output_directory()
else:
path = cpprefs.get_absolute_path(self.custom_output_directory.value)
h5_path = os.path.join(path, F_BATCH_DATA_H5)
else:
h5_path = outf
image_set_list = workspace.image_set_list
pipeline = workspace.pipeline
m = cpmeas.Measurements(copy=workspace.measurements,
filename=h5_path)
try:
assert isinstance(pipeline, cpp.Pipeline)
assert isinstance(m, cpmeas.Measurements)
orig_pipeline = pipeline
pipeline = pipeline.copy()
# this use of workspace.frame is okay, since we're called from
# prepare_run which happens in the main wx thread.
target_workspace = cpw.Workspace(pipeline, None, None, None,
m, image_set_list,
workspace.frame)
pipeline.prepare_to_create_batch(target_workspace, self.alter_path)
bizarro_self = pipeline.module(self.module_num)
bizarro_self.revision.value = version_number
if self.wants_default_output_directory:
bizarro_self.custom_output_directory.value = \
self.alter_path(cpprefs.get_default_output_directory())
bizarro_self.default_image_directory.value = \
self.alter_path(cpprefs.get_default_image_directory())
bizarro_self.batch_mode.value = True
pipeline.write_pipeline_measurement(m)
orig_pipeline.write_pipeline_measurement(m, user_pipeline=True)
#
# Write the path mappings to the batch measurements
#
m.write_path_mappings(
[(mapping.local_directory.value, mapping.remote_directory.value)
for mapping in self.mappings])
return h5_path
finally:
m.close()
def save_pipeline(self, workspace, outf=None):
'''Save the pipeline in Batch_data.mat
Save the pickled image_set_list state in a setting and put this
module in batch mode.
if outf is not None, it is used as a file object destination.
'''
from cellprofiler.utilities.version import version_number
if outf is None:
if self.wants_default_output_directory.value:
path = cpprefs.get_default_output_directory()
else:
path = cpprefs.get_absolute_path(self.custom_output_directory.value)
h5_path = os.path.join(path, F_BATCH_DATA_H5)
else:
h5_path = outf
image_set_list = workspace.image_set_list
pipeline = workspace.pipeline
m = cpmeas.Measurements(copy = workspace.measurements,
filename = h5_path)
try:
assert isinstance(pipeline, cpp.Pipeline)
assert isinstance(m, cpmeas.Measurements)
orig_pipeline = pipeline
pipeline = pipeline.copy()
# this use of workspace.frame is okay, since we're called from
# prepare_run which happens in the main wx thread.
target_workspace = cpw.Workspace(pipeline, None, None, None,
m, image_set_list,
workspace.frame)
pipeline.prepare_to_create_batch(target_workspace, self.alter_path)
bizarro_self = pipeline.module(self.module_num)
bizarro_self.revision.value = version_number
if self.wants_default_output_directory:
bizarro_self.custom_output_directory.value = \
self.alter_path(cpprefs.get_default_output_directory())
bizarro_self.default_image_directory.value = \
self.alter_path(cpprefs.get_default_image_directory())
bizarro_self.batch_mode.value = True
pipeline.write_pipeline_measurement(m)
orig_pipeline.write_pipeline_measurement(m, user_pipeline=True)
#
# Write the path mappings to the batch measurements
#
m.write_path_mappings(
[(mapping.local_directory.value, mapping.remote_directory.value)
for mapping in self.mappings])
return h5_path
finally:
m.close()
def __init__(self, *args, **kwargs):
super(RunMultplePipelinesDialog, self).__init__(*args, **kwargs)
#
# Main layout:
#
# # # # # # # # # # # # # # # # # # # #
# # #
# File chooser # select / #
# # add #
# # buttons #
# # # # # # # # # # # # # # # # # # # #
# Directory picker #
# # # # # # # # # # # # # # # # # # # #
# #
# Pipeline list box #
# #
# # # # # # # # # # # # # # # # # # # #
# OK / Cancel / Help buttons #
# # # # # # # # # # # # # # # # # # # #
self.sizer = wx.BoxSizer(wx.VERTICAL)
self.SetSizer(self.sizer)
self.top_sizer = wx.BoxSizer(wx.HORIZONTAL)
self.sizer.Add(self.top_sizer, 1, wx.EXPAND)
self.add_file_chooser(self.top_sizer)
self.add_directory_picker(self.sizer)
self.add_pipeline_list_box(self.sizer)
self.add_dialog_buttons(self.sizer)
self.hookup_events()
self.set_path(cpprefs.get_default_output_directory())
self.Layout()
def alter_path(self, path, **varargs):
if path == cpprefs.get_default_output_directory():
path = 'results'
else:
path = os.path.join('results', os.path.basename(path))
path = path.replace('\\', '/')
return path
def check_paths(self):
'''Check to make sure the default directories are remotely accessible'''
import wx
def check(path):
more = urllib.urlencode(dict(path=path))
url = ("/batchprofiler/cgi-bin/development/"
"CellProfiler_2.0/PathExists.py?%s") % more
conn = httplib.HTTPConnection("imageweb")
conn.request("GET",url)
result = conn.getresponse()
if result.status != httplib.OK:
raise RuntimeError("HTTP failed: %s" % result.reason)
body = result.read()
return body.find("OK") != -1
all_ok = True
for mapping in self.mappings:
path = mapping.remote_directory.value
if not check(path):
wx.MessageBox("Cannot find %s on the server." % path)
all_ok = False
for path, name in (
(cpprefs.get_default_image_directory(), "default image folder"),
(cpprefs.get_default_output_directory(), "default output folder")):
if not check(self.alter_path(path)):
wx.MessageBox("Cannot find the %s, \"%s\", on the server." %
(name, path))
all_ok = False
if all_ok:
wx.MessageBox("All paths are accessible")
def __init__(self, *args, **kwargs):
super(RunMultplePipelinesDialog, self).__init__(*args, **kwargs)
#
# Main layout:
#
# # # # # # # # # # # # # # # # # # # #
# # #
# File chooser # select / #
# # add #
# # buttons #
# # # # # # # # # # # # # # # # # # # #
# Directory picker #
# # # # # # # # # # # # # # # # # # # #
# #
# Pipeline list box #
# #
# # # # # # # # # # # # # # # # # # # #
# OK / Cancel / Help buttons #
# # # # # # # # # # # # # # # # # # # #
self.sizer = wx.BoxSizer(wx.VERTICAL)
self.SetSizer(self.sizer)
self.top_sizer = wx.BoxSizer(wx.HORIZONTAL)
self.sizer.Add(self.top_sizer, 1, wx.EXPAND)
self.add_file_chooser(self.top_sizer)
self.add_directory_picker(self.sizer)
self.add_pipeline_list_box(self.sizer)
self.add_dialog_buttons(self.sizer)
self.hookup_events()
self.set_path(cpprefs.get_default_output_directory())
self.Layout()
def test_04_02_alter_output_folder_path(self):
s = cps.DirectoryPath("whatever")
s.dir_choice = cps.DEFAULT_OUTPUT_FOLDER_NAME
s.alter_for_create_batch_files(TestDirectoryPath.fn_alter_path)
self.assertEqual(
s.get_absolute_path(),
cpprefs.get_default_output_directory())
def test_02_04_get_output_subfolder_path(self):
s = cps.DirectoryPath("whatever")
s.dir_choice = cps.DEFAULT_OUTPUT_SUBFOLDER_NAME
s.custom_path = "0"
self.assertEqual(
s.get_absolute_path(),
os.path.join(cpprefs.get_default_output_directory(), "0"))
def on_add(self, event):
for i in range(self.file_chooser.ItemCount):
if self.file_chooser.IsSelected(i):
path = os.path.join(self.directory_picker.Path,
self.file_chooser.GetItemText(i))
index = self.pipeline_list_view.InsertStringItem(
sys.maxint, path)
self.pipeline_list_view.SetStringItem(
index, P_INPUT_DIRECTORY_COLUMN,
cpprefs.get_default_image_directory())
self.pipeline_list_view.SetStringItem(
index, P_OUTPUT_DIRECTORY_COLUMN,
cpprefs.get_default_output_directory())
self.pipeline_list_view.SetStringItem(
index, P_OUTPUT_FILE_COLUMN,
cpprefs.get_output_file_name())
self.pipeline_list_view.SetItemColumnImage(
index, P_REMOVE_BUTTON_COLUMN, self.delete_bmp_idx)
self.file_chooser.Select(i, False)
self.pipeline_list_view.SetColumnWidth(P_FILENAME_COLUMN,
wx.LIST_AUTOSIZE)
self.pipeline_list_view.SetColumnWidth(P_INPUT_DIRECTORY_COLUMN,
wx.LIST_AUTOSIZE)
self.pipeline_list_view.SetColumnWidth(P_OUTPUT_DIRECTORY_COLUMN,
wx.LIST_AUTOSIZE)
self.pipeline_list_view.SetColumnWidth(P_OUTPUT_FILE_COLUMN,
wx.LIST_AUTOSIZE)
def check_paths(self):
'''Check to make sure the default directories are remotely accessible'''
import wx
def check(path):
more = urllib.urlencode(dict(path=path))
url = ("/batchprofiler/cgi-bin/development/"
"CellProfiler_2.0/PathExists.py?%s") % more
conn = httplib.HTTPConnection("imageweb")
conn.request("GET",url)
result = conn.getresponse()
if result.status != httplib.OK:
raise RuntimeError("HTTP failed: %s" % result.reason)
body = result.read()
return body.find("OK") != -1
all_ok = True
for mapping in self.mappings:
path = mapping.remote_directory.value
if not check(path):
wx.MessageBox("Cannot find %s on the server." % path)
all_ok = False
for path in (cpprefs.get_default_image_directory(),
cpprefs.get_default_output_directory()):
if not check(self.alter_path(path)):
wx.MessageBox("Cannot find %s on the server." % path)
all_ok = False
if all_ok:
wx.MessageBox("All paths are accessible")
def test_04_04_alter_output_subfolder_path(self):
s = cps.DirectoryPath("whatever")
s.dir_choice = cps.DEFAULT_OUTPUT_SUBFOLDER_NAME
s.custom_path = "0"
s.alter_for_create_batch_files(TestDirectoryPath.fn_alter_path)
self.assertEqual(s.get_absolute_path(),
os.path.join(cpprefs.get_default_output_directory(), "0altered"))
def test_04_04_alter_output_subfolder_path(self):
s = cps.DirectoryPath("whatever")
s.dir_choice = cps.DEFAULT_OUTPUT_SUBFOLDER_NAME
s.custom_path = "0"
s.alter_for_create_batch_files(TestDirectoryPath.fn_alter_path)
self.assertEqual(
s.get_absolute_path(),
os.path.join(cpprefs.get_default_output_directory(), "0altered"))
def create_settings(self):
'''Create the module settings and name the module'''
self.wants_default_output_directory = cps.Binary(
"Store batch files in default output folder?",
True,
doc="""
Check this box to store batch files in the Default Output folder. Uncheck
the box to enter the path to the folder that will be used to store
these files.""")
self.custom_output_directory = cps.Text(
"Output folder path",
cpprefs.get_default_output_directory(),
doc="""
Enter the path to the output folder.""")
# Worded this way not because I am windows-centric but because it's
# easier than listing every other OS in the universe except for VMS
self.remote_host_is_windows = cps.Binary(
"Are the cluster computers running Windows?",
False,
doc="""
Check this box if the cluster computers are running one of the Microsoft
Windows operating systems. If you check this box, <b>CreateBatchFiles</b> will
modify all paths to use the Windows file separator (backslash \). If you
leave the box unchecked, <b>CreateBatchFiles</b> will modify all paths to use
the Unix or Macintosh file separator (slash /).""")
self.batch_mode = cps.Binary("Hidden: in batch mode", False)
self.distributed_mode = cps.Binary("Hidden: in distributed mode",
False)
self.default_image_directory = cps.Setting(
"Hidden: default input folder at time of save",
cpprefs.get_default_image_directory())
self.revision = cps.Integer("Hidden: revision number", 0)
self.from_old_matlab = cps.Binary("Hidden: from old matlab", False)
self.acknowledge_old_matlab = cps.DoSomething(
"Could not update CP1.0 pipeline to be compatible with CP2.0. See module notes.",
"OK", self.clear_old_matlab)
self.mappings = []
self.add_mapping()
self.add_mapping_button = cps.DoSomething("",
"Add another path mapping",
self.add_mapping,
doc="""
Use this option if another path must be mapped because there is a difference
between how the local computer sees a folder location vs. how the cluster
computer sees the folder location.""")
self.check_path_button = cps.DoSomething(
"Press this button to check pathnames on the remote server",
"Check paths",
self.check_paths,
doc="""
his button will start a routine that will ask the
webserver to check whether the default input and default output
folders exist. It will also check whether all remote
path mappings exist.""")
def create_settings(self):
"""Create the module settings and name the module"""
self.wants_default_output_directory = cps.Binary(
"Store batch files in default output folder?",
True,
doc="""\
Select "*Yes*" to store batch files in the Default Output folder.
Select "*No*" to enter the path to the folder that will be used to
store these files. The Default Output folder can be set by clicking the "View output settings" button in the main CP window, or in CellProfiler Preferences. """
% globals(),
)
self.custom_output_directory = cps.Text(
"Output folder path",
cpprefs.get_default_output_directory(),
doc=
"Enter the path to the output folder. (Used only if not using the default output folder)",
)
# Worded this way not because I am windows-centric but because it's
# easier than listing every other OS in the universe except for VMS
self.remote_host_is_windows = cps.Binary(
"Are the cluster computers running Windows?",
False,
doc="""\
Select "*Yes*" if the cluster computers are running one of the
Microsoft Windows operating systems. In this case, **CreateBatchFiles**
will modify all paths to use the Windows file separator (backslash \\\\ ).
Select "*No*" for **CreateBatchFiles** to modify all paths to use the
Unix or Macintosh file separator (slash / ).""" % globals(),
)
self.batch_mode = cps.Binary("Hidden: in batch mode", False)
self.distributed_mode = cps.Binary("Hidden: in distributed mode",
False)
self.default_image_directory = cps.Setting(
"Hidden: default input folder at time of save",
cpprefs.get_default_image_directory(),
)
self.revision = cps.Integer("Hidden: revision number", 0)
self.from_old_matlab = cps.Binary("Hidden: from old matlab", False)
self.acknowledge_old_matlab = cps.DoSomething(
"Could not update CP1.0 pipeline to be compatible with CP2.0. See module notes.",
"OK",
self.clear_old_matlab,
)
self.mappings = []
self.add_mapping()
self.add_mapping_button = cps.DoSomething(
"",
"Add another path mapping",
self.add_mapping,
doc="""\
Use this option if another path must be mapped because there is a difference
between how the local computer sees a folder location vs. how the cluster
computer sees the folder location.""",
)
def image_directory(self):
"""Return the image directory
"""
if self.location == DIR_DEFAULT_IMAGE:
return preferences.get_default_image_directory()
elif self.location == DIR_DEFAULT_OUTPUT:
return preferences.get_default_output_directory()
else:
return preferences.get_absolute_path(self.location_other.value)
def image_directory(self):
"""Return the image directory
"""
if self.location == DIR_DEFAULT_IMAGE:
return preferences.get_default_image_directory()
elif self.location == DIR_DEFAULT_OUTPUT:
return preferences.get_default_output_directory()
else:
return preferences.get_absolute_path(self.location_other.value)
def save_default_folders_to_measurements(self):
from cellprofiler.pipeline import M_DEFAULT_INPUT_FOLDER
from cellprofiler.pipeline import M_DEFAULT_OUTPUT_FOLDER
from cellprofiler.preferences import get_default_image_directory
from cellprofiler.preferences import get_default_output_directory
self.measurements.add_experiment_measurement(
M_DEFAULT_INPUT_FOLDER, get_default_image_directory())
self.measurements.add_experiment_measurement(
M_DEFAULT_OUTPUT_FOLDER, get_default_output_directory())
def save_default_folders_to_measurements(self):
from cellprofiler.pipeline import M_DEFAULT_INPUT_FOLDER
from cellprofiler.pipeline import M_DEFAULT_OUTPUT_FOLDER
from cellprofiler.preferences import get_default_image_directory
from cellprofiler.preferences import get_default_output_directory
self.measurements.add_experiment_measurement(
M_DEFAULT_INPUT_FOLDER, get_default_image_directory())
self.measurements.add_experiment_measurement(
M_DEFAULT_OUTPUT_FOLDER, get_default_output_directory())
def create_settings(self):
'''Create the module settings and name the module'''
self.wants_default_output_directory = cps.Binary(
"Store batch files in default output folder?", True, doc="""
Select <i>%(YES)s</i> to store batch files in the Default Output folder. <br>
Select <i>%(NO)s</i> to enter the path to the folder that will be used to store
these files.""" % globals())
self.custom_output_directory = cps.Text(
"Output folder path",
cpprefs.get_default_output_directory(), doc="""
Enter the path to the output folder.""")
# Worded this way not because I am windows-centric but because it's
# easier than listing every other OS in the universe except for VMS
self.remote_host_is_windows = cps.Binary(
"Are the cluster computers running Windows?",
False, doc="""
Select <i>%(YES)s</i> if the cluster computers are running one of the Microsoft
Windows operating systems. In this case, <b>CreateBatchFiles</b> will
modify all paths to use the Windows file separator (backslash \). <br>
Select <i>%(NO)s</i> for <b>CreateBatchFiles</b> to modify all paths to use
the Unix or Macintosh file separator (slash /).""" % globals())
self.batch_mode = cps.Binary("Hidden: in batch mode", False)
self.distributed_mode = cps.Binary("Hidden: in distributed mode", False)
self.default_image_directory = cps.Setting("Hidden: default input folder at time of save",
cpprefs.get_default_image_directory())
self.revision = cps.Integer("Hidden: revision number", 0)
self.from_old_matlab = cps.Binary("Hidden: from old matlab", False)
self.acknowledge_old_matlab = cps.DoSomething(
"Could not update CP1.0 pipeline to be compatible with CP2.0. See module notes.", "OK",
self.clear_old_matlab)
self.mappings = []
self.add_mapping()
self.add_mapping_button = cps.DoSomething("",
"Add another path mapping", self.add_mapping, doc="""
Use this option if another path must be mapped because there is a difference
between how the local computer sees a folder location vs. how the cluster
computer sees the folder location.""")
self.go_to_website = cps.Binary(
"Launch BatchProfiler", True,
doc="""Launch BatchProfiler after creating the batch file. This
setting will launch a web browser to the BatchProfiler URL to
allow you to create batch jobs to run the analysis on a cluster.
""")
self.check_path_button = cps.DoSomething(
"Press this button to check pathnames on the remote server",
"Check paths", self.check_paths, doc="""
This button will start a routine that will ask the
webserver to check whether the default input and default output
folders exist. It will also check whether all remote
path mappings exist.""")
def save_pipeline(self, workspace, outf=None):
'''Save the pipeline in Batch_data.mat
Save the pickled image_set_list state in a setting and put this
module in batch mode.
if outf is not None, it is used as a file object destination.
'''
from cellprofiler.utilities.version import version_number
if outf is None:
if self.wants_default_output_directory.value:
path = cpprefs.get_default_output_directory()
else:
path = cpprefs.get_absolute_path(self.custom_output_directory.value)
h5_path = os.path.join(path, F_BATCH_DATA_H5)
else:
h5_path = outf
image_set_list = workspace.image_set_list
pipeline = workspace.pipeline
m = cpmeas.Measurements(copy = workspace.measurements,
filename = h5_path)
assert isinstance(image_set_list, cpi.ImageSetList)
assert isinstance(pipeline, cpp.Pipeline)
assert isinstance(m, cpmeas.Measurements)
pipeline = pipeline.copy()
target_workspace = cpw.Workspace(pipeline, None, None, None,
m, image_set_list,
workspace.frame)
pipeline.prepare_to_create_batch(target_workspace, self.alter_path)
bizarro_self = pipeline.module(self.module_num)
bizarro_self.revision.value = version_number
if self.wants_default_output_directory:
bizarro_self.custom_output_directory.value = \
self.alter_path(cpprefs.get_default_output_directory())
bizarro_self.default_image_directory.value = \
self.alter_path(cpprefs.get_default_image_directory())
bizarro_self.batch_mode.value = True
pipeline.write_pipeline_measurement(m)
del m
def save_pipeline(self, workspace, outf=None):
'''Save the pipeline in Batch_data.mat
Save the pickled image_set_list state in a setting and put this
module in batch mode.
if outf is not None, it is used as a file object destination.
'''
from cellprofiler.utilities.get_revision import version
if outf is None:
if self.wants_default_output_directory.value:
path = cpprefs.get_default_output_directory()
else:
path = cpprefs.get_absolute_path(self.custom_output_directory.value)
h5_path = os.path.join(path, F_BATCH_DATA_H5)
else:
h5_path = outf
image_set_list = workspace.image_set_list
pipeline = workspace.pipeline
m = cpmeas.Measurements(copy = workspace.measurements,
filename = h5_path)
assert isinstance(image_set_list, cpi.ImageSetList)
assert isinstance(pipeline, cpp.Pipeline)
assert isinstance(m, cpmeas.Measurements)
pipeline = pipeline.copy()
target_workspace = cpw.Workspace(pipeline, None, None, None,
m, image_set_list,
workspace.frame)
pipeline.prepare_to_create_batch(target_workspace, self.alter_path)
bizarro_self = pipeline.module(self.module_num)
bizarro_self.revision.value = version
if self.wants_default_output_directory:
bizarro_self.custom_output_directory.value = \
self.alter_path(cpprefs.get_default_output_directory())
bizarro_self.default_image_directory.value = \
self.alter_path(cpprefs.get_default_image_directory())
bizarro_self.batch_mode.value = True
pipeline.write_pipeline_measurement(m)
del m
def __init__(self, parent_sizer, panel, progress_panel, status_panel):
self.__panel = panel
self.__parent_sizer = parent_sizer
panel.AutoLayout = True
static_box = wx.StaticBox(panel, label="Folders")
panel.SetSizer(wx.BoxSizer(wx.VERTICAL))
static_box_sizer = wx.StaticBoxSizer(static_box, wx.VERTICAL)
panel.Sizer.Add(static_box_sizer, 1, wx.EXPAND)
self.__sizer = static_box_sizer
self.__image_folder_panel = wx.Panel(panel)
self.__image_folder_panel.AutoLayout = True
self.__image_edit_box = self.__make_folder_panel(
self.__image_folder_panel,
cpprefs.get_default_image_directory(),
lambda: cpprefs.get_recent_files(cpprefs.DEFAULT_IMAGE_DIRECTORY),
'Default Input Folder',
DEFAULT_IMAGE_FOLDER_HELP, [
cpprefs.set_default_image_directory,
self.__notify_pipeline_list_view_directory_change
],
refresh_action=self.refresh_input_directory)
self.__output_folder_panel = wx.Panel(panel)
self.__output_folder_panel.AutoLayout = True
self.__output_edit_box = self.__make_folder_panel(
self.__output_folder_panel, cpprefs.get_default_output_directory(),
lambda: cpprefs.get_recent_files(cpprefs.DEFAULT_OUTPUT_DIRECTORY),
'Default Output Folder', DEFAULT_OUTPUT_FOLDER_HELP, [
cpprefs.set_default_output_directory,
self.__notify_pipeline_list_view_directory_change
])
self.__odds_and_ends_panel = wx.Panel(panel)
self.__odds_and_ends_panel.AutoLayout = True
self.__make_odds_and_ends_panel()
self.__status_panel = status_panel
status_panel.Sizer = wx.BoxSizer()
self.__status_text = wx.StaticText(status_panel,
style=wx.SUNKEN_BORDER,
label=WELCOME_MESSAGE)
status_panel.Sizer.Add(self.__status_text, 1, wx.EXPAND)
self.__progress_panel = progress_panel
self.__progress_panel.AutoLayout = True
self.__make_progress_panel()
self.__sizer.AddMany([
(self.__image_folder_panel, 0, wx.EXPAND | wx.ALL, 1),
(self.__output_folder_panel, 0, wx.EXPAND | wx.ALL, 1),
(self.__odds_and_ends_panel, 0, wx.EXPAND | wx.ALL, 1)
])
self.show_status_text()
self.__errors = set()
self.__pipeline_list_view = None
self.__progress_watcher = None
def __init__(self, parent_sizer, panel, progress_panel, status_panel):
self.__panel = panel
self.__parent_sizer = parent_sizer
panel.AutoLayout = True
static_box = wx.StaticBox(panel, label="Folders")
panel.SetSizer(wx.BoxSizer(wx.VERTICAL))
static_box_sizer = wx.StaticBoxSizer(static_box, wx.VERTICAL)
panel.Sizer.Add(static_box_sizer, 1, wx.EXPAND)
self.__sizer = static_box_sizer
self.__image_folder_panel = wx.Panel(panel)
self.__image_folder_panel.AutoLayout = True
self.__image_edit_box = self.__make_folder_panel(
self.__image_folder_panel,
cpprefs.get_default_image_directory(),
lambda: cpprefs.get_recent_files(cpprefs.DEFAULT_IMAGE_DIRECTORY),
"Default Input Folder",
DEFAULT_IMAGE_FOLDER_HELP,
[cpprefs.set_default_image_directory, self.__notify_pipeline_list_view_directory_change],
refresh_action=self.refresh_input_directory,
)
self.__output_folder_panel = wx.Panel(panel)
self.__output_folder_panel.AutoLayout = True
self.__output_edit_box = self.__make_folder_panel(
self.__output_folder_panel,
cpprefs.get_default_output_directory(),
lambda: cpprefs.get_recent_files(cpprefs.DEFAULT_OUTPUT_DIRECTORY),
"Default Output Folder",
DEFAULT_OUTPUT_FOLDER_HELP,
[cpprefs.set_default_output_directory, self.__notify_pipeline_list_view_directory_change],
)
self.__odds_and_ends_panel = wx.Panel(panel)
self.__odds_and_ends_panel.AutoLayout = True
self.__make_odds_and_ends_panel()
self.__status_panel = status_panel
status_panel.Sizer = wx.BoxSizer()
self.__status_text = wx.StaticText(status_panel, style=wx.SUNKEN_BORDER, label=WELCOME_MESSAGE)
status_panel.Sizer.Add(self.__status_text, 1, wx.EXPAND)
self.__progress_panel = progress_panel
self.__progress_panel.AutoLayout = True
self.__make_progress_panel()
self.__sizer.AddMany(
[
(self.__image_folder_panel, 0, wx.EXPAND | wx.ALL, 1),
(self.__output_folder_panel, 0, wx.EXPAND | wx.ALL, 1),
(self.__odds_and_ends_panel, 0, wx.EXPAND | wx.ALL, 1),
]
)
self.show_status_text()
self.__errors = set()
self.__pipeline_list_view = None
self.__progress_watcher = None
def __init__(self,
can_overwrite=False,
image_set_start=None,
filename = None,
copy = None,
image_numbers = None):
"""Create a new measurements collection
can_overwrite - DEPRECATED and has no effect
image_set_start - the index of the first image set in the image set list
or None to start at the beginning
filename - store the measurement in an HDF5 file with this name
copy - initialize by copying measurements from here, either an HDF5Dict
or an H5py group or file.
"""
# XXX - allow saving of partial results
if filename is None:
dir = cpprefs.get_default_output_directory()
if not (os.path.exists(dir) and os.access(dir, os.W_OK)):
dir = None
fd, filename = tempfile.mkstemp(prefix='Cpmeasurements', suffix='.hdf5', dir=dir)
is_temporary = True
else:
is_temporary = False
if isinstance(copy, Measurements):
with copy.hdf5_dict.lock:
self.hdf5_dict = HDF5Dict(
filename,
is_temporary = is_temporary,
copy = copy.hdf5_dict.top_group,
image_numbers=image_numbers)
elif hasattr(copy, '__getitem__') and hasattr(copy, 'keys'):
self.hdf5_dict = HDF5Dict(
filename,
is_temporary = is_temporary,
copy = copy,
image_numbers=image_numbers)
else:
self.hdf5_dict = HDF5Dict(filename, is_temporary = is_temporary)
if is_temporary:
os.close(fd)
self.image_set_number = image_set_start or 1
self.image_set_start = image_set_start
self.__is_first_image = True
self.__initialized_explicitly = False
self.__relationships = set()
self.__relationship_names = set()
def __init__(self,
can_overwrite=False,
image_set_start=None,
filename = None,
copy = None):
"""Create a new measurements collection
can_overwrite - DEPRECATED and has no effect
image_set_start - the index of the first image set in the image set list
or None to start at the beginning
filename - store the measurement in an HDF5 file with this name
copy - initialize by copying measurements from here, either an HDF5Dict
or an H5py group or file.
"""
# XXX - allow saving of partial results
if filename is None:
dir = cpprefs.get_default_output_directory()
if not (os.path.exists(dir) and os.access(dir, os.W_OK)):
dir = None
fd, filename = tempfile.mkstemp(prefix='Cpmeasurements', suffix='.hdf5', dir=dir)
is_temporary = True
else:
is_temporary = False
if isinstance(copy, Measurements):
with copy.hdf5_dict.lock:
self.hdf5_dict = HDF5Dict(
filename,
is_temporary = is_temporary,
copy = copy.hdf5_dict.top_group)
elif hasattr(copy, '__getitem__') and hasattr(copy, 'keys'):
self.hdf5_dict = HDF5Dict(
filename,
is_temporary = is_temporary,
copy = copy)
else:
self.hdf5_dict = HDF5Dict(filename, is_temporary = is_temporary)
if is_temporary:
os.close(fd)
self.image_set_number = image_set_start or 1
self.image_set_start = image_set_start
self.__is_first_image = True
self.__initialized_explicitly = False
self.__relationships = set()
self.__relationship_names = set()
def test_03_07_metadata(self):
m = cpmeas.Measurements()
m.add_image_measurement("Metadata_Path", "2")
s = cps.DirectoryPath("whatever", allow_metadata=True)
for dir_choice, expected in (
(cps.DEFAULT_INPUT_SUBFOLDER_NAME, os.path.join(cpprefs.get_default_image_directory(), "0", "2")),
(cps.DEFAULT_OUTPUT_SUBFOLDER_NAME, os.path.join(cpprefs.get_default_output_directory(), "0", "2")),
(cps.ABSOLUTE_FOLDER_NAME, os.path.join(self.root_directory, "2")),
(cps.URL_FOLDER_NAME, "http://www.cellprofiler.org/2"),
):
s.dir_choice = dir_choice
if dir_choice in (cps.DEFAULT_INPUT_SUBFOLDER_NAME, cps.DEFAULT_OUTPUT_SUBFOLDER_NAME):
s.custom_path = "0" + os.path.sep.replace("\\", "\\\\") + "\\g<Path>"
elif dir_choice == cps.ABSOLUTE_FOLDER_NAME:
s.custom_path = self.root_directory + os.path.sep.replace("\\", "\\\\") + "\\g<Path>"
else:
s.custom_path = "http://www.cellprofiler.org/\\g<Path>"
self.assertEqual(s.get_absolute_path(m), expected)
def ask_for_output_dir(self):
'''
Ask for a destination for the downloaded files
'''
default_target = cpprefs.get_default_output_directory()
dialog = wx.DirDialog(None, "Choose an output directory",
default_target,
wx.DD_DEFAULT_STYLE | wx.DD_DIR_MUST_EXIST)
try:
if dialog.ShowModal() == wx.ID_CANCEL:
return False
target_directory = dialog.GetPath()
except Exception:
wx.LogError('Failed to open directory!')
raise
finally:
dialog.Destroy()
return target_directory
def upgrade_settings(self, setting_values, variable_revision_number,
module_name, from_matlab):
if from_matlab and variable_revision_number == 8:
batch_save_path, old_pathname, new_pathname = setting_values[:3]
if batch_save_path == '.':
wants_default_output_directory = cps.YES
batch_save_path = cpprefs.get_default_output_directory()
else:
wants_default_output_directory = cps.NO
old_pathnames = old_pathname.split(',')
new_pathnames = new_pathname.split(',')
if len(old_pathnames) != len(new_pathnames):
raise ValueError("Number of pathnames does not match. "
"%d local pathnames, but %d remote pathnames" %
(len(old_pathnames), len(new_pathnames)))
setting_values = [wants_default_output_directory, batch_save_path,
cps.NO, cps.NO, ""]
for old_pathname, new_pathname in zip(old_pathnames, new_pathnames):
setting_values += [old_pathname, new_pathname]
from_matlab = False
variable_revision_number = 1
if (not from_matlab) and variable_revision_number == 1:
setting_values = (setting_values[:5] +
[cpprefs.get_default_image_directory()] +
setting_values[5:])
variable_revision_number = 2
if (not from_matlab) and variable_revision_number == 2:
from cellprofiler.utilities.get_revision import version
setting_values = (setting_values[:6] +
[version] +
setting_values[6:])
variable_revision_number = 3
if (not from_matlab) and variable_revision_number == 3:
# Pickled image list is now the batch state
self.batch_state = np.array(zlib.compress(setting_values[4]))
setting_values = setting_values[:4]+setting_values[5:]
variable_revision_number = 4
if (not from_matlab) and variable_revision_number == 4:
setting_values = setting_values[:4] + [False] + setting_values[4:]
variable_revision_number = 5
return setting_values, variable_revision_number, from_matlab
def upgrade_settings(self, setting_values, variable_revision_number,
module_name, from_matlab):
if from_matlab and variable_revision_number == 8:
batch_save_path, old_pathname, new_pathname = setting_values[:3]
if batch_save_path == '.':
wants_default_output_directory = cps.YES
batch_save_path = cpprefs.get_default_output_directory()
else:
wants_default_output_directory = cps.NO
old_pathnames = old_pathname.split(',')
new_pathnames = new_pathname.split(',')
if len(old_pathnames) != len(new_pathnames):
raise ValueError("Number of pathnames does not match. "
"%d local pathnames, but %d remote pathnames" %
(len(old_pathnames), len(new_pathnames)))
setting_values = [wants_default_output_directory, batch_save_path,
cps.NO, cps.NO, ""]
for old_pathname, new_pathname in zip(old_pathnames, new_pathnames):
setting_values += [old_pathname, new_pathname]
from_matlab = False
variable_revision_number = 1
if (not from_matlab) and variable_revision_number == 1:
setting_values = (setting_values[:5] +
[cpprefs.get_default_image_directory()] +
setting_values[5:])
variable_revision_number = 2
if (not from_matlab) and variable_revision_number == 2:
from cellprofiler.utilities.get_revision import version
setting_values = (setting_values[:6] +
[version] +
setting_values[6:])
variable_revision_number = 3
if (not from_matlab) and variable_revision_number == 3:
# Pickled image list is now the batch state
self.batch_state = np.array(zlib.compress(setting_values[4]))
setting_values = setting_values[:4]+setting_values[5:]
variable_revision_number = 4
if (not from_matlab) and variable_revision_number == 4:
setting_values = setting_values[:4] + [False] + setting_values[4:]
variable_revision_number = 5
return setting_values, variable_revision_number, from_matlab
def save_pipeline(self, workspace, outf=None):
'''Save the pipeline in Batch_data.h5
Save the pickled image_set_list state in a setting and put this
module in batch mode.
if outf is not None, it is used as a file object destination.
'''
if outf is None:
path = cpprefs.get_default_output_directory()
h5_path = os.path.join(path, F_BATCH_DATA_H5)
else:
h5_path = outf
image_set_list = workspace.image_set_list
pipeline = workspace.pipeline
m = cpmeas.Measurements(copy=workspace.measurements, filename=h5_path)
try:
assert isinstance(pipeline, cpp.Pipeline)
assert isinstance(m, cpmeas.Measurements)
orig_pipeline = pipeline
pipeline = pipeline.copy()
# this use of workspace.frame is okay, since we're called from
# prepare_run which happens in the main wx thread.
target_workspace = cpw.Workspace(pipeline, None, None, None, m,
image_set_list, workspace.frame)
# Assuming all results go to the same place, output folder can be set
# in the script
pipeline.prepare_to_create_batch(target_workspace, self.alter_path)
self_copy = pipeline.module(self.module_num)
self_copy.revision.value = int(
re.sub(r"\.|rc\d{1}", "", cellprofiler.__version__))
self_copy.batch_mode.value = True
pipeline.write_pipeline_measurement(m)
orig_pipeline.write_pipeline_measurement(m, user_pipeline=True)
return h5_path
finally:
m.close()
def test_03_07_metadata(self):
m = cpmeas.Measurements()
m.add_image_measurement("Metadata_Path", "2")
s = cps.DirectoryPath("whatever", allow_metadata = True)
for dir_choice, expected in (
( cps.DEFAULT_INPUT_SUBFOLDER_NAME,
os.path.join(cpprefs.get_default_image_directory(), "0", "2")),
( cps.DEFAULT_OUTPUT_SUBFOLDER_NAME,
os.path.join(cpprefs.get_default_output_directory(), "0", "2")),
( cps.ABSOLUTE_FOLDER_NAME,
os.path.join(self.root_directory, "2")),
( cps.URL_FOLDER_NAME, "http://www.cellprofiler.org/2")):
s.dir_choice = dir_choice
if dir_choice in (cps.DEFAULT_INPUT_SUBFOLDER_NAME,
cps.DEFAULT_OUTPUT_SUBFOLDER_NAME):
s.custom_path = "0" + os.path.sep.replace('\\','\\\\') + "\\g<Path>"
elif dir_choice == cps.ABSOLUTE_FOLDER_NAME:
s.custom_path = self.root_directory + os.path.sep.replace('\\','\\\\') + "\\g<Path>"
else:
s.custom_path = "http://www.cellprofiler.org/\\g<Path>"
self.assertEqual(s.get_absolute_path(m), expected)
def create_settings(self):
'''Create the module settings and name the module'''
self.wants_default_output_directory = cps.Binary("Store batch files in default output folder?", True,doc="""
Do you want to store the batch files in the default output folder?
Check this box to store batch files in the Default Output folder. Uncheck
the box to enter the path to the folder that will be used to store
these files.""")
self.custom_output_directory = cps.Text("Output folder path",
cpprefs.get_default_output_directory(),doc="""
What is the path to the output folder?""")
# Worded this way not because I am windows-centric but because it's
# easier than listing every other OS in the universe except for VMS
self.remote_host_is_windows = cps.Binary("Are the cluster computers running Windows?",
False,doc="""
Check this box if the cluster computers are running one of the Microsoft
Windows operating systems. If you check this box, <b>CreateBatchFiles</b> will
modify all paths to use the Windows file separator (backslash \). If you
leave the box unchecked, <b>CreateBatchFiles</b> will modify all paths to use
the Unix or Macintosh file separator (slash,/).""")
self.batch_mode = cps.Binary("Hidden: in batch mode", False)
self.distributed_mode = cps.Binary("Hidden: in distributed mode", False)
self.default_image_directory = cps.Setting("Hidden: default input folder at time of save",
cpprefs.get_default_image_directory())
self.revision = cps.Integer("Hidden: SVN revision number", 0)
self.mappings = []
self.add_mapping()
self.add_mapping_button = cps.DoSomething("", "Add another path mapping",
self.add_mapping, doc="""
Use this option if another path must be mapped because there is a difference between how the local computer sees a folder location vs. how the cluster computer sees the folder location.""")
self.check_path_button = cps.DoSomething(
"Press this button to check pathnames on the remote server",
"Check paths", self.check_paths,
doc = """This button will start a routine that will ask the
webserver to check whether the default input and default output
folders exist. It will also check whether all remote
path mappings exist.""")
def create_settings(self):
'''Create the module settings and name the module'''
self.wants_default_output_directory = cps.Binary(
"Store batch files in default output folder?", True, doc="""\
Select "*%(YES)s*" to store batch files in the Default Output folder.
Select "*%(NO)s*" to enter the path to the folder that will be used to
store these files. The Default Output folder can be set by clicking the "View output settings" button in the main CP window, or in CellProfiler Preferences. """ % globals())
self.custom_output_directory = cps.Text(
"Output folder path",
cpprefs.get_default_output_directory(), doc="Enter the path to the output folder. (Used only if not using the default output folder)")
# Worded this way not because I am windows-centric but because it's
# easier than listing every other OS in the universe except for VMS
self.remote_host_is_windows = cps.Binary(
"Are the cluster computers running Windows?",
False, doc="""\
Select "*%(YES)s*" if the cluster computers are running one of the
Microsoft Windows operating systems. In this case, **CreateBatchFiles**
will modify all paths to use the Windows file separator (backslash \\\\ ).
Select "*%(NO)s*" for **CreateBatchFiles** to modify all paths to use the
Unix or Macintosh file separator (slash / ).""" % globals())
self.batch_mode = cps.Binary("Hidden: in batch mode", False)
self.distributed_mode = cps.Binary("Hidden: in distributed mode", False)
self.default_image_directory = cps.Setting("Hidden: default input folder at time of save",
cpprefs.get_default_image_directory())
self.revision = cps.Integer("Hidden: revision number", 0)
self.from_old_matlab = cps.Binary("Hidden: from old matlab", False)
self.acknowledge_old_matlab = cps.DoSomething(
"Could not update CP1.0 pipeline to be compatible with CP2.0. See module notes.", "OK",
self.clear_old_matlab)
self.mappings = []
self.add_mapping()
self.add_mapping_button = cps.DoSomething("",
"Add another path mapping", self.add_mapping, doc="""\
Use this option if another path must be mapped because there is a difference
between how the local computer sees a folder location vs. how the cluster
computer sees the folder location.""")
def __init__(self,panel):
self.__panel = panel
self.__sizer = wx.BoxSizer(wx.VERTICAL)
self.__image_folder_panel = wx.Panel(panel,-1)
self.__image_edit_box = self.__make_folder_panel(
self.__image_folder_panel,
cpprefs.get_default_image_directory(),
lambda : cpprefs.get_recent_files(cpprefs.DEFAULT_IMAGE_DIRECTORY),
'Default Input Folder',
DEFAULT_IMAGE_FOLDER_HELP,
[cpprefs.set_default_image_directory,
self.__notify_pipeline_list_view_directory_change],
refresh_action = self.refresh_input_directory)
self.__output_folder_panel = wx.Panel(panel,-1)
self.__output_edit_box = self.__make_folder_panel(
self.__output_folder_panel,
cpprefs.get_default_output_directory(),
lambda : cpprefs.get_recent_files(cpprefs.DEFAULT_OUTPUT_DIRECTORY),
'Default Output Folder',
DEFAULT_OUTPUT_FOLDER_HELP,
[cpprefs.set_default_output_directory,
self.__notify_pipeline_list_view_directory_change])
self.__odds_and_ends_panel = wx.Panel(panel,-1)
self.__make_odds_and_ends_panel()
self.__status_text = wx.StaticText(panel,-1,style=wx.SUNKEN_BORDER,label=WELCOME_MESSAGE)
self.__progress_panel = wx.Panel(panel, -1)
self.__make_progress_panel()
self.__sizer.AddMany([(self.__image_folder_panel,0,wx.EXPAND|wx.ALL,1),
(self.__output_folder_panel,0,wx.EXPAND|wx.ALL,1),
(self.__odds_and_ends_panel,0,wx.EXPAND|wx.ALL,1),
(self.__status_text,0,wx.EXPAND|wx.ALL, 4),
(self.__progress_panel, 0, wx.EXPAND | wx.BOTTOM, 2)])
panel.SetSizer(self.__sizer)
self.__sizer.Hide(self.__progress_panel)
self.__errors = set()
self.__pipeline_list_view = None
self.__progress_watcher = None
def on_add(self, event):
for i in range(self.file_chooser.ItemCount):
if self.file_chooser.IsSelected(i):
path = os.path.join(
self.directory_picker.Path,
self.file_chooser.GetItemText(i))
index = self.pipeline_list_view.InsertStringItem(
sys.maxint, path)
self.pipeline_list_view.SetStringItem(
index, P_INPUT_DIRECTORY_COLUMN,
cpprefs.get_default_image_directory())
self.pipeline_list_view.SetStringItem(
index, P_OUTPUT_DIRECTORY_COLUMN,
cpprefs.get_default_output_directory())
self.pipeline_list_view.SetStringItem(
index, P_OUTPUT_FILE_COLUMN,
cpprefs.get_output_file_name())
self.pipeline_list_view.SetItemColumnImage(
index, P_REMOVE_BUTTON_COLUMN, self.delete_bmp_idx)
self.file_chooser.Select(i, False)
self.pipeline_list_view.SetColumnWidth(P_FILENAME_COLUMN, wx.LIST_AUTOSIZE)
self.pipeline_list_view.SetColumnWidth(P_INPUT_DIRECTORY_COLUMN, wx.LIST_AUTOSIZE)
self.pipeline_list_view.SetColumnWidth(P_OUTPUT_DIRECTORY_COLUMN, wx.LIST_AUTOSIZE)
self.pipeline_list_view.SetColumnWidth(P_OUTPUT_FILE_COLUMN, wx.LIST_AUTOSIZE)
def prepare_run(self, workspace):
'''Invoke the image_set_list pickling mechanism and save the pipeline'''
pipeline = workspace.pipeline
if pipeline.test_mode:
return True
if self.batch_mode.value:
return True
else:
rynner = CPRynner()
if rynner is not None:
# Get parameters
max_tasks = int(cluster_tasks_per_node())
setup_script = cluster_setup_script()
# Set walltime
rynner.provider.walltime = str(
self.max_walltime.value) + ":00:00"
# save the pipeline
path = self.save_pipeline(workspace)
# Create the run data structure
file_list = pipeline.file_list
file_list = [
name.replace('file:///', '') for name in file_list
]
file_list = [name.replace('file:', '') for name in file_list]
file_list = [name.replace('%20', ' ') for name in file_list]
if len(file_list) == 0:
wx.MessageBox(
"No images found. Did you remember to add them to the Images module?",
caption="No images",
style=wx.OK | wx.ICON_INFORMATION)
return False
# Divide measurements to runs according to the number of cores on a node
n_images = len(file_list)
if not self.is_archive.value:
n_measurements = int(n_images /
self.n_images_per_measurement.value)
measurements_per_run = int(n_measurements / max_tasks) + 1
grouped_images = self.group_images(file_list,
n_measurements,
measurements_per_run,
self.type_first.value)
n_image_groups = max(zip(*grouped_images)[0]) + 1
# Add image files to uploads
uploads = [[name, 'run{}/images'.format(g)]
for g, name in grouped_images]
else:
if n_images > 1:
wx.MessageBox(
"Include only one image archive per run.",
caption="Image error",
style=wx.OK | wx.ICON_INFORMATION)
return False
uploads = [[file_list[0], 'images']]
n_measurements = self.measurements_in_archive.value
n_image_groups = max_tasks
# Also add the pipeline
uploads += [[path, '.']]
# The runs are downloaded in their separate folders. They can be processed later
output_dir = cpprefs.get_default_output_directory()
downloads = [['run{}'.format(g), output_dir]
for g in range(n_image_groups)]
# Create run scripts and add to uploads
for g in range(n_image_groups):
runscript_name = 'cellprofiler_run{}'.format(g)
local_script_path = os.path.join(
rynner.provider.script_dir, runscript_name)
if not self.is_archive.value:
n_measurements = len([
i for i in grouped_images if i[0] == g
]) / self.n_images_per_measurement.value
script = "cellprofiler -c -p ../Batch_data.h5 -o results -i images -f 1 -l {} 2>>../cellprofiler_output; rm -r images".format(
n_measurements)
else:
n_images_per_group = int(n_measurements / max_tasks)
n_additional_images = int(n_measurements % max_tasks)
if g < n_additional_images:
first = (n_images_per_group + 1) * g
last = (n_images_per_group + 1) * (g + 1)
else:
first = n_images_per_group * g + n_additional_images
last = n_images_per_group * (
g + 1) + n_additional_images
script = "mkdir images; cp ../images/* images; cellprofiler -c -p ../Batch_data.h5 -o results -i images -f {} -l {} 2>>../cellprofiler_output; rm -r images".format(
first, last)
with open(local_script_path, "w") as file:
file.write(script)
uploads += [[local_script_path, "run{}".format(g)]]
# Define the job to run
script = '{}; printf %s\\\\n {{0..{}}} | xargs -P 40 -n 1 -IX bash -c "cd runX ; ./cellprofiler_runX; ";'.format(
setup_script, n_image_groups - 1)
script = script.replace('\r\n', '\n')
script = script.replace(';;', ';')
print(script)
run = rynner.create_run(
jobname=self.runname.value.replace(' ', '_'),
script=script,
uploads=uploads,
downloads=downloads,
)
run['account'] = self.account.value
# Copy the pipeline and images accross
dialog = wx.GenericProgressDialog("Uploading",
"Uploading files",
style=wx.PD_APP_MODAL)
try:
self.upload(run, dialog)
# Submit the run
dialog.Update(dialog.GetRange() - 1, "Submitting")
success = CPRynner().submit(run)
dialog.Destroy()
if success:
wx.MessageBox(
"RunOnCluster submitted the run to the cluster",
caption="RunOnCluster: Batch job submitted",
style=wx.OK | wx.ICON_INFORMATION)
else:
wx.MessageBox("RunOnCluster failed to submit the run",
caption="RunOnCluster: Failure",
style=wx.OK | wx.ICON_INFORMATION)
except Exception as e:
dialog.Destroy()
raise e
return False
def test_02_02_get_default_output_folder_path(self):
s = cps.DirectoryPath("whatever")
s.dir_choice = cps.DEFAULT_OUTPUT_FOLDER_NAME
self.assertEqual(s.get_absolute_path(), cpprefs.get_default_output_directory())
def test_02_04_get_output_subfolder_path(self):
s = cps.DirectoryPath("whatever")
s.dir_choice = cps.DEFAULT_OUTPUT_SUBFOLDER_NAME
s.custom_path = "0"
self.assertEqual(s.get_absolute_path(),
os.path.join(cpprefs.get_default_output_directory(), "0"))
def __on_preferences_output_directory_event(self,event):
old_selection = self.__output_edit_box.Selection
if self.__output_edit_box.Value != cpprefs.get_default_output_directory():
self.__output_edit_box.Value = cpprefs.get_default_output_directory()
def upgrade_settings(self, setting_values, variable_revision_number,
module_name, from_matlab):
if from_matlab and variable_revision_number < 8:
# We never were able to convert from pre-8 to 8 in Matlab. Why may
# be lost to history, but my guess is there were conflicting ways
# to interpret settings previous to this point, so we decided not
# to try to automate it.
self.notes = ["The pipeline you loaded was from an old version of CellProfiler 1.0, "
"which could not be made compatible with this version of CellProfiler.",
"For reference, previous values were:"] + [str(x) for x in setting_values]
setting_values = [cps.NO,
"", cps.NO,
cps.NO, cps.NO,
"", 0, cps.YES]
variable_revision_number = 6
from_matlab = False
if from_matlab and variable_revision_number == 8:
batch_save_path, old_pathname, new_pathname = setting_values[:3]
if batch_save_path == '.':
wants_default_output_directory = cps.YES
batch_save_path = cpprefs.get_default_output_directory()
else:
wants_default_output_directory = cps.NO
old_pathnames = old_pathname.split(',')
new_pathnames = new_pathname.split(',')
if len(old_pathnames) != len(new_pathnames):
raise ValueError("Number of pathnames does not match. "
"%d local pathnames, but %d remote pathnames" %
(len(old_pathnames), len(new_pathnames)))
setting_values = [wants_default_output_directory, batch_save_path,
cps.NO, cps.NO, ""]
for old_pathname, new_pathname in zip(old_pathnames, new_pathnames):
setting_values += [old_pathname, new_pathname]
from_matlab = False
variable_revision_number = 1
if (not from_matlab) and variable_revision_number == 1:
setting_values = (setting_values[:5] +
[cpprefs.get_default_image_directory()] +
setting_values[5:])
variable_revision_number = 2
if (not from_matlab) and variable_revision_number == 2:
setting_values = (setting_values[:6] +
[int(re.sub(r"\.|rc\d{1}", "", cellprofiler.__version__))] +
setting_values[6:])
variable_revision_number = 3
if (not from_matlab) and variable_revision_number == 3:
# Pickled image list is now the batch state
self.batch_state = np.array(zlib.compress(setting_values[4]))
setting_values = setting_values[:4] + setting_values[5:]
variable_revision_number = 4
if (not from_matlab) and variable_revision_number == 4:
setting_values = setting_values[:4] + [False] + setting_values[4:]
variable_revision_number = 5
if (not from_matlab) and variable_revision_number == 5:
# added from_old_matlab
setting_values = setting_values[:7] + [False] + setting_values[7:]
variable_revision_number = 6
if (not from_matlab) and variable_revision_number == 6:
# added go_to_website
setting_values = setting_values[:8] + [False] + setting_values[8:]
variable_revision_number = 7
if variable_revision_number == 7:
setting_values = setting_values[:8] + setting_values[9:]
variable_revision_number = 8
return setting_values, variable_revision_number, from_matlab
def upgrade_settings(self, setting_values, variable_revision_number,
module_name, from_matlab):
if from_matlab and variable_revision_number < 8:
# We never were able to convert from pre-8 to 8 in Matlab. Why may
# be lost to history, but my guess is there were conflicting ways
# to interpret settings previous to this point, so we decided not
# to try to automate it.
self.notes = ["The pipeline you loaded was from an old version of CellProfiler 1.0, "
"which could not be made compatible with this version of CellProfiler.",
"For reference, previous values were:"] + [str(x) for x in setting_values]
setting_values = [cps.NO,
"", cps.NO,
cps.NO, cps.NO,
"", 0, cps.YES]
variable_revision_number = 6
from_matlab = False
if from_matlab and variable_revision_number == 8:
batch_save_path, old_pathname, new_pathname = setting_values[:3]
if batch_save_path == '.':
wants_default_output_directory = cps.YES
batch_save_path = cpprefs.get_default_output_directory()
else:
wants_default_output_directory = cps.NO
old_pathnames = old_pathname.split(',')
new_pathnames = new_pathname.split(',')
if len(old_pathnames) != len(new_pathnames):
raise ValueError("Number of pathnames does not match. "
"%d local pathnames, but %d remote pathnames" %
(len(old_pathnames), len(new_pathnames)))
setting_values = [wants_default_output_directory, batch_save_path,
cps.NO, cps.NO, ""]
for old_pathname, new_pathname in zip(old_pathnames, new_pathnames):
setting_values += [old_pathname, new_pathname]
from_matlab = False
variable_revision_number = 1
if (not from_matlab) and variable_revision_number == 1:
setting_values = (setting_values[:5] +
[cpprefs.get_default_image_directory()] +
setting_values[5:])
variable_revision_number = 2
if (not from_matlab) and variable_revision_number == 2:
from cellprofiler.utilities.version import version_number
setting_values = (setting_values[:6] +
[version_number] +
setting_values[6:])
variable_revision_number = 3
if (not from_matlab) and variable_revision_number == 3:
# Pickled image list is now the batch state
self.batch_state = np.array(zlib.compress(setting_values[4]))
setting_values = setting_values[:4]+setting_values[5:]
variable_revision_number = 4
if (not from_matlab) and variable_revision_number == 4:
setting_values = setting_values[:4] + [False] + setting_values[4:]
variable_revision_number = 5
if (not from_matlab) and variable_revision_number == 5:
# added from_old_matlab
setting_values = setting_values[:7] + [False] + setting_values[7:]
variable_revision_number = 6
return setting_values, variable_revision_number, from_matlab
def test_01_07_get_parts_from_output_subfolder_path(self):
s = cps.DirectoryPath("whatever")
dir_choice, custom_path = s.get_parts_from_path(
os.path.join(cpprefs.get_default_output_directory(), "2"))
self.assertEqual(dir_choice, cps.DEFAULT_OUTPUT_SUBFOLDER_NAME)
self.assertEqual(custom_path, "2")
def __on_preferences_output_directory_event(self, event):
old_selection = self.__output_edit_box.Selection
if self.__output_edit_box.Value != cpprefs.get_default_output_directory(
):
self.__output_edit_box.Value = cpprefs.get_default_output_directory(
)
def test_01_05_get_parts_from_output_folder_path(self):
s = cps.DirectoryPath("whatever")
dir_choice, custom_path = s.get_parts_from_path(cpprefs.get_default_output_directory())
self.assertEqual(dir_choice, cps.DEFAULT_OUTPUT_FOLDER_NAME)
