def add_mapping(self):
group = cps.SettingsGroup()
group.append("local_directory",
cps.Text("Local root path",
cpprefs.get_default_image_directory(),doc="""
What is the path to files on this computer?
This is the root path on the local machine (i.e., the computer setting up
the batch files). If <b>CreateBatchFiles</b> finds
any pathname that matches the local root path at the begining, it will replace the
start with the cluster root path.
<p>For example, if you have mapped the remote cluster machine like this:<br><br>
<tt>Z:\your_data\images</tt> (on a Windows machine, for instance)<br><br>
and the cluster machine sees the same folder like this:<br><br>
<tt>/server_name/your_name/your_data/images</tt><br><br>
you would enter <tt>Z:</tt> here and <t>/server_name/your_name/</tt>
for the cluster path in the next setting."""))
group.append("remote_directory",
cps.Text("Cluster root path",
cpprefs.get_default_image_directory(),doc="""
What is the path to files on the cluster? This is the cluster
root path, i.e., how the cluster machine sees the
top-most folder where your input/output files are stored.
<p>For example, if you have mapped the remote cluster machine like this:<br><br>
<tt>Z:\your_data\images</tt> (on a Windows machine, for instance)<br><br>
and the cluster machine sees the same folder like this:<br><br>
<tt>/server_name/your_name/your_data/images</tt><br><br>
you would enter <tt>Z:</tt> in the previous setting for the
local machine path and <t>/server_name/your_name/</tt> here. """))
group.append("remover",
cps.RemoveSettingButton("", "Remove this path mapping", self.mappings, group))
group.append("divider", cps.Divider(line=False))
self.mappings.append(group)
def add_mapping(self):
group = cps.SettingsGroup()
group.append("local_directory",
cps.Text(
"Local root path",
cpprefs.get_default_image_directory(),doc="""
Enter the path to files on this computer.
This is the root path on the local machine (i.e., the computer setting up
the batch files). If <b>CreateBatchFiles</b> finds
any pathname that matches the local root path at the begining, it will replace the
start with the cluster root path.
<p>For example, if you have mapped the remote cluster machine like this:<br><br>
<tt>Z:\your_data\images</tt> (on a Windows machine, for instance)<br><br>
and the cluster machine sees the same folder like this:<br><br>
<tt>/server_name/your_name/your_data/images</tt><br><br>
you would enter <tt>Z:</tt> here and <t>/server_name/your_name/</tt>
for the cluster path in the next setting."""))
group.append("remote_directory",
cps.Text(
"Cluster root path",
cpprefs.get_default_image_directory(),doc="""
Enter the path to files on the cluster. This is the cluster
root path, i.e., how the cluster machine sees the
top-most folder where your input/output files are stored.
<p>For example, if you have mapped the remote cluster machine like this:<br><br>
<tt>Z:\your_data\images</tt> (on a Windows machine, for instance)<br><br>
and the cluster machine sees the same folder like this:<br><br>
<tt>/server_name/your_name/your_data/images</tt><br><br>
you would enter <tt>Z:</tt> in the previous setting for the
local machine path and <t>/server_name/your_name/</tt> here. """))
group.append("remover",
cps.RemoveSettingButton("", "Remove this path mapping", self.mappings, group))
group.append("divider", cps.Divider(line=False))
self.mappings.append(group)
def test_03_02_pipeline_preferences(self):
#
# Walk the worker up through pipelines and preferences.
#
self.awthread = self.AWThread(self.announce_addr)
self.awthread.start()
self.set_work_socket()
self.awthread.ex(self.awthread.aw.do_job,
cpanalysis.WorkReply(
image_set_numbers = [1],
worker_runs_post_group = False,
wants_dictionary = True))
#
# The worker should ask for the pipeline and preferences next.
#
req = self.awthread.recv(self.work_socket)
self.assertIsInstance(req, cpanalysis.PipelinePreferencesRequest)
self.assertEqual(req.analysis_id, self.analysis_id)
maybe_download_example_image(["ExampleSBSImages"],
"Channel1-01-A-01.tif")
maybe_download_example_image(["ExampleHT29"],
"AS_09125_050116030001_D03f00d0.tif")
input_dir = os.path.normcase(
os.path.join(example_images_directory(), "ExampleSBSImages"))
output_dir = os.path.normcase(
os.path.join(example_images_directory(), "ExampleHT29"))
cpprefs.set_default_image_directory(input_dir)
input_dir = cpprefs.get_default_image_directory()
cpprefs.set_default_output_directory(output_dir)
output_dir = cpprefs.get_default_output_directory()
preferences = {cpprefs.DEFAULT_IMAGE_DIRECTORY:
cpprefs.config_read(cpprefs.DEFAULT_IMAGE_DIRECTORY),
cpprefs.DEFAULT_OUTPUT_DIRECTORY:
cpprefs.config_read(cpprefs.DEFAULT_OUTPUT_DIRECTORY)}
cpprefs.set_default_image_directory(example_images_directory())
cpprefs.set_default_output_directory(example_images_directory())
rep = cpanalysis.Reply(
pipeline_blob = np.array(GOOD_PIPELINE),
preferences = preferences)
req.reply(rep)
#
# Get the next request so that we know the worker has
# processed the preferences.
#
req = self.awthread.recv(self.work_socket)
self.assertEqual(cpprefs.get_default_image_directory(),
input_dir)
self.assertEqual(cpprefs.get_default_output_directory(),
output_dir)
self.assertIn(self.analysis_id,
self.awthread.aw.pipelines_and_preferences)
pipe, prefs = self.awthread.aw.pipelines_and_preferences[
self.analysis_id]
self.assertEqual(len(pipe.modules()), 7)
#
# Cancel and check for exit
#
req.reply(cpanalysis.ServerExited())
self.assertRaises(cpp.CancelledException, self.awthread.ecute)
def test_01_03_unicode_directory(self):
old = cpprefs.get_default_image_directory()
unicode_dir = u'P125 à 144 Crible Chimiothèque HBEC'
unicode_dir = tempfile.mkdtemp(prefix=unicode_dir)
cpprefs.set_default_image_directory(unicode_dir)
self.assertEqual(cpprefs.config_read(cpprefs.DEFAULT_IMAGE_DIRECTORY),
unicode_dir)
self.assertEqual(cpprefs.get_default_image_directory(), unicode_dir)
cpprefs.set_default_image_directory(old)
def test_01_03_unicode_directory(self):
old = cpprefs.get_default_image_directory()
unicode_dir = u'P125 à 144 Crible Chimiothèque HBEC'
unicode_dir = tempfile.mkdtemp(prefix=unicode_dir)
cpprefs.set_default_image_directory(unicode_dir)
self.assertEqual(cpprefs.config_read(cpprefs.DEFAULT_IMAGE_DIRECTORY),
unicode_dir)
self.assertEqual(cpprefs.get_default_image_directory(), unicode_dir)
cpprefs.set_default_image_directory(old)
def check_paths(self):
'''Check to make sure the default directories are remotely accessible'''
import wx
def check(path):
more = urllib.urlencode(dict(path=path))
url = ("/batchprofiler/cgi-bin/development/"
"CellProfiler_2.0/PathExists.py?%s") % more
conn = httplib.HTTPConnection("imageweb")
conn.request("GET",url)
result = conn.getresponse()
if result.status != httplib.OK:
raise RuntimeError("HTTP failed: %s" % result.reason)
body = result.read()
return body.find("OK") != -1
all_ok = True
for mapping in self.mappings:
path = mapping.remote_directory.value
if not check(path):
wx.MessageBox("Cannot find %s on the server." % path)
all_ok = False
for path in (cpprefs.get_default_image_directory(),
cpprefs.get_default_output_directory()):
if not check(self.alter_path(path)):
wx.MessageBox("Cannot find %s on the server." % path)
all_ok = False
if all_ok:
wx.MessageBox("All paths are accessible")
def run_create_webpage(self, image_paths, thumb_paths=None, metadata=None, alter_fn=None):
"""Run the create_webpage module, returning the resulting HTML document
image_paths - list of path / filename tuples. The function will
write an image to each of these and put images and
measurements into the workspace for each.
thumb_paths - if present a list of path / filename tuples. Same as above
metadata - a dictionary of feature / string values
alter_fn - function taking a CreateWebPage module, for you to
alter the module's settings
"""
np.random.seed(0)
module = C.CreateWebPage()
module.module_num = 1
module.orig_image_name.value = IMAGE_NAME
module.web_page_file_name.value = DEFAULT_HTML_FILE
if alter_fn is not None:
alter_fn(module)
pipeline = cpp.Pipeline()
def callback(caller, event):
self.assertFalse(isinstance(event, cpp.RunExceptionEvent))
pipeline.add_listener(callback)
pipeline.add_module(module)
images = [(IMAGE_NAME, image_paths)]
if thumb_paths:
images += [(THUMB_NAME, thumb_paths)]
self.assertEqual(len(image_paths), len(thumb_paths))
module.wants_thumbnails.value = True
module.thumbnail_image_name.value = THUMB_NAME
else:
module.wants_thumbnails.value = False
measurements = cpmeas.Measurements()
workspace = cpw.Workspace(pipeline, module, measurements, None, measurements, None, None)
for i in range(len(image_paths)):
image_number = i + 1
if metadata is not None:
for key in metadata.keys():
values = metadata[key]
feature = cpmeas.C_METADATA + "_" + key
measurements[cpmeas.IMAGE, feature, image_number] = values[i]
for image_name, paths in images:
pixel_data = np.random.uniform(size=(10, 13))
path_name, file_name = paths[i]
if path_name is None:
path_name = cpprefs.get_default_image_directory()
full_path = os.path.abspath(os.path.join(self.directory, path_name, file_name))
imsave(full_path, pixel_data)
path_feature = "_".join((C_PATH_NAME, image_name))
file_feature = "_".join((C_FILE_NAME, image_name))
measurements[cpmeas.IMAGE, path_feature, image_number] = os.path.split(full_path)[0]
measurements[cpmeas.IMAGE, file_feature, image_number] = file_name
module.post_run(workspace)
return measurements
def on_add(self, event):
for i in range(self.file_chooser.ItemCount):
if self.file_chooser.IsSelected(i):
path = os.path.join(self.directory_picker.Path,
self.file_chooser.GetItemText(i))
index = self.pipeline_list_view.InsertStringItem(
sys.maxint, path)
self.pipeline_list_view.SetStringItem(
index, P_INPUT_DIRECTORY_COLUMN,
cpprefs.get_default_image_directory())
self.pipeline_list_view.SetStringItem(
index, P_OUTPUT_DIRECTORY_COLUMN,
cpprefs.get_default_output_directory())
self.pipeline_list_view.SetStringItem(
index, P_OUTPUT_FILE_COLUMN,
cpprefs.get_output_file_name())
self.pipeline_list_view.SetItemColumnImage(
index, P_REMOVE_BUTTON_COLUMN, self.delete_bmp_idx)
self.file_chooser.Select(i, False)
self.pipeline_list_view.SetColumnWidth(P_FILENAME_COLUMN,
wx.LIST_AUTOSIZE)
self.pipeline_list_view.SetColumnWidth(P_INPUT_DIRECTORY_COLUMN,
wx.LIST_AUTOSIZE)
self.pipeline_list_view.SetColumnWidth(P_OUTPUT_DIRECTORY_COLUMN,
wx.LIST_AUTOSIZE)
self.pipeline_list_view.SetColumnWidth(P_OUTPUT_FILE_COLUMN,
wx.LIST_AUTOSIZE)
def test_02_03_get_input_subfolder_path(self):
s = cps.DirectoryPath("whatever")
s.dir_choice = cps.DEFAULT_INPUT_SUBFOLDER_NAME
s.custom_path = "2"
self.assertEqual(
s.get_absolute_path(),
os.path.join(cpprefs.get_default_image_directory(), "2"))
def test_05_03_http_image_zipfile(self):
# Make a zipfile using files accessed from the web
def alter_fn(module):
self.assertTrue(isinstance(module, C.CreateWebPage))
module.wants_zip_file.value = True
module.zipfile_name.value = ZIPFILE_NAME
module.directory_choice.dir_choice = C.ABSOLUTE_FOLDER_NAME
module.directory_choice.custom_path = cpprefs.get_default_image_directory()
url_root = "http://cellprofiler.org/svnmirror/ExampleImages/ExampleSBSImages/"
url_query = "?r=11710"
filenames = [ (url_root, fn + url_query) for fn in
("Channel1-01-A-01.tif", "Channel2-01-A-01.tif",
"Channel1-02-A-02.tif", "Channel2-02-A-02.tif")]
self.run_create_webpage(filenames, alter_fn = alter_fn)
zpath = os.path.join(cpprefs.get_default_image_directory(),
ZIPFILE_NAME)
with zipfile.ZipFile(zpath, "r") as zfile:
for _, filename in filenames:
fn = filename.split("?", 1)[0]
url = url_root + "/" + filename
svn_fd = urlopen(url)
with zfile.open(fn, "r") as zip_fd:
data = zip_fd.read()
offset = 0
while offset < len(data):
udata = svn_fd.read(len(data) - offset)
self.assertEqual(
udata, data[offset:(offset + len(udata))])
offset += len(udata)
svn_fd.close()
def test_05_03_http_image_zipfile(self):
# Make a zipfile using files accessed from the web
def alter_fn(module):
self.assertTrue(isinstance(module, C.CreateWebPage))
module.wants_zip_file.value = True
module.zipfile_name.value = ZIPFILE_NAME
module.directory_choice.dir_choice = C.ABSOLUTE_FOLDER_NAME
module.directory_choice.custom_path = cpprefs.get_default_image_directory(
)
url_root = "https://svn.broadinstitute.org/CellProfiler/trunk/ExampleImages/ExampleSBSImages/"
url_query = "?r=11710"
filenames = [(url_root, fn + url_query)
for fn in ("Channel1-01-A-01.tif", "Channel2-01-A-01.tif",
"Channel1-02-A-02.tif", "Channel2-02-A-02.tif")
]
self.run_create_webpage(filenames, alter_fn=alter_fn)
zpath = os.path.join(cpprefs.get_default_image_directory(),
ZIPFILE_NAME)
with zipfile.ZipFile(zpath, "r") as zfile:
for _, filename in filenames:
fn = filename.split("?", 1)[0]
url = url_root + "/" + filename
svn_fd = urlopen(url)
with zfile.open(fn, "r") as zip_fd:
data = zip_fd.read()
offset = 0
while offset < len(data):
udata = svn_fd.read(len(data) - offset)
self.assertEqual(udata,
data[offset:(offset + len(udata))])
offset += len(udata)
svn_fd.close()
def alter_fn(module):
self.assertTrue(isinstance(module, C.CreateWebPage))
module.wants_zip_file.value = True
module.zipfile_name.value = ZIPFILE_NAME
module.directory_choice.dir_choice = C.ABSOLUTE_FOLDER_NAME
module.directory_choice.custom_path = cpprefs.get_default_image_directory(
)
def test_04_01_alter_input_folder_path(self):
s = cps.DirectoryPath("whatever")
s.dir_choice = cps.DEFAULT_INPUT_FOLDER_NAME
s.alter_for_create_batch_files(TestDirectoryPath.fn_alter_path)
self.assertEqual(
s.get_absolute_path(),
cpprefs.get_default_image_directory())
def test_04_03_alter_input_subfolder_path(self):
s = cps.DirectoryPath("whatever")
s.dir_choice = cps.DEFAULT_INPUT_SUBFOLDER_NAME
s.custom_path = "2"
s.alter_for_create_batch_files(TestDirectoryPath.fn_alter_path)
self.assertEqual(s.get_absolute_path(),
os.path.join(cpprefs.get_default_image_directory(), "2altered"))
def check_paths(self):
'''Check to make sure the default directories are remotely accessible'''
import wx
def check(path):
more = urllib.urlencode(dict(path=path))
url = ("/batchprofiler/cgi-bin/development/"
"CellProfiler_2.0/PathExists.py?%s") % more
conn = httplib.HTTPConnection("imageweb")
conn.request("GET",url)
result = conn.getresponse()
if result.status != httplib.OK:
raise RuntimeError("HTTP failed: %s" % result.reason)
body = result.read()
return body.find("OK") != -1
all_ok = True
for mapping in self.mappings:
path = mapping.remote_directory.value
if not check(path):
wx.MessageBox("Cannot find %s on the server." % path)
all_ok = False
for path, name in (
(cpprefs.get_default_image_directory(), "default image folder"),
(cpprefs.get_default_output_directory(), "default output folder")):
if not check(self.alter_path(path)):
wx.MessageBox("Cannot find the %s, \"%s\", on the server." %
(name, path))
all_ok = False
if all_ok:
wx.MessageBox("All paths are accessible")
def test_04_03_alter_input_subfolder_path(self):
s = cps.DirectoryPath("whatever")
s.dir_choice = cps.DEFAULT_INPUT_SUBFOLDER_NAME
s.custom_path = "2"
s.alter_for_create_batch_files(TestDirectoryPath.fn_alter_path)
self.assertEqual(
s.get_absolute_path(),
os.path.join(cpprefs.get_default_image_directory(), "2altered"))
def create_settings(self):
'''Create the module settings and name the module'''
self.wants_default_output_directory = cps.Binary(
"Store batch files in default output folder?",
True,
doc="""
Check this box to store batch files in the Default Output folder. Uncheck
the box to enter the path to the folder that will be used to store
these files.""")
self.custom_output_directory = cps.Text(
"Output folder path",
cpprefs.get_default_output_directory(),
doc="""
Enter the path to the output folder.""")
# Worded this way not because I am windows-centric but because it's
# easier than listing every other OS in the universe except for VMS
self.remote_host_is_windows = cps.Binary(
"Are the cluster computers running Windows?",
False,
doc="""
Check this box if the cluster computers are running one of the Microsoft
Windows operating systems. If you check this box, <b>CreateBatchFiles</b> will
modify all paths to use the Windows file separator (backslash \). If you
leave the box unchecked, <b>CreateBatchFiles</b> will modify all paths to use
the Unix or Macintosh file separator (slash /).""")
self.batch_mode = cps.Binary("Hidden: in batch mode", False)
self.distributed_mode = cps.Binary("Hidden: in distributed mode",
False)
self.default_image_directory = cps.Setting(
"Hidden: default input folder at time of save",
cpprefs.get_default_image_directory())
self.revision = cps.Integer("Hidden: revision number", 0)
self.from_old_matlab = cps.Binary("Hidden: from old matlab", False)
self.acknowledge_old_matlab = cps.DoSomething(
"Could not update CP1.0 pipeline to be compatible with CP2.0. See module notes.",
"OK", self.clear_old_matlab)
self.mappings = []
self.add_mapping()
self.add_mapping_button = cps.DoSomething("",
"Add another path mapping",
self.add_mapping,
doc="""
Use this option if another path must be mapped because there is a difference
between how the local computer sees a folder location vs. how the cluster
computer sees the folder location.""")
self.check_path_button = cps.DoSomething(
"Press this button to check pathnames on the remote server",
"Check paths",
self.check_paths,
doc="""
his button will start a routine that will ask the
webserver to check whether the default input and default output
folders exist. It will also check whether all remote
path mappings exist.""")
def test_05_01_zipfiles(self):
# Test the zipfile function
def alter_fn(module):
self.assertTrue(isinstance(module, C.CreateWebPage))
module.wants_zip_file.value = True
module.zipfile_name.value = ZIPFILE_NAME
filenames = ["A%02d.png" % i for i in range(1, 3)]
self.run_create_webpage([(None, fn) for fn in filenames], alter_fn=alter_fn),
zpath = os.path.join(cpprefs.get_default_image_directory(), ZIPFILE_NAME)
with zipfile.ZipFile(zpath, "r") as zfile:
assert isinstance(zfile, zipfile.ZipFile)
for filename in filenames:
fpath = os.path.join(cpprefs.get_default_image_directory(), filename)
with open(fpath, "rb") as fd:
with zfile.open(filename, "r") as zfd:
self.assertEqual(fd.read(), zfd.read())
def image_directory(self):
"""Return the image directory
"""
if self.location == DIR_DEFAULT_IMAGE:
return preferences.get_default_image_directory()
elif self.location == DIR_DEFAULT_OUTPUT:
return preferences.get_default_output_directory()
else:
return preferences.get_absolute_path(self.location_other.value)
def create_settings(self):
"""Create the module settings and name the module"""
self.wants_default_output_directory = cps.Binary(
"Store batch files in default output folder?",
True,
doc="""\
Select "*Yes*" to store batch files in the Default Output folder.
Select "*No*" to enter the path to the folder that will be used to
store these files. The Default Output folder can be set by clicking the "View output settings" button in the main CP window, or in CellProfiler Preferences. """
% globals(),
)
self.custom_output_directory = cps.Text(
"Output folder path",
cpprefs.get_default_output_directory(),
doc=
"Enter the path to the output folder. (Used only if not using the default output folder)",
)
# Worded this way not because I am windows-centric but because it's
# easier than listing every other OS in the universe except for VMS
self.remote_host_is_windows = cps.Binary(
"Are the cluster computers running Windows?",
False,
doc="""\
Select "*Yes*" if the cluster computers are running one of the
Microsoft Windows operating systems. In this case, **CreateBatchFiles**
will modify all paths to use the Windows file separator (backslash \\\\ ).
Select "*No*" for **CreateBatchFiles** to modify all paths to use the
Unix or Macintosh file separator (slash / ).""" % globals(),
)
self.batch_mode = cps.Binary("Hidden: in batch mode", False)
self.distributed_mode = cps.Binary("Hidden: in distributed mode",
False)
self.default_image_directory = cps.Setting(
"Hidden: default input folder at time of save",
cpprefs.get_default_image_directory(),
)
self.revision = cps.Integer("Hidden: revision number", 0)
self.from_old_matlab = cps.Binary("Hidden: from old matlab", False)
self.acknowledge_old_matlab = cps.DoSomething(
"Could not update CP1.0 pipeline to be compatible with CP2.0. See module notes.",
"OK",
self.clear_old_matlab,
)
self.mappings = []
self.add_mapping()
self.add_mapping_button = cps.DoSomething(
"",
"Add another path mapping",
self.add_mapping,
doc="""\
Use this option if another path must be mapped because there is a difference
between how the local computer sees a folder location vs. how the cluster
computer sees the folder location.""",
)
def image_directory(self):
"""Return the image directory
"""
if self.location == DIR_DEFAULT_IMAGE:
return preferences.get_default_image_directory()
elif self.location == DIR_DEFAULT_OUTPUT:
return preferences.get_default_output_directory()
else:
return preferences.get_absolute_path(self.location_other.value)
def save_pipeline(self, workspace, outf=None):
'''Save the pipeline in Batch_data.mat
Save the pickled image_set_list state in a setting and put this
module in batch mode.
if outf is not None, it is used as a file object destination.
'''
from cellprofiler.utilities.get_revision import version
if self.wants_default_output_directory.value:
path = cpprefs.get_default_output_directory()
else:
path = cpprefs.get_absolute_path(self.custom_output_directory.value)
h5_path = os.path.join(path, F_BATCH_DATA_H5)
image_set_list = workspace.image_set_list
pipeline = workspace.pipeline
m = cpmeas.Measurements(copy = workspace.measurements,
filename = h5_path)
assert isinstance(image_set_list, cpi.ImageSetList)
assert isinstance(pipeline, cpp.Pipeline)
assert isinstance(m, cpmeas.Measurements)
pipeline = pipeline.copy()
target_workspace = cpw.Workspace(pipeline, None, None, None,
m, image_set_list,
workspace.frame)
pipeline.prepare_to_create_batch(target_workspace, self.alter_path)
bizarro_self = pipeline.module(self.module_num)
assert isinstance(bizarro_self, CreateBatchFiles)
state = image_set_list.save_state()
state = zlib.compress(state)
bizarro_self.revision.value = version
bizarro_self.batch_state = np.array(state)
if self.wants_default_output_directory:
bizarro_self.custom_output_directory.value = \
self.alter_path(cpprefs.get_default_output_directory())
bizarro_self.default_image_directory.value = \
self.alter_path(cpprefs.get_default_image_directory())
bizarro_self.batch_mode.value = True
pipeline.write_pipeline_measurement(m)
if outf is None:
mat_path = os.path.join(path, F_BATCH_DATA)
if os.path.exists(mat_path) and workspace.frame is not None:
import wx
if (wx.MessageBox("%s already exists. Do you want to overwrite it?"%
mat_path,
"Overwriting %s" % F_BATCH_DATA,
wx.YES_NO, workspace.frame) == wx.NO):
return
pipeline.save(mat_path, format=cpp.FMT_MATLAB) # Matlab... it's like the vestigial hole in the head of CP.
else:
pipeline.save(outf, format=cpp.FMT_NATIVE)
def save_default_folders_to_measurements(self):
from cellprofiler.pipeline import M_DEFAULT_INPUT_FOLDER
from cellprofiler.pipeline import M_DEFAULT_OUTPUT_FOLDER
from cellprofiler.preferences import get_default_image_directory
from cellprofiler.preferences import get_default_output_directory
self.measurements.add_experiment_measurement(
M_DEFAULT_INPUT_FOLDER, get_default_image_directory())
self.measurements.add_experiment_measurement(
M_DEFAULT_OUTPUT_FOLDER, get_default_output_directory())
def test_05_02_zipfile_and_metadata(self):
# Test the zipfile function with metadata substitution
def alter_fn(module):
self.assertTrue(isinstance(module, C.CreateWebPage))
module.wants_zip_file.value = True
module.zipfile_name.value = "\\g<FileName>"
filenames = ["A%02d.png" % i for i in range(1, 3)]
zipfiles = ["A%02d" % i for i in range(1, 3)]
self.run_create_webpage([(None, fn) for fn in filenames], metadata=dict(FileName=zipfiles), alter_fn=alter_fn)
for filename, zname in zip(filenames, zipfiles):
zpath = os.path.join(cpprefs.get_default_image_directory(), zname)
zpath += ".zip"
fpath = os.path.join(cpprefs.get_default_image_directory(), filename)
with zipfile.ZipFile(zpath, "r") as zfile:
with open(fpath, "rb") as fd:
with zfile.open(filename, "r") as zfd:
self.assertEqual(fd.read(), zfd.read())
def save_default_folders_to_measurements(self):
from cellprofiler.pipeline import M_DEFAULT_INPUT_FOLDER
from cellprofiler.pipeline import M_DEFAULT_OUTPUT_FOLDER
from cellprofiler.preferences import get_default_image_directory
from cellprofiler.preferences import get_default_output_directory
self.measurements.add_experiment_measurement(
M_DEFAULT_INPUT_FOLDER, get_default_image_directory())
self.measurements.add_experiment_measurement(
M_DEFAULT_OUTPUT_FOLDER, get_default_output_directory())
def create_settings(self):
'''Create the module settings and name the module'''
self.wants_default_output_directory = cps.Binary(
"Store batch files in default output folder?", True, doc="""
Select <i>%(YES)s</i> to store batch files in the Default Output folder. <br>
Select <i>%(NO)s</i> to enter the path to the folder that will be used to store
these files.""" % globals())
self.custom_output_directory = cps.Text(
"Output folder path",
cpprefs.get_default_output_directory(), doc="""
Enter the path to the output folder.""")
# Worded this way not because I am windows-centric but because it's
# easier than listing every other OS in the universe except for VMS
self.remote_host_is_windows = cps.Binary(
"Are the cluster computers running Windows?",
False, doc="""
Select <i>%(YES)s</i> if the cluster computers are running one of the Microsoft
Windows operating systems. In this case, <b>CreateBatchFiles</b> will
modify all paths to use the Windows file separator (backslash \). <br>
Select <i>%(NO)s</i> for <b>CreateBatchFiles</b> to modify all paths to use
the Unix or Macintosh file separator (slash /).""" % globals())
self.batch_mode = cps.Binary("Hidden: in batch mode", False)
self.distributed_mode = cps.Binary("Hidden: in distributed mode", False)
self.default_image_directory = cps.Setting("Hidden: default input folder at time of save",
cpprefs.get_default_image_directory())
self.revision = cps.Integer("Hidden: revision number", 0)
self.from_old_matlab = cps.Binary("Hidden: from old matlab", False)
self.acknowledge_old_matlab = cps.DoSomething(
"Could not update CP1.0 pipeline to be compatible with CP2.0. See module notes.", "OK",
self.clear_old_matlab)
self.mappings = []
self.add_mapping()
self.add_mapping_button = cps.DoSomething("",
"Add another path mapping", self.add_mapping, doc="""
Use this option if another path must be mapped because there is a difference
between how the local computer sees a folder location vs. how the cluster
computer sees the folder location.""")
self.go_to_website = cps.Binary(
"Launch BatchProfiler", True,
doc="""Launch BatchProfiler after creating the batch file. This
setting will launch a web browser to the BatchProfiler URL to
allow you to create batch jobs to run the analysis on a cluster.
""")
self.check_path_button = cps.DoSomething(
"Press this button to check pathnames on the remote server",
"Check paths", self.check_paths, doc="""
This button will start a routine that will ask the
webserver to check whether the default input and default output
folders exist. It will also check whether all remote
path mappings exist.""")
def prepare_run(self, workspace): '''Set up omero image providers inside the image_set_list''' pipeline = workspace.pipeline image_set_list = workspace.image_set_list if pipeline.in_batch_mode(): #TODO: Rewrite the OmeroImageProvider such that it can be used in batch mode #e.g. omero session keys could be used to attach to existing sessions to #keep OmeroImageProviders from creating a new session every time an image should be loaded return False if cpp.get_headless(): print 'OmeroLoadImages running in headless mode: image directory parameter will be used as omero object id' self.omero_object_id.set_value(int(cpp.get_default_image_directory())) print 'omero object id = %d'%self.omero_object_id.value print 'omero object type = %s'%self.omero_object.value self.create_omero_gateway() if self.omero_object == MS_IMAGE: omero_image_list = [self.omero_gateway.getImage(self.omero_object_id.value)] elif self.omero_object == MS_DATASET: #Get dataset without leaves(=images&pixels) dataset = self.omero_gateway.getDataset(self.omero_object_id.value, False) self.dataset_name = dataset.getName().getValue() omero_image_list = self.get_images_from_dataset(self.omero_object_id.value) elif self.omero_object == MS_PLATE: self.wells = self.get_wells_from_plate(self.omero_object_id.value) self.plate_name = self.wells[0].getPlate().getName().getValue() omero_image_list = [] for well in self.wells: for wellsample in well.iterateWellSamples(): omero_image_list.append(wellsample.getImage()) #get names and pixels from omero images pixels_list = [] for omero_image in omero_image_list: image_id = omero_image.getId().getValue() pixels_list += self.omero_gateway.getPixelsFromImage(image_id) #add images to image sets image_set_count = len(pixels_list) for i in range(0, image_set_count): image_set = image_set_list.get_image_set(i) pixels = pixels_list[i] pixels_id = pixels.getId().getValue() sizeZ = pixels.getSizeZ().getValue() sizeC = pixels.getSizeC().getValue() sizeT = pixels.getSizeT().getValue() for channel in self.channels: for z in range(0, sizeZ): for t in range(0, sizeT): c = int(channel.channel_number.value) self.save_image_set_info(image_set, channel.cpimage_name.value, P_OMERO, V_OMERO, self.omero_gateway, pixels_id, z, c, t) return True
def prepare_run(self, workspace):
'''Set up omero image providers inside the image_set_list'''
pipeline = workspace.pipeline
image_set_list = workspace.image_set_list
if pipeline.in_batch_mode():
# TODO: Rewrite the OmeroImageProvider such that it can be used in batch mode
# e.g., omero session keys could be used to attach to existing sessions to
# keep OmeroImageProviders from creating a new session every time an image should be loaded
return False
if cpp.get_headless():
print 'OmeroLoadImages running in headless mode: image directory parameter will be used as omero object id'
self.omero_object_id.set_value(int(cpp.get_default_image_directory()))
print 'omero object id = %d' % self.omero_object_id.value
print 'omero object type = %s' % self.omero_object.value
self.create_omero_gateway()
if self.omero_object == MS_IMAGE:
omero_image_list = [self.omero_gateway.getImage(self.omero_object_id.value)]
elif self.omero_object == MS_DATASET:
# Get dataset without leaves(=images&pixels)
dataset = self.omero_gateway.getDataset(self.omero_object_id.value, False)
self.dataset_name = dataset.getName().getValue()
omero_image_list = self.get_images_from_dataset(self.omero_object_id.value)
elif self.omero_object == MS_PLATE:
self.wells = self.get_wells_from_plate(self.omero_object_id.value)
self.plate_name = self.wells[0].getPlate().getName().getValue()
omero_image_list = []
for well in self.wells:
for wellsample in well.iterateWellSamples():
omero_image_list.append(wellsample.getImage())
# get names and pixels from omero images
pixels_list = []
for omero_image in omero_image_list:
image_id = omero_image.getId().getValue()
pixels_list += self.omero_gateway.getPixelsFromImage(image_id)
# add images to image sets
image_set_count = len(pixels_list)
for i in range(0, image_set_count):
image_set = image_set_list.get_image_set(i)
pixels = pixels_list[i]
pixels_id = pixels.getId().getValue()
sizeZ = pixels.getSizeZ().getValue()
sizeC = pixels.getSizeC().getValue()
sizeT = pixels.getSizeT().getValue()
for channel in self.channels:
for z in range(0, sizeZ):
for t in range(0, sizeT):
c = int(channel.channel_number.value)
self.save_image_set_info(image_set, channel.cpimage_name.value,
P_OMERO, V_OMERO,
self.omero_gateway, pixels_id, z, c, t)
return True
def test_05_01_zipfiles(self):
# Test the zipfile function
def alter_fn(module):
self.assertTrue(isinstance(module, C.CreateWebPage))
module.wants_zip_file.value = True
module.zipfile_name.value = ZIPFILE_NAME
filenames = ['A%02d.png' % i for i in range(1, 3)]
self.run_create_webpage([(None, fn) for fn in filenames],
alter_fn=alter_fn)
zpath = os.path.join(cpprefs.get_default_image_directory(), ZIPFILE_NAME)
with zipfile.ZipFile(zpath, "r") as zfile:
assert isinstance(zfile, zipfile.ZipFile)
for filename in filenames:
fpath = os.path.join(cpprefs.get_default_image_directory(),
filename)
with open(fpath, "rb") as fd:
with zfile.open(filename, "r") as zfd:
self.assertEqual(fd.read(), zfd.read())
def save_pipeline(self, workspace, outf=None):
'''Save the pipeline in Batch_data.mat
Save the pickled image_set_list state in a setting and put this
module in batch mode.
if outf is not None, it is used as a file object destination.
'''
from cellprofiler.utilities.version import version_number
if outf is None:
if self.wants_default_output_directory.value:
path = cpprefs.get_default_output_directory()
else:
path = cpprefs.get_absolute_path(self.custom_output_directory.value)
h5_path = os.path.join(path, F_BATCH_DATA_H5)
else:
h5_path = outf
image_set_list = workspace.image_set_list
pipeline = workspace.pipeline
m = cpmeas.Measurements(copy = workspace.measurements,
filename = h5_path)
try:
assert isinstance(pipeline, cpp.Pipeline)
assert isinstance(m, cpmeas.Measurements)
orig_pipeline = pipeline
pipeline = pipeline.copy()
# this use of workspace.frame is okay, since we're called from
# prepare_run which happens in the main wx thread.
target_workspace = cpw.Workspace(pipeline, None, None, None,
m, image_set_list,
workspace.frame)
pipeline.prepare_to_create_batch(target_workspace, self.alter_path)
bizarro_self = pipeline.module(self.module_num)
bizarro_self.revision.value = version_number
if self.wants_default_output_directory:
bizarro_self.custom_output_directory.value = \
self.alter_path(cpprefs.get_default_output_directory())
bizarro_self.default_image_directory.value = \
self.alter_path(cpprefs.get_default_image_directory())
bizarro_self.batch_mode.value = True
pipeline.write_pipeline_measurement(m)
orig_pipeline.write_pipeline_measurement(m, user_pipeline=True)
#
# Write the path mappings to the batch measurements
#
m.write_path_mappings(
[(mapping.local_directory.value, mapping.remote_directory.value)
for mapping in self.mappings])
return h5_path
finally:
m.close()
def save_pipeline(self, workspace, outf=None):
'''Save the pipeline in Batch_data.mat
Save the pickled image_set_list state in a setting and put this
module in batch mode.
if outf is not None, it is used as a file object destination.
'''
from cellprofiler.utilities.version import version_number
if outf is None:
if self.wants_default_output_directory.value:
path = cpprefs.get_default_output_directory()
else:
path = cpprefs.get_absolute_path(self.custom_output_directory.value)
h5_path = os.path.join(path, F_BATCH_DATA_H5)
else:
h5_path = outf
image_set_list = workspace.image_set_list
pipeline = workspace.pipeline
m = cpmeas.Measurements(copy=workspace.measurements,
filename=h5_path)
try:
assert isinstance(pipeline, cpp.Pipeline)
assert isinstance(m, cpmeas.Measurements)
orig_pipeline = pipeline
pipeline = pipeline.copy()
# this use of workspace.frame is okay, since we're called from
# prepare_run which happens in the main wx thread.
target_workspace = cpw.Workspace(pipeline, None, None, None,
m, image_set_list,
workspace.frame)
pipeline.prepare_to_create_batch(target_workspace, self.alter_path)
bizarro_self = pipeline.module(self.module_num)
bizarro_self.revision.value = version_number
if self.wants_default_output_directory:
bizarro_self.custom_output_directory.value = \
self.alter_path(cpprefs.get_default_output_directory())
bizarro_self.default_image_directory.value = \
self.alter_path(cpprefs.get_default_image_directory())
bizarro_self.batch_mode.value = True
pipeline.write_pipeline_measurement(m)
orig_pipeline.write_pipeline_measurement(m, user_pipeline=True)
#
# Write the path mappings to the batch measurements
#
m.write_path_mappings(
[(mapping.local_directory.value, mapping.remote_directory.value)
for mapping in self.mappings])
return h5_path
finally:
m.close()
def test_05_02_zipfile_and_metadata(self):
# Test the zipfile function with metadata substitution
def alter_fn(module):
self.assertTrue(isinstance(module, C.CreateWebPage))
module.wants_zip_file.value = True
module.zipfile_name.value = '\\g<FileName>'
filenames = ['A%02d.png' % i for i in range(1, 3)]
zipfiles = ['A%02d' % i for i in range(1, 3)]
self.run_create_webpage([(None, fn) for fn in filenames],
metadata=dict(FileName=zipfiles),
alter_fn=alter_fn)
for filename, zname in zip(filenames, zipfiles):
zpath = os.path.join(cpprefs.get_default_image_directory(), zname)
zpath += ".zip"
fpath = os.path.join(cpprefs.get_default_image_directory(),
filename)
with zipfile.ZipFile(zpath, "r") as zfile:
with open(fpath, "rb") as fd:
with zfile.open(filename, "r") as zfd:
self.assertEqual(fd.read(), zfd.read())
def read_html(self, html_path=None):
"""Read html file, assuming the default location
returns a DOM
"""
if html_path is None:
html_path = os.path.join(cpprefs.get_default_image_directory(), DEFAULT_HTML_FILE)
fd = open(html_path, "r")
try:
data = fd.read()
return DOM.parseString(data)
finally:
fd.close()
def __init__(self, parent_sizer, panel, progress_panel, status_panel):
self.__panel = panel
self.__parent_sizer = parent_sizer
panel.AutoLayout = True
static_box = wx.StaticBox(panel, label="Folders")
panel.SetSizer(wx.BoxSizer(wx.VERTICAL))
static_box_sizer = wx.StaticBoxSizer(static_box, wx.VERTICAL)
panel.Sizer.Add(static_box_sizer, 1, wx.EXPAND)
self.__sizer = static_box_sizer
self.__image_folder_panel = wx.Panel(panel)
self.__image_folder_panel.AutoLayout = True
self.__image_edit_box = self.__make_folder_panel(
self.__image_folder_panel,
cpprefs.get_default_image_directory(),
lambda: cpprefs.get_recent_files(cpprefs.DEFAULT_IMAGE_DIRECTORY),
'Default Input Folder',
DEFAULT_IMAGE_FOLDER_HELP, [
cpprefs.set_default_image_directory,
self.__notify_pipeline_list_view_directory_change
],
refresh_action=self.refresh_input_directory)
self.__output_folder_panel = wx.Panel(panel)
self.__output_folder_panel.AutoLayout = True
self.__output_edit_box = self.__make_folder_panel(
self.__output_folder_panel, cpprefs.get_default_output_directory(),
lambda: cpprefs.get_recent_files(cpprefs.DEFAULT_OUTPUT_DIRECTORY),
'Default Output Folder', DEFAULT_OUTPUT_FOLDER_HELP, [
cpprefs.set_default_output_directory,
self.__notify_pipeline_list_view_directory_change
])
self.__odds_and_ends_panel = wx.Panel(panel)
self.__odds_and_ends_panel.AutoLayout = True
self.__make_odds_and_ends_panel()
self.__status_panel = status_panel
status_panel.Sizer = wx.BoxSizer()
self.__status_text = wx.StaticText(status_panel,
style=wx.SUNKEN_BORDER,
label=WELCOME_MESSAGE)
status_panel.Sizer.Add(self.__status_text, 1, wx.EXPAND)
self.__progress_panel = progress_panel
self.__progress_panel.AutoLayout = True
self.__make_progress_panel()
self.__sizer.AddMany([
(self.__image_folder_panel, 0, wx.EXPAND | wx.ALL, 1),
(self.__output_folder_panel, 0, wx.EXPAND | wx.ALL, 1),
(self.__odds_and_ends_panel, 0, wx.EXPAND | wx.ALL, 1)
])
self.show_status_text()
self.__errors = set()
self.__pipeline_list_view = None
self.__progress_watcher = None
def __init__(self, parent_sizer, panel, progress_panel, status_panel):
self.__panel = panel
self.__parent_sizer = parent_sizer
panel.AutoLayout = True
static_box = wx.StaticBox(panel, label="Folders")
panel.SetSizer(wx.BoxSizer(wx.VERTICAL))
static_box_sizer = wx.StaticBoxSizer(static_box, wx.VERTICAL)
panel.Sizer.Add(static_box_sizer, 1, wx.EXPAND)
self.__sizer = static_box_sizer
self.__image_folder_panel = wx.Panel(panel)
self.__image_folder_panel.AutoLayout = True
self.__image_edit_box = self.__make_folder_panel(
self.__image_folder_panel,
cpprefs.get_default_image_directory(),
lambda: cpprefs.get_recent_files(cpprefs.DEFAULT_IMAGE_DIRECTORY),
"Default Input Folder",
DEFAULT_IMAGE_FOLDER_HELP,
[cpprefs.set_default_image_directory, self.__notify_pipeline_list_view_directory_change],
refresh_action=self.refresh_input_directory,
)
self.__output_folder_panel = wx.Panel(panel)
self.__output_folder_panel.AutoLayout = True
self.__output_edit_box = self.__make_folder_panel(
self.__output_folder_panel,
cpprefs.get_default_output_directory(),
lambda: cpprefs.get_recent_files(cpprefs.DEFAULT_OUTPUT_DIRECTORY),
"Default Output Folder",
DEFAULT_OUTPUT_FOLDER_HELP,
[cpprefs.set_default_output_directory, self.__notify_pipeline_list_view_directory_change],
)
self.__odds_and_ends_panel = wx.Panel(panel)
self.__odds_and_ends_panel.AutoLayout = True
self.__make_odds_and_ends_panel()
self.__status_panel = status_panel
status_panel.Sizer = wx.BoxSizer()
self.__status_text = wx.StaticText(status_panel, style=wx.SUNKEN_BORDER, label=WELCOME_MESSAGE)
status_panel.Sizer.Add(self.__status_text, 1, wx.EXPAND)
self.__progress_panel = progress_panel
self.__progress_panel.AutoLayout = True
self.__make_progress_panel()
self.__sizer.AddMany(
[
(self.__image_folder_panel, 0, wx.EXPAND | wx.ALL, 1),
(self.__output_folder_panel, 0, wx.EXPAND | wx.ALL, 1),
(self.__odds_and_ends_panel, 0, wx.EXPAND | wx.ALL, 1),
]
)
self.show_status_text()
self.__errors = set()
self.__pipeline_list_view = None
self.__progress_watcher = None
def read_html(self, html_path=None):
'''Read html file, assuming the default location
returns a DOM
'''
if html_path is None:
html_path = os.path.join(cpprefs.get_default_image_directory(),
DEFAULT_HTML_FILE)
fd = open(html_path, 'r')
try:
data = fd.read()
return DOM.parseString(data)
finally:
fd.close()
def estimate_absorbance(self):
"""Load an image and use it to estimate the absorbance of a stain
Returns a 3-tuple of the R/G/B absorbances
"""
from cellprofiler.modules.loadimages import LoadImagesImageProvider
import wx
dlg = wx.FileDialog(None, "Choose reference image",
cpprefs.get_default_image_directory())
dlg.Wildcard = (
"Image file (*.tif, *.tiff, *.bmp, *.png, *.gif, *.jpg)|"
"*.tif;*.tiff;*.bmp;*.png;*.gif;*.jpg")
if dlg.ShowModal() == wx.ID_OK:
lip = LoadImagesImageProvider("dummy", "", dlg.Path)
image = lip.provide_image(None).pixel_data
if image.ndim < 3:
wx.MessageBox(
"You must calibrate the absorbance using a color image",
"Error: not color image",
style=wx.OK | wx.ICON_ERROR,
)
return None
#
# Log-transform the image
#
eps = 1.0 / 256.0 / 2.0
log_image = np.log(image + eps)
data = [-log_image[:, :, i].flatten() for i in range(3)]
#
# Order channels by strength
#
sums = [np.sum(x) for x in data]
order = np.lexsort([sums])
#
# Calculate relative absorbance against the strongest.
# Fit Ax = y to find A where x is the strongest and y
# is each in turn.
#
strongest = data[order[-1]][:, np.newaxis]
absorbances = [lstsq(strongest, d)[0][0] for d in data]
#
# Normalize
#
absorbances = np.array(absorbances)
return absorbances / np.sqrt(np.sum(absorbances**2))
return None
def estimate_absorbance(self):
'''Load an image and use it to estimate the absorbance of a stain
Returns a 3-tuple of the R/G/B absorbances
'''
from cellprofiler.modules.loadimages import LoadImagesImageProvider
import wx
dlg = wx.FileDialog(
None, "Choose reference image",
cpprefs.get_default_image_directory())
dlg.Wildcard = ("Image file (*.tif, *.tiff, *.bmp, *.png, *.gif, *.jpg)|"
"*.tif;*.tiff;*.bmp;*.png;*.gif;*.jpg")
if dlg.ShowModal() == wx.ID_OK:
lip = LoadImagesImageProvider("dummy", "", dlg.Path)
image = lip.provide_image(None).pixel_data
if image.ndim < 3:
wx.MessageBox("You must calibrate the absorbance using a color image",
"Error: not color image",
style=wx.OK | wx.ICON_ERROR)
return None
#
# Log-transform the image
#
eps = 1.0 / 256.0 / 2.0
log_image = np.log(image + eps)
data = [- log_image[:, :, i].flatten() for i in range(3)]
#
# Order channels by strength
#
sums = [np.sum(x) for x in data]
order = np.lexsort([sums])
#
# Calculate relative absorbance against the strongest.
# Fit Ax = y to find A where x is the strongest and y
# is each in turn.
#
strongest = data[order[-1]][:, np.newaxis]
absorbances = [lstsq(strongest, d)[0][0] for d in data]
#
# Normalize
#
absorbances = np.array(absorbances)
return absorbances / np.sqrt(np.sum(absorbances ** 2))
return None
def test_04_03_metadata_filename(self):
'''Make two different webpages using metadata'''
def alter_fn(module):
self.assertTrue(isinstance(module, C.CreateWebPage))
module.web_page_file_name.value = '\\g<FileName>'
self.run_create_webpage([(None, 'A01.png'),(None, 'A02.png')],
metadata={'FileName':['foo','bar']},
alter_fn = alter_fn)
for file_name, image_name in (('foo.html', 'A01.png'),
('bar.html', 'A02.png')):
path = os.path.join(cpprefs.get_default_image_directory(), file_name)
dom = self.read_html(path)
imgs = dom.getElementsByTagName("img")
self.assertEqual(len(imgs), 1)
img = imgs[0]
self.assertTrue(img.hasAttribute("src"))
self.assertEqual(img.getAttribute("src"), image_name)
def test_03_07_metadata(self):
m = cpmeas.Measurements()
m.add_image_measurement("Metadata_Path", "2")
s = cps.DirectoryPath("whatever", allow_metadata=True)
for dir_choice, expected in (
(cps.DEFAULT_INPUT_SUBFOLDER_NAME, os.path.join(cpprefs.get_default_image_directory(), "0", "2")),
(cps.DEFAULT_OUTPUT_SUBFOLDER_NAME, os.path.join(cpprefs.get_default_output_directory(), "0", "2")),
(cps.ABSOLUTE_FOLDER_NAME, os.path.join(self.root_directory, "2")),
(cps.URL_FOLDER_NAME, "http://www.cellprofiler.org/2"),
):
s.dir_choice = dir_choice
if dir_choice in (cps.DEFAULT_INPUT_SUBFOLDER_NAME, cps.DEFAULT_OUTPUT_SUBFOLDER_NAME):
s.custom_path = "0" + os.path.sep.replace("\\", "\\\\") + "\\g<Path>"
elif dir_choice == cps.ABSOLUTE_FOLDER_NAME:
s.custom_path = self.root_directory + os.path.sep.replace("\\", "\\\\") + "\\g<Path>"
else:
s.custom_path = "http://www.cellprofiler.org/\\g<Path>"
self.assertEqual(s.get_absolute_path(m), expected)
def test_04_03_metadata_filename(self):
'''Make two different webpages using metadata'''
def alter_fn(module):
self.assertTrue(isinstance(module, C.CreateWebPage))
module.web_page_file_name.value = '\\g<FileName>'
self.run_create_webpage([(None, 'A01.png'), (None, 'A02.png')],
metadata={'FileName': ['foo', 'bar']},
alter_fn=alter_fn)
for file_name, image_name in (('foo.html', 'A01.png'),
('bar.html', 'A02.png')):
path = os.path.join(cpprefs.get_default_image_directory(), file_name)
dom = self.read_html(path)
imgs = dom.getElementsByTagName("img")
self.assertEqual(len(imgs), 1)
img = imgs[0]
self.assertTrue(img.hasAttribute("src"))
self.assertEqual(img.getAttribute("src"), image_name)
def save_pipeline(self, workspace, outf=None):
'''Save the pipeline in Batch_data.mat
Save the pickled image_set_list state in a setting and put this
module in batch mode.
if outf is not None, it is used as a file object destination.
'''
from cellprofiler.utilities.version import version_number
if outf is None:
if self.wants_default_output_directory.value:
path = cpprefs.get_default_output_directory()
else:
path = cpprefs.get_absolute_path(self.custom_output_directory.value)
h5_path = os.path.join(path, F_BATCH_DATA_H5)
else:
h5_path = outf
image_set_list = workspace.image_set_list
pipeline = workspace.pipeline
m = cpmeas.Measurements(copy = workspace.measurements,
filename = h5_path)
assert isinstance(image_set_list, cpi.ImageSetList)
assert isinstance(pipeline, cpp.Pipeline)
assert isinstance(m, cpmeas.Measurements)
pipeline = pipeline.copy()
target_workspace = cpw.Workspace(pipeline, None, None, None,
m, image_set_list,
workspace.frame)
pipeline.prepare_to_create_batch(target_workspace, self.alter_path)
bizarro_self = pipeline.module(self.module_num)
bizarro_self.revision.value = version_number
if self.wants_default_output_directory:
bizarro_self.custom_output_directory.value = \
self.alter_path(cpprefs.get_default_output_directory())
bizarro_self.default_image_directory.value = \
self.alter_path(cpprefs.get_default_image_directory())
bizarro_self.batch_mode.value = True
pipeline.write_pipeline_measurement(m)
del m
def upgrade_settings(self, setting_values, variable_revision_number,
module_name, from_matlab):
if from_matlab and variable_revision_number == 8:
batch_save_path, old_pathname, new_pathname = setting_values[:3]
if batch_save_path == '.':
wants_default_output_directory = cps.YES
batch_save_path = cpprefs.get_default_output_directory()
else:
wants_default_output_directory = cps.NO
old_pathnames = old_pathname.split(',')
new_pathnames = new_pathname.split(',')
if len(old_pathnames) != len(new_pathnames):
raise ValueError("Number of pathnames does not match. "
"%d local pathnames, but %d remote pathnames" %
(len(old_pathnames), len(new_pathnames)))
setting_values = [wants_default_output_directory, batch_save_path,
cps.NO, cps.NO, ""]
for old_pathname, new_pathname in zip(old_pathnames, new_pathnames):
setting_values += [old_pathname, new_pathname]
from_matlab = False
variable_revision_number = 1
if (not from_matlab) and variable_revision_number == 1:
setting_values = (setting_values[:5] +
[cpprefs.get_default_image_directory()] +
setting_values[5:])
variable_revision_number = 2
if (not from_matlab) and variable_revision_number == 2:
from cellprofiler.utilities.get_revision import version
setting_values = (setting_values[:6] +
[version] +
setting_values[6:])
variable_revision_number = 3
if (not from_matlab) and variable_revision_number == 3:
# Pickled image list is now the batch state
self.batch_state = np.array(zlib.compress(setting_values[4]))
setting_values = setting_values[:4]+setting_values[5:]
variable_revision_number = 4
if (not from_matlab) and variable_revision_number == 4:
setting_values = setting_values[:4] + [False] + setting_values[4:]
variable_revision_number = 5
return setting_values, variable_revision_number, from_matlab
def save_pipeline(self, workspace, outf=None):
'''Save the pipeline in Batch_data.mat
Save the pickled image_set_list state in a setting and put this
module in batch mode.
if outf is not None, it is used as a file object destination.
'''
from cellprofiler.utilities.get_revision import version
if outf is None:
if self.wants_default_output_directory.value:
path = cpprefs.get_default_output_directory()
else:
path = cpprefs.get_absolute_path(self.custom_output_directory.value)
h5_path = os.path.join(path, F_BATCH_DATA_H5)
else:
h5_path = outf
image_set_list = workspace.image_set_list
pipeline = workspace.pipeline
m = cpmeas.Measurements(copy = workspace.measurements,
filename = h5_path)
assert isinstance(image_set_list, cpi.ImageSetList)
assert isinstance(pipeline, cpp.Pipeline)
assert isinstance(m, cpmeas.Measurements)
pipeline = pipeline.copy()
target_workspace = cpw.Workspace(pipeline, None, None, None,
m, image_set_list,
workspace.frame)
pipeline.prepare_to_create_batch(target_workspace, self.alter_path)
bizarro_self = pipeline.module(self.module_num)
bizarro_self.revision.value = version
if self.wants_default_output_directory:
bizarro_self.custom_output_directory.value = \
self.alter_path(cpprefs.get_default_output_directory())
bizarro_self.default_image_directory.value = \
self.alter_path(cpprefs.get_default_image_directory())
bizarro_self.batch_mode.value = True
pipeline.write_pipeline_measurement(m)
del m
def upgrade_settings(self, setting_values, variable_revision_number,
module_name, from_matlab):
if from_matlab and variable_revision_number == 8:
batch_save_path, old_pathname, new_pathname = setting_values[:3]
if batch_save_path == '.':
wants_default_output_directory = cps.YES
batch_save_path = cpprefs.get_default_output_directory()
else:
wants_default_output_directory = cps.NO
old_pathnames = old_pathname.split(',')
new_pathnames = new_pathname.split(',')
if len(old_pathnames) != len(new_pathnames):
raise ValueError("Number of pathnames does not match. "
"%d local pathnames, but %d remote pathnames" %
(len(old_pathnames), len(new_pathnames)))
setting_values = [wants_default_output_directory, batch_save_path,
cps.NO, cps.NO, ""]
for old_pathname, new_pathname in zip(old_pathnames, new_pathnames):
setting_values += [old_pathname, new_pathname]
from_matlab = False
variable_revision_number = 1
if (not from_matlab) and variable_revision_number == 1:
setting_values = (setting_values[:5] +
[cpprefs.get_default_image_directory()] +
setting_values[5:])
variable_revision_number = 2
if (not from_matlab) and variable_revision_number == 2:
from cellprofiler.utilities.get_revision import version
setting_values = (setting_values[:6] +
[version] +
setting_values[6:])
variable_revision_number = 3
if (not from_matlab) and variable_revision_number == 3:
# Pickled image list is now the batch state
self.batch_state = np.array(zlib.compress(setting_values[4]))
setting_values = setting_values[:4]+setting_values[5:]
variable_revision_number = 4
if (not from_matlab) and variable_revision_number == 4:
setting_values = setting_values[:4] + [False] + setting_values[4:]
variable_revision_number = 5
return setting_values, variable_revision_number, from_matlab
def test_03_07_metadata(self):
m = cpmeas.Measurements()
m.add_image_measurement("Metadata_Path", "2")
s = cps.DirectoryPath("whatever", allow_metadata = True)
for dir_choice, expected in (
( cps.DEFAULT_INPUT_SUBFOLDER_NAME,
os.path.join(cpprefs.get_default_image_directory(), "0", "2")),
( cps.DEFAULT_OUTPUT_SUBFOLDER_NAME,
os.path.join(cpprefs.get_default_output_directory(), "0", "2")),
( cps.ABSOLUTE_FOLDER_NAME,
os.path.join(self.root_directory, "2")),
( cps.URL_FOLDER_NAME, "http://www.cellprofiler.org/2")):
s.dir_choice = dir_choice
if dir_choice in (cps.DEFAULT_INPUT_SUBFOLDER_NAME,
cps.DEFAULT_OUTPUT_SUBFOLDER_NAME):
s.custom_path = "0" + os.path.sep.replace('\\','\\\\') + "\\g<Path>"
elif dir_choice == cps.ABSOLUTE_FOLDER_NAME:
s.custom_path = self.root_directory + os.path.sep.replace('\\','\\\\') + "\\g<Path>"
else:
s.custom_path = "http://www.cellprofiler.org/\\g<Path>"
self.assertEqual(s.get_absolute_path(m), expected)
def create_settings(self):
'''Create the module settings and name the module'''
self.wants_default_output_directory = cps.Binary("Store batch files in default output folder?", True,doc="""
Do you want to store the batch files in the default output folder?
Check this box to store batch files in the Default Output folder. Uncheck
the box to enter the path to the folder that will be used to store
these files.""")
self.custom_output_directory = cps.Text("Output folder path",
cpprefs.get_default_output_directory(),doc="""
What is the path to the output folder?""")
# Worded this way not because I am windows-centric but because it's
# easier than listing every other OS in the universe except for VMS
self.remote_host_is_windows = cps.Binary("Are the cluster computers running Windows?",
False,doc="""
Check this box if the cluster computers are running one of the Microsoft
Windows operating systems. If you check this box, <b>CreateBatchFiles</b> will
modify all paths to use the Windows file separator (backslash \). If you
leave the box unchecked, <b>CreateBatchFiles</b> will modify all paths to use
the Unix or Macintosh file separator (slash,/).""")
self.batch_mode = cps.Binary("Hidden: in batch mode", False)
self.distributed_mode = cps.Binary("Hidden: in distributed mode", False)
self.default_image_directory = cps.Setting("Hidden: default input folder at time of save",
cpprefs.get_default_image_directory())
self.revision = cps.Integer("Hidden: SVN revision number", 0)
self.mappings = []
self.add_mapping()
self.add_mapping_button = cps.DoSomething("", "Add another path mapping",
self.add_mapping, doc="""
Use this option if another path must be mapped because there is a difference between how the local computer sees a folder location vs. how the cluster computer sees the folder location.""")
self.check_path_button = cps.DoSomething(
"Press this button to check pathnames on the remote server",
"Check paths", self.check_paths,
doc = """This button will start a routine that will ask the
webserver to check whether the default input and default output
folders exist. It will also check whether all remote
path mappings exist.""")
def create_settings(self):
'''Create the module settings and name the module'''
self.wants_default_output_directory = cps.Binary(
"Store batch files in default output folder?", True, doc="""\
Select "*%(YES)s*" to store batch files in the Default Output folder.
Select "*%(NO)s*" to enter the path to the folder that will be used to
store these files. The Default Output folder can be set by clicking the "View output settings" button in the main CP window, or in CellProfiler Preferences. """ % globals())
self.custom_output_directory = cps.Text(
"Output folder path",
cpprefs.get_default_output_directory(), doc="Enter the path to the output folder. (Used only if not using the default output folder)")
# Worded this way not because I am windows-centric but because it's
# easier than listing every other OS in the universe except for VMS
self.remote_host_is_windows = cps.Binary(
"Are the cluster computers running Windows?",
False, doc="""\
Select "*%(YES)s*" if the cluster computers are running one of the
Microsoft Windows operating systems. In this case, **CreateBatchFiles**
will modify all paths to use the Windows file separator (backslash \\\\ ).
Select "*%(NO)s*" for **CreateBatchFiles** to modify all paths to use the
Unix or Macintosh file separator (slash / ).""" % globals())
self.batch_mode = cps.Binary("Hidden: in batch mode", False)
self.distributed_mode = cps.Binary("Hidden: in distributed mode", False)
self.default_image_directory = cps.Setting("Hidden: default input folder at time of save",
cpprefs.get_default_image_directory())
self.revision = cps.Integer("Hidden: revision number", 0)
self.from_old_matlab = cps.Binary("Hidden: from old matlab", False)
self.acknowledge_old_matlab = cps.DoSomething(
"Could not update CP1.0 pipeline to be compatible with CP2.0. See module notes.", "OK",
self.clear_old_matlab)
self.mappings = []
self.add_mapping()
self.add_mapping_button = cps.DoSomething("",
"Add another path mapping", self.add_mapping, doc="""\
Use this option if another path must be mapped because there is a difference
between how the local computer sees a folder location vs. how the cluster
computer sees the folder location.""")
def __init__(self,panel):
self.__panel = panel
self.__sizer = wx.BoxSizer(wx.VERTICAL)
self.__image_folder_panel = wx.Panel(panel,-1)
self.__image_edit_box = self.__make_folder_panel(
self.__image_folder_panel,
cpprefs.get_default_image_directory(),
lambda : cpprefs.get_recent_files(cpprefs.DEFAULT_IMAGE_DIRECTORY),
'Default Input Folder',
DEFAULT_IMAGE_FOLDER_HELP,
[cpprefs.set_default_image_directory,
self.__notify_pipeline_list_view_directory_change],
refresh_action = self.refresh_input_directory)
self.__output_folder_panel = wx.Panel(panel,-1)
self.__output_edit_box = self.__make_folder_panel(
self.__output_folder_panel,
cpprefs.get_default_output_directory(),
lambda : cpprefs.get_recent_files(cpprefs.DEFAULT_OUTPUT_DIRECTORY),
'Default Output Folder',
DEFAULT_OUTPUT_FOLDER_HELP,
[cpprefs.set_default_output_directory,
self.__notify_pipeline_list_view_directory_change])
self.__odds_and_ends_panel = wx.Panel(panel,-1)
self.__make_odds_and_ends_panel()
self.__status_text = wx.StaticText(panel,-1,style=wx.SUNKEN_BORDER,label=WELCOME_MESSAGE)
self.__progress_panel = wx.Panel(panel, -1)
self.__make_progress_panel()
self.__sizer.AddMany([(self.__image_folder_panel,0,wx.EXPAND|wx.ALL,1),
(self.__output_folder_panel,0,wx.EXPAND|wx.ALL,1),
(self.__odds_and_ends_panel,0,wx.EXPAND|wx.ALL,1),
(self.__status_text,0,wx.EXPAND|wx.ALL, 4),
(self.__progress_panel, 0, wx.EXPAND | wx.BOTTOM, 2)])
panel.SetSizer(self.__sizer)
self.__sizer.Hide(self.__progress_panel)
self.__errors = set()
self.__pipeline_list_view = None
self.__progress_watcher = None
def on_add(self, event):
for i in range(self.file_chooser.ItemCount):
if self.file_chooser.IsSelected(i):
path = os.path.join(
self.directory_picker.Path,
self.file_chooser.GetItemText(i))
index = self.pipeline_list_view.InsertStringItem(
sys.maxint, path)
self.pipeline_list_view.SetStringItem(
index, P_INPUT_DIRECTORY_COLUMN,
cpprefs.get_default_image_directory())
self.pipeline_list_view.SetStringItem(
index, P_OUTPUT_DIRECTORY_COLUMN,
cpprefs.get_default_output_directory())
self.pipeline_list_view.SetStringItem(
index, P_OUTPUT_FILE_COLUMN,
cpprefs.get_output_file_name())
self.pipeline_list_view.SetItemColumnImage(
index, P_REMOVE_BUTTON_COLUMN, self.delete_bmp_idx)
self.file_chooser.Select(i, False)
self.pipeline_list_view.SetColumnWidth(P_FILENAME_COLUMN, wx.LIST_AUTOSIZE)
self.pipeline_list_view.SetColumnWidth(P_INPUT_DIRECTORY_COLUMN, wx.LIST_AUTOSIZE)
self.pipeline_list_view.SetColumnWidth(P_OUTPUT_DIRECTORY_COLUMN, wx.LIST_AUTOSIZE)
self.pipeline_list_view.SetColumnWidth(P_OUTPUT_FILE_COLUMN, wx.LIST_AUTOSIZE)
def test_01_01_default_directory_none(self):
print cpprefs.get_default_image_directory()
self.assertTrue(True);
def upgrade_settings(self, setting_values, variable_revision_number,
module_name, from_matlab):
if from_matlab and variable_revision_number < 8:
# We never were able to convert from pre-8 to 8 in Matlab. Why may
# be lost to history, but my guess is there were conflicting ways
# to interpret settings previous to this point, so we decided not
# to try to automate it.
self.notes = ["The pipeline you loaded was from an old version of CellProfiler 1.0, "
"which could not be made compatible with this version of CellProfiler.",
"For reference, previous values were:"] + [str(x) for x in setting_values]
setting_values = [cps.NO,
"", cps.NO,
cps.NO, cps.NO,
"", 0, cps.YES]
variable_revision_number = 6
from_matlab = False
if from_matlab and variable_revision_number == 8:
batch_save_path, old_pathname, new_pathname = setting_values[:3]
if batch_save_path == '.':
wants_default_output_directory = cps.YES
batch_save_path = cpprefs.get_default_output_directory()
else:
wants_default_output_directory = cps.NO
old_pathnames = old_pathname.split(',')
new_pathnames = new_pathname.split(',')
if len(old_pathnames) != len(new_pathnames):
raise ValueError("Number of pathnames does not match. "
"%d local pathnames, but %d remote pathnames" %
(len(old_pathnames), len(new_pathnames)))
setting_values = [wants_default_output_directory, batch_save_path,
cps.NO, cps.NO, ""]
for old_pathname, new_pathname in zip(old_pathnames, new_pathnames):
setting_values += [old_pathname, new_pathname]
from_matlab = False
variable_revision_number = 1
if (not from_matlab) and variable_revision_number == 1:
setting_values = (setting_values[:5] +
[cpprefs.get_default_image_directory()] +
setting_values[5:])
variable_revision_number = 2
if (not from_matlab) and variable_revision_number == 2:
from cellprofiler.utilities.version import version_number
setting_values = (setting_values[:6] +
[version_number] +
setting_values[6:])
variable_revision_number = 3
if (not from_matlab) and variable_revision_number == 3:
# Pickled image list is now the batch state
self.batch_state = np.array(zlib.compress(setting_values[4]))
setting_values = setting_values[:4]+setting_values[5:]
variable_revision_number = 4
if (not from_matlab) and variable_revision_number == 4:
setting_values = setting_values[:4] + [False] + setting_values[4:]
variable_revision_number = 5
if (not from_matlab) and variable_revision_number == 5:
# added from_old_matlab
setting_values = setting_values[:7] + [False] + setting_values[7:]
variable_revision_number = 6
return setting_values, variable_revision_number, from_matlab
def test_01_04_get_parts_from_image_folder_path(self):
s = cps.DirectoryPath("whatever")
dir_choice, custom_path = s.get_parts_from_path(
cpprefs.get_default_image_directory())
self.assertEqual(dir_choice, cps.DEFAULT_INPUT_FOLDER_NAME)
def test_01_06_get_parts_from_image_subfolder_path(self):
s = cps.DirectoryPath("whatever")
dir_choice, custom_path = s.get_parts_from_path(
os.path.join(cpprefs.get_default_image_directory(), "1"))
self.assertEqual(dir_choice, cps.DEFAULT_INPUT_SUBFOLDER_NAME)
self.assertEqual(custom_path, "1")
def test_02_01_get_default_input_folder_path(self):
s = cps.DirectoryPath("whatever")
s.dir_choice = cps.DEFAULT_INPUT_FOLDER_NAME
self.assertEqual(s.get_absolute_path(), cpprefs.get_default_image_directory())
def __on_preferences_image_directory_event(self, event):
if self.__image_edit_box.Value != cpprefs.get_default_image_directory():
self.__image_edit_box.Value = cpprefs.get_default_image_directory()
def __on_preferences_image_directory_event(self, event):
if self.__image_edit_box.Value != cpprefs.get_default_image_directory(
):
self.__image_edit_box.Value = cpprefs.get_default_image_directory()
def test_01_01_default_directory_none(self):
print cpprefs.get_default_image_directory()
self.assertTrue(True)