def test_01_03_literal(self):
f1 = cps.Filter.FilterPredicate("foo", "Foo", lambda a, b: a == b,
[cps.Filter.LITERAL_PREDICATE])
f = cps.Filter("", [f1], 'foo "x"')
self.assertTrue(f.evaluate("x"))
self.assertFalse(f.evaluate("y"))
f = cps.Filter("", [f1], 'foo "y"')
self.assertTrue(f.evaluate("y"))
self.assertFalse(f.evaluate("x"))
def test_01_02_compound(self):
f2 = cps.Filter.FilterPredicate("bar", "Bar", lambda: "y", [])
f1 = cps.Filter.FilterPredicate("foo", "Foo", lambda a, b: a == b(),
[f2])
f = cps.Filter("", [f1], "foo bar")
self.assertFalse(f.evaluate("x"))
self.assertTrue(f.evaluate("y"))
def test_02_05_build_escaped_symbol(self):
ugly = '(\\")'
expected = '\\(\\\\\\"\\)'
f1 = cps.Filter.FilterPredicate(ugly, "Foo", lambda a,b: a==b, [])
f = cps.Filter("", [f1])
f.build([f1])
self.assertEqual(f.value, '\\(\\\\\\"\\)')
def test_01_01_simple(self):
filters = [
cps.Filter.FilterPredicate("foo", "Foo", lambda a: a == "x", [])
]
f = cps.Filter("", filters, "foo")
self.assertTrue(f.evaluate("x"))
self.assertFalse(f.evaluate("y"))
def test_01_07_or(self):
f1 = cps.Filter.FilterPredicate("eq", "Foo", lambda a, b: a == b,
[cps.Filter.LITERAL_PREDICATE])
f = cps.Filter("", [f1], 'or (eq "x") (eq "y")')
self.assertTrue(f.evaluate("x"))
self.assertTrue(f.evaluate("y"))
self.assertFalse(f.evaluate("z"))
def test_02_04_build_escaped_literal(self):
f1 = cps.Filter.FilterPredicate("foo", "Foo", lambda a,b: a==b,
[cps.Filter.LITERAL_PREDICATE])
f = cps.Filter("", [f1])
f.build([f1, '"12\\'])
self.assertEqual(f.value, 'foo "\\"12\\\\"')
tokens = f.parse()
self.assertEqual(tokens[1], '"12\\')
def test_02_06_parse_escaped_symbol(self):
ugly = '(\\")'
encoded_ugly = '\\(\\\\\\"\\)'
f1 = cps.Filter.FilterPredicate(ugly, "Foo", lambda a, b: a == b, [])
f = cps.Filter("", [f1], encoded_ugly)
result = f.parse()
self.assertEqual(len(result), 1)
self.assertEqual(result[0].symbol, ugly)
def create_settings(self):
self.workspace = None
module_explanation = [
"To begin creating your project, use the %s module to compile" %
self.module_name,
"a list of files and/or folders that you want to analyze. You can also specify a set of rules",
"to include only the desired files in your selected folders."
]
self.set_notes([" ".join(module_explanation)])
self.path_list_display = cps.PathListDisplay()
predicates = [
FilePredicate(),
DirectoryPredicate(),
ExtensionPredicate()
]
self.filter_choice = cps.Choice("Filter images?",
FILTER_CHOICE_ALL,
value=FILTER_CHOICE_IMAGES,
doc="""
The <b>Images</b> module will pass all the files specified in the file list
panel downstream to have a meaningful name assigned to it (so other modules can
access it) or optionally, to define the relationships between images and associated
metadata. Enabling file filtering will allow you to specify a subset of the files
from the file list panel by defining rules to filter the files. This approach is
useful if, for example, you drag-and-dropped a folder onto the file list panel
which contains a mixture of images that you want to analyze and other files that
you want to ignore.
<p>Several options are available for this setting:
<ul>
<li><i>%(FILTER_CHOICE_NONE)s:</i> Do not enable filtering; all files in the File list
panel will be passed to downstream modules for processing. This option can be
selected if you are sure that only images are specified in the list.</li>
<li><i>%(FILTER_CHOICE_IMAGES)s:</i> Only image files will be passed to downstream
modules. The permissible image formats are provided by a library called Bio-Formats; see
<a href="http://www.openmicroscopy.org/site/support/bio-formats5/supported-formats.html">here</a> for the formats available.</li>
<li><i>%(FILTER_CHOICE_CUSTOM)s:</i> Specify custom rules for selecting a subset of
the files from the File list panel. This approach is useful if, for example, you
drag-and-dropped a folder onto the File list panel which contains a mixture of images
that you want to analyze and other files that you want to ignore.</li>
</ul></p>""" % globals())
self.filter = cps.Filter("Select the rule criteria",
predicates,
FILTER_DEFAULT,
doc="""
Specify a set of rules to narrow down the files to be analyzed.
<p>%(FILTER_RULES_BUTTONS_HELP)s</p>""" % globals())
self.update_button = cps.PathListRefreshButton(
"",
"Apply filters to the file list",
doc="""
<i>(Only displayed if filtering based on rules)</i><br>
Re-display the file list, removing or graying out the files
that do not pass the current filter.
""")
def test_01_06_parentheses(self):
f1 = cps.Filter.FilterPredicate("eq", "Foo", lambda a, b: a == b,
[cps.Filter.LITERAL_PREDICATE])
f2 = cps.Filter.FilterPredicate("ne", "Bar", lambda a, b: a != b,
[cps.Filter.LITERAL_PREDICATE])
f = cps.Filter("", [f1, f2], 'and (eq "x") (ne "y")')
self.assertTrue(f.evaluate("x"))
self.assertFalse(f.evaluate("y"))
self.assertFalse(f.evaluate("z"))
def create_settings(self):
self.workspace = None
module_explanation = [
"To begin creating your project, use the %s module to compile" %
self.module_name,
"a list of files and/or folders that you want to analyze. You can also specify a set of rules",
"to include only the desired files in your selected folders."
]
self.set_notes([" ".join(module_explanation)])
self.path_list_display = cps.PathListDisplay()
predicates = [
FilePredicate(),
DirectoryPredicate(),
ExtensionPredicate()
]
self.wants_filter = cps.Binary(
"Filter images based on matching rules?",
True,
doc="""
Check this setting to display and use rules to filter files for processing.
<p>By default, <b>Images</b> will pass all the files specified in the file list
panel downstream to have a meaningful name assigned to it (so other modules can access it) or optionally, to
define the relationships between images and associated metadata. Selecting this option will allow you to specify
a subset of the files from the file list panel by defining rules to filter the files. This
approach is useful if, for example, you drag-and-dropped a folder onto the file list panel
which contains a mixture of images that you want to analyze and other files that you want to ignore.</p>"""
)
self.filter = cps.Filter(
"Select the rule criteria",
predicates,
'and (extension does isimage) (directory doesnot startwith ".")',
doc="""
Specify a set of rules to narrow down the files to be analyzed.
<p>%(FILTER_RULES_BUTTONS_HELP)s</p>""" % globals())
self.update_button = cps.PathListRefreshButton(
"",
"Apply filters to the file list",
doc="""
<i>(Only displayed if filtering based on rules)</i><br>
Re-display the file list, removing or graying out the files
that do not pass the current filter.
""")
def test_02_03_build_nested(self):
f1 = cps.Filter.FilterPredicate("foo", "Foo", lambda a, b: a == b,
[cps.Filter.LITERAL_PREDICATE])
f = cps.Filter("", [f1])
f.build([cps.Filter.OR_PREDICATE, [f1, "bar"], [f1, u"baz"]])
self.assertEqual(f.text, 'or (foo "bar") (foo "baz")')
def test_02_02_build_literal(self):
f1 = cps.Filter.FilterPredicate("foo", "Foo", lambda a, b: a == b,
[cps.Filter.LITERAL_PREDICATE])
f = cps.Filter("", [f1])
f.build([f1, "bar"])
self.assertEqual(f.text, 'foo "bar"')
def test_02_01_build_one(self):
f1 = cps.Filter.FilterPredicate("foo", "Foo", lambda a: a == "foo", [])
f = cps.Filter("", [f1])
f.build([f1])
self.assertEqual(f.text, "foo")
def test_01_05_literal_with_quote(self):
f1 = cps.Filter.FilterPredicate("foo", "Foo", lambda a, b: a == b,
[cps.Filter.LITERAL_PREDICATE])
f = cps.Filter("", [f1], 'foo "\\""')
self.assertTrue(f.evaluate("\""))
self.assertFalse(f.evaluate("/"))
def add_extraction_method(self, can_remove=True):
group = cps.SettingsGroup()
self.extraction_methods.append(group)
if can_remove:
group.append("divider", cps.Divider())
group.append(
"extraction_method",
cps.Choice("Extraction method",
X_ALL_EXTRACTION_METHODS,
doc="""
The <b>Metadata</b> module can extract metadata from the images one of three ways:
<ul>
<li><i>%(X_AUTOMATIC_EXTRACTION)s</i>: From the structure of the file format itself.</li>
<li><i>%(X_MANUAL_EXTRACTION)s</i>: From the file nomenclature and/or location, often
structure according to the user or acquistion software vendor.</li>
<li><i>%(X_IMPORTED_EXTRACTION)s</i>: From a comma-delimited list (csv) of information;
provided as one type of metadata per column, and one row of metadata per image. This is a
convenient way for you to add data from your own sources to the output generated by
CellProfiler.</li>
</ul>
For more details on how metadata is used downstream from this module, see the help for
the <b>NamesAndTypes</b> or <b>Groups</b> module, or <i>%(METADATA_HELP_REF)s</i>"""
% globals()))
group.append(
"source",
cps.Choice("Source", [XM_FILE_NAME, XM_FOLDER_NAME],
doc="""
Do you want to extract metadata from the image's file
name or from its folder name?"""))
group.append(
"file_regexp",
cps.RegexpText(
"Regular expression",
'^(?P<Plate>.*)_(?P<Well>[A-P][0-9]{2})_s(?P<Site>[0-9])_w(?P<ChannelNumber>[0-9])',
get_example_fn=self.example_file_fn,
doc="""
<a name='regular_expression'><i>(Used only if you want to extract
metadata from the file name)</i><br>
The regular expression to extract the metadata from the file name
is entered here. Note that this field is available whether you have
selected <i>Text-Regular expressions</i> to load the files or not.
Please see the general module help for more information on
construction of a regular expression.</a>
<p>Clicking the magnifying glass icon to the right will bring up a
tool for checking the accuracy of your regular expression. The
regular expression syntax can be used to name different parts of
your expression. The syntax <i>(?P<fieldname>expr)</i> will
extract whatever matches <i>expr</i> and assign it to the
measurement,<i>fieldname</i> for the image.
<p>For instance, a researcher uses plate names composed of a string
of letters and numbers, followed by an underscore, then the well,
followed by another underscore, followed by an "s" and a digit
representing the site taken within the well (e.g., <i>TE12345_A05_s1.tif</i>).
The following regular expression will capture the plate, well, and
site in the fields "Plate", "Well", and "Site":<br><br>
<table border = "1">
<tr><td colspan = "2">^(?P<Plate>.*)_(?P<Well>[A-P][0-9]{1,2})_s(?P<Site>[0-9])</td></tr>
<tr><td>^</td><td>Start only at beginning of the file name</td></tr>
<tr><td>(?P<Plate></td><td>Name the captured field <i>Plate</i></td></tr>
<tr><td>.*</td><td>Capture as many characters as follow</td></tr>
<tr><td>_</td><td>Discard the underbar separating plate from well</td></tr>
<tr><td>(?P<Well></td><td>Name the captured field <i>Well</i></td></tr>
<tr><td>[A-P]</td><td>Capture exactly one letter between A and P</td></tr>
<tr><td>[0-9]{1,2}</td><td>Capture one or two digits that follow</td></tr>
<tr><td>_s</td><td>Discard the underbar followed by <i>s</i> separating well from site</td></tr>
<tr><td>(?P<Site></td><td>Name the captured field <i>Site</i></td></tr>
<tr><td>[0-9]</td><td>Capture one digit following</td></tr>
</table>
<p>The regular expression can be typed in the upper text box, with
a sample file name given in the lower text box. Provided the syntax
is correct, the corresponding fields will be highlighted in the same
color in the two boxes. Press <i>Submit</i> to enter the typed
regular expression.</p>
<p>You can create metadata tags for any portion of the filename or path, but if you are
specifying metadata for multiple images in a single <b>LoadImages</b> module, an image cycle can
only have one set of values for each metadata tag. This means that you can only
specify the metadata tags which have the same value across all images listed in the module. For example,
in the example above, you might load two wavelengths of data, one named <i>TE12345_A05_s1_w1.tif</i>
and the other <i>TE12345_A05_s1_w2.tif</i>, where the number following the <i>w</i> is the wavelength.
In this case, a "Wavelength" tag <i>should not</i> be included in the regular expression
because while the "Plate", "Well" and "Site" metadata is identical for both images, the wavelength metadata is not.</p>
<p>Note that if you use the special fieldnames <i><WellColumn></i> and
<i><WellRow></i> together, LoadImages will automatically create a <i><Well></i>
metadata field by joining the two fieldname values together. For example,
if <i><WellRow></i> is "A" and <i><WellColumn></i> is "01", a field
<i><Well></i> will be "A01". This is useful if your well row and column names are
separated from each other in the filename, but you want to retain the standard
well nomenclature.</p>"""))
group.append(
"folder_regexp",
cps.RegexpText("Regular expression",
'(?P<Date>[0-9]{4}_[0-9]{2}_[0-9]{2})$',
get_example_fn=self.example_directory_fn,
doc="""
<i>(Used only if you want to extract metadata from the path)</i><br>
Enter the regular expression for extracting the metadata from the
path. Note that this field is available whether you have selected
<i>Text-Regular expressions</i> to load the files or not.
<p>Clicking the magnifying glass icon to the right will bring up a
tool that will allow you to check the accuracy of your regular
expression. The regular expression syntax can be used to
name different parts of your expression. The syntax
<i>(?<fieldname>expr)</i> will extract whatever matches
<i>expr</i> and assign it to the image's <i>fieldname</i> measurement.
<p>For instance, a researcher uses folder names with the date and
subfolders containing the images with the run ID
(e.g., <i>./2009_10_02/1234/</i>) The following regular expression
will capture the plate, well, and site in the fields
<i>Date</i> and <i>Run</i>:<br>
<table border = "1">
<tr><td colspan = "2">.*[\\\/](?P<Date>.*)[\\\\/](?P<Run>.*)$</td></tr>
<tr><td>.*[\\\\/]</td><td>Skip characters at the beginning of the pathname until either a slash (/) or
backslash (\\) is encountered (depending on the operating system)</td></tr>
<tr><td>(?P<Date></td><td>Name the captured field <i>Date</i></td></tr>
<tr><td>.*</td><td>Capture as many characters that follow</td></tr>
<tr><td>[\\\\/]</td><td>Discard the slash/backslash character</td></tr>
<tr><td>(?P<Run></td><td>Name the captured field <i>Run</i></td></tr>
<tr><td>.*</td><td>Capture as many characters as follow</td></tr>
<tr><td>$</td><td>The <i>Run</i> field must be at the end of the path string, i.e., the
last folder on the path. This also means that the Date field contains the parent
folder of the Date folder.</td></tr>
</table></p>"""))
group.append(
"filter_choice",
cps.Choice("Extract metadata from:",
[F_ALL_IMAGES, F_FILTERED_IMAGES],
doc="""
Do you want to extract metadata from all of the images
chosen by the <b>Images</b> module or on a subset of the images?
<p>This setting controls how different image types (e.g., an image
of the GFP stain and a brightfield image) have different metadata
extracted. There are two choices:<br>
<ul>
<li><i>%(F_ALL_IMAGES)s</i>: Extract metadata from all images specified in
<b>Images</b>. This is the simplest choice and the appropriate one if you have
only one kind of image (or only one image). CellProfiler will
extract metadata from all images using the same method per iteration.</li>
<li><i>%(F_FILTERED_IMAGES)s</i>: Extract metadata depending on specific file
attributes. This is the appropriate choice if more than one image was taken of each
imaging site. You can specify distinctive criteria for each image subset with
matching metadata.</li>
</ul></p>""" % globals()))
group.append(
"filter",
cps.Filter(
"",
[FilePredicate(),
DirectoryPredicate(),
ExtensionPredicate()],
'and (file does contain "")',
doc="""
Check this setting to display and use rules to select files for metadata extraction.
<p>%(FILTER_RULES_BUTTONS_HELP)s</p>""" % globals()))
group.append(
"csv_location",
cps.Pathname(
"Metadata file location:",
wildcard="Metadata files (*.csv)|*.csv|All files (*.*)|*.*"))
group.append(
"csv_joiner",
cps.Joiner(
"Match file and image metadata",
allow_none=False,
doc="""Match columns in your .csv file to image metadata items
<hr>
This setting controls how rows in your .csv file are matched to
different images. The setting displays the columns in your
.csv file in one of its columns and the metadata in your images
in the other, including the metadata extracted by previous
metadata extractors in this module.
"""))
group.append(
"wants_case_insensitive",
cps.Binary(
"Case insensitive matching",
False,
doc="""This setting controls whether row matching takes the
metadata case into account when matching. If this setting is
not checked, metadata entries that only differ by case
(for instance, "A01" and "a01") will not match. If this setting
is checked, then metadata entries that only differ by case
will match. Check this setting if your CSV metadata is not being
applied because the case does not match."""))
group.append(
"update_metadata",
cps.DoSomething(
"Update",
"Update metadata",
lambda: self.do_update_metadata(group),
doc="""Press this button to automatically extract metadata from
your image files."""))
group.can_remove = can_remove
if can_remove:
group.append(
"remover",
cps.RemoveSettingButton('Remove above extraction method',
'Remove', self.extraction_methods,
group))
