def getListOfExperiments(self):
settings = ConfigSettings(SECTION_REGISTRY)
settings.read(self.oBatchSettings.settingsFilename)
settings.set_section(SECTION_NAME_GENERAL)
settings.set2('pathin', self.oBatchSettings.baseInDir)
settings.set2('pathout', self.oBatchSettings.baseOutDir)
imagecontainer = ImageContainer()
imagecontainer.import_from_settings(settings)
if self.oBatchSettings.plates is None:
plates = imagecontainer.plates
else:
plates = self.oBatchSettings.plates
lstExperiments = []
for plate in plates:
meta_data = imagecontainer.get_meta_data(plate)
positions = meta_data.positions
if self.oBatchSettings.omit_processed_positions:
finished_pos = [os.path.splitext(x)[0].split('__')[0] for x in
os.listdir(os.path.join(self.oBatchSettings.baseOutDir,
plate, 'log', '_finished'))]
positions = filter(lambda x: x not in finished_pos, positions)
lstExperiments.extend([(x,y) for x,y in zip([plate for i in positions],
positions) ])
return lstExperiments
def core_helper(plate,
settings_dict,
imagecontainer,
position,
version,
redirect=True,
debug=False):
"""Embeds analysis of a positon in a single function"""
# see http://stackoverflow.com/questions/3288595/
# multiprocessing-using-pool-map-on-a-function-defined-in-a-class
logger = logging.getLogger(str(os.getpid()))
import numpy
reload(numpy.core._dotblas)
try:
settings = ConfigSettings()
settings.from_dict(settings_dict)
settings.set('General', 'constrain_positions', True)
settings.set('General', 'positions', position)
environ = CecogEnvironment(version, redirect=redirect, debug=debug)
if debug:
environ.pprint()
analyzer = AnalyzerCore(plate, settings, imagecontainer)
post_hdf5_link_list = analyzer.processPositions()
return plate, position, copy.deepcopy(post_hdf5_link_list)
except Exception as e:
errortxt = "plate: %s, position: %s\n" % (plate, position)
errortxt = "".join([errortxt] + \
traceback.format_exception(*sys.exc_info()))
logger.error(errortxt)
raise e.__class__(errortxt)
def core_helper(plate, settings_dict, imagecontainer, position, version,
mode="r+", redirect=True):
"""Embedds analysis of a positon in a single function"""
# see http://stackoverflow.com/questions/3288595/
# multiprocessing-using-pool-map-on-a-function-defined-in-a-class
logger = logging.getLogger(str(os.getpid()))
try:
# FIXME numpy 1.11 does not have ._dotblas
import numpy.core._dotblas
reload(numpy.core._dotblas)
except ImportError as e:
pass
try:
settings = ConfigSettings()
settings.from_dict(settings_dict)
settings.set('General', 'constrain_positions', True)
settings.set('General', 'positions', position)
environ = CecogEnvironment(version, redirect=redirect)
analyzer = PlateAnalyzer(plate, settings, imagecontainer, mode=mode)
analyzer()
return plate, position
except Exception as e:
errortxt = "Plate: %s, Site: %s\n" %(plate, position)
errortxt = "".join([errortxt] + \
traceback.format_exception(*sys.exc_info()))
logger.error(errortxt)
raise type(e)(errortxt)
def core_helper(plate, settings_dict, imagecontainer, position, version,
redirect=True, debug=False):
"""Embeds analysis of a positon in a single function"""
# see http://stackoverflow.com/questions/3288595/
# multiprocessing-using-pool-map-on-a-function-defined-in-a-class
logger = logging.getLogger(str(os.getpid()))
import numpy
reload(numpy.core._dotblas)
try:
settings = ConfigSettings()
settings.from_dict(settings_dict)
settings.set('General', 'constrain_positions', True)
settings.set('General', 'positions', position)
environ = CecogEnvironment(version, redirect=redirect, debug=debug)
if debug:
environ.pprint()
analyzer = AnalyzerCore(plate, settings, imagecontainer)
post_hdf5_link_list = analyzer.processPositions()
return plate, position, copy.deepcopy(post_hdf5_link_list)
except Exception as e:
errortxt = "plate: %s, position: %s\n" %(plate, position)
errortxt = "".join([errortxt] + \
traceback.format_exception(*sys.exc_info()))
logger.error(errortxt)
raise e.__class__(errortxt)
def core_helper(plate_id, settings_str, imagecontainer, position):
settings = ConfigSettings(SECTION_REGISTRY)
settings.from_string(settings_str)
settings.set(SECTION_NAME_GENERAL, 'constrain_positions', True)
settings.set(SECTION_NAME_GENERAL, 'positions', position)
try:
analyzer = AnalyzerCore(plate_id, settings, imagecontainer)
result = analyzer.processPositions()
except Exception:
traceback.print_exc()
raise
return plate_id, position, copy.deepcopy(result['post_hdf5_link_list'])
def core_helper(plate_id, settings_str, imagecontainer, position):
settings = ConfigSettings(SECTION_REGISTRY)
settings.from_string(settings_str)
settings.set(SECTION_NAME_GENERAL, 'constrain_positions', True)
settings.set(SECTION_NAME_GENERAL, 'positions', position)
try:
analyzer = AnalyzerCore(plate_id, settings, imagecontainer)
result = analyzer.processPositions()
except Exception:
traceback.print_exc()
raise
return plate_id, position, copy.deepcopy(result['post_hdf5_link_list'])
def core_helper(plate,
settings_dict,
imagecontainer,
position,
version,
mode="r+",
redirect=True):
"""Embedds analysis of a positon in a single function"""
# see http://stackoverflow.com/questions/3288595/
# multiprocessing-using-pool-map-on-a-function-defined-in-a-class
logger = logging.getLogger(str(os.getpid()))
try:
# FIXME numpy 1.11 does not have ._dotblas
import numpy.core._dotblas
reload(numpy.core._dotblas)
except ImportError as e:
pass
try:
settings = ConfigSettings()
settings.from_dict(settings_dict)
settings.set('General', 'constrain_positions', True)
settings.set('General', 'positions', position)
environ = CecogEnvironment(version, redirect=redirect)
analyzer = PlateAnalyzer(plate, settings, imagecontainer, mode=mode)
analyzer()
return plate, position
except Exception as e:
errortxt = "Plate: %s, Site: %s\n" % (plate, position)
errortxt = "".join([errortxt] + \
traceback.format_exception(*sys.exc_info()))
logger.error(errortxt)
raise type(e)(errortxt)
logger.info("*************************************************" + '*'*len(version))
logger.info("*** CellCognition - Batch Analyzer - Version %s ***" %version)
logger.info("*************************************************" + '*'*len(version))
logger.info('argv: %s' % sys.argv)
environ = CecogEnvironment(version)
if options.settings is None:
parser.error('Settings filename required.')
environ = CecogEnvironment(version, redirect=False, debug=False)
filename_settings = os.path.abspath(options.settings)
# read the settings data from file
settings = ConfigSettings()
settings.read(filename_settings)
settings.set_section(SECTION_NAME_GENERAL)
index = options.cluster_index
batch_size = options.batch_size
position_list = options.position_list
create_images = options.create_images
multiple_plates = options.multiple_plates
path_input = options.input
path_output = options.output
if path_input is not None:
settings.set2('pathin', path_input)
logger.info('Overwrite input path by %s' % path_input)
def set(self, section_name, trait_name, value):
ConfigSettings.set(self, section_name, trait_name, value)
if not self._parent is None and self._notify_change:
self._parent.settings_changed(True)
def __init__(self, parent, section_registry):
self._parent = parent
self._notify_change = True
ConfigSettings.__init__(self, section_registry)
def __init__(self, parent):
self._parent = parent
self._notify_change = True
ConfigSettings.__init__(self)
def __init__(self, configfile):
self.img_height = None
self.img_width = None
self.settings = ConfigSettings()
self.settings.read(configfile)
class SettingsMapper(object):
"""Map parameters from a ConfigSettings instance to groups to fit
the API"""
CHANNEL_CLASSES = (PrimaryChannel, SecondaryChannel, TertiaryChannel)
FEATURES = {'featurecategory_intensity': ['normbase', 'normbase2'],
'featurecategory_haralick': ['haralick', 'haralick2'],
'featurecategory_stat_geom': ['levelset'],
'featurecategory_granugrey': ['granulometry'],
'featurecategory_basicshape': ['roisize',
'circularity',
'irregularity',
'irregularity2',
'axes'],
'featurecategory_convhull': ['convexhull'],
'featurecategory_distance': ['distance'],
'featurecategory_moments': ['moments']}
def __init__(self, configfile):
self.img_height = None
self.img_width = None
self.settings = ConfigSettings()
self.settings.read(configfile)
def __call__(self, section, param):
return self.settings.get(section, param)
def setImageSize(self, width, height):
self.img_width = width
self.img_height = height
@property
def img_size(self):
return self.img_width, self.img_height
def featureParams(self, ch_name="Primary"):
f_categories = list()
f_cat_params = dict()
# unfortunateley some classes expect empty list and dict
if ch_name.lower() in CH_VIRTUAL:
return f_categories, f_cat_params
for cat, feature in self.FEATURES.iteritems():
featopt = '%s_%s' %(ch_name, cat)
if self('FeatureExtraction', featopt):
if "haralick" in cat:
try:
f_cat_params['haralick_categories'].extend(feature)
except KeyError:
assert isinstance(feature, list)
f_cat_params['haralick_categories'] = feature
else:
f_categories += feature
if f_cat_params.has_key("haralick_categories"):
f_cat_params['haralick_distances'] = (1, 2, 4, 8)
return f_categories, f_cat_params
def zsliceParams(self, chname):
self.settings.set_section('ObjectDetection')
if self("ObjectDetection", "%s_%s" %(chname.lower(), 'zslice_selection')):
par = self("ObjectDetection", "%s_%s" %(chname.lower(), 'zslice_selection_slice'))
elif self("ObjectDetection", "%s_%s" %(chname.lower(), 'zslice_projection')):
method = self("ObjectDetection", "%s_%s" %(chname.lower(), 'zslice_projection_method'))
begin = self("ObjectDetection", "%s_%s" %(chname.lower(), 'zslice_projection_begin'))
end = self("ObjectDetection", "%s_%s" %(chname.lower(), 'zslice_projection_end'))
step = self("ObjectDetection", "%s_%s" %(chname.lower(), 'zslice_projection_step'))
par = (method, begin, end, step)
return par
def registrationShift(self):
xs = [0]
ys = [0]
for prefix in (SecondaryChannel.PREFIX, TertiaryChannel.PREFIX):
if self('General','process_%s' %prefix):
reg_x = self('ObjectDetection', '%s_channelregistration_x' %prefix)
reg_y = self('ObjectDetection', '%s_channelregistration_y' %prefix)
xs.append(reg_x)
ys.append(reg_y)
diff_x = []
diff_y = []
for i in range(len(xs)):
for j in range(i, len(xs)):
diff_x.append(abs(xs[i]-xs[j]))
diff_y.append(abs(ys[i]-ys[j]))
if self('General', 'crop_image'):
y0 = self('General', 'crop_image_y0')
y1 = self('General', 'crop_image_y1')
x0 = self('General', 'crop_image_x0')
x1 = self('General', 'crop_image_x1')
self.img_width = x1 - x0
self.img_height = y1 - y0
if self.img_height is None or self.img_width is None:
raise RuntimeError("Images size is not set. Use self.setImageSize(*size)")
# new image size after registration of all images
image_size = (self.img_width - max(diff_x),
self.img_height - max(diff_y))
return (max(xs), max(ys)), image_size
def channelParams(self, chname="Primary", color=None):
f_cats, f_params = self.featureParams(chname)
shift, size = self.registrationShift()
params = {'strChannelId': color,
'channelRegistration': (self(
'ObjectDetection', '%s_channelregistration_x' %chname),
self(
'ObjectDetection', '%s_channelregistration_y' %chname)),
'oZSliceOrProjection': self.zsliceParams(chname),
'new_image_size': size,
'registration_start': shift,
'fNormalizeMin': self('ObjectDetection', '%s_normalizemin' %chname),
'fNormalizeMax': self('ObjectDetection', '%s_normalizemax' %chname),
'lstFeatureCategories': f_cats,
'dctFeatureParameters': f_params}
return params
def channelRegions(self):
"""Return a dict of channel region pairs according to the classifier."""
regions = OrderedDict()
for ch_cls in self.CHANNEL_CLASSES:
name = ch_cls.NAME
if not ch_cls.is_virtual():
region = self( \
"Classification", "%s_classification_regionname" %(name))
# no plugins loaded
if region not in (None, ""):
regions[name] = region
else:
regions2 = OrderedDict()
for ch_cls2 in self.CHANNEL_CLASSES:
if ch_cls2.is_virtual():
continue
name2 = ch_cls2.NAME
if self("Classification", "merge_%s" %name2):
regions2[name2] = self("Classification", "%s_%s_region" %(name, name2))
if regions2:
regions[name] = regions2
return regions
def __init__(self, configfile):
self.img_height = None
self.img_width = None
self.settings = ConfigSettings()
self.settings.read(configfile)
class SettingsMapper(object):
"""Map parameters from a ConfigSettings instance to groups to fit
the API"""
CHANNEL_CLASSES = (PrimaryChannel, SecondaryChannel, TertiaryChannel)
FEATURES = {
'featurecategory_intensity': ['normbase', 'normbase2'],
'featurecategory_haralick': ['haralick', 'haralick2'],
'featurecategory_stat_geom': ['levelset'],
'featurecategory_granugrey': ['granulometry'],
'featurecategory_basicshape':
['roisize', 'circularity', 'irregularity', 'irregularity2', 'axes'],
'featurecategory_convhull': ['convexhull'],
'featurecategory_distance': ['distance'],
'featurecategory_moments': ['moments']
}
def __init__(self, configfile):
self.img_height = None
self.img_width = None
self.settings = ConfigSettings()
self.settings.read(configfile)
def __call__(self, section, param):
return self.settings.get(section, param)
def setImageSize(self, width, height):
self.img_width = width
self.img_height = height
@property
def img_size(self):
return self.img_width, self.img_height
def featureParams(self, ch_name="Primary"):
f_categories = list()
f_cat_params = dict()
# unfortunateley some classes expect empty list and dict
if ch_name.lower() in CH_VIRTUAL:
return f_categories, f_cat_params
for cat, feature in self.FEATURES.iteritems():
featopt = '%s_%s' % (ch_name, cat)
if self('FeatureExtraction', featopt):
if "haralick" in cat:
try:
f_cat_params['haralick_categories'].extend(feature)
except KeyError:
assert isinstance(feature, list)
f_cat_params['haralick_categories'] = feature
else:
f_categories += feature
if f_cat_params.has_key("haralick_categories"):
f_cat_params['haralick_distances'] = (1, 2, 4, 8)
return f_categories, f_cat_params
def zsliceParams(self, chname):
self.settings.set_section('ObjectDetection')
if self("ObjectDetection",
"%s_%s" % (chname.lower(), 'zslice_selection')):
par = self("ObjectDetection",
"%s_%s" % (chname.lower(), 'zslice_selection_slice'))
elif self("ObjectDetection",
"%s_%s" % (chname.lower(), 'zslice_projection')):
method = self(
"ObjectDetection",
"%s_%s" % (chname.lower(), 'zslice_projection_method'))
begin = self("ObjectDetection",
"%s_%s" % (chname.lower(), 'zslice_projection_begin'))
end = self("ObjectDetection",
"%s_%s" % (chname.lower(), 'zslice_projection_end'))
step = self("ObjectDetection",
"%s_%s" % (chname.lower(), 'zslice_projection_step'))
par = (method, begin, end, step)
return par
def registrationShift(self):
xs = [0]
ys = [0]
for prefix in (SecondaryChannel.PREFIX, TertiaryChannel.PREFIX):
if self('General', 'process_%s' % prefix):
reg_x = self('ObjectDetection',
'%s_channelregistration_x' % prefix)
reg_y = self('ObjectDetection',
'%s_channelregistration_y' % prefix)
xs.append(reg_x)
ys.append(reg_y)
diff_x = []
diff_y = []
for i in range(len(xs)):
for j in range(i, len(xs)):
diff_x.append(abs(xs[i] - xs[j]))
diff_y.append(abs(ys[i] - ys[j]))
if self('General', 'crop_image'):
y0 = self('General', 'crop_image_y0')
y1 = self('General', 'crop_image_y1')
x0 = self('General', 'crop_image_x0')
x1 = self('General', 'crop_image_x1')
self.img_width = x1 - x0
self.img_height = y1 - y0
if self.img_height is None or self.img_width is None:
raise RuntimeError(
"Images size is not set. Use self.setImageSize(*size)")
# new image size after registration of all images
image_size = (self.img_width - max(diff_x),
self.img_height - max(diff_y))
return (max(xs), max(ys)), image_size
def channelParams(self, chname="Primary", color=None):
f_cats, f_params = self.featureParams(chname)
shift, size = self.registrationShift()
params = {
'strChannelId':
color,
'channelRegistration':
(self('ObjectDetection', '%s_channelregistration_x' % chname),
self('ObjectDetection', '%s_channelregistration_y' % chname)),
'oZSliceOrProjection':
self.zsliceParams(chname),
'new_image_size':
size,
'registration_start':
shift,
'fNormalizeMin':
self('ObjectDetection', '%s_normalizemin' % chname),
'fNormalizeMax':
self('ObjectDetection', '%s_normalizemax' % chname),
'lstFeatureCategories':
f_cats,
'dctFeatureParameters':
f_params
}
return params
def channelRegions(self):
"""Return a dict of channel region pairs according to the classifier."""
regions = OrderedDict()
for ch_cls in self.CHANNEL_CLASSES:
name = ch_cls.NAME
if not ch_cls.is_virtual():
region = self( \
"Classification", "%s_classification_regionname" %(name))
# no plugins loaded
if region not in (None, ""):
regions[name] = region
else:
regions2 = OrderedDict()
for ch_cls2 in self.CHANNEL_CLASSES:
if ch_cls2.is_virtual():
continue
name2 = ch_cls2.NAME
if self("Classification", "merge_%s" % name2):
regions2[name2] = self("Classification",
"%s_%s_region" % (name, name2))
if regions2:
regions[name] = regions2
return regions
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.setLevel(logging.INFO)
logger.info("*************************************************" + '*'*len(VERSION))
logger.info("*** CellCognition - Batch Analyzer - Version %s ***" % VERSION)
logger.info("*************************************************" + '*'*len(VERSION))
logger.info('argv: %s' % sys.argv)
if options.settings is None:
parser.error('Settings filename required.')
filename_settings = os.path.abspath(options.settings)
# read the settings data from file
settings = ConfigSettings(SECTION_REGISTRY)
settings.read(filename_settings)
settings.set_section(SECTION_NAME_GENERAL)
index = options.cluster_index
batch_size = options.batch_size
position_list = options.position_list
create_images = options.create_images
multiple_plates = options.multiple_plates
path_input = options.input
path_output = options.output
if path_input is not None:
settings.set2('pathin', path_input)
logger.info('Overwrite input path by %s' % path_input)
formatter = logging.Formatter('%(asctime)s %(levelname)-6s %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.setLevel(logging.ERROR)
print "*"*(len(version) + 53)
print "*** CellCognition - Batch Script - Version %s ***" %version
print "*"*(len(version) + 53)
print "SGE job item index: environment variable '%s'" %str(index)
print 'cmd: %s' %" ".join(sys.argv)
environ = CecogEnvironment(version, redirect=False, debug=False)
settingsfile = os.path.abspath(args.settings)
# read the settings data from file
settings = ConfigSettings()
settings.read(settingsfile)
imagecontainer = ImageContainer()
imagecontainer.import_from_settings(settings)
if settings('General', 'constrain_positions'):
positions = settings('General', 'positions').split(POSSEP)
if not settings('General', 'has_multiple_plates'):
plate = os.path.split(settings("General", "pathin"))[1]
positions = ['%s%s%s' % (plate, PLATESEP, p) for p in positions]
else:
positions = list()
for plate in imagecontainer.plates:
imagecontainer.set_plate(plate)
meta_data = imagecontainer.get_meta_data()
