def load_images(ipl): """ These images are loaded: paths_true (paths within single label objects) paths_false (paths of merged objects which cross the merging site) featureims_true featureims_false :param ipl: :return: """ paths_true = IPL() paths_false = IPL() featureims_true = IPL() featureims_false = IPL() params = ipl.get_params() ipl.logging('Loading true paths ...') # Paths within labels (true paths) paths_true.data_from_file(filepath=params['intermedfolder'] + params['pathstruefile'], skeys='path', recursive_search=True, nodata=True) ipl.logging('Loading false paths ...') # Paths of merges (false paths) paths_false.data_from_file(filepath=params['intermedfolder'] + params['pathsfalsefile'], skeys='path', recursive_search=True, nodata=True) ipl.logging('Loading features for true paths ...') # Load features for true paths featureims_true.data_from_file(filepath=params['intermedfolder'] + params['featureimsfile'], nodata=True) featureims_true.delete_items(params['largeobjmnames'][0]) ipl.logging('Loading features for false paths ...') # Load features for false paths featureims_false.data_from_file(filepath=params['intermedfolder'] + params['featureimsfile'], nodata=True) featureims_false.delete_items(params['largeobjname']) return (paths_true, paths_false, featureims_true, featureims_false)
def split_in_xyz(ipl): ipl.logging('Datastructure\n---\n{}', ipl.datastructure2string()) reskeys = ('0', '1') ipl_split = IPL() ipl_split['z'] = ipl.anytask(lib.split, 2, axis=0, result_keys=reskeys, return_only=True, rtrntype=IPL) ipl_split['y'] = ipl.anytask(lib.split, 2, axis=1, result_keys=reskeys, return_only=True, rtrntype=IPL) ipl_split['x'] = ipl.anytask(lib.split, 2, axis=2, result_keys=reskeys, return_only=True, rtrntype=IPL) ipl_split = ipl_split.switch_levels(1, 2) ipl.logging('Split sample datastructure\n---\n{}', ipl_split.datastructure2string()) return ipl_split
def find_border_contacts_image_iteration(ipl): params = ipl.get_params() thisparams = params['find_border_contacts'] if thisparams['return_bordercontact_images']: bordercontacts = IPL() for d, k, v, kl in ipl.data_iterator(yield_short_kl=True): if k == params['largeobjname']: if thisparams['return_bordercontact_images']: ipl[kl].setlogger(ipl.getlogger()) [ipl[kl], bordercontacts[kl]] = find_border_contacts( ipl[kl], (params['largeobjname'], params['largeobjmnames'][0]), thisparams) else: ipl[kl].setlogger(ipl.getlogger()) ipl[kl] = find_border_contacts( ipl[kl], (params['largeobjname'], params['largeobjmnames'][0]), thisparams) if thisparams['return_bordercontact_images']: return bordercontacts else: return None
def run_paths_of_merges(yamlfile, logging=True): ipl = IPL(yaml=yamlfile) ipl.set_indent(1) params = rdict(data=ipl.get_params()) if logging: ipl.startlogger(filename=params['resultfolder'] + 'paths_of_merges.log', type='w', name='PathsOfMerges') else: ipl.startlogger() try: # # Copy the script file and the parameters to the scriptsfolder # copy(inspect.stack()[0][1], params['scriptsfolder']) # copy(yamlfile, params['scriptsfolder'] + 'paths_of_merges.parameters.yml') # ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) paths_of_merges(ipl, params['debug']) # ipl.logging('\nFinal datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) # ipl.write(filepath=params['intermedfolder'] + params['largeobjfile']) ipl.logging('') ipl.stoplogger() except: ipl.errout('Unexpected error')
def paths_within_labels(hfp, key, locmaxkeys, disttransfkey, ignore=[]): params = hfp.get_params() thisparams = params['paths_within_labels'] if type(locmaxkeys) is str: locmaxkeys = (locmaxkeys, ) # This results in the format: # more_keys = (locmaxkeys[0], ..., locmaxkeys[n], disttransfkey) more_keys = locmaxkeys + (disttransfkey, ) paths = IPL() for k in locmaxkeys: paths['pathsim', k] = np.zeros(hfp[key].shape) for lbl, lblim, more_ims, bounds in hfp.label_image_bounds_iterator( key=key, background=0, more_keys=more_keys, maskvalue=0, value=0): if lbl in ignore: continue # The format of more_ims is: # more_ims = {locmaxkeys[0]: locmax_1, # ... # locmaxkeys[n]: locmax_n, # disttransfkey: disttransf} hfp.logging('======================\nWorking on label = {}', lbl) # hfp.logging('more_ims structure:\n---\n{}', more_ims.datastructure2string()) hfp.logging('bounds = {}', bounds) for i in xrange(0, len(locmaxkeys)): hfp.logging('locmaxkey = {}', locmaxkeys[i]) if np.amax(more_ims[locmaxkeys[i]]) > 0: ps, pathsim = find_shortest_path(hfp, thisparams['penaltypower'], bounds, more_ims[disttransfkey], more_ims[locmaxkeys[i]]) # Only store the path if the path-calculation successfully determined a path # Otherwise an empty list would be stored if ps: hfp.logging('Number of paths found: {}', len(ps)) paths[key, locmaxkeys[i], 'path', lbl] = ps paths[key, locmaxkeys[i], 'pathsim'] = pathsim paths['pathsim', locmaxkeys[i]][bounds][ pathsim > 0] = pathsim[pathsim > 0] for k in locmaxkeys: paths['overlay', k] = np.array([ paths['pathsim', k], hfp[key].astype(np.float32) / np.amax(hfp[key]), vigra.filters.multiBinaryDilation(hfp[k].astype(np.uint8), 5) ]) return paths
def make_feature_arrays(ipl): params = ipl.get_params() thisparams = rdict(params['random_forest']) targetfile = params['resultfolder'] + params['resultsfile'] # Load the necessary images load_images(ipl) ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) result = IPL() evaluation = rdict() for d, k, v, kl in ipl.data_iterator(yield_short_kl=True): if k == '0': ipl.logging( '===============================\nWorking on group: {}', kl) # TODO: Implement copy full logger ipl[kl].set_logger(ipl.get_logger()) # Load the image data into memory ipl[kl].populate() # def shp(x): # return x.shape # print ipl[kl]['0', 'true'] # print ipl[kl].dss(function=shp) ipl[kl]['0', 'true'] = libip.rf_make_feature_array(ipl[kl]['0', 'true']) ipl.logging( "Computed feature array for ['0', 'true'] with shape {}", ipl[kl]['0', 'true'].shape) ipl[kl]['0', 'false'] = libip.rf_make_feature_array(ipl[kl]['0', 'false']) ipl.logging( "Computed feature array for ['0', 'false'] with shape {}", ipl[kl]['0', 'false'].shape) ipl[kl]['1', 'true'] = libip.rf_make_feature_array(ipl[kl]['1', 'true']) ipl.logging( "Computed feature array for ['1', 'true'] with shape {}", ipl[kl]['1', 'true'].shape) ipl[kl]['1', 'false'] = libip.rf_make_feature_array(ipl[kl]['1', 'false']) ipl.logging( "Computed feature array for ['1', 'false'] with shape {}", ipl[kl]['1', 'false'].shape) ipl.write(filepath=params['intermedfolder'] + 'feature_arrays.h5', keys=[kl])
def get_features(paths, featureimage, featurelist, max_paths_per_label, ipl=None): newfeats = IPL() # TODO: Selection of a limited amount of paths should be random keylist = range(0, max_paths_per_label) keylist = [str(x) for x in keylist] # Iterate over all paths, yielding a list of one path per label object until no paths are left for i, keys, vals in paths.simultaneous_iterator( max_count_per_item=max_paths_per_label, keylist=keylist): if ipl is not None: ipl.logging('Working in iteration = {}', i) # Create a working image image = np.zeros(featureimage.shape, dtype=np.uint32) # And fill it with one path per label object c = 1 for curk, curv in (dict(zip(keys, vals))).iteritems(): curv = lib.swapaxes(curv, 0, 1) lib.positions2value(image, curv, c) c += 1 # Extract the region features of the working image newnewfeats = IPL( data=vigra.analysis.extractRegionFeatures( featureimage, image, ignoreLabel=0, features=featurelist ) ) for k, v in newnewfeats.iteritems(): newnewfeats[k] = newnewfeats[k][1:] if k in newfeats: try: newfeats[k] = np.concatenate((newfeats[k], newnewfeats[k])) except ValueError: pass else: newfeats[k] = newnewfeats[k] return newfeats
def merge_adjacent_objects_image_iteration(ipl): params = ipl.get_params() thisparams = params['merge_adjacent_objects'] merged = IPL() for d, k, v, kl in ipl.data_iterator(yield_short_kl=True): ipl.logging('Working on image: {}', kl + [k]) if k == params['largeobjname']: data = IPL(data={k: v}) data.setlogger(ipl.getlogger()) merged[kl] = merge_adjacent_objects(data, k, thisparams) return merged
def run_random_forest(yamlfile, logging=True, make_only_feature_array=False, debug=False, write=True): ipl = IPL(yaml=yamlfile) ipl.set_indent(1) params = rdict(data=ipl.get_params()) if logging: ipl.startlogger(filename=params['resultfolder'] + 'random_forest.log', type='w', name='RandomForest') else: ipl.startlogger() try: # # Copy the script file and the parameters to the scriptsfolder # copy(inspect.stack()[0][1], params['scriptsfolder']) # copy(yamlfile, params['scriptsfolder'] + 'random_forest.parameters.yml') # ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) if make_only_feature_array: make_feature_arrays(ipl) else: result = IPL() result['result'], result['evaluation'] = random_forest(ipl, debug=debug) # ipl.logging('\nFinal datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) if write: result.write(filepath=params['resultfolder'] + params['resultsfile']) ipl.logging('') ipl.stoplogger() except: ipl.errout('Unexpected error')
def get_features(paths, featureimage, featurelist, max_paths_per_label, hfp=None): newfeats = IPL() keylist = range(0, max_paths_per_label) keylist = [str(x) for x in keylist] for i, keys, vals in paths.simultaneous_iterator( max_count_per_item=max_paths_per_label, keylist=keylist): if hfp is not None: hfp.logging('Working in iteration = {}', i) image = np.zeros(featureimage.shape, dtype=np.uint32) c = 1 for curk, curv in (dict(zip(keys, vals))).iteritems(): curv = lib.swapaxes(curv, 0, 1) lib.positions2value(image, curv, c) c += 1 newnewfeats = IPL(data=vigra.analysis.extractRegionFeatures( featureimage, image, ignoreLabel=0, features=featurelist)) for k, v in newnewfeats.iteritems(): newnewfeats[k] = newnewfeats[k][1:] if k in newfeats: try: newfeats[k] = np.concatenate((newfeats[k], newnewfeats[k])) except ValueError: pass else: newfeats[k] = newnewfeats[k] return newfeats
def load_images(filepath, skeys=None, recursive_search=False, logger=None): if logger is not None: logger.logging('Loading data from \n{}', filepath) else: print 'Loading data from \n{}'.format(filepath) data = IPL() data.data_from_file(filepath=filepath, skeys=skeys, recursive_search=recursive_search, nodata=True) return data
def random_forest_iteration(features): result = IPL() for k, v in features.iteritems(): # Make the feature arrays v['0']['true'] = make_feature_array(v['0']['true']) v['0']['false'] = make_feature_array(v['0']['false']) v['1']['true'] = make_feature_array(v['1']['true']) v['1']['false'] = make_feature_array(v['1']['false']) result[k, '0'] = random_forest(v['0'], v['1']) result[k, '1'] = random_forest(v['1'], v['0']) return result
def make_features_paths(paths, disttransf_images, feature_images, features_out): paths.astype(np.uint32) feature_images.astype(np.float32) # features_out = Hdf5ImageProcessing() features_out['disttransf'] = IPL(data=paths) features_out['disttransf'].anytask(vigra.analysis.extractRegionFeatures, ignoreLabel=0, reverse_order=True, reciprocal=False, keys=('paths_true', 'paths_false'), indict=disttransf_images) features_out['raw'] = paths features_out['raw'].anytask(vigra.analysis.extractRegionFeatures, ignoreLabel=0, reverse_order=True, reciprocal=False, keys=('paths_true', 'paths_false'), indict=feature_images)
def compute_features(image, general_params, subfeature_params): ff = FeatureFunctions() result = IPL() for k, v in subfeature_params.iteritems(): if v: result[k] = getattr(ff, v.pop('func'))(image, general_params, v.pop('params')) if len(v) > 0: result[k] = compute_features(result[k], general_params, v) else: result[k] = image return result
def run_remove_small_objects(yamlfile): ipl = IPL(yaml=yamlfile, yamlspec={ 'path': 'datafolder', 'filename': 'labelsfile', 'skeys': 'labelsname' }, recursive_search=True, nodata=True) # Set indentation of the logging ipl.set_indent(1) params = ipl.get_params() ipl.startlogger(filename=params['resultfolder'] + 'remove_small_objects.log', type='w', name='RemoveSmallObjects') try: # # Copy the script file and the parameters to the scriptsfolder # copy(inspect.stack()[0][1], params['scriptsfolder']) # copy(yamlfile, params['scriptsfolder'] + 'remove_small_objects.parameters.yml') ipl.logging('\nipl datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) remove_small_objects(ipl) ipl.logging('\nFinal datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) # ipl.write(filepath=params['intermedfolder'] + params['largeobjfile']) ipl.logging('') ipl.stoplogger() except: ipl.errout('Unexpected error')
def features_of_paths_image_iteration(ipl, disttransf_images, feature_images): params = ipl.get_params() thisparams = params['features_of_paths'] features = IPL() for d, k, v, kl in ipl.data_iterator(yield_short_kl=True): if k == 'true': ipl.logging('Key list: {}', kl) ipl[kl].setlogger(ipl.getlogger()) features[kl] = features_of_paths( ipl[kl], disttransf_images[kl], feature_images[kl], thisparams ) return features
def paths_within_labels_image_iteration(ipl): params = ipl.get_params() thisparams = params['paths_within_labels'] paths = IPL() for d, k, v, kl in ipl.data_iterator(yield_short_kl=True): if k == params['locmaxbordernames'][0]: ipl[kl].setlogger(ipl.getlogger()) paths[kl] = paths_within_labels( ipl[kl], params['largeobjname'], (params['locmaxbordernames'][0], params['locmaxnames'][0]), params['locmaxbordernames'][2], thisparams, ignore=thisparams['ignore']) return paths
def paths_of_partners_image_iteration(ipl): params = ipl.get_params() thisparams = params['paths_of_partners'] paths = IPL() for d, k, v, kl in ipl.data_iterator(yield_short_kl=True): if k == params['locmaxbordernames'][1]: ipl[kl].setlogger(ipl.getlogger()) paths[kl] = paths_of_partners( ipl[kl], params['largeobjmnames'][0], params['largeobjname'], params['largeobjmnames'][4], (params['locmaxbordernames'][1], params['locmaxnames'][1]), params['locmaxbordernames'][3], thisparams, thisparams['ignore']) # (ipl, 'largeobjm', 'largeobj', # 'change_hash', ('border_locmax_m', 'locmaxm'), # 'disttransfm', thisparams['ignore']) return paths
import matplotlib.pylab as lab import processing_lib as lib # TODO: Visualize Distancetransform along paths of true and false merges # TODO: Also intensity values or raw data and probability map # TODO: Do not forget to save the result for the thesis! __author__ = 'jhennies' if __name__ == '__main__': yamlfile = os.path.dirname(os.path.abspath(__file__)) + '/parameters.yml' hfp = IPL(yaml=yamlfile, yamlspec={ 'path': 'intermedfolder', 'filename': 'pathstruefile' }, tkeys='true', castkey=None) params = hfp.get_params() hfp.data_from_file(filepath=params['intermedfolder'] + params['pathsfalsefile'], tkeys='false', castkey=None) hfp.data_from_file(filepath=params['intermedfolder'] + params['locmaxfile'], skeys=('disttransf', 'disttransfm'), tkeys=('disttransf', 'disttransfm'))
result[k, '0'] = random_forest(v['0'], v['1']) result[k, '1'] = random_forest(v['1'], v['0']) return result if __name__ == '__main__': resultsfolder = '/mnt/localdata02/jhennies/neuraldata/results/cremi_2016/161110_random_forest_of_paths/' yamlfile = resultsfolder + '/parameters.yml' features = IPL(yaml=yamlfile, yamlspec={ 'path': 'intermedfolder', 'filename': 'featurefile' }) params = features.get_params() thisparams = params['random_forest'] features.startlogger(filename=params['resultfolder'] + 'random_forest.log', type='w') try: # Copy the script file and the parameters to the scriptsfolder copy(inspect.stack()[0][1], params['scriptsfolder']) copy(yamlfile, params['scriptsfolder'] + 'random_forest.parameters.yml') # Write script and parameters to the logfile features.code2log(inspect.stack()[0][1])
# cremi = IPL(filepath='/mnt/localdata02/jhennies/neuraldata/cremi_2016/sample_B_20160501.hdf') # # cremi.logging('Datastructure:\n---\n{}', cremi.datastructure2string()) # # images = IPL(data={ # 'raw': cremi['volumes', 'raw'], # 'neuron_ids': cremi['volumes', 'labels', 'neuron_ids'] # }) # # images.logging('Datastructure:\n---\n{}', images.datastructure2string()) # # images.write('/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splB.raw_neurons.crop.h5') cremi = IPL( filepath= '/mnt/localdata02/jhennies/neuraldata/cremi_2016/sample_C_20160501.hdf' ) cremi.logging('Datastructure:\n---\n{}', cremi.datastructure2string()) images = IPL( data={ 'raw': cremi['volumes', 'raw'], 'neuron_ids': cremi['volumes', 'labels', 'neuron_ids'] }) images.logging('Datastructure:\n---\n{}', images.datastructure2string()) images.write( '/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splC.raw_neurons.crop.h5' )
from hdf5_image_processing import Hdf5ImageProcessingLib as IPL import os import numpy as np __author__ = 'jhennies' if __name__ == '__main__': yamlfile = os.path.dirname(os.path.abspath(__file__)) + '/parameters.yml' ipl = IPL(yaml=yamlfile) ipl.logging('Parameters: {}', ipl.get_params()) params = ipl.get_params() ipl.data_from_file(filepath=params['datafolder'] + 'cremi.splA.raw_neurons.crop.h5', skeys='raw', tkeys='raw') ipl.crop_bounding_rect(np.s_[10:110, 200:712, 200:712], keys='raw') ipl.write(filepath=params['datafolder'] + 'cremi.splA.raw_neurons.crop.crop_10-200-200_110-712-712.h5')
keys=locmaxnames) # Local maxima hfp.logging('Discovering mountains ...') hfp.extended_local_maxima(neighborhood=26, keys=locmaxnames) if __name__ == '__main__': yamlfile = os.path.dirname(os.path.abspath(__file__)) + '/parameters.yml' hfp = IPL(yaml=yamlfile, yamlspec={ 'path': 'intermedfolder', 'filename': 'locmaxborderfile', 'skeys': { 'locmaxbordernames': (2, 3) } }, tkeys=('disttransf', 'disttransfm'), castkey=None) params = hfp.get_params() thisparams = params['localmax_on_disttransf'] hfp.startlogger(filename=params['resultfolder'] + 'localmax_on_disttransf.log', type='w') try: # Copy the script file and the parameters to the scriptsfolder copy(inspect.stack()[0][1], params['scriptsfolder']) copy(yamlfile,
'===============================\nWorking on image: {}', kl + [k]) ipl[kl].setlogger(ipl.getlogger()) ipl[kl] = accumulate_small_objects(ipl[kl], k, thisparams) if __name__ == '__main__': resultsfolder = '/mnt/localdata02/jhennies/neuraldata/results/cremi_2016/161110_random_forest_of_paths/' yamlfile = resultsfolder + '/parameters.yml' ipl = IPL(yaml=yamlfile, yamlspec={ 'path': 'datafolder', 'filename': 'labelsfile' }) params = ipl.get_params() ipl.startlogger(filename=params['resultfolder'] + 'remove_small_objects.log', type='a') try: # Create folder for scripts if not os.path.exists(params['scriptsfolder']): os.makedirs(params['scriptsfolder']) else: if params['overwriteresults']: ipl.logging(
# sample_a.logging('Sample A datastructure\n---\n{}', sample_a.datastructure2string()) # # reskeys = ('0', '1') # split_sample_a = IPL() # split_sample_a['z'] = sample_a.anytask(lib.split, 2, axis=0, result_keys=reskeys, return_only=True) # split_sample_a['y'] = sample_a.anytask(lib.split, 2, axis=1, result_keys=reskeys, return_only=True) # split_sample_a['x'] = sample_a.anytask(lib.split, 2, axis=2, result_keys=reskeys, return_only=True) # # split_sample_a = split_sample_a.switch_levels(1, 2) # sample_a.logging('Split sample A datastructure\n---\n{}', split_sample_a.datastructure2string()) # # split_sample_a.write('/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splA.raw_neurons.crop.split_xyz.h5') # Sample B sample = IPL( filepath= '/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splB.raw_neurons.crop.h5' ) sample.logging('Sample B datastructure\n---\n{}', sample.datastructure2string()) reskeys = ('0', '1') split_sample = IPL() split_sample['z'] = sample.anytask(lib.split, 2, axis=0, result_keys=reskeys, return_only=True) split_sample['y'] = sample.anytask(lib.split, 2, axis=1,
(params['locmaxbordernames'][2], params['locmaxbordernames'][3]), thisparams) if __name__ == '__main__': resultsfolder = '/mnt/localdata02/jhennies/neuraldata/results/cremi_2016/161110_random_forest_of_paths/' yamlfile = resultsfolder + '/parameters.yml' ipl = IPL(yaml=yamlfile, yamlspec={ 'path': 'intermedfolder', 'filename': 'locmaxborderfile', 'skeys': { 'locmaxbordernames': (2, 3) } }, recursive_search=True) params = ipl.get_params() thisparams = params['localmax_on_disttransf'] ipl.startlogger(filename=params['resultfolder'] + 'localmax_on_disttransf.log', type='w') try: # Copy the script file and the parameters to the scriptsfolder copy(inspect.stack()[0][1], params['scriptsfolder']) copy(yamlfile,
data = IPL(data={k: v}) data.setlogger(ipl.getlogger()) merged[kl] = merge_adjacent_objects(data, k, thisparams) return merged if __name__ == '__main__': resultsfolder = '/mnt/localdata02/jhennies/neuraldata/results/cremi_2016/161110_random_forest_of_paths/' yamlfile = resultsfolder + '/parameters.yml' ipl = IPL(yaml=yamlfile, yamlspec={ 'path': 'intermedfolder', 'filename': 'largeobjfile' }) params = ipl.get_params() ipl.startlogger(filename=params['resultfolder'] + 'merge_adjacent_objects.log', type='w') try: # Copy the script file and the parameters to the scriptsfolder copy(inspect.stack()[0][1], params['scriptsfolder']) copy(yamlfile, params['scriptsfolder'] + 'merge_adjacent_objects.parameters.yml') # Write script and parameters to the logfile ipl.code2log(inspect.stack()[0][1])
hfp.rename_entry(key, targetnames[0]) for k, v in change_hash.iteritems(): for x in v: if x != k: hfp.logging('Setting {} to {}!', x, k) hfp.filter_values(x, type='eq', setto=k, keys=targetnames[0]) if __name__ == '__main__': yamlfile = os.path.dirname(os.path.abspath(__file__)) + '/parameters.yml' hfp = IPL(yaml=yamlfile, yamlspec={ 'path': 'intermedfolder', 'filename': 'largeobjfile', 'skeys': 'largeobjname' }, tkeys='largeobj', castkey=None) params = hfp.get_params() hfp.startlogger(filename=params['resultfolder'] + 'merge_adjacent_objects.log', type='w') try: # Copy the script file and the parameters to the scriptsfolder copy(inspect.stack()[0][1], params['scriptsfolder']) copy(yamlfile, params['scriptsfolder'] + 'merge_adjacent_objects.parameters.yml') # Write script and parameters to the logfile
def merge_adjacent_objects(ipl, key, thisparams): """ :param ipl: ipl.get_params(): merge_adjacent_objects seed numberbysize numberbyrandom largeobjmnames - 'largeobj_merged' - 'mergeids_small' - 'mergeids_random' - 'mergeids_all' - 'change_hash' :param key: the source key for calculation """ numberbysize = thisparams['numberbysize'] numberbyrandom = thisparams['numberbyrandom'] targetnames = params['largeobjmnames'] # Get only the relevant labels labels = lib.unique(ipl[key]) ipl.logging('labels = {}', labels) # Seed the randomize function random.seed(thisparams['seed']) ipl.astype(np.uint32, keys=key) (grag, rag) = graphs.gridRegionAdjacencyGraph(ipl[key], ignoreLabel=0) edge_ids = rag.edgeIds() # ipl.logging('Edge ids: {}', edge_ids) # Type 1: # Select edges by size (smallest edges) ipl.logging('Number of edgeLengths = {}', len(rag.edgeLengths())) edgelen_ids = dict(zip(edge_ids, rag.edgeLengths())) # ifp.logging('edgelen_ids = {}', edgelen_ids) sorted_edgelens = np.sort(rag.edgeLengths()) # smallest_merge_lens = sorted_edgelens[0:numberbysize] ipl.logging('Lengths selected for merging: {}', smallest_merge_lens) # smallest_merge_ids = [] for x in smallest_merge_lens: edge_id = edgelen_ids.keys()[edgelen_ids.values().index(x)] smallest_merge_ids.append(edge_id) edgelen_ids.pop(edge_id) # edge_ids = edgelen_ids.keys() ipl.logging('Edge IDs selected for merging due to size: {}', smallest_merge_ids) # Type 2: # Randomly choose edges random_merge_ids = random.sample(edge_ids, numberbyrandom) ipl.logging('Edge IDs randomly selected for merging: {}', random_merge_ids) # Now get the label ids smallest_merge_labelids_u = [ rag.uId(rag.edgeFromId(x)) for x in smallest_merge_ids ] smallest_merge_labelids_v = [ rag.vId(rag.edgeFromId(x)) for x in smallest_merge_ids ] smallest_merge_labelids = list( zip(smallest_merge_labelids_u, smallest_merge_labelids_v)) random_merge_labelids_u = [ rag.uId(rag.edgeFromId(x)) for x in random_merge_ids ] random_merge_labelids_v = [ rag.vId(rag.edgeFromId(x)) for x in random_merge_ids ] random_merge_labelids = list( zip(random_merge_labelids_u, random_merge_labelids_v)) ipl.logging('Label IDs selected for merging by size: {}', smallest_merge_labelids) ipl.logging('Label IDs randomly selected for merging: {}', random_merge_labelids) # Concatenate all_merge_labelids = smallest_merge_labelids + random_merge_labelids # Sort ipl.logging('all_merge_labelids = {}', all_merge_labelids) all_merge_labelids = [sorted(x) for x in all_merge_labelids] all_merge_labelids = sorted(all_merge_labelids) ipl.logging('all_merge_labelids = {}', all_merge_labelids) # Store this for later use ipl[targetnames[1]] = smallest_merge_labelids ipl[targetnames[2]] = random_merge_labelids ipl[targetnames[3]] = all_merge_labelids # Create change hash list change_hash = IPL(data=dict( zip(np.unique(all_merge_labelids), [[ x, ] for x in np.unique(all_merge_labelids)]))) for i in xrange(0, 3): prev_change_hash = IPL(data=change_hash) for x in all_merge_labelids: ipl.logging('Adding {} and {}', *x) change_hash[x[0]] += change_hash[x[1]] change_hash[x[0]] = list(np.unique(change_hash[x[0]])) change_hash[x[1]] += change_hash[x[0]] change_hash[x[1]] = list(np.unique(change_hash[x[1]])) # This removes the redundancy from the hash def reduce(hash): br = False for k, v in hash.iteritems(): for x in v: if x != k: if x in hash.keys(): del hash[x] reduce(hash) br = True break else: br = False if br: break reduce(change_hash) # Change the list in the hash to np-arrays for better storage in h5 files for k, v in change_hash.iteritems(): change_hash[k] = np.array(v) # And now we have a perfect change list which we just need to iterate over and change the labels in the image ipl.logging('change_hash after change:') ipl.logging(change_hash) ipl[targetnames[4]] = change_hash # Create the merged image # ipl.deepcopy_entry('largeobj', targetnames[0]) ipl.rename_entry(key, targetnames[0]) for k, v in change_hash.iteritems(): for x in v: if x != k: ipl.logging('Setting {} to {}!', x, k) ipl.filter_values(x, type='eq', setto=k, keys=targetnames[0]) return ipl
def random_forest(ipl, debug=False): params = ipl.get_params() thisparams = rdict(params['random_forest']) targetfile = params['resultfolder'] + params['resultsfile'] # Load the necessary images load_images(ipl) ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) result = IPL() new_eval = rdict() evaluation = rdict() for d, k, v, kl in ipl.data_iterator(yield_short_kl=True): if k == '0': ipl.logging('===============================\nWorking on group: {}', kl) # TODO: Implement copy full logger ipl[kl].set_logger(ipl.get_logger()) # Note: # Due to the feature input being a dictionary organized by the feature images where # the feature values come from # # [kl] # '0'|'1' # 'true'|'false' # [featureims] # 'Sum': [s1, ..., sN] # 'Variance': [v1, ..., vN] # ... # [Pathlength]: [l1, ..., lN] # # the exact order in which items are iterated over by data_iterator() is not known. # # Solution: # Iterate over it once and store the keylist in an array (which conserves the order) # When accumulating the featrues for each of the four corresponding subsets, namely # training and testing set with true and false paths each, i.e. # ['0'|'1']['true'|'false'], # the the keylist is used, thus maintaining the correct order in every subset. # # And that is what is happening here: keylist = [] for d2, k2, v2, kl2 in ipl[kl]['0', 'true'].data_iterator(): if type(v2) is not type(ipl[kl]['0', 'true']): keylist.append(kl2) # Load the image data into memory ipl[kl].populate() # ipl[kl]['0', 'true'] = libip.rf_make_feature_array(ipl[kl]['0', 'true']) # ipl.logging("Computed feature array for ['0', 'true'] with shape {}", ipl[kl]['0', 'true'].shape) # ipl[kl]['0', 'false'] = libip.rf_make_feature_array(ipl[kl]['0', 'false']) # ipl.logging("Computed feature array for ['0', 'false'] with shape {}", ipl[kl]['0', 'false'].shape) # ipl[kl]['1', 'true'] = libip.rf_make_feature_array(ipl[kl]['1', 'true']) # ipl.logging("Computed feature array for ['1', 'true'] with shape {}", ipl[kl]['1', 'true'].shape) # ipl[kl]['1', 'false'] = libip.rf_make_feature_array(ipl[kl]['1', 'false']) # ipl.logging("Computed feature array for ['1', 'false'] with shape {}", ipl[kl]['1', 'false'].shape) ipl[kl]['0', 'true'] = libip.rf_make_feature_array_with_keylist(ipl[kl]['0', 'true'], keylist) ipl.logging("Computed feature array for ['0', 'true'] with shape {}", ipl[kl]['0', 'true'].shape) ipl[kl]['0', 'false'] = libip.rf_make_feature_array_with_keylist(ipl[kl]['0', 'false'], keylist) ipl.logging("Computed feature array for ['0', 'false'] with shape {}", ipl[kl]['0', 'false'].shape) ipl[kl]['1', 'true'] = libip.rf_make_feature_array_with_keylist(ipl[kl]['1', 'true'], keylist) ipl.logging("Computed feature array for ['1', 'true'] with shape {}", ipl[kl]['1', 'true'].shape) ipl[kl]['1', 'false'] = libip.rf_make_feature_array_with_keylist(ipl[kl]['1', 'false'], keylist) ipl.logging("Computed feature array for ['1', 'false'] with shape {}", ipl[kl]['1', 'false'].shape) # print '...' # print ipl[kl]['0'] result[kl + ['0']] = libip.random_forest(ipl[kl]['0'], ipl[kl]['1'], debug=debug) result[kl + ['1']] = libip.random_forest(ipl[kl]['1'], ipl[kl]['0'], debug=debug) new_eval[kl + ['0']] = libip.new_eval([x[0] for x in result[kl]['0']], [x[1] for x in result[kl]['0']]) new_eval[kl + ['1']] = libip.new_eval([x[0] for x in result[kl]['1']], [x[1] for x in result[kl]['1']]) evaluation[kl + ['0']] = libip.evaluation(result[kl]['0']) evaluation[kl + ['1']] = libip.evaluation(result[kl]['1']) ipl.logging('+++ RESULTS +++') ipl.logging("[kl]['0']") for i in result[kl]['0']: ipl.logging('{}', i) # for key, value in evaluation[kl]['0'].iteritems(): # ipl.logging('{} = {}', key, value) for key, value in new_eval[kl]['0'].iteritems(): ipl.logging('{} = {}', key, value) ipl.logging('+++') ipl.logging("[kl]['1']") for i in result[kl]['1']: ipl.logging('{}', i) # for key, value in evaluation[kl]['1'].iteritems(): # ipl.logging('{} = {}', key, value) for key, value in new_eval[kl]['1'].iteritems(): ipl.logging('{} = {}', key, value) # # Write the result to file # ipl.write(filepath=targetfile, keys=[kl]) # Free memory ipl[kl] = None return IPL(data=result), IPL(data=evaluation)