def make_feature_arrays(ipl): params = ipl.get_params() thisparams = rdict(params['random_forest']) targetfile = params['resultfolder'] + params['resultsfile'] # Load the necessary images load_images(ipl) ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) result = IPL() evaluation = rdict() for d, k, v, kl in ipl.data_iterator(yield_short_kl=True): if k == '0': ipl.logging( '===============================\nWorking on group: {}', kl) # TODO: Implement copy full logger ipl[kl].set_logger(ipl.get_logger()) # Load the image data into memory ipl[kl].populate() # def shp(x): # return x.shape # print ipl[kl]['0', 'true'] # print ipl[kl].dss(function=shp) ipl[kl]['0', 'true'] = libip.rf_make_feature_array(ipl[kl]['0', 'true']) ipl.logging( "Computed feature array for ['0', 'true'] with shape {}", ipl[kl]['0', 'true'].shape) ipl[kl]['0', 'false'] = libip.rf_make_feature_array(ipl[kl]['0', 'false']) ipl.logging( "Computed feature array for ['0', 'false'] with shape {}", ipl[kl]['0', 'false'].shape) ipl[kl]['1', 'true'] = libip.rf_make_feature_array(ipl[kl]['1', 'true']) ipl.logging( "Computed feature array for ['1', 'true'] with shape {}", ipl[kl]['1', 'true'].shape) ipl[kl]['1', 'false'] = libip.rf_make_feature_array(ipl[kl]['1', 'false']) ipl.logging( "Computed feature array for ['1', 'false'] with shape {}", ipl[kl]['1', 'false'].shape) ipl.write(filepath=params['intermedfolder'] + 'feature_arrays.h5', keys=[kl])
def find_border_contacts(yparams): params = yparams.get_params() thisparams = rdict(params['find_border_contacts']) # targetfile = params['intermedfolder'] + params['featureimsfile'] data = ipl() for sourcekey, source in thisparams['sources'].iteritems(): # Load the necessary images # 1. Determine the settings for fetching the data try: recursive_search = False recursive_search = thisparams['skwargs', 'default', 'recursive_search'] recursive_search = thisparams['skwargs', sourcekey, 'recursive_search'] except KeyError: pass if len(source) > 2: skeys = source[2] else: skeys = None # 2. Load the data data[sourcekey] = load_images(params[source[0]] + params[source[1]], skeys=skeys, recursive_search=recursive_search, logger=yparams) # TODO: Get rid of this at some point! Probable re-implement the whole data-loading step data['disttransf'].reduce_from_leaves(iterate=False) data['disttransf'].reduce_from_leaves(iterate=False) # Set targetfile targetfile = params[thisparams['target'][0]] \ + params[thisparams['target'][1]] yparams.logging('\nInitial datastructure: \n\n{}', data.datastructure2string(maxdepth=4)) for d, k, v, kl in data['disttransf'].data_iterator(yield_short_kl=True, leaves_only=True): yparams.logging( '===============================\nWorking on image: {}', kl + [k]) # # TODO: Implement copy full logger # data[kl].set_logger(data.get_logger()) # We need: the distance transform of the MERGED labels (i.e. segmentation) and the # corresponding segmentation data['segmentation'][kl][k] = libip.find_border_contacts_arr( data['segmentation'][kl][k], data['disttransf'][kl][k], tkey=params['borderctname'], debug=params['debug']) # Write the result to file data['segmentation'].write(filepath=targetfile, keys=[kl + [k]])
def evaluation(x): def tp(l): return len([x for x in l if (x[0] + x[1] == 0)]) def fp(l): return len([x for x in l if (x[0] + x[1] == 1 and x[0] == 0)]) def fn(l): return len([x for x in l if (x[0] + x[1] == 1 and x[0] == 1)]) def tn(l): return len([x for x in l if (x[0] + x[1] == 2)]) def p(l): return len([x for x in l if (x[0] == 0)]) def n(l): return len([x for x in l if (x[0] == 1)]) def recall(l): return float(tp(l)) / (tp(l) + fn(l)) def precision(l): return float(tp(l)) / (tp(l) + fp(l)) def f1(l): return float(2 * tp(l)) / (2 * tp(l) + fp(l) + fn(l)) def accuracy(l): return float(tp(l) + tn(l)) / (p(l) + n(l)) return rdict(data={ 'precision': precision(x), 'recall': recall(x), 'f1': f1(x), 'accuracy': accuracy(x) })
def features_of_paths(ipl): params = ipl.get_params() thisparams = rdict(params['features_of_paths']) targetfile = params['intermedfolder'] + params['featuresfile'] # Load the necessary images paths_true, paths_false, featureims_true, featureims_false = load_images( ipl) ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) for d, k, v, kl in paths_true.data_iterator(yield_short_kl=True): if k == 'path': ipl.logging( '===============================\nWorking on group: {}', kl) # # TODO: Implement copy full logger # ipl[kl].set_logger(ipl.get_logger()) # # Load the image data into memory # ipl[kl].populate() ipl[kl] = libip.features_of_paths(ipl, paths_true[kl][k], paths_false[kl][k], featureims_true[kl], featureims_false[kl], kl) # Write the result to file ipl.write(filepath=targetfile, keys=[kl]) # # Free memory ipl[kl] = None
def compute_selected_features(ipl, params): thisparams = rdict(data=params['compute_feature_images']) targetfile = params['intermedfolder'] + params['featureimsfile'] maxd = ipl.maxdepth() for d, k, v, kl in ipl.data_iterator(yield_short_kl=True): if d == maxd: ipl.logging('----------------------\nWorking on image: {}', k) ipl[kl].populate(k) if k in [params['rawdataname'], params['probsname'], params['largeobjname'], params['largeobjmnames'][0]]: general_params = thisparams.dcp() del general_params['features'] if k == params['rawdataname']: subfeature_params = thisparams['features']['rawdata'] ipl[kl][k] = compute_features(ipl[kl][k], general_params, subfeature_params) elif k == params['probsname']: subfeature_params = thisparams['features']['probs'] ipl[kl][k] = compute_features(ipl[kl][k], general_params, subfeature_params) elif k == params['largeobjname']: subfeature_params = thisparams['features']['largeobj'] ipl[kl][k] = compute_features(ipl[kl][k], general_params, subfeature_params) elif k == params['largeobjmnames'][0]: subfeature_params = thisparams['features']['largeobjm'] ipl[kl][k] = compute_features(ipl[kl][k], general_params, subfeature_params) ipl.write(filepath=targetfile, keys=[kl + [k]]) ipl[kl][k] = None
def run_compute_feature_images(yamlfile): ipl = IPL(yaml=yamlfile) ipl.set_indent(1) params = rdict(data=ipl.get_params()) ipl.startlogger(filename=params['resultfolder'] + 'compute_feature_images.log', type='w', name='ComputeFeatureImages') try: # # Copy the script file and the parameters to the scriptsfolder # copy(inspect.stack()[0][1], params['scriptsfolder']) # copy(yamlfile, params['scriptsfolder'] + 'compute_feature_images.parameters.yml') # ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) compute_feature_images(ipl) # ipl.logging('\nFinal datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) # ipl.write(filepath=params['intermedfolder'] + params['largeobjfile']) ipl.logging('') ipl.stoplogger() except: ipl.errout('Unexpected error')
def compute_feature_images(ipl): params = ipl.get_params() thisparams = rdict(params['compute_feature_images']) # targetfile = params['intermedfolder'] + params['featureimsfile'] # Load the necessary images load_images(ipl, thisparams['features'].keys()) ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) maxd = ipl.maxdepth() for d, k, v, kl in ipl.data_iterator(maxdepth=ipl.maxdepth() - 1): if d == maxd - 1: ipl.logging('===============================\nWorking on group: {}', kl) # TODO: Implement copy full logger ipl[kl].set_logger(ipl.get_logger()) # # Load the image data into memory # ipl[kl].populate(k) compute_selected_features(ipl.subset(kl), params)
def merge_adjacent_objects(ipl): params = ipl.get_params() thisparams = rdict(params['merge_adjacent_objects']) targetfile = params['intermedfolder'] + params['largeobjmfile'] # Load the necessary images load_images(ipl, params['intermedfolder'], params['largeobjfile'], params['largeobjname']) ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) for d, k, v, kl in ipl.data_iterator(yield_short_kl=True): if k == params['largeobjname']: ipl.logging('===============================\nWorking on image: {}', kl + [k]) ipl[kl].set_logger(ipl.get_logger()) # Load the image data into memory ipl[kl].populate(k) ipl[kl] = libip.merge_adjacent_objects( ipl[kl], k, thisparams['numberbysize'], thisparams['numberbyrandom'], thisparams['seed'], targetnames=params['largeobjmnames'], algorithm=thisparams['algorithm'] ) # Write the result to file ipl.write(filepath=targetfile, keys=[kl]) # Free memory ipl[kl] = None
def run_paths_of_merges(yamlfile, logging=True): ipl = IPL(yaml=yamlfile) ipl.set_indent(1) params = rdict(data=ipl.get_params()) if logging: ipl.startlogger(filename=params['resultfolder'] + 'paths_of_merges.log', type='w', name='PathsOfMerges') else: ipl.startlogger() try: # # Copy the script file and the parameters to the scriptsfolder # copy(inspect.stack()[0][1], params['scriptsfolder']) # copy(yamlfile, params['scriptsfolder'] + 'paths_of_merges.parameters.yml') # ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) paths_of_merges(ipl, params['debug']) # ipl.logging('\nFinal datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) # ipl.write(filepath=params['intermedfolder'] + params['largeobjfile']) ipl.logging('') ipl.stoplogger() except: ipl.errout('Unexpected error')
def run_find_border_contacts(yamlfile, logging=True): ipl = IPL(yaml=yamlfile) ipl.set_indent(1) params = rdict(data=ipl.get_params()) if logging: ipl.startlogger(filename=params['resultfolder'] + 'find_border_contacts.log', type='w', name='FindBorderContacts') else: ipl.startlogger() try: # # Copy the script file and the parameters to the scriptsfolder # copy(inspect.stack()[0][1], params['scriptsfolder']) # copy(yamlfile, params['scriptsfolder'] + 'find_border_contacts.parameters.yml') # ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) find_border_contacts(ipl) # ipl.logging('\nFinal datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) # ipl.write(filepath=params['intermedfolder'] + params['largeobjfile']) ipl.logging('') ipl.stoplogger() except: ipl.errout('Unexpected error')
def compute_feature_images(ipl): params = ipl.get_params() thisparams = rdict(params['compute_feature_images']) # targetfile = params['intermedfolder'] + params['featureimsfile'] # Load the necessary images load_images(ipl, thisparams['features'].keys()) ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) maxd = ipl.maxdepth() for d, k, v, kl in ipl.data_iterator(maxdepth=ipl.maxdepth() - 1): if d == maxd - 1: ipl.logging( '===============================\nWorking on group: {}', kl) # TODO: Implement copy full logger ipl[kl].set_logger(ipl.get_logger()) # # Load the image data into memory # ipl[kl].populate(k) compute_selected_features(ipl.subset(kl), params)
def features_of_paths(ipl): params = ipl.get_params() thisparams = rdict(params['features_of_paths']) targetfile = params['intermedfolder'] + params['featuresfile'] # Load the necessary images paths_true, paths_false, featureims_true, featureims_false = load_images(ipl) ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) for d, k, v, kl in paths_true.data_iterator(yield_short_kl=True): if k == 'path': ipl.logging('===============================\nWorking on group: {}', kl) # # TODO: Implement copy full logger # ipl[kl].set_logger(ipl.get_logger()) # # Load the image data into memory # ipl[kl].populate() ipl[kl] = libip.features_of_paths( ipl, paths_true[kl][k], paths_false[kl][k], featureims_true[kl], featureims_false[kl], kl ) # Write the result to file ipl.write(filepath=targetfile, keys=[kl]) # # Free memory ipl[kl] = None
def compute_feature_images(yparams): params = yparams.get_params() thisparams = rdict(params['compute_feature_images']) # targetfile = params['intermedfolder'] + params['featureimsfile'] general_params = thisparams['general_params'] for sourcekey, source in thisparams['sources'].iteritems(): # Load the necessary images # 1. Determine the settings for fetching the data try: recursive_search = False recursive_search = thisparams['skwargs', 'default', 'recursive_search'] recursive_search = thisparams['skwargs', sourcekey, 'recursive_search'] except KeyError: pass if len(source) > 2: skeys = source[2] else: skeys = None # 2. Load the data data = load_images(params[source[0]] + params[source[1]], skeys=skeys, recursive_search=recursive_search, logger=yparams) # Set targetfile targetfile = params[thisparams['targets', sourcekey][0]] \ + params[thisparams['targets', sourcekey][1]] yparams.logging('\nInitial datastructure: \n\n{}', data.datastructure2string(maxdepth=3)) for d, k, v, kl in data.data_iterator(yield_short_kl=True, leaves_only=True): yparams.logging( '===============================\nWorking on image: {}', kl + [k]) # # TODO: Implement copy full logger # data[kl].set_logger(data.get_logger()) # Load the image data into memory data[kl].populate(k) # compute_selected_features(data.subset(kl), params) subfeature_params = thisparams['features', sourcekey].dcp() data[kl][k] = compute_features(data[kl][k], general_params, subfeature_params) # Write the result to file data.write(filepath=targetfile, keys=[kl + [k]]) # Free memory data[kl][k] = None
def make_feature_arrays(ipl): params = ipl.get_params() thisparams = rdict(params['random_forest']) targetfile = params['resultfolder'] + params['resultsfile'] # Load the necessary images load_images(ipl) ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) result = IPL() evaluation = rdict() for d, k, v, kl in ipl.data_iterator(yield_short_kl=True): if k == '0': ipl.logging('===============================\nWorking on group: {}', kl) # TODO: Implement copy full logger ipl[kl].set_logger(ipl.get_logger()) # Load the image data into memory ipl[kl].populate() # def shp(x): # return x.shape # print ipl[kl]['0', 'true'] # print ipl[kl].dss(function=shp) ipl[kl]['0', 'true'] = libip.rf_make_feature_array(ipl[kl]['0', 'true']) ipl.logging("Computed feature array for ['0', 'true'] with shape {}", ipl[kl]['0', 'true'].shape) ipl[kl]['0', 'false'] = libip.rf_make_feature_array(ipl[kl]['0', 'false']) ipl.logging("Computed feature array for ['0', 'false'] with shape {}", ipl[kl]['0', 'false'].shape) ipl[kl]['1', 'true'] = libip.rf_make_feature_array(ipl[kl]['1', 'true']) ipl.logging("Computed feature array for ['1', 'true'] with shape {}", ipl[kl]['1', 'true'].shape) ipl[kl]['1', 'false'] = libip.rf_make_feature_array(ipl[kl]['1', 'false']) ipl.logging("Computed feature array for ['1', 'false'] with shape {}", ipl[kl]['1', 'false'].shape) ipl.write(filepath=params['intermedfolder'] + 'feature_arrays.h5', keys=[kl])
def run_random_forest(yamlfile, logging=True, make_only_feature_array=False, debug=False, write=True): ipl = IPL(yaml=yamlfile) ipl.set_indent(1) params = rdict(data=ipl.get_params()) if logging: ipl.startlogger(filename=params['resultfolder'] + 'random_forest.log', type='w', name='RandomForest') else: ipl.startlogger() try: # # Copy the script file and the parameters to the scriptsfolder # copy(inspect.stack()[0][1], params['scriptsfolder']) # copy(yamlfile, params['scriptsfolder'] + 'random_forest.parameters.yml') # ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) if make_only_feature_array: make_feature_arrays(ipl) else: result = IPL() result['result'], result['evaluation'] = random_forest(ipl, debug=debug) # ipl.logging('\nFinal datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) if write: result.write(filepath=params['resultfolder'] + params['resultsfile']) ipl.logging('') ipl.stoplogger() except: ipl.errout('Unexpected error')
def compute_paths(ipl): params = ipl.get_params() thisparams = rdict(params['compute_paths']) targetfile = params['intermedfolder'] + params['pathsfile'] # Load the necessary images load_images(ipl) ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) maxd = ipl.maxdepth() for d, k, v, kl in ipl.data_iterator(maxdepth=ipl.maxdepth() - 1): if d == maxd - 1: ipl.logging( '===============================\nWorking on group: {}', kl) # TODO: Implement copy full logger ipl[kl].set_logger(ipl.get_logger()) # Load the image data into memory ipl[kl].populate() # The first step would be to just compute the paths, regardless whether they cross a # label barrier or not ipl[kl] = libip.compute_paths_with_class( ipl[kl], params['largeobjmnames'][0], params['borderctname'], 'disttransf', params['largeobjname'], thisparams, ignore=thisparams['ignorelabels'], max_end_count=thisparams['max_end_count'], max_end_count_seed=thisparams['max_end_count_seed'], debug=params['debug']) # Write the result to file ipl.write(filepath=targetfile, keys=[kl]) # Free memory ipl[kl] = None
def paths_of_merges(ipl, debug=False): params = ipl.get_params() thisparams = rdict(params['paths_of_merges']) targetfile = params['intermedfolder'] + params['pathsfalsefile'] # Load the necessary images load_images(ipl) ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) maxd = ipl.maxdepth() for d, k, v, kl in ipl.data_iterator(maxdepth=ipl.maxdepth() - 1): if d == maxd - 1: ipl.logging( '===============================\nWorking on group: {}', kl) # TODO: Implement copy full logger ipl[kl].set_logger(ipl.get_logger()) # Load the image data into memory ipl[kl].populate() ipl[kl] = libip.paths_of_labelpairs( ipl[kl], params['largeobjmnames'][0], params['largeobjname'], params['largeobjmnames'][4], params['borderctname'], 'disttransf', thisparams, ignore=thisparams['ignorelabels'], max_end_count=thisparams['max_end_count'], max_end_count_seed=thisparams['max_end_count_seed'], debug=debug) # Write the result to file ipl.write(filepath=targetfile, keys=[kl]) # Free memory ipl[kl] = None
def paths_of_merges(ipl): params = ipl.get_params() thisparams = rdict(params['paths_of_merges']) targetfile = params['intermedfolder'] + params['pathsfalsefile'] # Load the necessary images load_images(ipl) ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) maxd = ipl.maxdepth() for d, k, v, kl in ipl.data_iterator(maxdepth=ipl.maxdepth() - 1): if d == maxd - 1: ipl.logging('===============================\nWorking on group: {}', kl) # TODO: Implement copy full logger ipl[kl].set_logger(ipl.get_logger()) # Load the image data into memory ipl[kl].populate() ipl[kl] = libip.paths_of_labelpairs( ipl[kl], params['largeobjmnames'][0], params['largeobjname'], params['largeobjmnames'][4], params['borderctname'], 'disttransf', thisparams, ignore=thisparams['ignorelabels'], max_end_count=thisparams['max_end_count'], max_end_count_seed=thisparams['max_end_count_seed'], debug=False ) # Write the result to file ipl.write(filepath=targetfile, keys=[kl]) # Free memory ipl[kl] = None
def compute_selected_features(ipl, params): thisparams = rdict(data=params['compute_feature_images']) targetfile = params['intermedfolder'] + params['featureimsfile'] maxd = ipl.maxdepth() for d, k, v, kl in ipl.data_iterator(yield_short_kl=True): if d == maxd: ipl.logging('----------------------\nWorking on image: {}', k) ipl[kl].populate(k) if k in [ params['rawdataname'], params['probsname'], params['largeobjname'], params['largeobjmnames'][0] ]: general_params = thisparams.dcp() del general_params['features'] if k == params['rawdataname']: subfeature_params = thisparams['features']['rawdata'] ipl[kl][k] = compute_features(ipl[kl][k], general_params, subfeature_params) elif k == params['probsname']: subfeature_params = thisparams['features']['probs'] ipl[kl][k] = compute_features(ipl[kl][k], general_params, subfeature_params) elif k == params['largeobjname']: subfeature_params = thisparams['features']['largeobj'] ipl[kl][k] = compute_features(ipl[kl][k], general_params, subfeature_params) elif k == params['largeobjmnames'][0]: subfeature_params = thisparams['features']['largeobjm'] ipl[kl][k] = compute_features(ipl[kl][k], general_params, subfeature_params) ipl.write(filepath=targetfile, keys=[kl + [k]]) ipl[kl][k] = None
def merge_adjacent_objects(ipl): params = ipl.get_params() thisparams = rdict(params['merge_adjacent_objects']) targetfile = params['intermedfolder'] + params['largeobjmfile'] # Load the necessary images load_images(ipl, params['intermedfolder'], params['largeobjfile'], params['largeobjname']) ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) for d, k, v, kl in ipl.data_iterator(yield_short_kl=True): if k == params['largeobjname']: ipl.logging( '===============================\nWorking on image: {}', kl + [k]) ipl[kl].set_logger(ipl.get_logger()) # Load the image data into memory ipl[kl].populate(k) ipl[kl] = libip.merge_adjacent_objects( ipl[kl], k, thisparams['numberbysize'], thisparams['numberbyrandom'], thisparams['seed'], targetnames=params['largeobjmnames'], algorithm=thisparams['algorithm']) # Write the result to file ipl.write(filepath=targetfile, keys=[kl]) # Free memory ipl[kl] = None
def features_of_paths(yparams): params = yparams.get_params() thisparams = rdict(params['features_of_paths']) featureims = ipl() # Load feature images feature_sources = thisparams['sources', 'featureims'] feature_skwargs = thisparams['skwargs', 'featureims'] for sourcekey, source in feature_sources.iteritems(): # Load the necessary images # 1. Determine the settings for fetching the data try: recursive_search = False recursive_search = feature_skwargs['default', 'recursive_search'] recursive_search = feature_skwargs[sourcekey, 'recursive_search'] except KeyError: pass if len(source) > 2: skeys = source[2] else: skeys = None # 2. Load the data yparams.logging('skeys = {}', skeys) yparams.logging('recursive_search = {}', recursive_search) featureims[sourcekey] = load_images(params[source[0]] + params[source[1]], skeys=skeys, recursive_search=recursive_search, logger=yparams) yparams.logging('\nFeatureims datastructure: \n\n{}', featureims.datastructure2string()) # Load true and false paths paths = ipl() paths_sources = thisparams['sources', 'paths'] paths_skwargs = thisparams['skwargs', 'paths'] for sourcekey, source in paths_sources.iteritems(): # Load the necessary images # 1. Determine the settings for fetching the data try: recursive_search = False recursive_search = paths_skwargs['default', 'recursive_search'] recursive_search = paths_skwargs[sourcekey, 'recursive_search'] except KeyError: pass if len(source) > 2: skeys = source[2] else: skeys = None # 2. Load the data yparams.logging('skeys = {}', skeys) yparams.logging('recursive_search = {}', recursive_search) paths[sourcekey] = load_images(params[source[0]] + params[source[1]], skeys=skeys, recursive_search=recursive_search, logger=yparams) yparams.logging('\nPaths datastructure: \n\n{}', paths.datastructure2string(maxdepth=4)) # Load the segmentation image datastructure (We just require the datastructure, not the data # itself) try: recursive_search = False recursive_search = thisparams['skwargs', 'segmentation', 'recursive_search'] except KeyError: pass if len(thisparams['sources', 'segmentation']) > 2: skeys = thisparams['sources', 'segmentation'][2] else: skeys = None segmentation = load_images( params[thisparams['sources', 'segmentation'][0]] + params[thisparams['sources', 'segmentation'][1]], skeys=skeys, recursive_search=recursive_search, logger=yparams) yparams.logging('\nSegmentation datastructure: \n\n{}', segmentation.datastructure2string(maxdepth=4)) # data['contacts'].reduce_from_leafs(iterate=True) # data['disttransf'].reduce_from_leafs(iterate=True) # Set targetfile featuresfile = params[thisparams['target'][0]] \ + params[thisparams['target'][1]] pathlistfile = params[thisparams['pathlist'][0]] \ + params[thisparams['pathlist'][1]] pathlist = ipl() for d, k, v, kl in segmentation.data_iterator(yield_short_kl=True, leaves_only=True): yparams.logging( '===============================\nWorking on image: {}', kl + [k]) # # TODO: Implement copy full logger # data[kl].set_logger(data.get_logger()) # Bild an input featureims dict for the path computation infeatims = ipl() sourcelist = thisparams['sources', 'featureims'].dcp() if 'segmentation' in sourcelist: infeatims['segmentation'] = featureims['segmentation'][kl][k] sourcelist.pop('segmentation') for source in sourcelist: infeatims[source] = featureims[source][kl] infeatims.populate() # Bild an input dict for true paths intruepaths = paths['truepaths'][kl][k]['truepaths'] infalsepaths = paths['falsepaths'][kl][k]['falsepaths'] intruepaths.populate() infalsepaths.populate() yparams.logging('\nInfeatims datastructure: \n\n{}', infeatims.datastructure2string()) yparams.logging('\nIntruepaths datastructure: \n\n{}', intruepaths.datastructure2string(maxdepth=3)) yparams.logging('\nInfalsepaths datastructure: \n\n{}', infalsepaths.datastructure2string(maxdepth=3)) features = ipl() features[kl + [k]], pathlist[kl + [k]] = libip.features_of_paths( yparams, intruepaths, infalsepaths, infeatims, infeatims, kl, return_pathlist=True) yparams.logging( '\nPathlist datastructure: \n\n{}', pathlist.datastructure2string(function=type, leaves_only=False)) # Write the result to file features.write(filepath=featuresfile) # pathlist.astype(np.uint8) # pathlist.write(filepath=pathlistfile) with open(pathlistfile, 'wb') as f: pickle.dump(pathlist, f)
def random_forest(ipl, debug=False): params = ipl.get_params() thisparams = rdict(params['random_forest']) targetfile = params['resultfolder'] + params['resultsfile'] # Load the necessary images load_images(ipl) ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) result = IPL() new_eval = rdict() evaluation = rdict() for d, k, v, kl in ipl.data_iterator(yield_short_kl=True): if k == '0': ipl.logging( '===============================\nWorking on group: {}', kl) # TODO: Implement copy full logger ipl[kl].set_logger(ipl.get_logger()) # Load the image data into memory ipl[kl].populate() # def shp(x): # return x.shape # print ipl[kl]['0', 'true'] # print ipl[kl].dss(function=shp) ipl[kl]['0', 'true'] = libip.rf_make_feature_array(ipl[kl]['0', 'true']) ipl.logging( "Computed feature array for ['0', 'true'] with shape {}", ipl[kl]['0', 'true'].shape) ipl[kl]['0', 'false'] = libip.rf_make_feature_array(ipl[kl]['0', 'false']) ipl.logging( "Computed feature array for ['0', 'false'] with shape {}", ipl[kl]['0', 'false'].shape) ipl[kl]['1', 'true'] = libip.rf_make_feature_array(ipl[kl]['1', 'true']) ipl.logging( "Computed feature array for ['1', 'true'] with shape {}", ipl[kl]['1', 'true'].shape) ipl[kl]['1', 'false'] = libip.rf_make_feature_array(ipl[kl]['1', 'false']) ipl.logging( "Computed feature array for ['1', 'false'] with shape {}", ipl[kl]['1', 'false'].shape) # print '...' # print ipl[kl]['0'] result[kl + ['0']] = libip.random_forest(ipl[kl]['0'], ipl[kl]['1'], debug=debug) result[kl + ['1']] = libip.random_forest(ipl[kl]['1'], ipl[kl]['0'], debug=debug) new_eval[kl + ['0']] = libip.new_eval([x[0] for x in result[kl]['0']], [x[1] for x in result[kl]['0']]) new_eval[kl + ['1']] = libip.new_eval([x[0] for x in result[kl]['1']], [x[1] for x in result[kl]['1']]) evaluation[kl + ['0']] = libip.evaluation(result[kl]['0']) evaluation[kl + ['1']] = libip.evaluation(result[kl]['1']) ipl.logging('+++ RESULTS +++') ipl.logging("[kl]['0']") # for i in result[kl]['0']: # ipl.logging('{}', i) for key, value in evaluation[kl]['0'].iteritems(): ipl.logging('{} = {}', key, value) for key, value in new_eval[kl]['0'].iteritems(): ipl.logging('{} = {}', key, value) ipl.logging('+++') ipl.logging("[kl]['1']") # for i in result[kl]['1']: # ipl.logging('{}', i) for key, value in evaluation[kl]['1'].iteritems(): ipl.logging('{} = {}', key, value) for key, value in new_eval[kl]['1'].iteritems(): ipl.logging('{} = {}', key, value) # # Write the result to file # ipl.write(filepath=targetfile, keys=[kl]) # Free memory ipl[kl] = None return IPL(data=result), IPL(data=evaluation)
def compute_paths(yparams): params = yparams.get_params() thisparams = rdict(params['compute_paths']) data = ipl() for sourcekey, source in thisparams['sources'].iteritems(): # Load the necessary images # 1. Determine the settings for fetching the data try: recursive_search = False recursive_search = thisparams['skwargs', 'default', 'recursive_search'] recursive_search = thisparams['skwargs', sourcekey, 'recursive_search'] except KeyError: pass if len(source) > 2: skeys = source[2] else: skeys = None # 2. Load the data yparams.logging('skeys = {}', skeys) yparams.logging('recursive_search = {}', recursive_search) data[sourcekey] = load_images(params[source[0]] + params[source[1]], skeys=skeys, recursive_search=recursive_search, logger=yparams) data['contacts'].reduce_from_leafs(iterate=True) data['disttransf'].reduce_from_leafs(iterate=True) # Set targetfile targetfile = params[thisparams['target'][0]] \ + params[thisparams['target'][1]] yparams.logging('\nInitial datastructure: \n\n{}', data.datastructure2string(maxdepth=3)) for d, k, v, kl in data['segmentation'].data_iterator(yield_short_kl=True, leaves_only=True): yparams.logging( '===============================\nWorking on image: {}', kl + [k]) # # TODO: Implement copy full logger # data[kl].set_logger(data.get_logger()) # prepare the dict for the path computation indata = ipl() indata['segmentation'] = np.array(data['segmentation'][kl][k]) indata['contacts'] = np.array(data['contacts'][kl][k]) indata['groundtruth'] = np.array( data['groundtruth'][kl][params['gtruthname']]) indata['disttransf'] = np.array(data['disttransf'][kl][k]) yparams.logging('Input datastructure: \n\n{}', indata.datastructure2string()) # Compute the paths sorted into their respective class paths = ipl() paths[kl + [k]] = libip.compute_paths_with_class( indata, 'segmentation', 'contacts', 'disttransf', 'groundtruth', thisparams, ignore=thisparams['ignorelabels'], max_end_count=thisparams['max_end_count'], max_end_count_seed=thisparams['max_end_count_seed'], debug=params['debug']) # Write the result to file paths.write(filepath=targetfile)
def random_forest(ipl, debug=False): params = ipl.get_params() thisparams = rdict(params['random_forest']) targetfile = params['resultfolder'] + params['resultsfile'] # Load the necessary images load_images(ipl) ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) result = IPL() new_eval = rdict() evaluation = rdict() for d, k, v, kl in ipl.data_iterator(yield_short_kl=True): if k == '0': ipl.logging('===============================\nWorking on group: {}', kl) # TODO: Implement copy full logger ipl[kl].set_logger(ipl.get_logger()) # Note: # Due to the feature input being a dictionary organized by the feature images where # the feature values come from # # [kl] # '0'|'1' # 'true'|'false' # [featureims] # 'Sum': [s1, ..., sN] # 'Variance': [v1, ..., vN] # ... # [Pathlength]: [l1, ..., lN] # # the exact order in which items are iterated over by data_iterator() is not known. # # Solution: # Iterate over it once and store the keylist in an array (which conserves the order) # When accumulating the featrues for each of the four corresponding subsets, namely # training and testing set with true and false paths each, i.e. # ['0'|'1']['true'|'false'], # the the keylist is used, thus maintaining the correct order in every subset. # # And that is what is happening here: keylist = [] for d2, k2, v2, kl2 in ipl[kl]['0', 'true'].data_iterator(): if type(v2) is not type(ipl[kl]['0', 'true']): keylist.append(kl2) # Load the image data into memory ipl[kl].populate() # ipl[kl]['0', 'true'] = libip.rf_make_feature_array(ipl[kl]['0', 'true']) # ipl.logging("Computed feature array for ['0', 'true'] with shape {}", ipl[kl]['0', 'true'].shape) # ipl[kl]['0', 'false'] = libip.rf_make_feature_array(ipl[kl]['0', 'false']) # ipl.logging("Computed feature array for ['0', 'false'] with shape {}", ipl[kl]['0', 'false'].shape) # ipl[kl]['1', 'true'] = libip.rf_make_feature_array(ipl[kl]['1', 'true']) # ipl.logging("Computed feature array for ['1', 'true'] with shape {}", ipl[kl]['1', 'true'].shape) # ipl[kl]['1', 'false'] = libip.rf_make_feature_array(ipl[kl]['1', 'false']) # ipl.logging("Computed feature array for ['1', 'false'] with shape {}", ipl[kl]['1', 'false'].shape) ipl[kl]['0', 'true'] = libip.rf_make_feature_array_with_keylist(ipl[kl]['0', 'true'], keylist) ipl.logging("Computed feature array for ['0', 'true'] with shape {}", ipl[kl]['0', 'true'].shape) ipl[kl]['0', 'false'] = libip.rf_make_feature_array_with_keylist(ipl[kl]['0', 'false'], keylist) ipl.logging("Computed feature array for ['0', 'false'] with shape {}", ipl[kl]['0', 'false'].shape) ipl[kl]['1', 'true'] = libip.rf_make_feature_array_with_keylist(ipl[kl]['1', 'true'], keylist) ipl.logging("Computed feature array for ['1', 'true'] with shape {}", ipl[kl]['1', 'true'].shape) ipl[kl]['1', 'false'] = libip.rf_make_feature_array_with_keylist(ipl[kl]['1', 'false'], keylist) ipl.logging("Computed feature array for ['1', 'false'] with shape {}", ipl[kl]['1', 'false'].shape) # print '...' # print ipl[kl]['0'] result[kl + ['0']] = libip.random_forest(ipl[kl]['0'], ipl[kl]['1'], debug=debug) result[kl + ['1']] = libip.random_forest(ipl[kl]['1'], ipl[kl]['0'], debug=debug) new_eval[kl + ['0']] = libip.new_eval([x[0] for x in result[kl]['0']], [x[1] for x in result[kl]['0']]) new_eval[kl + ['1']] = libip.new_eval([x[0] for x in result[kl]['1']], [x[1] for x in result[kl]['1']]) evaluation[kl + ['0']] = libip.evaluation(result[kl]['0']) evaluation[kl + ['1']] = libip.evaluation(result[kl]['1']) ipl.logging('+++ RESULTS +++') ipl.logging("[kl]['0']") for i in result[kl]['0']: ipl.logging('{}', i) # for key, value in evaluation[kl]['0'].iteritems(): # ipl.logging('{} = {}', key, value) for key, value in new_eval[kl]['0'].iteritems(): ipl.logging('{} = {}', key, value) ipl.logging('+++') ipl.logging("[kl]['1']") for i in result[kl]['1']: ipl.logging('{}', i) # for key, value in evaluation[kl]['1'].iteritems(): # ipl.logging('{} = {}', key, value) for key, value in new_eval[kl]['1'].iteritems(): ipl.logging('{} = {}', key, value) # # Write the result to file # ipl.write(filepath=targetfile, keys=[kl]) # Free memory ipl[kl] = None return IPL(data=result), IPL(data=evaluation)
def compute_paths(yparams): all_params = yparams.get_params() # Zero'th layer: # -------------- zeroth = rdict(all_params['compute_paths']) if 'default' in zeroth: zeroth_defaults = zeroth.pop('default') else: zeroth_defaults = ipl() for exp_lbl, experiment in zeroth.iteritems(): # First layer # ----------- # An experiment is now selected and performed yparams.logging( 'Performing experiment {}\n==============================\n', exp_lbl) first = zeroth_defaults.dcp() first.merge(experiment) if 'default' in first: first_defaults = first.pop('default') else: first_defaults = ipl() statistics = rdict() for exp_class_lbl in ['truepaths', 'falsepaths']: # Final layer # ----------- # The true or false paths for the current experiment are here computed, respectively yparams.logging( 'Computing {}...\n------------------------------\n', exp_class_lbl) final = first_defaults.dcp() final.merge(first[exp_class_lbl]) exp_sources = final['sources'] exp_params = final['params'] exp_target = final['target'] # Load the necessary images data = ipl() for datakey, content in exp_sources.iteritems(): data[datakey] = load_images(all_params[content[0]] + all_params[content[1]], skeys=content[2]['skeys'], recursive_search=False, logger=yparams) yparams.logging('\nInitial datastructure: \n\n{}', data.datastructure2string(maxdepth=4)) yparams.logging('experiment_params: \n{}', exp_params) # Compute the paths # ----------------- paths = ipl() for_class = False if exp_class_lbl == 'truepaths': for_class = True paths[exp_lbl][exp_class_lbl], statistics[exp_lbl][ exp_class_lbl] = libip.compute_paths_for_class( data, 'segm', 'conts', 'dt', 'gt', exp_params, for_class=for_class, ignore=[], debug=all_params['debug'], logger=yparams) yparams.logging('\nPaths datastructure after running {}: \n\n{}', exp_class_lbl, paths.datastructure2string()) def val(x): return x yparams.logging( '\nStatistics after {}: \n\n{}', exp_class_lbl, simplify_statistics( statistics[exp_lbl]).datastructure2string(function=val)) # Save the result to disk # ----------------------- targetfile = all_params[exp_target[0]] + all_params[exp_target[1]] paths.write(filepath=targetfile) def val(x): return x yparams.logging( '\nStatistics after full experiment: \n\n{}', simplify_statistics( statistics[exp_lbl]).datastructure2string(function=val))
def random_forest(ipl, debug=False): params = ipl.get_params() thisparams = rdict(params['random_forest']) targetfile = params['resultfolder'] + params['resultsfile'] # Load the necessary images load_images(ipl) ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3)) result = IPL() new_eval = rdict() evaluation = rdict() for d, k, v, kl in ipl.data_iterator(yield_short_kl=True): if k == '0': ipl.logging('===============================\nWorking on group: {}', kl) # TODO: Implement copy full logger ipl[kl].set_logger(ipl.get_logger()) # Load the image data into memory ipl[kl].populate() # def shp(x): # return x.shape # print ipl[kl]['0', 'true'] # print ipl[kl].dss(function=shp) ipl[kl]['0', 'true'] = libip.rf_make_feature_array(ipl[kl]['0', 'true']) ipl.logging("Computed feature array for ['0', 'true'] with shape {}", ipl[kl]['0', 'true'].shape) ipl[kl]['0', 'false'] = libip.rf_make_feature_array(ipl[kl]['0', 'false']) ipl.logging("Computed feature array for ['0', 'false'] with shape {}", ipl[kl]['0', 'false'].shape) ipl[kl]['1', 'true'] = libip.rf_make_feature_array(ipl[kl]['1', 'true']) ipl.logging("Computed feature array for ['1', 'true'] with shape {}", ipl[kl]['1', 'true'].shape) ipl[kl]['1', 'false'] = libip.rf_make_feature_array(ipl[kl]['1', 'false']) ipl.logging("Computed feature array for ['1', 'false'] with shape {}", ipl[kl]['1', 'false'].shape) # print '...' # print ipl[kl]['0'] result[kl + ['0']] = libip.random_forest(ipl[kl]['0'], ipl[kl]['1'], debug=debug) result[kl + ['1']] = libip.random_forest(ipl[kl]['1'], ipl[kl]['0'], debug=debug) new_eval[kl + ['0']] = libip.new_eval([x[0] for x in result[kl]['0']], [x[1] for x in result[kl]['0']]) new_eval[kl + ['1']] = libip.new_eval([x[0] for x in result[kl]['1']], [x[1] for x in result[kl]['1']]) evaluation[kl + ['0']] = libip.evaluation(result[kl]['0']) evaluation[kl + ['1']] = libip.evaluation(result[kl]['1']) ipl.logging('+++ RESULTS +++') ipl.logging("[kl]['0']") # for i in result[kl]['0']: # ipl.logging('{}', i) for key, value in evaluation[kl]['0'].iteritems(): ipl.logging('{} = {}', key, value) for key, value in new_eval[kl]['0'].iteritems(): ipl.logging('{} = {}', key, value) ipl.logging('+++') ipl.logging("[kl]['1']") # for i in result[kl]['1']: # ipl.logging('{}', i) for key, value in evaluation[kl]['1'].iteritems(): ipl.logging('{} = {}', key, value) for key, value in new_eval[kl]['1'].iteritems(): ipl.logging('{} = {}', key, value) # # Write the result to file # ipl.write(filepath=targetfile, keys=[kl]) # Free memory ipl[kl] = None return IPL(data=result), IPL(data=evaluation)
def random_forest(yparams, debug=False): all_params = yparams.get_params() # Zero'th layer: # -------------- zeroth = rdict(all_params['random_forest']) if 'default' in zeroth: zeroth_defaults = zeroth.pop('default') else: zeroth_defaults = ipl() # pathlist = ipl() # pathlistfile = zeroth_defaults['targets', 'pathlist'] # pathlistfile = all_params[pathlistfile[0]] + all_params[pathlistfile[1]] # yparams.logging('\nDatastructure of pathlistin:\n\n{}', pathlistin.datastructure2string()) for exp_lbl, experiment in zeroth.iteritems(): # First layer # ----------- # An experiment is now selected and performed yparams.logging('\n\nPerforming experiment {}\n==============================', exp_lbl) final = zeroth_defaults.dcp() final.merge(experiment) exp_sources = final['sources'] exp_params = final['params'] exp_targets = final['targets'] exp_source_kl = [exp_lbl] if len(exp_sources['train']) == 4: exp_source_kl = exp_sources['train'][3] exp_predict_kl = ['predict'] if len(exp_sources['predict']) == 4: exp_predict_kl = exp_sources['predict'][3] if type(exp_source_kl) is str: exp_source_kl = [exp_source_kl] if type(exp_predict_kl) is str: exp_predict_kl = [exp_predict_kl] # Get the pathlist stored in features_of_paths pathlist_source = exp_sources.pop('pathlist') pathlistfile = all_params[pathlist_source[0]] \ + all_params[pathlist_source[1]] with open(pathlistfile, 'r') as f: pathlistin = pickle.load(f) if len(pathlist_source) > 2: if 'skeys' in pathlist_source[2]: pathlistin = pathlistin.subset(*pathlist_source[2]['skeys']) print 'I was here...' yparams.logging('pathlistin.datastructure: \n{}\n', pathlistin.datastructure2string(maxdepth=4)) pathlistout = ipl() # Load training data if 'train' in exp_sources.keys(): truesource = exp_sources['train'] falsesource = exp_sources['train'] else: truesource = exp_sources['traintrue'] falsesource = exp_sources['trainfalse'] truetrainfeats = load_data( all_params[truesource[0]] + all_params[truesource[1]], logger=yparams, **truesource[2] ).subset('truepaths', search=True) falsetrainfeats = load_data( all_params[falsesource[0]] + all_params[falsesource[1]], logger=yparams, **falsesource[2] ).subset('falsepaths', search=True) yparams.logging('\ntruetrainfeats.datastructure: \n{}\n', truetrainfeats.datastructure2string(maxdepth=4)) yparams.logging('\nfalsetrainfeats.datastructure: \n{}\n', falsetrainfeats.datastructure2string(maxdepth=4)) # Load prediction data predictsource = exp_sources['predict'] predictfeats = load_data( all_params[predictsource[0]] + all_params[predictsource[1]], logger=yparams, **predictsource[2] ) yparams.logging('\npredictfeats.datastructure: \n{}\n', predictfeats.datastructure2string(maxdepth=4)) # Load the data into memory truetrainfeats.populate() falsetrainfeats.populate() predictfeats.populate() # Concatenate the different sources # 1. Of training data plo_true = ipl() truetrainfeats, plo_true['truepaths'] = libip.rf_combine_sources_new( truetrainfeats[exp_source_kl]['truepaths'].dcp(), pathlistin[exp_source_kl]['truepaths'].dcp() ) plo_false = ipl() falsetrainfeats, plo_false['falsepaths'] = libip.rf_combine_sources_new( falsetrainfeats[exp_source_kl]['falsepaths'].dcp(), pathlistin[exp_source_kl]['falsepaths'].dcp() ) pathlistout[exp_source_kl + ['train']] = plo_true + plo_false # 2. Of prediction data ipf_true = ipl() plo_true = ipl() ipf_true['truepaths'], plo_true['truepaths'] = libip.rf_combine_sources_new( predictfeats[exp_predict_kl]['truepaths'].dcp(), pathlistin[exp_predict_kl]['truepaths'].dcp() ) ipf_false = ipl() plo_false = ipl() ipf_false['falsepaths'], plo_false['falsepaths'] = libip.rf_combine_sources_new( predictfeats[exp_predict_kl]['falsepaths'].dcp(), pathlistin[exp_predict_kl]['falsepaths'].dcp() ) inpredictfeats = ipf_true + ipf_false pathlistout[exp_source_kl, 'predict'] = plo_true + plo_false # Note: # Due to the feature input being a dictionary organized by the feature images where # the feature values come from # # [source] # 'truepaths'|'falsepaths' # [featureims] # 'Sum': [s1, ..., sN] # 'Variance': [v1, ..., vN] # ... # [Pathlength]: [l1, ..., lN] # # the exact order in which items are iterated over by data_iterator() is not known. # # Solution: # Iterate over it once and store the keylist in an array (which conserves the order) # When accumulating the features for each of the four corresponding subsets, namely # training and testing set with true and false paths each, i.e. # ['0'|'1']['truefeats'|'falsefeats'], # the the keylist is used, thus maintaining the correct order in every subset. # # And that is what is happening here: # # 1. Get the keylist of a full feature list, e.g. one of true paths # example_kl = None # for d2, k2, v2, kl2 in truetrainfeats.data_iterator(): # if k2 == 'truepaths': # example_kl = kl2 # break # 2. Get the keylist order of the feature space feature_space_list = [] for d2, k2, v2, kl2 in truetrainfeats.data_iterator(): if type(v2) is not type(truetrainfeats): feature_space_list.append(kl2) intrain = ipl() intrain['true'] = libip.rf_make_feature_array_with_keylist(truetrainfeats, feature_space_list) yparams.logging("Computed feature array for train['true'] with shape {}", intrain['true'].shape) intrain['false'] = libip.rf_make_feature_array_with_keylist(falsetrainfeats, feature_space_list) yparams.logging("Computed feature array for train['false'] with shape {}", intrain['false'].shape) inpredictfeats['true'] = libip.rf_make_feature_array_with_keylist(inpredictfeats['truepaths'], feature_space_list) yparams.logging("Computed feature array for predict['true'] with shape {}", inpredictfeats['true'].shape) inpredictfeats['false'] = libip.rf_make_feature_array_with_keylist(inpredictfeats['falsepaths'], feature_space_list) yparams.logging("Computed feature array for predict['false'] with shape {}", inpredictfeats['false'].shape) # Classify result = ipl() result[exp_lbl] = libip.random_forest( intrain, inpredictfeats, debug=debug, balance=exp_params['balance_classes'], logger=yparams ) # Evaluate new_eval = ipl() # print [x[0] for x in result[kl]] # print [x[1] for x in result[kl]] new_eval[exp_lbl] = libip.new_eval([x[0] for x in result[exp_lbl]], [x[1] for x in result[exp_lbl]]) yparams.logging('+++ RESULTS +++') yparams.logging("[kl]") # for i in result[kl]: # yparams.logging('{}', i) for key, value in new_eval[exp_lbl].iteritems(): yparams.logging('{} = {}', key, value) pass sys.exit() params = yparams.get_params() thisparams = rdict(params['random_forest']) # Get the pathlist stored in features_of_paths pathlistfile = params[thisparams['pathlistin'][0]] \ + params[thisparams['pathlistin'][1]] with open(pathlistfile, 'r') as f: pathlistin = pickle.load(f) pathlistout = ipl() yparams.logging('\nDatastructure of pathlistin:\n\n{}', pathlistin.datastructure2string()) # for i in xrange(0, len(thisparams['sources'])): for d, k, v, kl in thisparams['sources'].data_iterator(yield_short_kl=True): if k == 'predict': yparams.logging('===============================\nWorking on group: {}', kl) # Get parameters (currently only 'balance_classes') and set defaults balance_classes = False if 'balance_classes' in thisparams['sources'][kl].keys(): balance_classes = thisparams['sources'][kl]['balance_classes'] # Load training data if 'train' in thisparams['sources'][kl].keys(): truesource = thisparams['sources'][kl]['train'] falsesource = thisparams['sources'][kl]['train'] else: truesource = thisparams['sources'][kl]['traintrue'] falsesource = thisparams['sources'][kl]['trainfalse'] truetrainfeats = load_data( params[truesource[0]] + params[truesource[1]], logger=yparams, **truesource[2] ).subset('true', search=True) falsetrainfeats = load_data( params[falsesource[0]] + params[falsesource[1]], logger=yparams, **falsesource[2] ).subset('false', search=True) # # The plus operator is overloaded to perform a merging operation on RecursiveDicts # trainfeats = truetrainfeats + falsetrainfeats # yparams.logging( # '\nDatastructure of truetrainfeats\n\n{}', # truetrainfeats.datastructure2string(maxdepth=3) # ) # yparams.logging( # '\nDatastructure of falsetrainfeats\n\n{}', # falsetrainfeats.datastructure2string(maxdepth=3) # ) # Load prediction data predictsource = thisparams['sources'][kl]['predict'] predictfeats = load_data( params[predictsource[0]] + params[predictsource[1]], logger=yparams, **predictsource[2] ) # Note: # Due to the feature input being a dictionary organized by the feature images where # the feature values come from # # [source] # 'true'|'false' # [featureims] # 'Sum': [s1, ..., sN] # 'Variance': [v1, ..., vN] # ... # [Pathlength]: [l1, ..., lN] # # the exact order in which items are iterated over by data_iterator() is not known. # # Solution: # Iterate over it once and store the keylist in an array (which conserves the order) # When accumulating the featrues for each of the four corresponding subsets, namely # training and testing set with true and false paths each, i.e. # ['0'|'1']['true'|'false'], # the the keylist is used, thus maintaining the correct order in every subset. # # And that is what is happening here: # 1. Get the keylist of a full feature list, e.g. one of true paths example_kl = None for d2, k2, v2, kl2 in truetrainfeats.data_iterator(): if k2 == 'true': example_kl = kl2 break # 2. Get the keylist order of the feature space feature_space_list = [] for d2, k2, v2, kl2 in truetrainfeats[example_kl].data_iterator(): if type(v2) is not type(truetrainfeats[example_kl]): feature_space_list.append(kl2) # yparams.logging('feature_space_list[i] = {}', feature_space_list, listed=True) # Load the data into memory truetrainfeats.populate() falsetrainfeats.populate() predictfeats.populate() truetrainfeats, plo_true = libip.rf_combine_sources( truetrainfeats, search_for='true', pathlist=pathlistin ) falsetrainfeats, plo_false = libip.rf_combine_sources( falsetrainfeats, search_for='false', pathlist=pathlistin ) pathlistout[kl + ['train']] = plo_true + plo_false ipf_true, plo_true = libip.rf_combine_sources( predictfeats, search_for='true', pathlist=pathlistin ) ipf_false, plo_false = libip.rf_combine_sources( predictfeats, search_for='false', pathlist=pathlistin ) inpredictfeats = ipf_true + ipf_false pathlistout[kl + ['predict']] = plo_true + plo_false # yparams.logging( # '\nDatastructure of truetrainfeats\n\n{}', # truetrainfeats.datastructure2string(maxdepth=3) # ) # yparams.logging( # '\nDatastructure of falsetrainfeats\n\n{}', # falsetrainfeats.datastructure2string(maxdepth=3) # ) intrain = ipl() intrain['true'] = libip.rf_make_feature_array_with_keylist(truetrainfeats['true'], feature_space_list) yparams.logging("Computed feature array for train['true'] with shape {}", intrain['true'].shape) intrain['false'] = libip.rf_make_feature_array_with_keylist(falsetrainfeats['false'], feature_space_list) yparams.logging("Computed feature array for train['false'] with shape {}", intrain['false'].shape) inpredictfeats['true'] = libip.rf_make_feature_array_with_keylist(inpredictfeats['true'], feature_space_list) yparams.logging("Computed feature array for predict['true'] with shape {}", inpredictfeats['true'].shape) inpredictfeats['false'] = libip.rf_make_feature_array_with_keylist(inpredictfeats['false'], feature_space_list) yparams.logging("Computed feature array for predict['false'] with shape {}", inpredictfeats['false'].shape) # Classify result = ipl() result[kl] = libip.random_forest( intrain, inpredictfeats, debug=debug, balance=balance_classes, logger=yparams ) # Evaluate new_eval = ipl() # print [x[0] for x in result[kl]] # print [x[1] for x in result[kl]] new_eval[kl] = libip.new_eval([x[0] for x in result[kl]], [x[1] for x in result[kl]]) yparams.logging('+++ RESULTS +++') yparams.logging("[kl]") # for i in result[kl]: # yparams.logging('{}', i) for key, value in new_eval[kl].iteritems(): yparams.logging('{} = {}', key, value)
def features_of_paths(yparams): all_params = yparams.get_params() # Zero'th layer: # -------------- zeroth = rdict(all_params['features_of_paths']) if 'default' in zeroth: zeroth_defaults = zeroth.pop('default') else: zeroth_defaults = ipl() pathlist = ipl() pathlistfile = zeroth_defaults['targets', 'pathlist'] pathlistfile = all_params[pathlistfile[0]] + all_params[pathlistfile[1]] for exp_lbl, experiment in zeroth.iteritems(): # First layer # ----------- # An experiment is now selected and performed yparams.logging( '\n\nPerforming experiment {}\n==============================', exp_lbl) final = zeroth_defaults.dcp() final.merge(experiment) exp_sources = final['sources'] exp_params = final['params'] exp_targets = final['targets'] def val(x): return x yparams.logging('exp_sources = \n{}', exp_sources.datastructure2string(function=val)) yparams.logging('exp_params = \n{}', exp_sources.datastructure2string(function=val)) yparams.logging('exp_targets = \n{}', exp_targets.datastructure2string(function=val)) # Load feature images # ------------------- featureims = ipl() for k, v in exp_sources['featureims'].iteritems(): skeys = None if 'skeys' in v[2]: skeys = v[2]['skeys'] featureims[k] = load_images(all_params[v[0]] + all_params[v[1]], skeys=skeys, logger=yparams) yparams.logging('\nFeatureims datastructure: \n\n{}', featureims.datastructure2string(maxdepth=4)) for exp_class_lbl, exp_class_src in exp_sources['paths'].iteritems(): yparams.logging('\nWorking on {}\n------------------------------', exp_class_lbl) # Load paths # ---------- skeys = None if 'skeys' in exp_class_src[2]: skeys = exp_class_src[2]['skeys'] paths = load_images(all_params[exp_class_src[0]] + all_params[exp_class_src[1]], skeys=skeys, logger=yparams) yparams.logging('\nPaths datastructure: \n\n{}', paths.datastructure2string(maxdepth=4)) # Iterate over the segmentation images for d, k, v, kl in paths[exp_class_src[2]['skeys'] [0]].data_iterator(leaves_only=True, yield_short_kl=True, maxdepth=3): yparams.logging( '\nImage keylist: {}\n..............................', kl + [k]) segm_kl = kl + [k] imgs_kl = kl yparams.logging('segm_kl = {}', segm_kl) yparams.logging('imgs_kl = {}', imgs_kl) # Bild an input featureims dict for the path computation infeatims = ipl() sourcelist = exp_sources['featureims'].dcp() if 'segmentation' in sourcelist: infeatims['segmentation'] = featureims['segmentation'][ segm_kl] sourcelist.pop('segmentation') for source in sourcelist: infeatims[source] = featureims[source][imgs_kl] infeatims.populate() # Bild an input dict for true paths inpaths = v.dcp() inpaths.populate() features = ipl() features[exp_lbl][[exp_class_lbl] + kl + [k]], pathlist[ exp_lbl][[exp_class_lbl] + kl + [k]] = libip.get_features( inpaths, np.array(np.array( infeatims.yield_an_item()).shape)[0:3], infeatims, list(exp_params['features']), exp_params['max_paths_per_label'], ipl=yparams, anisotropy=exp_params['anisotropy'], return_pathlist=True) yparams.logging('\nFeatures datastructure: \n\n{}', features.datastructure2string(maxdepth=4)) # Write the result to file features.write( filepath=all_params[exp_targets['features'][0]] + all_params[exp_targets['features'][1]]) with open(pathlistfile, 'wb') as f: pickle.dump(pathlist, f)