def load_images(dict_main): """ These images are loaded: paths_true (paths within single label objects) paths_false (paths of merged objects which cross the merging site) featureims_true featureims_false :param dict_main: :return: """ paths_true = ipl() paths_false = ipl() featureims_true = ipl() featureims_false = ipl() params = dict_main.get_params() dict_main.logging('Loading true paths ...') # Paths within labels (true paths) paths_true.data_from_file( filepath=params['intermedfolder'] + params['pathstruefile'], skeys=[['x', '0', 'path']], recursive_search=False, nodata=True ) dict_main.logging('Loading false paths ...') # Paths of merges (false paths) paths_false.data_from_file( filepath=params['intermedfolder'] + params['pathsfalsefile'], skeys=[['x', '0', 'path']], recursive_search=False, nodata=True ) dict_main.logging('Loading features for true paths ...') # Load features for true paths featureims_true.data_from_file( filepath=params['intermedfolder'] + params['featureimsfile'], nodata=True, skeys=[['x', '0']] ) featureims_true.delete_items(params['largeobjmnames'][0]) dict_main.logging('Loading features for false paths ...') # Load features for false paths featureims_false.data_from_file( filepath=params['intermedfolder'] + params['featureimsfile'], nodata=True, skeys=[['x', '0']] ) featureims_false.delete_items(params['largeobjname']) return (paths_true, paths_false, featureims_true, featureims_false)
def load_images(dict_main): """ These images are loaded: paths_true (paths within single label objects) paths_false (paths of merged objects which cross the merging site) featureims_true featureims_false :param dict_main: :return: """ paths_true = ipl() paths_false = ipl() featureims_true = ipl() featureims_false = ipl() params = dict_main.get_params() dict_main.logging('Loading true paths ...') # Paths within labels (true paths) paths_true.data_from_file(filepath=params['intermedfolder'] + params['pathstruefile'], skeys=[['x', '0', 'path']], recursive_search=False, nodata=True) dict_main.logging('Loading false paths ...') # Paths of merges (false paths) paths_false.data_from_file(filepath=params['intermedfolder'] + params['pathsfalsefile'], skeys=[['x', '0', 'path']], recursive_search=False, nodata=True) dict_main.logging('Loading features for true paths ...') # Load features for true paths featureims_true.data_from_file(filepath=params['intermedfolder'] + params['featureimsfile'], nodata=True, skeys=[['x', '0']]) featureims_true.delete_items(params['largeobjmnames'][0]) dict_main.logging('Loading features for false paths ...') # Load features for false paths featureims_false.data_from_file(filepath=params['intermedfolder'] + params['featureimsfile'], nodata=True, skeys=[['x', '0']]) featureims_false.delete_items(params['largeobjname']) return (paths_true, paths_false, featureims_true, featureims_false)
def find_border_contacts(yparams): params = yparams.get_params() thisparams = rdict(params['find_border_contacts']) # targetfile = params['intermedfolder'] + params['featureimsfile'] data = ipl() for sourcekey, source in thisparams['sources'].iteritems(): # Load the necessary images # 1. Determine the settings for fetching the data try: recursive_search = False recursive_search = thisparams['skwargs', 'default', 'recursive_search'] recursive_search = thisparams['skwargs', sourcekey, 'recursive_search'] except KeyError: pass if len(source) > 2: skeys = source[2] else: skeys = None # 2. Load the data data[sourcekey] = load_images(params[source[0]] + params[source[1]], skeys=skeys, recursive_search=recursive_search, logger=yparams) # TODO: Get rid of this at some point! Probable re-implement the whole data-loading step data['disttransf'].reduce_from_leaves(iterate=False) data['disttransf'].reduce_from_leaves(iterate=False) # Set targetfile targetfile = params[thisparams['target'][0]] \ + params[thisparams['target'][1]] yparams.logging('\nInitial datastructure: \n\n{}', data.datastructure2string(maxdepth=4)) for d, k, v, kl in data['disttransf'].data_iterator(yield_short_kl=True, leaves_only=True): yparams.logging( '===============================\nWorking on image: {}', kl + [k]) # # TODO: Implement copy full logger # data[kl].set_logger(data.get_logger()) # We need: the distance transform of the MERGED labels (i.e. segmentation) and the # corresponding segmentation data['segmentation'][kl][k] = libip.find_border_contacts_arr( data['segmentation'][kl][k], data['disttransf'][kl][k], tkey=params['borderctname'], debug=params['debug']) # Write the result to file data['segmentation'].write(filepath=targetfile, keys=[kl + [k]])
def split_in_xyz(data, dims): reskeys = ('0', '1') split_data = ipl() if 'z' in dims: split_data['z'] = data.anytask(lib.split, 2, axis=0, result_keys=reskeys, return_only=True, rtrntype=ipl) if 'y' in dims: split_data['y'] = data.anytask(lib.split, 2, axis=1, result_keys=reskeys, return_only=True, rtrntype=ipl) if 'x' in dims: split_data['x'] = data.anytask(lib.split, 2, axis=2, result_keys=reskeys, return_only=True, rtrntype=ipl) split_data = split_data.switch_levels(1, 2) return split_data
def load_images(filepath, skeys=None, recursive_search=False, logger=None): if logger is not None: logger.logging('Loading data from \n{}', filepath) else: print 'Loading data from \n{}'.format(filepath) data = ipl() data.data_from_file(filepath=filepath, skeys=skeys, recursive_search=recursive_search, nodata=True) return data
def compute_paths(yparams): all_params = yparams.get_params() # Zero'th layer: # -------------- zeroth = rdict(all_params['compute_paths']) if 'default' in zeroth: zeroth_defaults = zeroth.pop('default') else: zeroth_defaults = ipl() for exp_lbl, experiment in zeroth.iteritems(): # First layer # ----------- # An experiment is now selected and performed yparams.logging( 'Performing experiment {}\n==============================\n', exp_lbl) first = zeroth_defaults.dcp() first.merge(experiment) if 'default' in first: first_defaults = first.pop('default') else: first_defaults = ipl() statistics = rdict() for exp_class_lbl in ['truepaths', 'falsepaths']: # Final layer # ----------- # The true or false paths for the current experiment are here computed, respectively yparams.logging( 'Computing {}...\n------------------------------\n', exp_class_lbl) final = first_defaults.dcp() final.merge(first[exp_class_lbl]) exp_sources = final['sources'] exp_params = final['params'] exp_target = final['target'] # Load the necessary images data = ipl() for datakey, content in exp_sources.iteritems(): data[datakey] = load_images(all_params[content[0]] + all_params[content[1]], skeys=content[2]['skeys'], recursive_search=False, logger=yparams) yparams.logging('\nInitial datastructure: \n\n{}', data.datastructure2string(maxdepth=4)) yparams.logging('experiment_params: \n{}', exp_params) # Compute the paths # ----------------- paths = ipl() for_class = False if exp_class_lbl == 'truepaths': for_class = True paths[exp_lbl][exp_class_lbl], statistics[exp_lbl][ exp_class_lbl] = libip.compute_paths_for_class( data, 'segm', 'conts', 'dt', 'gt', exp_params, for_class=for_class, ignore=[], debug=all_params['debug'], logger=yparams) yparams.logging('\nPaths datastructure after running {}: \n\n{}', exp_class_lbl, paths.datastructure2string()) def val(x): return x yparams.logging( '\nStatistics after {}: \n\n{}', exp_class_lbl, simplify_statistics( statistics[exp_lbl]).datastructure2string(function=val)) # Save the result to disk # ----------------------- targetfile = all_params[exp_target[0]] + all_params[exp_target[1]] paths.write(filepath=targetfile) def val(x): return x yparams.logging( '\nStatistics after full experiment: \n\n{}', simplify_statistics( statistics[exp_lbl]).datastructure2string(function=val))
rename_entry_args = ('data', 'labels') files = dict(zip(sourcefiles, targetfiles)) for source, target in files.iteritems(): sourcefilepath = path + source print 'sourcefilepath = {}'.format(sourcefilepath) targetfilepath = path + target print 'targetfilepath = {}'.format(targetfilepath) if not debug: # Load data data = ipl(filepath=sourcefilepath, nodata=False) # Print shape def shp(x): return x.shape print 'data.dss(function=shp) = {}'.format(data.dss(function=shp)) # Swap axes data.swapaxes(*swap_axes_args) # Re-name entry data.rename_entry(*rename_entry_args) # Print shape again print 'data.dss(function=shp) = {}'.format(data.dss(function=shp))
def features_of_paths(yparams): params = yparams.get_params() thisparams = rdict(params['features_of_paths']) featureims = ipl() # Load feature images feature_sources = thisparams['sources', 'featureims'] feature_skwargs = thisparams['skwargs', 'featureims'] for sourcekey, source in feature_sources.iteritems(): # Load the necessary images # 1. Determine the settings for fetching the data try: recursive_search = False recursive_search = feature_skwargs['default', 'recursive_search'] recursive_search = feature_skwargs[sourcekey, 'recursive_search'] except KeyError: pass if len(source) > 2: skeys = source[2] else: skeys = None # 2. Load the data yparams.logging('skeys = {}', skeys) yparams.logging('recursive_search = {}', recursive_search) featureims[sourcekey] = load_images(params[source[0]] + params[source[1]], skeys=skeys, recursive_search=recursive_search, logger=yparams) yparams.logging('\nFeatureims datastructure: \n\n{}', featureims.datastructure2string()) # Load true and false paths paths = ipl() paths_sources = thisparams['sources', 'paths'] paths_skwargs = thisparams['skwargs', 'paths'] for sourcekey, source in paths_sources.iteritems(): # Load the necessary images # 1. Determine the settings for fetching the data try: recursive_search = False recursive_search = paths_skwargs['default', 'recursive_search'] recursive_search = paths_skwargs[sourcekey, 'recursive_search'] except KeyError: pass if len(source) > 2: skeys = source[2] else: skeys = None # 2. Load the data yparams.logging('skeys = {}', skeys) yparams.logging('recursive_search = {}', recursive_search) paths[sourcekey] = load_images(params[source[0]] + params[source[1]], skeys=skeys, recursive_search=recursive_search, logger=yparams) yparams.logging('\nPaths datastructure: \n\n{}', paths.datastructure2string(maxdepth=4)) # Load the segmentation image datastructure (We just require the datastructure, not the data # itself) try: recursive_search = False recursive_search = thisparams['skwargs', 'segmentation', 'recursive_search'] except KeyError: pass if len(thisparams['sources', 'segmentation']) > 2: skeys = thisparams['sources', 'segmentation'][2] else: skeys = None segmentation = load_images( params[thisparams['sources', 'segmentation'][0]] + params[thisparams['sources', 'segmentation'][1]], skeys=skeys, recursive_search=recursive_search, logger=yparams) yparams.logging('\nSegmentation datastructure: \n\n{}', segmentation.datastructure2string(maxdepth=4)) # data['contacts'].reduce_from_leafs(iterate=True) # data['disttransf'].reduce_from_leafs(iterate=True) # Set targetfile featuresfile = params[thisparams['target'][0]] \ + params[thisparams['target'][1]] pathlistfile = params[thisparams['pathlist'][0]] \ + params[thisparams['pathlist'][1]] pathlist = ipl() for d, k, v, kl in segmentation.data_iterator(yield_short_kl=True, leaves_only=True): yparams.logging( '===============================\nWorking on image: {}', kl + [k]) # # TODO: Implement copy full logger # data[kl].set_logger(data.get_logger()) # Bild an input featureims dict for the path computation infeatims = ipl() sourcelist = thisparams['sources', 'featureims'].dcp() if 'segmentation' in sourcelist: infeatims['segmentation'] = featureims['segmentation'][kl][k] sourcelist.pop('segmentation') for source in sourcelist: infeatims[source] = featureims[source][kl] infeatims.populate() # Bild an input dict for true paths intruepaths = paths['truepaths'][kl][k]['truepaths'] infalsepaths = paths['falsepaths'][kl][k]['falsepaths'] intruepaths.populate() infalsepaths.populate() yparams.logging('\nInfeatims datastructure: \n\n{}', infeatims.datastructure2string()) yparams.logging('\nIntruepaths datastructure: \n\n{}', intruepaths.datastructure2string(maxdepth=3)) yparams.logging('\nInfalsepaths datastructure: \n\n{}', infalsepaths.datastructure2string(maxdepth=3)) features = ipl() features[kl + [k]], pathlist[kl + [k]] = libip.features_of_paths( yparams, intruepaths, infalsepaths, infeatims, infeatims, kl, return_pathlist=True) yparams.logging( '\nPathlist datastructure: \n\n{}', pathlist.datastructure2string(function=type, leaves_only=False)) # Write the result to file features.write(filepath=featuresfile) # pathlist.astype(np.uint8) # pathlist.write(filepath=pathlistfile) with open(pathlistfile, 'wb') as f: pickle.dump(pathlist, f)
def features_of_paths(yparams): all_params = yparams.get_params() # Zero'th layer: # -------------- zeroth = rdict(all_params['features_of_paths']) if 'default' in zeroth: zeroth_defaults = zeroth.pop('default') else: zeroth_defaults = ipl() pathlist = ipl() pathlistfile = zeroth_defaults['targets', 'pathlist'] pathlistfile = all_params[pathlistfile[0]] + all_params[pathlistfile[1]] for exp_lbl, experiment in zeroth.iteritems(): # First layer # ----------- # An experiment is now selected and performed yparams.logging( '\n\nPerforming experiment {}\n==============================', exp_lbl) final = zeroth_defaults.dcp() final.merge(experiment) exp_sources = final['sources'] exp_params = final['params'] exp_targets = final['targets'] def val(x): return x yparams.logging('exp_sources = \n{}', exp_sources.datastructure2string(function=val)) yparams.logging('exp_params = \n{}', exp_sources.datastructure2string(function=val)) yparams.logging('exp_targets = \n{}', exp_targets.datastructure2string(function=val)) # Load feature images # ------------------- featureims = ipl() for k, v in exp_sources['featureims'].iteritems(): skeys = None if 'skeys' in v[2]: skeys = v[2]['skeys'] featureims[k] = load_images(all_params[v[0]] + all_params[v[1]], skeys=skeys, logger=yparams) yparams.logging('\nFeatureims datastructure: \n\n{}', featureims.datastructure2string(maxdepth=4)) for exp_class_lbl, exp_class_src in exp_sources['paths'].iteritems(): yparams.logging('\nWorking on {}\n------------------------------', exp_class_lbl) # Load paths # ---------- skeys = None if 'skeys' in exp_class_src[2]: skeys = exp_class_src[2]['skeys'] paths = load_images(all_params[exp_class_src[0]] + all_params[exp_class_src[1]], skeys=skeys, logger=yparams) yparams.logging('\nPaths datastructure: \n\n{}', paths.datastructure2string(maxdepth=4)) # Iterate over the segmentation images for d, k, v, kl in paths[exp_class_src[2]['skeys'] [0]].data_iterator(leaves_only=True, yield_short_kl=True, maxdepth=3): yparams.logging( '\nImage keylist: {}\n..............................', kl + [k]) segm_kl = kl + [k] imgs_kl = kl yparams.logging('segm_kl = {}', segm_kl) yparams.logging('imgs_kl = {}', imgs_kl) # Bild an input featureims dict for the path computation infeatims = ipl() sourcelist = exp_sources['featureims'].dcp() if 'segmentation' in sourcelist: infeatims['segmentation'] = featureims['segmentation'][ segm_kl] sourcelist.pop('segmentation') for source in sourcelist: infeatims[source] = featureims[source][imgs_kl] infeatims.populate() # Bild an input dict for true paths inpaths = v.dcp() inpaths.populate() features = ipl() features[exp_lbl][[exp_class_lbl] + kl + [k]], pathlist[ exp_lbl][[exp_class_lbl] + kl + [k]] = libip.get_features( inpaths, np.array(np.array( infeatims.yield_an_item()).shape)[0:3], infeatims, list(exp_params['features']), exp_params['max_paths_per_label'], ipl=yparams, anisotropy=exp_params['anisotropy'], return_pathlist=True) yparams.logging('\nFeatures datastructure: \n\n{}', features.datastructure2string(maxdepth=4)) # Write the result to file features.write( filepath=all_params[exp_targets['features'][0]] + all_params[exp_targets['features'][1]]) with open(pathlistfile, 'wb') as f: pickle.dump(pathlist, f)
pathvals = np.swapaxes(pathvals, 0, 1) print x plt.plot(x, pathvals) # plt.show() print folder + 'plots/' + name + kl2str(kl) + '.png' print kl2str(kl) print kl lab.savefig(folder + 'plots/' + name + kl2str(kl) + '.png') plt.clf() if __name__ == '__main__': yamlfile = os.path.dirname(os.path.abspath(__file__)) + '/parameters_ref.yml' dict_main = ipl(yaml=yamlfile) paths_true, paths_false, featureims_true, featureims_false = load_images(dict_main) dict_main.startlogger() dict_main.logging('\npaths_true datastructure\n---\n{}', paths_true.datastructure2string(maxdepth=2)) dict_main.logging('\npaths_false datastructure\n---\n{}', paths_false.datastructure2string(maxdepth=2)) dict_main.logging('\nfeatureims_true datastructure\n---\n{}', featureims_true.datastructure2string(maxdepth=2)) dict_main.logging('\nfeatureims_false datastructure\n---\n{}', featureims_false.datastructure2string(maxdepth=2)) params = dict_main.get_params() plot_paths(paths_true, featureims_true, params['intermedfolder'], '_true_') plot_paths(paths_false, featureims_false, params['intermedfolder'], '_false_') sys.exit()
def random_forest(yparams, debug=False): all_params = yparams.get_params() # Zero'th layer: # -------------- zeroth = rdict(all_params['random_forest']) if 'default' in zeroth: zeroth_defaults = zeroth.pop('default') else: zeroth_defaults = ipl() # pathlist = ipl() # pathlistfile = zeroth_defaults['targets', 'pathlist'] # pathlistfile = all_params[pathlistfile[0]] + all_params[pathlistfile[1]] # yparams.logging('\nDatastructure of pathlistin:\n\n{}', pathlistin.datastructure2string()) for exp_lbl, experiment in zeroth.iteritems(): # First layer # ----------- # An experiment is now selected and performed yparams.logging('\n\nPerforming experiment {}\n==============================', exp_lbl) final = zeroth_defaults.dcp() final.merge(experiment) exp_sources = final['sources'] exp_params = final['params'] exp_targets = final['targets'] exp_source_kl = [exp_lbl] if len(exp_sources['train']) == 4: exp_source_kl = exp_sources['train'][3] exp_predict_kl = ['predict'] if len(exp_sources['predict']) == 4: exp_predict_kl = exp_sources['predict'][3] if type(exp_source_kl) is str: exp_source_kl = [exp_source_kl] if type(exp_predict_kl) is str: exp_predict_kl = [exp_predict_kl] # Get the pathlist stored in features_of_paths pathlist_source = exp_sources.pop('pathlist') pathlistfile = all_params[pathlist_source[0]] \ + all_params[pathlist_source[1]] with open(pathlistfile, 'r') as f: pathlistin = pickle.load(f) if len(pathlist_source) > 2: if 'skeys' in pathlist_source[2]: pathlistin = pathlistin.subset(*pathlist_source[2]['skeys']) print 'I was here...' yparams.logging('pathlistin.datastructure: \n{}\n', pathlistin.datastructure2string(maxdepth=4)) pathlistout = ipl() # Load training data if 'train' in exp_sources.keys(): truesource = exp_sources['train'] falsesource = exp_sources['train'] else: truesource = exp_sources['traintrue'] falsesource = exp_sources['trainfalse'] truetrainfeats = load_data( all_params[truesource[0]] + all_params[truesource[1]], logger=yparams, **truesource[2] ).subset('truepaths', search=True) falsetrainfeats = load_data( all_params[falsesource[0]] + all_params[falsesource[1]], logger=yparams, **falsesource[2] ).subset('falsepaths', search=True) yparams.logging('\ntruetrainfeats.datastructure: \n{}\n', truetrainfeats.datastructure2string(maxdepth=4)) yparams.logging('\nfalsetrainfeats.datastructure: \n{}\n', falsetrainfeats.datastructure2string(maxdepth=4)) # Load prediction data predictsource = exp_sources['predict'] predictfeats = load_data( all_params[predictsource[0]] + all_params[predictsource[1]], logger=yparams, **predictsource[2] ) yparams.logging('\npredictfeats.datastructure: \n{}\n', predictfeats.datastructure2string(maxdepth=4)) # Load the data into memory truetrainfeats.populate() falsetrainfeats.populate() predictfeats.populate() # Concatenate the different sources # 1. Of training data plo_true = ipl() truetrainfeats, plo_true['truepaths'] = libip.rf_combine_sources_new( truetrainfeats[exp_source_kl]['truepaths'].dcp(), pathlistin[exp_source_kl]['truepaths'].dcp() ) plo_false = ipl() falsetrainfeats, plo_false['falsepaths'] = libip.rf_combine_sources_new( falsetrainfeats[exp_source_kl]['falsepaths'].dcp(), pathlistin[exp_source_kl]['falsepaths'].dcp() ) pathlistout[exp_source_kl + ['train']] = plo_true + plo_false # 2. Of prediction data ipf_true = ipl() plo_true = ipl() ipf_true['truepaths'], plo_true['truepaths'] = libip.rf_combine_sources_new( predictfeats[exp_predict_kl]['truepaths'].dcp(), pathlistin[exp_predict_kl]['truepaths'].dcp() ) ipf_false = ipl() plo_false = ipl() ipf_false['falsepaths'], plo_false['falsepaths'] = libip.rf_combine_sources_new( predictfeats[exp_predict_kl]['falsepaths'].dcp(), pathlistin[exp_predict_kl]['falsepaths'].dcp() ) inpredictfeats = ipf_true + ipf_false pathlistout[exp_source_kl, 'predict'] = plo_true + plo_false # Note: # Due to the feature input being a dictionary organized by the feature images where # the feature values come from # # [source] # 'truepaths'|'falsepaths' # [featureims] # 'Sum': [s1, ..., sN] # 'Variance': [v1, ..., vN] # ... # [Pathlength]: [l1, ..., lN] # # the exact order in which items are iterated over by data_iterator() is not known. # # Solution: # Iterate over it once and store the keylist in an array (which conserves the order) # When accumulating the features for each of the four corresponding subsets, namely # training and testing set with true and false paths each, i.e. # ['0'|'1']['truefeats'|'falsefeats'], # the the keylist is used, thus maintaining the correct order in every subset. # # And that is what is happening here: # # 1. Get the keylist of a full feature list, e.g. one of true paths # example_kl = None # for d2, k2, v2, kl2 in truetrainfeats.data_iterator(): # if k2 == 'truepaths': # example_kl = kl2 # break # 2. Get the keylist order of the feature space feature_space_list = [] for d2, k2, v2, kl2 in truetrainfeats.data_iterator(): if type(v2) is not type(truetrainfeats): feature_space_list.append(kl2) intrain = ipl() intrain['true'] = libip.rf_make_feature_array_with_keylist(truetrainfeats, feature_space_list) yparams.logging("Computed feature array for train['true'] with shape {}", intrain['true'].shape) intrain['false'] = libip.rf_make_feature_array_with_keylist(falsetrainfeats, feature_space_list) yparams.logging("Computed feature array for train['false'] with shape {}", intrain['false'].shape) inpredictfeats['true'] = libip.rf_make_feature_array_with_keylist(inpredictfeats['truepaths'], feature_space_list) yparams.logging("Computed feature array for predict['true'] with shape {}", inpredictfeats['true'].shape) inpredictfeats['false'] = libip.rf_make_feature_array_with_keylist(inpredictfeats['falsepaths'], feature_space_list) yparams.logging("Computed feature array for predict['false'] with shape {}", inpredictfeats['false'].shape) # Classify result = ipl() result[exp_lbl] = libip.random_forest( intrain, inpredictfeats, debug=debug, balance=exp_params['balance_classes'], logger=yparams ) # Evaluate new_eval = ipl() # print [x[0] for x in result[kl]] # print [x[1] for x in result[kl]] new_eval[exp_lbl] = libip.new_eval([x[0] for x in result[exp_lbl]], [x[1] for x in result[exp_lbl]]) yparams.logging('+++ RESULTS +++') yparams.logging("[kl]") # for i in result[kl]: # yparams.logging('{}', i) for key, value in new_eval[exp_lbl].iteritems(): yparams.logging('{} = {}', key, value) pass sys.exit() params = yparams.get_params() thisparams = rdict(params['random_forest']) # Get the pathlist stored in features_of_paths pathlistfile = params[thisparams['pathlistin'][0]] \ + params[thisparams['pathlistin'][1]] with open(pathlistfile, 'r') as f: pathlistin = pickle.load(f) pathlistout = ipl() yparams.logging('\nDatastructure of pathlistin:\n\n{}', pathlistin.datastructure2string()) # for i in xrange(0, len(thisparams['sources'])): for d, k, v, kl in thisparams['sources'].data_iterator(yield_short_kl=True): if k == 'predict': yparams.logging('===============================\nWorking on group: {}', kl) # Get parameters (currently only 'balance_classes') and set defaults balance_classes = False if 'balance_classes' in thisparams['sources'][kl].keys(): balance_classes = thisparams['sources'][kl]['balance_classes'] # Load training data if 'train' in thisparams['sources'][kl].keys(): truesource = thisparams['sources'][kl]['train'] falsesource = thisparams['sources'][kl]['train'] else: truesource = thisparams['sources'][kl]['traintrue'] falsesource = thisparams['sources'][kl]['trainfalse'] truetrainfeats = load_data( params[truesource[0]] + params[truesource[1]], logger=yparams, **truesource[2] ).subset('true', search=True) falsetrainfeats = load_data( params[falsesource[0]] + params[falsesource[1]], logger=yparams, **falsesource[2] ).subset('false', search=True) # # The plus operator is overloaded to perform a merging operation on RecursiveDicts # trainfeats = truetrainfeats + falsetrainfeats # yparams.logging( # '\nDatastructure of truetrainfeats\n\n{}', # truetrainfeats.datastructure2string(maxdepth=3) # ) # yparams.logging( # '\nDatastructure of falsetrainfeats\n\n{}', # falsetrainfeats.datastructure2string(maxdepth=3) # ) # Load prediction data predictsource = thisparams['sources'][kl]['predict'] predictfeats = load_data( params[predictsource[0]] + params[predictsource[1]], logger=yparams, **predictsource[2] ) # Note: # Due to the feature input being a dictionary organized by the feature images where # the feature values come from # # [source] # 'true'|'false' # [featureims] # 'Sum': [s1, ..., sN] # 'Variance': [v1, ..., vN] # ... # [Pathlength]: [l1, ..., lN] # # the exact order in which items are iterated over by data_iterator() is not known. # # Solution: # Iterate over it once and store the keylist in an array (which conserves the order) # When accumulating the featrues for each of the four corresponding subsets, namely # training and testing set with true and false paths each, i.e. # ['0'|'1']['true'|'false'], # the the keylist is used, thus maintaining the correct order in every subset. # # And that is what is happening here: # 1. Get the keylist of a full feature list, e.g. one of true paths example_kl = None for d2, k2, v2, kl2 in truetrainfeats.data_iterator(): if k2 == 'true': example_kl = kl2 break # 2. Get the keylist order of the feature space feature_space_list = [] for d2, k2, v2, kl2 in truetrainfeats[example_kl].data_iterator(): if type(v2) is not type(truetrainfeats[example_kl]): feature_space_list.append(kl2) # yparams.logging('feature_space_list[i] = {}', feature_space_list, listed=True) # Load the data into memory truetrainfeats.populate() falsetrainfeats.populate() predictfeats.populate() truetrainfeats, plo_true = libip.rf_combine_sources( truetrainfeats, search_for='true', pathlist=pathlistin ) falsetrainfeats, plo_false = libip.rf_combine_sources( falsetrainfeats, search_for='false', pathlist=pathlistin ) pathlistout[kl + ['train']] = plo_true + plo_false ipf_true, plo_true = libip.rf_combine_sources( predictfeats, search_for='true', pathlist=pathlistin ) ipf_false, plo_false = libip.rf_combine_sources( predictfeats, search_for='false', pathlist=pathlistin ) inpredictfeats = ipf_true + ipf_false pathlistout[kl + ['predict']] = plo_true + plo_false # yparams.logging( # '\nDatastructure of truetrainfeats\n\n{}', # truetrainfeats.datastructure2string(maxdepth=3) # ) # yparams.logging( # '\nDatastructure of falsetrainfeats\n\n{}', # falsetrainfeats.datastructure2string(maxdepth=3) # ) intrain = ipl() intrain['true'] = libip.rf_make_feature_array_with_keylist(truetrainfeats['true'], feature_space_list) yparams.logging("Computed feature array for train['true'] with shape {}", intrain['true'].shape) intrain['false'] = libip.rf_make_feature_array_with_keylist(falsetrainfeats['false'], feature_space_list) yparams.logging("Computed feature array for train['false'] with shape {}", intrain['false'].shape) inpredictfeats['true'] = libip.rf_make_feature_array_with_keylist(inpredictfeats['true'], feature_space_list) yparams.logging("Computed feature array for predict['true'] with shape {}", inpredictfeats['true'].shape) inpredictfeats['false'] = libip.rf_make_feature_array_with_keylist(inpredictfeats['false'], feature_space_list) yparams.logging("Computed feature array for predict['false'] with shape {}", inpredictfeats['false'].shape) # Classify result = ipl() result[kl] = libip.random_forest( intrain, inpredictfeats, debug=debug, balance=balance_classes, logger=yparams ) # Evaluate new_eval = ipl() # print [x[0] for x in result[kl]] # print [x[1] for x in result[kl]] new_eval[kl] = libip.new_eval([x[0] for x in result[kl]], [x[1] for x in result[kl]]) yparams.logging('+++ RESULTS +++') yparams.logging("[kl]") # for i in result[kl]: # yparams.logging('{}', i) for key, value in new_eval[kl].iteritems(): yparams.logging('{} = {}', key, value)
# Parameters skey = 'labels' for id in ids: source = sourcename.format(id) sourcefilepath = path + source print 'sourcefilepath = {}'.format(sourcefilepath) targetfilepath = path + targetfile if not debug: # Load data data = ipl() data.data_from_file(filepath=sourcefilepath, skeys=skey, recursive_search=True, nodata=True) data.rename_entry(skey, id, search=True) # Print shape def shp(x): return x.shape print 'data.dss(function=shp) = {}'.format(data.dss(function=shp)) # Write to target data.write(filepath=targetfilepath)
def compute_paths(yparams): params = yparams.get_params() thisparams = rdict(params['compute_paths']) data = ipl() for sourcekey, source in thisparams['sources'].iteritems(): # Load the necessary images # 1. Determine the settings for fetching the data try: recursive_search = False recursive_search = thisparams['skwargs', 'default', 'recursive_search'] recursive_search = thisparams['skwargs', sourcekey, 'recursive_search'] except KeyError: pass if len(source) > 2: skeys = source[2] else: skeys = None # 2. Load the data yparams.logging('skeys = {}', skeys) yparams.logging('recursive_search = {}', recursive_search) data[sourcekey] = load_images(params[source[0]] + params[source[1]], skeys=skeys, recursive_search=recursive_search, logger=yparams) data['contacts'].reduce_from_leafs(iterate=True) data['disttransf'].reduce_from_leafs(iterate=True) # Set targetfile targetfile = params[thisparams['target'][0]] \ + params[thisparams['target'][1]] yparams.logging('\nInitial datastructure: \n\n{}', data.datastructure2string(maxdepth=3)) for d, k, v, kl in data['segmentation'].data_iterator(yield_short_kl=True, leaves_only=True): yparams.logging( '===============================\nWorking on image: {}', kl + [k]) # # TODO: Implement copy full logger # data[kl].set_logger(data.get_logger()) # prepare the dict for the path computation indata = ipl() indata['segmentation'] = np.array(data['segmentation'][kl][k]) indata['contacts'] = np.array(data['contacts'][kl][k]) indata['groundtruth'] = np.array( data['groundtruth'][kl][params['gtruthname']]) indata['disttransf'] = np.array(data['disttransf'][kl][k]) yparams.logging('Input datastructure: \n\n{}', indata.datastructure2string()) # Compute the paths sorted into their respective class paths = ipl() paths[kl + [k]] = libip.compute_paths_with_class( indata, 'segmentation', 'contacts', 'disttransf', 'groundtruth', thisparams, ignore=thisparams['ignorelabels'], max_end_count=thisparams['max_end_count'], max_end_count_seed=thisparams['max_end_count_seed'], debug=params['debug']) # Write the result to file paths.write(filepath=targetfile)
print x plt.plot(x, pathvals) # plt.show() print folder + 'plots/' + name + kl2str(kl) + '.png' print kl2str(kl) print kl lab.savefig(folder + 'plots/' + name + kl2str(kl) + '.png') plt.clf() if __name__ == '__main__': yamlfile = os.path.dirname( os.path.abspath(__file__)) + '/parameters_ref.yml' dict_main = ipl(yaml=yamlfile) paths_true, paths_false, featureims_true, featureims_false = load_images( dict_main) dict_main.startlogger() dict_main.logging('\npaths_true datastructure\n---\n{}', paths_true.datastructure2string(maxdepth=2)) dict_main.logging('\npaths_false datastructure\n---\n{}', paths_false.datastructure2string(maxdepth=2)) dict_main.logging('\nfeatureims_true datastructure\n---\n{}', featureims_true.datastructure2string(maxdepth=2)) dict_main.logging('\nfeatureims_false datastructure\n---\n{}', featureims_false.datastructure2string(maxdepth=2))