def split_in_xyz(ipl):
ipl.logging('Datastructure\n---\n{}', ipl.datastructure2string())
reskeys = ('0', '1')
ipl_split = IPL()
ipl_split['z'] = ipl.anytask(lib.split,
2,
axis=0,
result_keys=reskeys,
return_only=True,
rtrntype=IPL)
ipl_split['y'] = ipl.anytask(lib.split,
2,
axis=1,
result_keys=reskeys,
return_only=True,
rtrntype=IPL)
ipl_split['x'] = ipl.anytask(lib.split,
2,
axis=2,
result_keys=reskeys,
return_only=True,
rtrntype=IPL)
ipl_split = ipl_split.switch_levels(1, 2)
ipl.logging('Split sample datastructure\n---\n{}',
ipl_split.datastructure2string())
return ipl_split
def compute_faces(ipl, key):
faces = IPL(data={key: ipl[key]})
shp = faces[key].shape
ipl.logging('Computing faces ...')
faces.get_faces_with_neighbors(keys=key, rtrntype=IPL)
# startpoints = ipl['faces', keys[0]].keys()
additionalinfo = IPL()
startpoints = IPL(data={'xyf': shp[2],
'xyb': shp[2],
'xzf': shp[1],
'xzb': shp[1],
'yzf': shp[0],
'yzb': shp[0]})
areas = IPL(data={'xyf': np.s_[shp[2]:shp[2] + shp[0], shp[2]:shp[2] + shp[1]],
'xyb': np.s_[shp[2]:shp[2] + shp[0], shp[2]:shp[2] + shp[1]],
'xzf': np.s_[shp[1]:shp[1] + shp[0], shp[1]:shp[1] + shp[2]],
'xzb': np.s_[shp[1]:shp[1] + shp[0], shp[1]:shp[1] + shp[2]],
'yzf': np.s_[shp[0]:shp[0] + shp[1], shp[0]:shp[0] + shp[2]],
'yzb': np.s_[shp[0]:shp[0] + shp[1], shp[0]:shp[0] + shp[2]]})
additionalinfo[key, 'startpoints'] = startpoints
additionalinfo[key, 'areas'] = areas
return (faces, additionalinfo)
def get_features(paths, featureimage, featurelist, max_paths_per_label, hfp=None):
newfeats = IPL()
keylist = range(0, max_paths_per_label)
keylist = [str(x) for x in keylist]
for i, keys, vals in paths.simultaneous_iterator(max_count_per_item=max_paths_per_label, keylist=keylist):
if hfp is not None:
hfp.logging("Working in iteration = {}", i)
image = np.zeros(featureimage.shape, dtype=np.uint32)
c = 1
for curk, curv in (dict(zip(keys, vals))).iteritems():
curv = lib.swapaxes(curv, 0, 1)
lib.positions2value(image, curv, c)
c += 1
newnewfeats = IPL(
data=vigra.analysis.extractRegionFeatures(featureimage, image, ignoreLabel=0, features=featurelist)
)
for k, v in newnewfeats.iteritems():
newnewfeats[k] = newnewfeats[k][1:]
if k in newfeats:
try:
newfeats[k] = np.concatenate((newfeats[k], newnewfeats[k]))
except ValueError:
pass
else:
newfeats[k] = newnewfeats[k]
return newfeats
def get_features(paths, featureimages, featurelist, max_paths_per_label, ipl=None):
newfeats = IPL()
keylist = range(0, max_paths_per_label)
keylist = [str(x) for x in keylist]
# Iterate over all paths, yielding a list of one path per label object until no paths are left
for i, keys, vals in paths.simultaneous_iterator(
max_count_per_item=max_paths_per_label,
keylist=keylist):
if ipl is not None:
ipl.logging('Working in iteration = {}', i)
ipl.logging('Keys: {}', keys)
if not keys:
continue
# Create a working image
image = np.zeros(np.array(featureimages.yield_an_item()).shape, dtype=np.uint32)
# And fill it with one path per label object
c = 1
for curk, curv in (dict(zip(keys, vals))).iteritems():
curv = np.array(curv)
curv = lib.swapaxes(curv, 0, 1)
lib.positions2value(image, curv, c)
c += 1
# TODO: If this loop iterated over the parameter list it would be more broadly applicable
for d, k, v, kl in featureimages.data_iterator():
if type(v) is not IPL:
# Extract the region features of the working image
# TODO: Extract feature 'Count' manually due to anisotropy
newnewfeats = IPL(
data=vigra.analysis.extractRegionFeatures(
np.array(v).astype(np.float32),
image, ignoreLabel=0,
features=featurelist
)
)
# Append to the recently computed list of features
for nk, nv in newnewfeats.iteritems():
nv = nv[1:]
if newfeats.inkeys(kl+[nk]):
try:
newfeats[kl + [nk]] = np.concatenate((newfeats[kl + [nk]], nv))
except ValueError:
pass
else:
newfeats[kl + [nk]] = nv
return newfeats
def split_in_xyz(ipl):
ipl.logging('Datastructure\n---\n{}', ipl.datastructure2string())
reskeys = ('0', '1')
ipl_split = IPL()
ipl_split['z'] = ipl.anytask(lib.split, 2, axis=0, result_keys=reskeys, return_only=True, rtrntype=IPL)
ipl_split['y'] = ipl.anytask(lib.split, 2, axis=1, result_keys=reskeys, return_only=True, rtrntype=IPL)
ipl_split['x'] = ipl.anytask(lib.split, 2, axis=2, result_keys=reskeys, return_only=True, rtrntype=IPL)
ipl_split = ipl_split.switch_levels(1, 2)
ipl.logging('Split sample datastructure\n---\n{}', ipl_split.datastructure2string())
return ipl_split
def load_images(filepath, skeys=None, recursive_search=False, logger=None):
if logger is not None:
logger.logging('Loading data from \n{}', filepath)
else:
print 'Loading data from \n{}'.format(filepath)
data = IPL()
data.data_from_file(filepath=filepath,
skeys=skeys,
recursive_search=recursive_search,
nodata=True)
return data
def get_features(paths, featureimage, featurelist, max_paths_per_label, ipl=None):
newfeats = IPL()
# TODO: Selection of a limited amount of paths should be random
keylist = range(0, max_paths_per_label)
keylist = [str(x) for x in keylist]
# Iterate over all paths, yielding a list of one path per label object until no paths are left
for i, keys, vals in paths.simultaneous_iterator(
max_count_per_item=max_paths_per_label,
keylist=keylist):
if ipl is not None:
ipl.logging('Working in iteration = {}', i)
# Create a working image
image = np.zeros(featureimage.shape, dtype=np.uint32)
# And fill it with one path per label object
c = 1
for curk, curv in (dict(zip(keys, vals))).iteritems():
curv = lib.swapaxes(curv, 0, 1)
lib.positions2value(image, curv, c)
c += 1
# Extract the region features of the working image
newnewfeats = IPL(
data=vigra.analysis.extractRegionFeatures(
featureimage,
image, ignoreLabel=0,
features=featurelist
)
)
for k, v in newnewfeats.iteritems():
newnewfeats[k] = newnewfeats[k][1:]
if k in newfeats:
try:
newfeats[k] = np.concatenate((newfeats[k], newnewfeats[k]))
except ValueError:
pass
else:
newfeats[k] = newnewfeats[k]
return newfeats
def merge_adjacent_objects_image_iteration(ipl):
params = ipl.get_params()
thisparams = params['merge_adjacent_objects']
merged = IPL()
for d, k, v, kl in ipl.data_iterator(yield_short_kl=True):
ipl.logging('Working on image: {}', kl + [k])
if k == params['largeobjname']:
data = IPL(data={k: v})
data.setlogger(ipl.getlogger())
merged[kl] = merge_adjacent_objects(data, k, thisparams)
return merged
def merge_adjacent_objects_image_iteration(ipl):
params = ipl.get_params()
thisparams = params['merge_adjacent_objects']
merged = IPL()
for d, k, v, kl in ipl.data_iterator(yield_short_kl=True):
ipl.logging('Working on image: {}', kl + [k])
if k == params['largeobjname']:
data = IPL(data={k: v})
data.setlogger(ipl.getlogger())
merged[kl] = merge_adjacent_objects(data, k, thisparams)
return merged
def get_features(paths,
featureimage,
featurelist,
max_paths_per_label,
hfp=None):
newfeats = IPL()
keylist = range(0, max_paths_per_label)
keylist = [str(x) for x in keylist]
for i, keys, vals in paths.simultaneous_iterator(
max_count_per_item=max_paths_per_label, keylist=keylist):
if hfp is not None:
hfp.logging('Working in iteration = {}', i)
image = np.zeros(featureimage.shape, dtype=np.uint32)
c = 1
for curk, curv in (dict(zip(keys, vals))).iteritems():
curv = lib.swapaxes(curv, 0, 1)
lib.positions2value(image, curv, c)
c += 1
newnewfeats = IPL(data=vigra.analysis.extractRegionFeatures(
featureimage, image, ignoreLabel=0, features=featurelist))
for k, v in newnewfeats.iteritems():
newnewfeats[k] = newnewfeats[k][1:]
if k in newfeats:
try:
newfeats[k] = np.concatenate((newfeats[k], newnewfeats[k]))
except ValueError:
pass
else:
newfeats[k] = newnewfeats[k]
return newfeats
def load_images(ipl):
"""
These images are loaded:
paths_true (paths within single label objects)
paths_false (paths of merged objects which cross the merging site)
featureims_true
featureims_false
:param ipl:
:return:
"""
paths_true = IPL()
paths_false = IPL()
featureims = IPL()
params = ipl.get_params()
ipl.logging('Loading true paths from:\n{} ...', params['intermedfolder'] + params['pathsfile'])
# Paths within labels (true paths)
paths_true.data_from_file(
filepath=params['intermedfolder'] + params['pathsfile'],
skeys='truepaths',
recursive_search=True, nodata=True
)
ipl.logging('Loading false paths from:\n{} ...', params['intermedfolder'] + params['pathsfile'])
# Paths of merges (false paths)
paths_false.data_from_file(
filepath=params['intermedfolder'] + params['pathsfile'],
skeys='falsepaths',
recursive_search=True, nodata=True
)
ipl.logging('Loading feature images from:\n{} ...', params['intermedfolder'] + params['featureimsfile'])
# Load features for true paths
featureims.data_from_file(
filepath=params['intermedfolder'] + params['featureimsfile'],
nodata=True
)
return (paths_true, paths_false, featureims)
def run_random_forest(yamlfile, logging=True, make_only_feature_array=False, debug=False, write=True):
ipl = IPL(yaml=yamlfile)
ipl.set_indent(1)
params = rdict(data=ipl.get_params())
if logging:
ipl.startlogger(filename=params['resultfolder'] + 'random_forest.log', type='w', name='RandomForest')
else:
ipl.startlogger()
try:
# # Copy the script file and the parameters to the scriptsfolder
# copy(inspect.stack()[0][1], params['scriptsfolder'])
# copy(yamlfile, params['scriptsfolder'] + 'random_forest.parameters.yml')
# ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3))
if make_only_feature_array:
make_feature_arrays(ipl)
else:
result = IPL()
result['result'], result['evaluation'] = random_forest(ipl, debug=debug)
# ipl.logging('\nFinal datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3))
if write:
result.write(filepath=params['resultfolder'] + params['resultsfile'])
ipl.logging('')
ipl.stoplogger()
except:
ipl.errout('Unexpected error')
keys=locmaxnames)
# Local maxima
hfp.logging('Discovering mountains ...')
hfp.extended_local_maxima(neighborhood=26, keys=locmaxnames)
if __name__ == '__main__':
yamlfile = os.path.dirname(os.path.abspath(__file__)) + '/parameters.yml'
hfp = IPL(yaml=yamlfile,
yamlspec={
'path': 'intermedfolder',
'filename': 'locmaxborderfile',
'skeys': {
'locmaxbordernames': (2, 3)
}
},
tkeys=('disttransf', 'disttransfm'),
castkey=None)
params = hfp.get_params()
thisparams = params['localmax_on_disttransf']
hfp.startlogger(filename=params['resultfolder'] +
'localmax_on_disttransf.log',
type='w')
try:
# Copy the script file and the parameters to the scriptsfolder
copy(inspect.stack()[0][1], params['scriptsfolder'])
copy(yamlfile,
if __name__ == "__main__":
infiles = [
'/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splA.train.probs.crop.h5',
'/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splA.train.raw_neurons.crop.h5'
]
outfiles = [
'/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splA.train.probs.crop.crop_x10_110_y200_712_z200_712.split_xyz.h5',
'/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splA.train.raw_neurons.crop.crop_x10_110_y200_712_z200_712.split_xyz.h5'
]
for i in xrange(0, len(infiles)):
ipl = IPL(
filepath=infiles[i]
)
ipl.logging('Datastructure\n---\n{}', ipl.datastructure2string())
ipl.crop_bounding_rect(bounds=np.s_[10:110, 200:712, 200:712])
def shape(image):
return image.shape
print ipl.datastructure2string(function=shape)
ipl_split = split_in_xyz(ipl)
ipl_split.write(filepath=outfiles[i])
# # Sample A
# sample_a = IPL(
from hdf5_image_processing import Hdf5ImageProcessingLib as IPL
import processing_lib as lib
# Sample A probs
probs_a = IPL(
filepath=
'/mnt/localdata01/jhennies/neuraldata/cremi_2016/sample_A_train_betas/sample_A_train_mcseg_beta_0.5.h5'
)
probs_a.logging('Probs A datastructure\n---\n{}',
probs_a.datastructure2string())
probs_a.anytask(lib.swapaxes, 0, 2)
probs_a.write(
'/mnt/localdata01/jhennies/neuraldata/cremi_2016/sample_A_train_betas/cremi.splA.train.seg_beta_0.5.crop.h5'
)
reskeys = ('0', '1')
split_probs_a = IPL()
split_probs_a['z'] = probs_a.anytask(lib.split,
2,
axis=0,
result_keys=reskeys,
return_only=True,
rtrntype=IPL)
split_probs_a['y'] = probs_a.anytask(lib.split,
2,
axis=1,
result_keys=reskeys,
return_only=True,
if thisparams['return_bordercontact_images']:
return bordercontacts
else:
return None
if __name__ == '__main__':
resultsfolder = '/mnt/localdata02/jhennies/neuraldata/results/cremi_2016/161110_random_forest_of_paths/'
yamlfile = resultsfolder + '/parameters.yml'
ipl = IPL(
yaml=yamlfile,
yamlspec={'path': 'intermedfolder', 'filename': 'largeobjfile', 'skeys': 'largeobjname'},
recursive_search=True
)
params = ipl.get_params()
thisparams = params['find_border_contacts']
ipl.data_from_file(params['intermedfolder'] + params['largeobjmfile'],
skeys=params['largeobjmnames'][0], recursive_search=True, integrate=True)
ipl.startlogger(filename=params['resultfolder'] + 'find_border_contacts.log', type='w')
try:
# Copy the script file and the parameters to the scriptsfolder
copy(inspect.stack()[0][1], params['scriptsfolder'])
copy(yamlfile, params['scriptsfolder'] + 'find_border_contacts.parameters.yml')
# Write script and parameters to the logfile
ipl.code2log(inspect.stack()[0][1])
def merge_adjacent_objects(ipl, key, thisparams):
"""
:param ipl:
ipl.get_params():
merge_adjacent_objects
seed
numberbysize
numberbyrandom
largeobjmnames
- 'largeobj_merged'
- 'mergeids_small'
- 'mergeids_random'
- 'mergeids_all'
- 'change_hash'
:param key: the source key for calculation
"""
numberbysize = thisparams['numberbysize']
numberbyrandom = thisparams['numberbyrandom']
targetnames = params['largeobjmnames']
# Get only the relevant labels
labels = lib.unique(ipl[key])
ipl.logging('labels = {}', labels)
# Seed the randomize function
random.seed(thisparams['seed'])
ipl.astype(np.uint32, keys=key)
(grag, rag) = graphs.gridRegionAdjacencyGraph(ipl[key], ignoreLabel=0)
edge_ids = rag.edgeIds()
# ipl.logging('Edge ids: {}', edge_ids)
# Type 1:
# Select edges by size (smallest edges)
ipl.logging('Number of edgeLengths = {}', len(rag.edgeLengths()))
edgelen_ids = dict(zip(edge_ids, rag.edgeLengths()))
# ifp.logging('edgelen_ids = {}', edgelen_ids)
sorted_edgelens = np.sort(rag.edgeLengths())
#
smallest_merge_lens = sorted_edgelens[0:numberbysize]
ipl.logging('Lengths selected for merging: {}', smallest_merge_lens)
#
smallest_merge_ids = []
for x in smallest_merge_lens:
edge_id = edgelen_ids.keys()[edgelen_ids.values().index(x)]
smallest_merge_ids.append(edge_id)
edgelen_ids.pop(edge_id)
#
edge_ids = edgelen_ids.keys()
ipl.logging('Edge IDs selected for merging due to size: {}',
smallest_merge_ids)
# Type 2:
# Randomly choose edges
random_merge_ids = random.sample(edge_ids, numberbyrandom)
ipl.logging('Edge IDs randomly selected for merging: {}', random_merge_ids)
# Now get the label ids
smallest_merge_labelids_u = [
rag.uId(rag.edgeFromId(x)) for x in smallest_merge_ids
]
smallest_merge_labelids_v = [
rag.vId(rag.edgeFromId(x)) for x in smallest_merge_ids
]
smallest_merge_labelids = list(
zip(smallest_merge_labelids_u, smallest_merge_labelids_v))
random_merge_labelids_u = [
rag.uId(rag.edgeFromId(x)) for x in random_merge_ids
]
random_merge_labelids_v = [
rag.vId(rag.edgeFromId(x)) for x in random_merge_ids
]
random_merge_labelids = list(
zip(random_merge_labelids_u, random_merge_labelids_v))
ipl.logging('Label IDs selected for merging by size: {}',
smallest_merge_labelids)
ipl.logging('Label IDs randomly selected for merging: {}',
random_merge_labelids)
# Concatenate
all_merge_labelids = smallest_merge_labelids + random_merge_labelids
# Sort
ipl.logging('all_merge_labelids = {}', all_merge_labelids)
all_merge_labelids = [sorted(x) for x in all_merge_labelids]
all_merge_labelids = sorted(all_merge_labelids)
ipl.logging('all_merge_labelids = {}', all_merge_labelids)
# Store this for later use
ipl[targetnames[1]] = smallest_merge_labelids
ipl[targetnames[2]] = random_merge_labelids
ipl[targetnames[3]] = all_merge_labelids
# Create change hash list
change_hash = IPL(data=dict(
zip(np.unique(all_merge_labelids), [[
x,
] for x in np.unique(all_merge_labelids)])))
for i in xrange(0, 3):
prev_change_hash = IPL(data=change_hash)
for x in all_merge_labelids:
ipl.logging('Adding {} and {}', *x)
change_hash[x[0]] += change_hash[x[1]]
change_hash[x[0]] = list(np.unique(change_hash[x[0]]))
change_hash[x[1]] += change_hash[x[0]]
change_hash[x[1]] = list(np.unique(change_hash[x[1]]))
# This removes the redundancy from the hash
def reduce(hash):
br = False
for k, v in hash.iteritems():
for x in v:
if x != k:
if x in hash.keys():
del hash[x]
reduce(hash)
br = True
break
else:
br = False
if br:
break
reduce(change_hash)
# Change the list in the hash to np-arrays for better storage in h5 files
for k, v in change_hash.iteritems():
change_hash[k] = np.array(v)
# And now we have a perfect change list which we just need to iterate over and change the labels in the image
ipl.logging('change_hash after change:')
ipl.logging(change_hash)
ipl[targetnames[4]] = change_hash
# Create the merged image
# ipl.deepcopy_entry('largeobj', targetnames[0])
ipl.rename_entry(key, targetnames[0])
for k, v in change_hash.iteritems():
for x in v:
if x != k:
ipl.logging('Setting {} to {}!', x, k)
ipl.filter_values(x, type='eq', setto=k, keys=targetnames[0])
return ipl
# 'disttransfm', thisparams['ignore'])
return paths
if __name__ == '__main__':
resultsfolder = '/mnt/localdata02/jhennies/neuraldata/results/cremi_2016/161110_random_forest_of_paths/'
yamlfile = resultsfolder + '/parameters.yml'
ipl = IPL(yaml=yamlfile,
yamlspec={
'path': 'intermedfolder',
'filename': 'locmaxborderfile',
'skeys': {
'locmaxbordernames': (1, 3)
}
},
recursive_search=True)
params = ipl.get_params()
thisparams = params['paths_of_partners']
ipl.startlogger(filename=params['resultfolder'] + 'paths_of_partners.log',
type='w')
ipl.data_from_file(params['intermedfolder'] + params['largeobjfile'],
skeys=params['largeobjname'],
recursive_search=True,
integrate=True)
ipl.data_from_file(params['intermedfolder'] + params['largeobjmfile'],
skeys=(params['largeobjmnames'][0],
params['largeobjmnames'][4]),
# cremi = IPL(filepath='/mnt/localdata02/jhennies/neuraldata/cremi_2016/sample_B_20160501.hdf')
#
# cremi.logging('Datastructure:\n---\n{}', cremi.datastructure2string())
#
# images = IPL(data={
# 'raw': cremi['volumes', 'raw'],
# 'neuron_ids': cremi['volumes', 'labels', 'neuron_ids']
# })
#
# images.logging('Datastructure:\n---\n{}', images.datastructure2string())
#
# images.write('/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splB.raw_neurons.crop.h5')
cremi = IPL(
filepath=
'/mnt/localdata02/jhennies/neuraldata/cremi_2016/sample_C_20160501.hdf'
)
cremi.logging('Datastructure:\n---\n{}', cremi.datastructure2string())
images = IPL(
data={
'raw': cremi['volumes', 'raw'],
'neuron_ids': cremi['volumes', 'labels', 'neuron_ids']
})
images.logging('Datastructure:\n---\n{}', images.datastructure2string())
images.write(
'/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splC.raw_neurons.crop.h5'
)
c += 1
del (hfp['faces'])
if thisparams['return_bordercontact_images']:
return bordercontacts
if __name__ == '__main__':
yamlfile = os.path.dirname(os.path.abspath(__file__)) + '/parameters.yml'
hfp = IPL(
yaml=yamlfile,
yamlspec={'path': 'intermedfolder', 'filename': 'largeobjfile', 'skeys': 'largeobjname'},
tkeys='largeobj',
castkey=None
)
params = hfp.get_params()
thisparams = params['find_border_contacts']
hfp.data_from_file(params['intermedfolder'] + params['largeobjmfile'],
skeys=params['largeobjmnames'][0], tkeys='largeobjm')
hfp.startlogger(filename=params['resultfolder'] + 'find_orphans.log', type='w')
try:
# Copy the script file and the parameters to the scriptsfolder
copy(inspect.stack()[0][1], params['scriptsfolder'])
copy(yamlfile, params['scriptsfolder'] + 'find_orphans.parameters.yml')
# Write script and parameters to the logfile
hfp.code2log(inspect.stack()[0][1])
def run_remove_small_objects(yamlfile):
ipl = IPL(yaml=yamlfile,
yamlspec={
'path': 'datafolder',
'filename': 'labelsfile',
'skeys': 'labelsname'
},
recursive_search=True,
nodata=True)
# Set indentation of the logging
ipl.set_indent(1)
params = ipl.get_params()
ipl.startlogger(filename=params['resultfolder'] +
'remove_small_objects.log',
type='w',
name='RemoveSmallObjects')
try:
# # Copy the script file and the parameters to the scriptsfolder
# copy(inspect.stack()[0][1], params['scriptsfolder'])
# copy(yamlfile, params['scriptsfolder'] + 'remove_small_objects.parameters.yml')
ipl.logging('\nipl datastructure: \n\n{}',
ipl.datastructure2string(maxdepth=3))
remove_small_objects(ipl)
ipl.logging('\nFinal datastructure: \n\n{}',
ipl.datastructure2string(maxdepth=3))
# ipl.write(filepath=params['intermedfolder'] + params['largeobjfile'])
ipl.logging('')
ipl.stoplogger()
except:
ipl.errout('Unexpected error')
def run_random_forest(yamlfile, logging=True, make_only_feature_array=False, debug=False, write=True):
ipl = IPL(yaml=yamlfile)
ipl.set_indent(1)
params = rdict(data=ipl.get_params())
if logging:
ipl.startlogger(filename=params['resultfolder'] + 'random_forest.log', type='w', name='RandomForest')
else:
ipl.startlogger()
try:
# # Copy the script file and the parameters to the scriptsfolder
# copy(inspect.stack()[0][1], params['scriptsfolder'])
# copy(yamlfile, params['scriptsfolder'] + 'random_forest.parameters.yml')
# ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3))
if make_only_feature_array:
make_feature_arrays(ipl)
else:
result = IPL()
result['result'], result['evaluation'] = random_forest(ipl, debug=debug)
# ipl.logging('\nFinal datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3))
if write:
result.write(filepath=params['resultfolder'] + params['resultsfile'])
ipl.logging('')
ipl.stoplogger()
except:
ipl.errout('Unexpected error')
hfp[key].astype(np.float32) / np.amax(hfp[key]),
vigra.filters.multiBinaryDilation(hfp[k].astype(np.uint8), 5)
])
return paths
if __name__ == '__main__':
yamlfile = os.path.dirname(os.path.abspath(__file__)) + '/parameters.yml'
hfp = IPL(yaml=yamlfile,
yamlspec={
'path': 'intermedfolder',
'filename': 'locmaxborderfile',
'skeys': {
'locmaxbordernames': (0, 2)
}
},
tkeys=('border_locmax', 'disttransf'),
castkey=None)
params = hfp.get_params()
thisparams = params['paths_within_labels']
hfp.startlogger(filename=params['resultfolder'] +
'paths_within_labels.log',
type='w')
hfp.data_from_file(params['intermedfolder'] + params['largeobjfile'],
skeys=params['largeobjname'],
tkeys='largeobj')
hfp.data_from_file(params['intermedfolder'] + params['locmaxfile'],
skeys=params['locmaxnames'][0],
tkeys='locmax')
def run_compute_feature_images(yamlfile):
ipl = IPL(yaml=yamlfile)
ipl.set_indent(1)
params = rdict(data=ipl.get_params())
ipl.startlogger(filename=params['resultfolder'] + 'compute_feature_images.log', type='w', name='ComputeFeatureImages')
try:
# # Copy the script file and the parameters to the scriptsfolder
# copy(inspect.stack()[0][1], params['scriptsfolder'])
# copy(yamlfile, params['scriptsfolder'] + 'compute_feature_images.parameters.yml')
# ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3))
compute_feature_images(ipl)
# ipl.logging('\nFinal datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3))
# ipl.write(filepath=params['intermedfolder'] + params['largeobjfile'])
ipl.logging('')
ipl.stoplogger()
except:
ipl.errout('Unexpected error')
(params['locmaxbordernames'][0], params['locmaxnames'][0]),
params['locmaxbordernames'][2],
thisparams, ignore=thisparams['ignore'])
return paths
if __name__ == '__main__':
resultsfolder = '/mnt/localdata02/jhennies/neuraldata/results/cremi_2016/161110_random_forest_of_paths/'
yamlfile = resultsfolder + '/parameters.yml'
ipl = IPL(
yaml=yamlfile,
yamlspec={'path': 'intermedfolder', 'filename': 'largeobjfile', 'skeys': 'largeobjname'},
recursive_search=True
)
params = ipl.get_params()
thisparams = params['paths_within_labels']
ipl.startlogger(filename=params['resultfolder'] + 'paths_within_labels.log', type='w')
ipl.data_from_file(params['intermedfolder'] + params['locmaxfile'],
skeys=params['locmaxnames'][0],
recursive_search=True,
integrate=True)
ipl.data_from_file(params['intermedfolder'] + params['locmaxborderfile'],
skeys=(params['locmaxbordernames'][0], params['locmaxbordernames'][2]),
recursive_search=True,
integrate=True)
# Done: Visualize Distancetransform along paths of true and false merges
# Done: Also intensity values of raw data
# TODO: And probability map
# Done: Do not forget to save the result for the thesis!
__author__ = 'jhennies'
if __name__ == '__main__':
yamlfile = os.path.dirname(os.path.abspath(__file__)) + '/parameters.yml'
hfp = IPL(
yaml=yamlfile,
yamlspec={'path': 'intermedfolder', 'filename': 'pathstruefile'},
castkey=None
)
# hfp.logging('datastructure:\n---\n{}', hfp.datastructure2string())
params = hfp.get_params()
hfp['true', 'border'] = IPL(data=hfp['largeobj', 'border_locmax', 'path'])
hfp['true', 'locmax'] = IPL(data=hfp['largeobj', 'locmax', 'path'])
hfp.pop('largeobj')
hfp.data_from_file(filepath=params['intermedfolder'] + params['pathsfalsefile'])
hfp['false', 'border'] = IPL(data=hfp['largeobjm', 'border_locmax_m', 'path'])
hfp['false', 'locmax'] = IPL(data=hfp['largeobjm', 'locmaxm', 'path'])
hfp.pop('largeobjm')
def merge_adjacent_objects(hfp):
params = hfp.get_params()
thisparams = params['merge_adjacent_objects']
numberbysize = thisparams['numberbysize']
numberbyrandom = thisparams['numberbyrandom']
targetnames = params['largeobjmnames']
# Get only the relevant labels
labels = lib.unique(hfp['largeobj'])
hfp.logging('labels = {}', labels)
# Seed the randomize function
random.seed(thisparams['seed'])
hfp.astype(np.uint32, keys='largeobj')
(grag, rag) = graphs.gridRegionAdjacencyGraph(hfp['largeobj'], ignoreLabel=0)
edge_ids = rag.edgeIds()
# hfp.logging('Edge ids: {}', edge_ids)
# Type 1:
# Select edges by size (smallest edges)
hfp.logging('Number of edgeLengths = {}', len(rag.edgeLengths()))
edgelen_ids = dict(zip(edge_ids, rag.edgeLengths()))
# ifp.logging('edgelen_ids = {}', edgelen_ids)
sorted_edgelens = np.sort(rag.edgeLengths())
#
smallest_merge_lens = sorted_edgelens[0:numberbysize]
hfp.logging('Lengths selected for merging: {}', smallest_merge_lens)
#
smallest_merge_ids = []
for x in smallest_merge_lens:
edge_id = edgelen_ids.keys()[edgelen_ids.values().index(x)]
smallest_merge_ids.append(edge_id)
edgelen_ids.pop(edge_id)
#
edge_ids = edgelen_ids.keys()
hfp.logging('Edge IDs selected for merging due to size: {}', smallest_merge_ids)
# Type 2:
# Randomly choose edges
random_merge_ids = random.sample(edge_ids, numberbyrandom)
hfp.logging('Edge IDs randomly selected for merging: {}', random_merge_ids)
# Now get the label ids
smallest_merge_labelids_u = [rag.uId(rag.edgeFromId(x)) for x in smallest_merge_ids]
smallest_merge_labelids_v = [rag.vId(rag.edgeFromId(x)) for x in smallest_merge_ids]
smallest_merge_labelids = list(zip(smallest_merge_labelids_u, smallest_merge_labelids_v))
random_merge_labelids_u = [rag.uId(rag.edgeFromId(x)) for x in random_merge_ids]
random_merge_labelids_v = [rag.vId(rag.edgeFromId(x)) for x in random_merge_ids]
random_merge_labelids = list(zip(random_merge_labelids_u, random_merge_labelids_v))
hfp.logging('Label IDs selected for merging by size: {}', smallest_merge_labelids)
hfp.logging('Label IDs randomly selected for merging: {}', random_merge_labelids)
# Concatenate
all_merge_labelids = smallest_merge_labelids + random_merge_labelids
# Sort
hfp.logging('all_merge_labelids = {}', all_merge_labelids)
all_merge_labelids = [sorted(x) for x in all_merge_labelids]
all_merge_labelids = sorted(all_merge_labelids)
hfp.logging('all_merge_labelids = {}', all_merge_labelids)
# Store this for later use
hfp[targetnames[1]] = smallest_merge_labelids
hfp[targetnames[2]] = random_merge_labelids
hfp[targetnames[3]] = all_merge_labelids
# Create change hash list
change_hash = IPL(data=dict(zip(np.unique(all_merge_labelids), [[x,] for x in np.unique(all_merge_labelids)])))
for i in xrange(0, 3):
prev_change_hash = IPL(data=change_hash)
for x in all_merge_labelids:
hfp.logging('Adding {} and {}', *x)
change_hash[x[0]] += change_hash[x[1]]
change_hash[x[0]] = list(np.unique(change_hash[x[0]]))
change_hash[x[1]] += change_hash[x[0]]
change_hash[x[1]] = list(np.unique(change_hash[x[1]]))
# This removes the redundancy from the hash
def reduce(hash):
br = False
for k, v in hash.iteritems():
for x in v:
if x != k:
if x in hash.keys():
del hash[x]
reduce(hash)
br = True
break
else:
br = False
if br:
break
reduce(change_hash)
# And now we have a perfect change list which we just need to iterate over and change the labels in the image
hfp.logging('change_hash after change:')
hfp.logging(change_hash)
hfp[targetnames[4]] = change_hash
# Create the merged image
# hfp.deepcopy_entry('largeobj', targetnames[0])
hfp.rename_entry('largeobj', targetnames[0])
for k, v in change_hash.iteritems():
for x in v:
if x != k:
hfp.logging('Setting {} to {}!', x, k)
hfp.filter_values(x, type='eq', setto=k, keys=targetnames[0])
if thisparams['return_bordercontact_images']:
return bordercontacts
else:
return None
if __name__ == '__main__':
resultsfolder = '/mnt/localdata02/jhennies/neuraldata/results/cremi_2016/161110_random_forest_of_paths/'
yamlfile = resultsfolder + '/parameters.yml'
ipl = IPL(yaml=yamlfile,
yamlspec={
'path': 'intermedfolder',
'filename': 'largeobjfile',
'skeys': 'largeobjname'
},
recursive_search=True)
params = ipl.get_params()
thisparams = params['find_border_contacts']
ipl.data_from_file(params['intermedfolder'] + params['largeobjmfile'],
skeys=params['largeobjmnames'][0],
recursive_search=True,
integrate=True)
ipl.startlogger(filename=params['resultfolder'] +
'find_border_contacts.log',
type='w')
try:
def run_find_border_contacts(yamlfile, logging=True):
ipl = IPL(yaml=yamlfile)
ipl.set_indent(1)
params = rdict(data=ipl.get_params())
if logging:
ipl.startlogger(filename=params['resultfolder'] + 'find_border_contacts.log', type='w', name='FindBorderContacts')
else:
ipl.startlogger()
try:
# # Copy the script file and the parameters to the scriptsfolder
# copy(inspect.stack()[0][1], params['scriptsfolder'])
# copy(yamlfile, params['scriptsfolder'] + 'find_border_contacts.parameters.yml')
# ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3))
find_border_contacts(ipl)
# ipl.logging('\nFinal datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3))
# ipl.write(filepath=params['intermedfolder'] + params['largeobjfile'])
ipl.logging('')
ipl.stoplogger()
except:
ipl.errout('Unexpected error')
from hdf5_image_processing import Hdf5ImageProcessingLib as IPL
import os
import numpy as np
__author__ = 'jhennies'
if __name__ == '__main__':
yamlfile = os.path.dirname(os.path.abspath(__file__)) + '/parameters.yml'
ipl = IPL(
yaml=yamlfile
)
ipl.logging('Parameters: {}', ipl.get_params())
params = ipl.get_params()
ipl.data_from_file(filepath=params['datafolder'] + 'cremi.splA.raw_neurons.crop.h5',
skeys='raw',
tkeys='raw')
ipl.crop_bounding_rect(np.s_[10:110, 200:712, 200:712], keys='raw')
ipl.write(filepath=params['datafolder'] + 'cremi.splA.raw_neurons.crop.crop_10-200-200_110-712-712.h5')
data = IPL(data={k: v})
data.setlogger(ipl.getlogger())
merged[kl] = merge_adjacent_objects(data, k, thisparams)
return merged
if __name__ == '__main__':
resultsfolder = '/mnt/localdata02/jhennies/neuraldata/results/cremi_2016/161110_random_forest_of_paths/'
yamlfile = resultsfolder + '/parameters.yml'
ipl = IPL(yaml=yamlfile,
yamlspec={
'path': 'intermedfolder',
'filename': 'largeobjfile'
})
params = ipl.get_params()
ipl.startlogger(filename=params['resultfolder'] +
'merge_adjacent_objects.log',
type='w')
try:
# Copy the script file and the parameters to the scriptsfolder
copy(inspect.stack()[0][1], params['scriptsfolder'])
copy(yamlfile,
params['scriptsfolder'] + 'merge_adjacent_objects.parameters.yml')
# Write script and parameters to the logfile
ipl.code2log(inspect.stack()[0][1])
def merge_adjacent_objects(
ipl, key, numberbysize=0, numberbyrandom=0, seed=None,
targetnames=('largeobjm', 'mergeids_small', 'mergeids_random', 'change_hash'),
algorithm='standard'
):
"""
:param ipl:
ipl.get_params():
merge_adjacent_objects
seed
numberbysize
numberbyrandom
largeobjmnames
- 'largeobj_merged'
- 'mergeids_small'
- 'mergeids_random'
- 'mergeids_all'
- 'change_hash'
:param key: the source key for calculation
"""
# This removes the redundancy from the hash
def reduce_hash(hash):
br = False
for k, v in hash.iteritems():
for x in v:
if x != k:
if x in hash.keys():
del hash[x]
reduce_hash(hash)
br = True
break
else:
br = False
if br:
break
if algorithm == 'pairs':
# Get only the relevant labels
labels = lib.unique(ipl[key])
ipl.logging('labels = {}', labels)
# Seed the randomize function
if seed:
random.seed(seed)
else:
random.seed()
ipl.astype(np.uint32, keys=key)
(grag, rag) = graphs.gridRegionAdjacencyGraph(ipl[key], ignoreLabel=0)
edge_ids = rag.edgeIds()
merge_ids = []
used_ids = ()
# Calculate the merges such that only pairs are found
for i in xrange(0, numberbyrandom):
c = 0
finished = False
while not finished:
edge_id = random.choice(edge_ids)
uid = rag.uId(rag.edgeFromId(edge_id))
vid = rag.vId(rag.edgeFromId(edge_id))
c += 1
if c > 50:
ipl.logging('merge_adjacent_objects: Warning: Not finding any more pairs!')
finished = True
elif uid not in used_ids and vid not in used_ids:
finished = True
used_ids += (uid, vid)
merge_ids.append((uid, vid))
ipl.logging('Label IDs randomly selected for merging: {}', merge_ids)
# Sort
merge_ids = [sorted(x) for x in merge_ids]
merge_ids = sorted(merge_ids)
ipl.logging('merge_ids = {}', merge_ids)
# Store this for later use
ipl[targetnames[2]] = merge_ids
ipl[targetnames[3]] = merge_ids
# # Create change hash list
# change_hash = IPL(data=dict(zip(np.unique(merge_ids), [[x, ] for x in np.unique(merge_ids)])))
# for i in xrange(0, 3):
# prev_change_hash = IPL(data=change_hash)
# for x in merge_ids:
# ipl.logging('Adding {} and {}', *x)
# change_hash[x[0]] += change_hash[x[1]]
# change_hash[x[0]] = list(np.unique(change_hash[x[0]]))
# change_hash[x[1]] += change_hash[x[0]]
# change_hash[x[1]] = list(np.unique(change_hash[x[1]]))
#
# reduce_hash(change_hash)
# # Change the list in the hash to np-arrays for better storage in h5 files
# for k, v in change_hash.iteritems():
# change_hash[k] = np.array(v)
# # And now we have a perfect change list which we just need to iterate over and change the labels in the image
# ipl.logging('change_hash after change:')
us = [x[0] for x in merge_ids]
change_hash = IPL(data=dict(zip(us, merge_ids)))
ipl.logging('change_hash: {}', change_hash)
ipl[targetnames[4]] = change_hash
elif algorithm == 'standard':
# Get only the relevant labels
labels = lib.unique(ipl[key])
ipl.logging('labels = {}', labels)
# Seed the randomize function
random.seed(seed)
ipl.astype(np.uint32, keys=key)
(grag, rag) = graphs.gridRegionAdjacencyGraph(ipl[key], ignoreLabel=0)
edge_ids = rag.edgeIds()
# ipl.logging('Edge ids: {}', edge_ids)
# Type 1:
# Select edges by size (smallest edges)
ipl.logging('Number of edgeLengths = {}', len(rag.edgeLengths()))
edgelen_ids = dict(zip(edge_ids, rag.edgeLengths()))
# ifp.logging('edgelen_ids = {}', edgelen_ids)
sorted_edgelens = np.sort(rag.edgeLengths())
#
smallest_merge_lens = sorted_edgelens[0:numberbysize]
ipl.logging('Lengths selected for merging: {}', smallest_merge_lens)
#
smallest_merge_ids = []
for x in smallest_merge_lens:
edge_id = edgelen_ids.keys()[edgelen_ids.values().index(x)]
smallest_merge_ids.append(edge_id)
edgelen_ids.pop(edge_id)
#
edge_ids = edgelen_ids.keys()
ipl.logging('Edge IDs selected for merging due to size: {}', smallest_merge_ids)
# Type 2:
# Randomly choose edges
random_merge_ids = random.sample(edge_ids, numberbyrandom)
ipl.logging('Edge IDs randomly selected for merging: {}', random_merge_ids)
# Now get the label ids
smallest_merge_labelids_u = [rag.uId(rag.edgeFromId(x)) for x in smallest_merge_ids]
smallest_merge_labelids_v = [rag.vId(rag.edgeFromId(x)) for x in smallest_merge_ids]
smallest_merge_labelids = list(zip(smallest_merge_labelids_u, smallest_merge_labelids_v))
random_merge_labelids_u = [rag.uId(rag.edgeFromId(x)) for x in random_merge_ids]
random_merge_labelids_v = [rag.vId(rag.edgeFromId(x)) for x in random_merge_ids]
random_merge_labelids = list(zip(random_merge_labelids_u, random_merge_labelids_v))
ipl.logging('Label IDs selected for merging by size: {}', smallest_merge_labelids)
ipl.logging('Label IDs randomly selected for merging: {}', random_merge_labelids)
# Concatenate
all_merge_labelids = smallest_merge_labelids + random_merge_labelids
# Sort
ipl.logging('all_merge_labelids = {}', all_merge_labelids)
all_merge_labelids = [sorted(x) for x in all_merge_labelids]
all_merge_labelids = sorted(all_merge_labelids)
ipl.logging('all_merge_labelids = {}', all_merge_labelids)
# Store this for later use
ipl[targetnames[1]] = smallest_merge_labelids
ipl[targetnames[2]] = random_merge_labelids
ipl[targetnames[3]] = all_merge_labelids
# Create change hash list
change_hash = IPL(data=dict(zip(np.unique(all_merge_labelids), [[x,] for x in np.unique(all_merge_labelids)])))
for i in xrange(0, 3):
prev_change_hash = IPL(data=change_hash)
for x in all_merge_labelids:
ipl.logging('Adding {} and {}', *x)
change_hash[x[0]] += change_hash[x[1]]
change_hash[x[0]] = list(np.unique(change_hash[x[0]]))
change_hash[x[1]] += change_hash[x[0]]
change_hash[x[1]] = list(np.unique(change_hash[x[1]]))
# This removes the redundancy from the hash
reduce_hash(change_hash)
# Change the list in the hash to np-arrays for better storage in h5 files
for k, v in change_hash.iteritems():
change_hash[k] = np.array(v)
# And now we have a perfect change list which we just need to iterate over and change the labels in the image
ipl.logging('change_hash after change:')
ipl.logging(change_hash)
ipl[targetnames[4]] = change_hash
# Create the merged image
# ipl.deepcopy_entry('largeobj', targetnames[0])
ipl.rename_entry(key, targetnames[0])
for k, v in change_hash.iteritems():
for x in v:
if x != k:
ipl.logging('Setting {} to {}!', x, k)
ipl.filter_values(x, type='eq', setto=k, keys=targetnames[0])
return ipl
from hdf5_image_processing import Hdf5ImageProcessingLib as IPL
import processing_lib as lib
# Sample A probs
probs_a = IPL(
filepath='/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splA.probs_cantorV1.h5'
)
probs_a.logging('Probs A datastructure\n---\n{}', probs_a.datastructure2string())
probs_a.anytask(lib.swapaxes, 0, 2)
probs_a.write('/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splA.train.probs.crop.h5')
reskeys = ('0', '1')
split_probs_a = IPL()
split_probs_a['z'] = probs_a.anytask(lib.split, 2, axis=0, result_keys=reskeys, return_only=True, rtrntype=IPL)
split_probs_a['y'] = probs_a.anytask(lib.split, 2, axis=1, result_keys=reskeys, return_only=True, rtrntype=IPL)
split_probs_a['x'] = probs_a.anytask(lib.split, 2, axis=2, result_keys=reskeys, return_only=True, rtrntype=IPL)
split_probs_a = split_probs_a.switch_levels(1, 2)
probs_a.logging('Split sample A datastructure\n---\n{}', split_probs_a.datastructure2string())
split_probs_a.write('/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splA.train.probs.crop.split_xyz.h5')
for d, k, v, kl in ipl.data_iterator(yield_short_kl=True):
if k == params['locmaxbordernames'][2]:
ipl[kl].setlogger(ipl.getlogger())
ipl[kl] = localmax_on_disttransf(ipl[kl], (params['locmaxbordernames'][2], params['locmaxbordernames'][3]), thisparams)
if __name__ == '__main__':
resultsfolder = '/mnt/localdata02/jhennies/neuraldata/results/cremi_2016/161110_random_forest_of_paths/'
yamlfile = resultsfolder + '/parameters.yml'
ipl = IPL(
yaml=yamlfile,
yamlspec={'path': 'intermedfolder', 'filename': 'locmaxborderfile', 'skeys': {'locmaxbordernames': (2, 3)}},
recursive_search=True
)
params = ipl.get_params()
thisparams = params['localmax_on_disttransf']
ipl.startlogger(filename=params['resultfolder'] + 'localmax_on_disttransf.log', type='w')
try:
# Copy the script file and the parameters to the scriptsfolder
copy(inspect.stack()[0][1], params['scriptsfolder'])
copy(yamlfile, params['scriptsfolder'] + 'localmax_on_disttransf.parameters.yml')
# Write script and parameters to the logfile
ipl.code2log(inspect.stack()[0][1])
ipl.logging('')
ipl.yaml2log()
if __name__ == "__main__":
# cremi = IPL(filepath='/mnt/localdata02/jhennies/neuraldata/cremi_2016/sample_B_20160501.hdf')
#
# cremi.logging('Datastructure:\n---\n{}', cremi.datastructure2string())
#
# images = IPL(data={
# 'raw': cremi['volumes', 'raw'],
# 'neuron_ids': cremi['volumes', 'labels', 'neuron_ids']
# })
#
# images.logging('Datastructure:\n---\n{}', images.datastructure2string())
#
# images.write('/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splB.raw_neurons.crop.h5')
cremi = IPL(filepath='/mnt/localdata02/jhennies/neuraldata/cremi_2016/sample_C_20160501.hdf')
cremi.logging('Datastructure:\n---\n{}', cremi.datastructure2string())
images = IPL(data={
'raw': cremi['volumes', 'raw'],
'neuron_ids': cremi['volumes', 'labels', 'neuron_ids']
})
images.logging('Datastructure:\n---\n{}', images.datastructure2string())
images.write('/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splC.raw_neurons.crop.h5')
for fk, fv in feature_images.iteritems():
features["false", k, fk] = get_features(
v, fv, thisparams["features"], thisparams["max_paths_per_label"], hfp=hfp
)
# for i, k, ks, vs in hfp['paths_true'].simultaneous_iterator():
# pass
if __name__ == "__main__":
yamlfile = os.path.dirname(os.path.abspath(__file__)) + "/parameters.yml"
yamlfile = os.path.dirname(os.path.abspath(__file__)) + "/parameters.yml"
hfp = IPL(yaml=yamlfile, yamlspec={"path": "intermedfolder", "filename": "pathstruefile"}, castkey=None)
# hfp.logging('datastructure:\n---\n{}', hfp.datastructure2string())
params = hfp.get_params()
hfp["true", "border"] = IPL(data=hfp["largeobj", "border_locmax", "path"])
hfp["true", "locmax"] = IPL(data=hfp["largeobj", "locmax", "path"])
hfp.pop("largeobj")
hfp.data_from_file(filepath=params["intermedfolder"] + params["pathsfalsefile"])
hfp["false", "border"] = IPL(data=hfp["largeobjm", "border_locmax_m", "path"])
hfp["false", "locmax"] = IPL(data=hfp["largeobjm", "locmaxm", "path"])
hfp.pop("largeobjm")
hfp.pop("pathsim")
hfp.pop("overlay")
v['1']['false'] = make_feature_array(v['1']['false'])
result[k, '0'] = random_forest(v['0'], v['1'])
result[k, '1'] = random_forest(v['1'], v['0'])
return result
if __name__ == '__main__':
resultsfolder = '/mnt/localdata02/jhennies/neuraldata/results/cremi_2016/161110_random_forest_of_paths/'
yamlfile = resultsfolder + '/parameters.yml'
features = IPL(
yaml=yamlfile,
yamlspec={'path': 'intermedfolder', 'filename': 'featurefile'}
)
params = features.get_params()
thisparams = params['random_forest']
features.startlogger(filename=params['resultfolder'] + 'random_forest.log', type='w')
try:
# Copy the script file and the parameters to the scriptsfolder
copy(inspect.stack()[0][1], params['scriptsfolder'])
copy(yamlfile, params['scriptsfolder'] + 'random_forest.parameters.yml')
# Write script and parameters to the logfile
features.code2log(inspect.stack()[0][1])
features.logging('')
features.yaml2log()
features.logging('')
if k == params['labelsname']:
ipl.logging('===============================\nWorking on image: {}', kl + [k])
ipl[kl].setlogger(ipl.getlogger())
ipl[kl] = accumulate_small_objects(ipl[kl], k, thisparams)
if __name__ == '__main__':
resultsfolder = '/mnt/localdata02/jhennies/neuraldata/results/cremi_2016/161110_random_forest_of_paths/'
yamlfile = resultsfolder + '/parameters.yml'
ipl = IPL(
yaml=yamlfile,
yamlspec={'path': 'datafolder', 'filename': 'labelsfile'}
)
params = ipl.get_params()
ipl.startlogger(filename=params['resultfolder']+'remove_small_objects.log', type='a')
try:
# Create folder for scripts
if not os.path.exists(params['scriptsfolder']):
os.makedirs(params['scriptsfolder'])
else:
if params['overwriteresults']:
ipl.logging('remove_small_objects: Warning: Scriptsfolder already exists and content will be overwritten...\n')
else:
raise IOError('remove_small_objects: Error: Scriptsfolder already exists!')
# sample_a.logging('Sample A datastructure\n---\n{}', sample_a.datastructure2string())
#
# reskeys = ('0', '1')
# split_sample_a = IPL()
# split_sample_a['z'] = sample_a.anytask(lib.split, 2, axis=0, result_keys=reskeys, return_only=True)
# split_sample_a['y'] = sample_a.anytask(lib.split, 2, axis=1, result_keys=reskeys, return_only=True)
# split_sample_a['x'] = sample_a.anytask(lib.split, 2, axis=2, result_keys=reskeys, return_only=True)
#
# split_sample_a = split_sample_a.switch_levels(1, 2)
# sample_a.logging('Split sample A datastructure\n---\n{}', split_sample_a.datastructure2string())
#
# split_sample_a.write('/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splA.raw_neurons.crop.split_xyz.h5')
# Sample B
sample = IPL(
filepath='/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splB.raw_neurons.crop.h5'
)
sample.logging('Sample B datastructure\n---\n{}', sample.datastructure2string())
reskeys = ('0', '1')
split_sample = IPL()
split_sample['z'] = sample.anytask(lib.split, 2, axis=0, result_keys=reskeys, return_only=True)
split_sample['y'] = sample.anytask(lib.split, 2, axis=1, result_keys=reskeys, return_only=True)
split_sample['x'] = sample.anytask(lib.split, 2, axis=2, result_keys=reskeys, return_only=True)
split_sample = split_sample.switch_levels(1, 2)
sample.logging('Split sample B datastructure\n---\n{}', split_sample.datastructure2string())
split_sample.write('/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splB.raw_neurons.crop.split_xyz.h5')
if k == params['largeobjname']:
data = IPL(data={k: v})
data.setlogger(ipl.getlogger())
merged[kl] = merge_adjacent_objects(data, k, thisparams)
return merged
if __name__ == '__main__':
resultsfolder = '/mnt/localdata02/jhennies/neuraldata/results/cremi_2016/161110_random_forest_of_paths/'
yamlfile = resultsfolder + '/parameters.yml'
ipl = IPL(
yaml=yamlfile,
yamlspec={'path': 'intermedfolder', 'filename': 'largeobjfile'}
)
params = ipl.get_params()
ipl.startlogger(filename=params['resultfolder'] + 'merge_adjacent_objects.log', type='w')
try:
# Copy the script file and the parameters to the scriptsfolder
copy(inspect.stack()[0][1], params['scriptsfolder'])
copy(yamlfile, params['scriptsfolder'] + 'merge_adjacent_objects.parameters.yml')
# Write script and parameters to the logfile
ipl.code2log(inspect.stack()[0][1])
ipl.logging('')
ipl.yaml2log()
ipl.logging('')
import matplotlib.pylab as lab
import processing_lib as lib
# TODO: Visualize Distancetransform along paths of true and false merges
# TODO: Also intensity values or raw data and probability map
# TODO: Do not forget to save the result for the thesis!
__author__ = 'jhennies'
if __name__ == '__main__':
yamlfile = os.path.dirname(os.path.abspath(__file__)) + '/parameters.yml'
hfp = IPL(yaml=yamlfile,
yamlspec={
'path': 'intermedfolder',
'filename': 'pathstruefile'
},
tkeys='true',
castkey=None)
params = hfp.get_params()
hfp.data_from_file(filepath=params['intermedfolder'] +
params['pathsfalsefile'],
tkeys='false',
castkey=None)
hfp.data_from_file(filepath=params['intermedfolder'] +
params['locmaxfile'],
skeys=('disttransf', 'disttransfm'),
tkeys=('disttransf', 'disttransfm'))
ignoreLabel=0,
reverse_order=True,
reciprocal=False,
keys=('paths_true', 'paths_false'),
indict=disttransf_images)
if __name__ == '__main__':
yamlfile = os.path.dirname(os.path.abspath(__file__)) + '/parameters.yml'
# TODO: Insert code here
hfp = Hdf5ImageProcessingLib(yaml=yamlfile,
yamlspec={
'path': 'intermedfolder',
'filename': 'pathstruefile'
},
tkeys='true',
castkey=None)
params = hfp.get_params()
hfp.logging('params = {}', params)
hfp.data_from_file(filepath=params['intermedfolder'] +
params['pathsfalsefile'],
tkeys='false',
castkey=None)
hfp.startlogger(filename=params['intermedfolder'] +
'features_of_paths.log',
type='a')
try:
'===============================\nWorking on image: {}',
kl + [k])
ipl[kl].setlogger(ipl.getlogger())
ipl[kl] = accumulate_small_objects(ipl[kl], k, thisparams)
if __name__ == '__main__':
resultsfolder = '/mnt/localdata02/jhennies/neuraldata/results/cremi_2016/161110_random_forest_of_paths/'
yamlfile = resultsfolder + '/parameters.yml'
ipl = IPL(yaml=yamlfile,
yamlspec={
'path': 'datafolder',
'filename': 'labelsfile'
})
params = ipl.get_params()
ipl.startlogger(filename=params['resultfolder'] +
'remove_small_objects.log',
type='a')
try:
# Create folder for scripts
if not os.path.exists(params['scriptsfolder']):
os.makedirs(params['scriptsfolder'])
else:
if params['overwriteresults']:
ipl.logging(
ipl[kl], disttransf_images[kl], feature_images[kl],
thisparams
)
return features
if __name__ == '__main__':
resultsfolder = '/mnt/localdata02/jhennies/neuraldata/results/cremi_2016/161110_random_forest_of_paths/'
yamlfile = resultsfolder + '/parameters.yml'
ipl = IPL(
yaml=yamlfile,
yamlspec={'path': 'intermedfolder', 'filename': 'pathstruefile'},
skeys='path', recursive_search=True
)
# ipl.logging('datastructure:\n---\n{}', ipl.datastructure2string())
params = ipl.get_params()
ipl.rename_layer('largeobj', 'true')
# ipl['true', 'border'] = IPL(data=ipl['largeobj', 'border_locmax', 'path'])
# ipl['true', 'locmax'] = IPL(data=ipl['largeobj', 'locmax', 'path'])
# ipl.pop('largeobj')
ipl.data_from_file(filepath=params['intermedfolder'] + params['pathsfalsefile'],
skeys='path', recursive_search=True, integrate=True)
ipl.rename_layer('largeobjm', 'false')
ipl.remove_layer('path')
def run_paths_of_merges(yamlfile, logging=True):
ipl = IPL(yaml=yamlfile)
ipl.set_indent(1)
params = rdict(data=ipl.get_params())
if logging:
ipl.startlogger(filename=params['resultfolder'] +
'paths_of_merges.log',
type='w',
name='PathsOfMerges')
else:
ipl.startlogger()
try:
# # Copy the script file and the parameters to the scriptsfolder
# copy(inspect.stack()[0][1], params['scriptsfolder'])
# copy(yamlfile, params['scriptsfolder'] + 'paths_of_merges.parameters.yml')
# ipl.logging('\nInitial datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3))
paths_of_merges(ipl, params['debug'])
# ipl.logging('\nFinal datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3))
# ipl.write(filepath=params['intermedfolder'] + params['largeobjfile'])
ipl.logging('')
ipl.stoplogger()
except:
ipl.errout('Unexpected error')
def load_images(ipl):
"""
These images are loaded:
paths_true (paths within single label objects)
paths_false (paths of merged objects which cross the merging site)
featureims_true
featureims_false
:param ipl:
:return:
"""
paths_true = IPL()
paths_false = IPL()
featureims_true = IPL()
featureims_false = IPL()
params = ipl.get_params()
ipl.logging('Loading true paths ...')
# Paths within labels (true paths)
paths_true.data_from_file(
filepath=params['intermedfolder'] + params['pathstruefile'],
skeys='path',
recursive_search=True, nodata=True
)
ipl.logging('Loading false paths ...')
# Paths of merges (false paths)
paths_false.data_from_file(
filepath=params['intermedfolder'] + params['pathsfalsefile'],
skeys='path',
recursive_search=True, nodata=True
)
ipl.logging('Loading features for true paths ...')
# Load features for true paths
featureims_true.data_from_file(
filepath=params['intermedfolder'] + params['featureimsfile'],
nodata=True
)
featureims_true.delete_items(params['largeobjmnames'][0])
ipl.logging('Loading features for false paths ...')
# Load features for false paths
featureims_false.data_from_file(
filepath=params['intermedfolder'] + params['featureimsfile'],
nodata=True
)
featureims_false.delete_items(params['largeobjname'])
return (paths_true, paths_false, featureims_true, featureims_false)
# hfp.deepcopy_entry('largeobj', targetnames[0])
hfp.rename_entry('largeobj', targetnames[0])
for k, v in change_hash.iteritems():
for x in v:
if x != k:
hfp.logging('Setting {} to {}!', x, k)
hfp.filter_values(x, type='eq', setto=k, keys=targetnames[0])
if __name__ == '__main__':
yamlfile = os.path.dirname(os.path.abspath(__file__)) + '/parameters.yml'
hfp = IPL(
yaml=yamlfile,
yamlspec={'path': 'datafolder', 'filename': 'largeobjfile', 'skeys': 'largeobjname'},
tkeys='largeobj',
castkey=None
)
params = hfp.get_params()
hfp.startlogger(filename=params['resultfolder'] + 'merge_adjacent_objects.log', type='w')
try:
# Copy the script file and the parameters to the scriptsfolder
copy(inspect.stack()[0][1], params['scriptsfolder'])
copy(yamlfile, params['scriptsfolder'] + 'merge_adjacent_objects.parameters.yml')
# Write script and parameters to the logfile
hfp.code2log(inspect.stack()[0][1])
hfp.logging('')
hfp.yaml2log()
hfp.logging('')
if __name__ == "__main__":
infiles = [
'/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splA.train.probs.crop.h5',
'/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splA.train.raw_neurons.crop.h5'
]
outfiles = [
'/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splA.train.probs.crop.crop_x10_110_y200_712_z200_712.split_xyz.h5',
'/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splA.train.raw_neurons.crop.crop_x10_110_y200_712_z200_712.split_xyz.h5'
]
for i in xrange(0, len(infiles)):
ipl = IPL(filepath=infiles[i])
ipl.logging('Datastructure\n---\n{}', ipl.datastructure2string())
ipl.crop_bounding_rect(bounds=np.s_[10:110, 200:712, 200:712])
def shape(image):
return image.shape
print ipl.datastructure2string(function=shape)
ipl_split = split_in_xyz(ipl)
ipl_split.write(filepath=outfiles[i])
# # Sample A
# sample_a = IPL(
from hdf5_image_processing import Hdf5ImageProcessingLib as IPL
import os
import numpy as np
__author__ = 'jhennies'
if __name__ == '__main__':
yamlfile = os.path.dirname(os.path.abspath(__file__)) + '/parameters.yml'
ipl = IPL(yaml=yamlfile)
ipl.logging('Parameters: {}', ipl.get_params())
params = ipl.get_params()
ipl.data_from_file(filepath=params['datafolder'] +
'cremi.splA.raw_neurons.crop.h5',
skeys='raw',
tkeys='raw')
ipl.crop_bounding_rect(np.s_[10:110, 200:712, 200:712], keys='raw')
ipl.write(filepath=params['datafolder'] +
'cremi.splA.raw_neurons.crop.crop_10-200-200_110-712-712.h5')
thisparams['features'],
thisparams['max_paths_per_label'],
hfp=hfp)
# for i, k, ks, vs in hfp['paths_true'].simultaneous_iterator():
# pass
if __name__ == '__main__':
yamlfile = os.path.dirname(os.path.abspath(__file__)) + '/parameters.yml'
yamlfile = os.path.dirname(os.path.abspath(__file__)) + '/parameters.yml'
hfp = IPL(yaml=yamlfile,
yamlspec={
'path': 'intermedfolder',
'filename': 'pathstruefile'
},
castkey=None)
# hfp.logging('datastructure:\n---\n{}', hfp.datastructure2string())
params = hfp.get_params()
hfp['true', 'border'] = IPL(data=hfp['largeobj', 'border_locmax', 'path'])
hfp['true', 'locmax'] = IPL(data=hfp['largeobj', 'locmax', 'path'])
hfp.pop('largeobj')
hfp.data_from_file(filepath=params['intermedfolder'] +
params['pathsfalsefile'])
hfp['false', 'border'] = IPL(data=hfp['largeobjm', 'border_locmax_m',
'path'])
hfp['false', 'locmax'] = IPL(data=hfp['largeobjm', 'locmaxm', 'path'])
result[k, '0'] = random_forest(v['0'], v['1'])
result[k, '1'] = random_forest(v['1'], v['0'])
return result
if __name__ == '__main__':
resultsfolder = '/mnt/localdata02/jhennies/neuraldata/results/cremi_2016/161110_random_forest_of_paths/'
yamlfile = resultsfolder + '/parameters.yml'
features = IPL(yaml=yamlfile,
yamlspec={
'path': 'intermedfolder',
'filename': 'featurefile'
})
params = features.get_params()
thisparams = params['random_forest']
features.startlogger(filename=params['resultfolder'] + 'random_forest.log',
type='w')
try:
# Copy the script file and the parameters to the scriptsfolder
copy(inspect.stack()[0][1], params['scriptsfolder'])
copy(yamlfile,
params['scriptsfolder'] + 'random_forest.parameters.yml')
# Write script and parameters to the logfile
features.code2log(inspect.stack()[0][1])
# sample_a.logging('Sample A datastructure\n---\n{}', sample_a.datastructure2string())
#
# reskeys = ('0', '1')
# split_sample_a = IPL()
# split_sample_a['z'] = sample_a.anytask(lib.split, 2, axis=0, result_keys=reskeys, return_only=True)
# split_sample_a['y'] = sample_a.anytask(lib.split, 2, axis=1, result_keys=reskeys, return_only=True)
# split_sample_a['x'] = sample_a.anytask(lib.split, 2, axis=2, result_keys=reskeys, return_only=True)
#
# split_sample_a = split_sample_a.switch_levels(1, 2)
# sample_a.logging('Split sample A datastructure\n---\n{}', split_sample_a.datastructure2string())
#
# split_sample_a.write('/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splA.raw_neurons.crop.split_xyz.h5')
# Sample B
sample = IPL(
filepath=
'/mnt/localdata02/jhennies/neuraldata/cremi_2016/cremi.splB.raw_neurons.crop.h5'
)
sample.logging('Sample B datastructure\n---\n{}',
sample.datastructure2string())
reskeys = ('0', '1')
split_sample = IPL()
split_sample['z'] = sample.anytask(lib.split,
2,
axis=0,
result_keys=reskeys,
return_only=True)
split_sample['y'] = sample.anytask(lib.split,
2,
axis=1,
def run_remove_small_objects(yamlfile):
ipl = IPL(
yaml=yamlfile,
yamlspec={'path': 'datafolder', 'filename': 'labelsfile', 'skeys': 'labelsname'},
recursive_search=True,
nodata=True
)
# Set indentation of the logging
ipl.set_indent(1)
params = ipl.get_params()
ipl.startlogger(filename=params['resultfolder'] + 'remove_small_objects.log', type='w', name='RemoveSmallObjects')
try:
# # Copy the script file and the parameters to the scriptsfolder
# copy(inspect.stack()[0][1], params['scriptsfolder'])
# copy(yamlfile, params['scriptsfolder'] + 'remove_small_objects.parameters.yml')
ipl.logging('\nipl datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3))
remove_small_objects(ipl)
ipl.logging('\nFinal datastructure: \n\n{}', ipl.datastructure2string(maxdepth=3))
# ipl.write(filepath=params['intermedfolder'] + params['largeobjfile'])
ipl.logging('')
ipl.stoplogger()
except:
ipl.errout('Unexpected error')
#
# ifp.errout('Unexpected error', traceback)
#
# try:
# hfp.write(filepath=params['intermedfolder'] + params['pathsfalsefile'])
# except:
# pass
if __name__ == '__main__':
yamlfile = os.path.dirname(os.path.abspath(__file__)) + '/parameters.yml'
hfp = IPL(
yaml=yamlfile,
yamlspec={'path': 'intermedfolder', 'filename': 'locmaxborderfile', 'skeys': {'locmaxbordernames': (1, 3)}},
tkeys=('border_locmax_m', 'disttransfm'),
castkey=None
)
params = hfp.get_params()
thisparams = params['paths_of_partners']
hfp.startlogger(filename=params['resultfolder'] + 'paths_of_partners.log', type='w')
hfp.data_from_file(params['intermedfolder'] + params['largeobjfile'],
skeys=params['largeobjname'],
tkeys='largeobj')
hfp.data_from_file(params['intermedfolder'] + params['largeobjmfile'],
skeys=(params['largeobjmnames'][0], params['largeobjmnames'][4]),
tkeys=('largeobjm', 'change_hash'))
hfp.data_from_file(params['intermedfolder'] + params['locmaxfile'],
skeys=params['locmaxnames'][0],
tkeys='locmaxm')
