def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
preproc = topGroup
for opPre in self._o.innerOperators:
mst = opPre._prepData[0]
if mst is not None:
#The values to be saved for sigma and filter are the
#values of the last valid preprocess
#!These may differ from the current settings!
deleteIfPresent(preproc, "sigma")
deleteIfPresent(preproc, "filter")
deleteIfPresent(preproc, "watershed_source")
deleteIfPresent(preproc, "invert_watershed_source")
deleteIfPresent(preproc, "graph")
preproc.create_dataset("sigma", data=opPre.initialSigma)
preproc.create_dataset("filter", data=opPre.initialFilter)
ws_source = str(opPre.WatershedSource.value)
assert isinstance(
ws_source, str
), "WatershedSource was {}, but it should be a string.".format(
ws_source)
preproc.create_dataset("watershed_source", data=ws_source)
preproc.create_dataset("invert_watershed_source",
data=opPre.InvertWatershedSource.value)
preprocgraph = getOrCreateGroup(preproc, "graph")
mst.saveH5G(preprocgraph)
opPre._unsavedData = False
def serialize(self, group):
if not self.shouldSerialize(group):
return
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
innerops = mainOperator.innerOperators
for i, op in enumerate(innerops):
gr = getOrCreateGroup(group, str(i))
for cropKey in op.Annotations.value.keys():
crop_gr = getOrCreateGroup(gr, str(cropKey))
labels_gr = getOrCreateGroup(crop_gr, str("labels"))
for t in op.Annotations.value[cropKey]["labels"].keys():
t_gr = getOrCreateGroup(labels_gr, str(t))
for oid in op.Annotations.value[cropKey]["labels"][t].keys(
):
l = op.Annotations.value[cropKey]["labels"][t][oid]
dset = list(l)
if len(dset) > 0:
t_gr.create_dataset(name=str(oid), data=dset)
divisions_gr = getOrCreateGroup(crop_gr, str("divisions"))
dset = []
for trackid in op.Annotations.value[cropKey]["divisions"].keys(
):
(children, t_parent
) = op.Annotations.value[cropKey]["divisions"][trackid]
dset.append([trackid, children[0], children[1], t_parent])
if len(dset) > 0:
divisions_gr.create_dataset(name=str(i), data=dset)
self.dirty = False
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
preproc = topGroup
for opPre in self._o.innerOperators:
mst = opPre._prepData[0]
if mst is not None:
#The values to be saved for sigma and filter are the
#values of the last valid preprocess
#!These may differ from the current settings!
deleteIfPresent(preproc, "sigma")
deleteIfPresent(preproc, "filter")
deleteIfPresent(preproc, "watershed_source")
deleteIfPresent(preproc, "invert_watershed_source")
deleteIfPresent(preproc, "graph")
preproc.create_dataset("sigma",data= opPre.initialSigma)
preproc.create_dataset("filter",data= opPre.initialFilter)
ws_source = str(opPre.WatershedSource.value)
assert isinstance( ws_source, str ), "WatershedSource was {}, but it should be a string.".format( ws_source )
preproc.create_dataset("watershed_source", data=ws_source)
preproc.create_dataset("invert_watershed_source", data=opPre.InvertWatershedSource.value)
preprocgraph = getOrCreateGroup(preproc, "graph")
mst.saveH5G(preprocgraph)
opPre._unsavedData = False
def _serializeLabels(self, topGroup):
with Tracer(traceLogger):
# Delete all labels from the file
deleteIfPresent(topGroup, 'LabelSets')
labelSetDir = topGroup.create_group('LabelSets')
numImages = len(self.mainOperator.NonzeroLabelBlocks)
for imageIndex in range(numImages):
# Create a group for this image
labelGroupName = 'labels{:03d}'.format(imageIndex)
labelGroup = labelSetDir.create_group(labelGroupName)
# Get a list of slicings that contain labels
nonZeroBlocks = self.mainOperator.NonzeroLabelBlocks[
imageIndex].value
for blockIndex, slicing in enumerate(nonZeroBlocks):
# Read the block from the label output
block = self.mainOperator.LabelImages[imageIndex][
slicing].wait()
# Store the block as a new dataset
blockName = 'block{:04d}'.format(blockIndex)
labelGroup.create_dataset(blockName, data=block)
# Add the slice this block came from as an attribute of the dataset
labelGroup[blockName].attrs[
'blockSlice'] = self.slicingToString(slicing)
self._dirtyFlags[Section.Labels] = False
def _serializeClassifiers(self, topGroup):
with Tracer(traceLogger):
deleteIfPresent(topGroup, 'Classifiers')
self._dirtyFlags[Section.Classifiers] = False
if not self.mainOperator.Classifiers.ready():
return
classifiers = self.mainOperator.Classifiers
topGroup.require_group("Classifiers")
for i in range(len(classifiers)):
classifier_forests = classifiers[i].value
# Classifier can be None if there isn't any training data yet.
if classifier_forests is None:
return
for forest in classifier_forests:
if forest is None:
return
# Due to non-shared hdf5 dlls, vigra can't write directly to our open hdf5 group.
# Instead, we'll use vigra to write the classifier to a temporary file.
tmpDir = tempfile.mkdtemp()
cachePath = os.path.join(tmpDir, 'tmp_classifier_cache.h5').replace('\\', '/')
for j, forest in enumerate(classifier_forests):
forest.writeHDF5( cachePath, 'ClassifierForests/Forest{:04d}'.format(j) )
# Open the temp file and copy to our project group
with h5py.File(cachePath, 'r') as cacheFile:
grouppath = "Classifiers/Classifier%d"%i
topGroup.copy(cacheFile['ClassifierForests'], grouppath)
os.remove(cachePath)
os.removedirs(tmpDir)
def _serializeLabels(self, topGroup):
with Tracer(traceLogger):
# Delete all labels from the file
deleteIfPresent(topGroup, 'LabelSets')
labelSetDir = topGroup.create_group('LabelSets')
numImages = len(self.mainOperator.NonzeroLabelBlocks)
for imageIndex in range(numImages):
# Create a group for this image
labelGroupName = 'labels{:03d}'.format(imageIndex)
labelGroup = labelSetDir.create_group(labelGroupName)
# Get a list of slicings that contain labels
nonZeroBlocks = self.mainOperator.NonzeroLabelBlocks[imageIndex].value
for blockIndex, slicing in enumerate(nonZeroBlocks):
# Read the block from the label output
block = self.mainOperator.LabelImages[imageIndex][slicing].wait()
# Store the block as a new dataset
blockName = 'block{:04d}'.format(blockIndex)
labelGroup.create_dataset(blockName, data=block)
# Add the slice this block came from as an attribute of the dataset
labelGroup[blockName].attrs['blockSlice'] = self.slicingToString(slicing)
self._dirtyFlags[Section.Labels] = False
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
getOrCreateGroup(topGroup, "local_data")
deleteIfPresent(topGroup, "Role Names")
role_names = [name.encode("utf-8") for name in self.topLevelOperator.DatasetRoles.value]
topGroup.create_dataset("Role Names", data=role_names)
# Access the info group
infoDir = getOrCreateGroup(topGroup, "infos")
# Delete all infos
infoDir.clear()
# Rebuild the list of infos
roleNames = self.topLevelOperator.DatasetRoles.value
internal_datasets_to_keep = set()
for laneIndex, multislot in enumerate(self.topLevelOperator.DatasetGroup):
laneGroupName = "lane{:04d}".format(laneIndex)
laneGroup = infoDir.create_group(laneGroupName)
for roleIndex, slot in enumerate(multislot):
infoGroup = laneGroup.create_group(roleNames[roleIndex])
if slot.ready():
datasetInfo = slot.value
if isinstance(datasetInfo, ProjectInternalDatasetInfo):
internal_datasets_to_keep.add(hdf5File[datasetInfo.inner_path])
for k, v in datasetInfo.to_json_data().items():
if v is not None:
infoGroup.create_dataset(k, data=v)
if self.local_data_path.as_posix() in hdf5File:
for dataset in hdf5File[self.local_data_path.as_posix()].values():
if dataset not in internal_datasets_to_keep:
del hdf5File[dataset.name]
self._dirty = False
def serialize(self, group):
if not self.shouldSerialize(group):
return
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
innerops = mainOperator.innerOperators
for i, op in enumerate(innerops):
gr = getOrCreateGroup(group, str(i))
labels_gr = getOrCreateGroup(gr, str("labels"))
if "labels" in op.Annotations.value.keys():
for t in op.Annotations.value["labels"].keys():
t_gr = getOrCreateGroup(labels_gr, str(t))
for oid in op.Annotations.value["labels"][t].keys():
l = op.Annotations.value["labels"][t][oid]
dset = list(l)
if len(dset) > 0:
t_gr.create_dataset(name=str(oid), data=dset)
divisions_gr = getOrCreateGroup(gr, str("divisions"))
dset = []
if "divisions" in op.Annotations.value.keys():
for trackid in op.Annotations.value["divisions"].keys():
(children, t_parent) = op.Annotations.value["divisions"][trackid]
dset.append([trackid, children[0], children[1], t_parent])
if len(dset) > 0:
divisions_gr.create_dataset(name=str(i), data=dset)
self.dirty = False
def serialize(self, group):
if not self.shouldSerialize(group):
return
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
for i in range(len(mainOperator)):
subgroup = getOrCreateGroup(group, "{:04}".format(i))
cleanBlockRois = self.blockslot[i].value
for roi in cleanBlockRois:
region_features_arr = self.slot[i](*roi).wait()
assert region_features_arr.shape == (1, )
region_features = region_features_arr[0]
roi_grp = subgroup.create_group(name=str(roi))
logger.debug('Saving region features into group: "{}"'.format(
roi_grp.name))
for key, val in region_features.iteritems():
plugin_group = getOrCreateGroup(roi_grp, key)
for featname, featval in val.iteritems():
plugin_group.create_dataset(name=featname,
data=featval)
self.dirty = False
def serialize(self, group):
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
innerops = mainOperator.innerOperators
for i, op in enumerate(innerops):
dset = []
for trackid in op.divisions.keys():
(children, t_parent) = op.divisions[trackid]
dset.append([trackid, children[0], children[1], t_parent])
if len(dset) > 0:
group.create_dataset(name=str(i), data=dset)
self.dirty = False
def serialize(self, group):
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
innerops = mainOperator.innerOperators
for i, op in enumerate(innerops):
dset = []
for trackid in op.divisions.keys():
(children, t_parent) = op.divisions[trackid]
dset.append([trackid, children[0], children[1], t_parent])
if len(dset) > 0:
group.create_dataset(name=str(i), data=dset)
self.dirty = False
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
split_settings_grp = getOrCreateGroup(topGroup, "split_settings")
for laneIndex in range(len( self._topLevelOperator )):
lane_grp = getOrCreateGroup(split_settings_grp, "{}".format( laneIndex ))
opLaneView = self._topLevelOperator.getLane(laneIndex)
if opLaneView.AnnotationFilepath.ready():
annotation_filepath = opLaneView.AnnotationFilepath.value
deleteIfPresent( lane_grp, "annotation_filepath" )
lane_grp.create_dataset("annotation_filepath", data=annotation_filepath)
# Now save the regular the carving data.
super( SplitBodyCarvingSerializer, self )._serializeToHdf5( topGroup, hdf5File, projectFilePath )
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
with Tracer(traceLogger):
# Can't store anything without both scales and features
if not self.topLevelOperator.Scales.ready() \
or not self.topLevelOperator.FeatureIds.ready():
return
# Delete previous entries if they exist
deleteIfPresent(topGroup, 'Scales')
deleteIfPresent(topGroup, 'FeatureIds')
deleteIfPresent(topGroup, 'SelectionMatrix')
deleteIfPresent(topGroup, 'FeatureListFilename')
# Store the new values (as numpy arrays)
topGroup.create_dataset('Scales', data=self.topLevelOperator.Scales.value)
topGroup.create_dataset('FeatureIds', data=self.topLevelOperator.FeatureIds.value)
if self.topLevelOperator.SelectionMatrix.ready():
topGroup.create_dataset('SelectionMatrix', data=self.topLevelOperator.SelectionMatrix.value)
if self.topLevelOperator.FeatureListFilename.ready():
fname = str(self.topLevelOperator.FeatureListFilename.value)
if fname:
topGroup.create_dataset('FeatureListFilename', data=fname)
self._dirty = False
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
split_settings_grp = getOrCreateGroup(topGroup, "split_settings")
for laneIndex in range(len( self._topLevelOperator )):
lane_grp = getOrCreateGroup(split_settings_grp, "{}".format( laneIndex ))
opLaneView = self._topLevelOperator.getLane(laneIndex)
if opLaneView.AnnotationFilepath.ready():
annotation_filepath = opLaneView.AnnotationFilepath.value
deleteIfPresent( lane_grp, "annotation_filepath" )
lane_grp.create_dataset("annotation_filepath", data=annotation_filepath)
# Now save the regular the carving data.
super( SplitBodyCarvingSerializer, self )._serializeToHdf5( topGroup, hdf5File, projectFilePath )
self.__dirty = False
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
# Can't store anything without both scales and features
if not self.topLevelOperator.Scales.ready() \
or not self.topLevelOperator.FeatureIds.ready():
return
# Delete previous entries if they exist
deleteIfPresent(topGroup, 'Scales')
deleteIfPresent(topGroup, 'FeatureIds')
deleteIfPresent(topGroup, 'SelectionMatrix')
deleteIfPresent(topGroup, 'FeatureListFilename')
# Store the new values (as numpy arrays)
topGroup.create_dataset('Scales', data=self.topLevelOperator.Scales.value)
feature_ids = list(map(lambda s: s.encode('utf-8'), self.topLevelOperator.FeatureIds.value))
topGroup.create_dataset('FeatureIds', data=feature_ids)
if self.topLevelOperator.SelectionMatrix.ready():
topGroup.create_dataset('SelectionMatrix', data=self.topLevelOperator.SelectionMatrix.value)
if self.topLevelOperator.FeatureListFilename.ready():
fnames = []
for slot in self.topLevelOperator.FeatureListFilename:
fnames.append(slot.value)
if fnames:
fnames = map(lambda s: s.encode('utf-8'), fnames)
topGroup.create_dataset('FeatureListFilename', data=fnames)
self._dirty = False
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
# Can't store anything without both scales and features
if not self.topLevelOperator.Scales.ready() \
or not self.topLevelOperator.FeatureIds.ready():
return
# Delete previous entries if they exist
deleteIfPresent(topGroup, 'Scales')
deleteIfPresent(topGroup, 'FeatureIds')
deleteIfPresent(topGroup, 'SelectionMatrix')
deleteIfPresent(topGroup, 'FeatureListFilename')
# Store the new values (as numpy arrays)
topGroup.create_dataset('Scales', data=self.topLevelOperator.Scales.value)
topGroup.create_dataset('FeatureIds', data=self.topLevelOperator.FeatureIds.value)
if self.topLevelOperator.SelectionMatrix.ready():
topGroup.create_dataset('SelectionMatrix', data=self.topLevelOperator.SelectionMatrix.value)
if self.topLevelOperator.FeatureListFilename.ready():
fname = str(self.topLevelOperator.FeatureListFilename.value)
if fname:
topGroup.create_dataset('FeatureListFilename', data=fname)
self._dirty = False
def serialize(self, group):
if not self.shouldSerialize(group):
return
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
for i, op in enumerate(mainOperator.innerOperators):
ts = op._processedTimeSteps
if len(ts) > 0:
subgroup = getOrCreateGroup(group, str(i))
subgroup.create_dataset(name='timesteps', data=list(ts))
src = op._mem_h5
subgroup.copy(src['/TranslationVectors'], subgroup, name='data')
self.dirty = False
def serialize(self, group):
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
innerops = mainOperator.innerOperators
for i, op in enumerate(innerops):
gr = getOrCreateGroup(group, str(i))
for t in op.labels.keys():
t_gr = getOrCreateGroup(gr, str(t))
for oid in op.labels[t].keys():
l = op.labels[t][oid]
dset = list(l)
if len(dset) > 0:
t_gr.create_dataset(name=str(oid), data=dset)
self.dirty = False
def serialize(self, group):
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
innerops = mainOperator.innerOperators
for i, op in enumerate(innerops):
gr = getOrCreateGroup(group, str(i))
for t in op.labels.keys():
t_gr = getOrCreateGroup(gr, str(t))
for oid in op.labels[t].keys():
l = op.labels[t][oid]
dset = list(l)
if len(dset) > 0:
t_gr.create_dataset(name=str(oid), data=dset)
self.dirty = False
def serialize(self, group):
if not self.shouldSerialize(group):
return
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
innerops = mainOperator.innerOperators
for i, op in enumerate(innerops):
gr = getOrCreateGroup(group, str(i))
for t in list(op.disappearances.keys()):
t_gr = getOrCreateGroup(gr, str(t))
for oid in list(op.disappearances[t].keys()):
oid_gr = getOrCreateGroup(t_gr, str(oid))
for track in list(op.disappearances[t][oid].keys()):
app = op.disappearances[t][oid][track]
if app:
oid_gr.create_dataset(name=str(track), data=app)
self.dirty = False
def serialize(self, group):
if not self.shouldSerialize(group):
return
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
innerops = mainOperator.innerOperators
for i, op in enumerate(innerops):
gr = getOrCreateGroup(group, str(i))
for t in list(op.disappearances.keys()):
t_gr = getOrCreateGroup(gr, str(t))
for oid in list(op.disappearances[t].keys()):
oid_gr = getOrCreateGroup(t_gr, str(oid))
for track in list(op.disappearances[t][oid].keys()):
app = op.disappearances[t][oid][track]
if app:
oid_gr.create_dataset(name=str(track), data=app)
self.dirty = False
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
# Can't store anything without both scales and features
if not self.topLevelOperator.Scales.ready() \
or not self.topLevelOperator.FeatureIds.ready():
return
# Delete previous entries if they exist
deleteIfPresent(topGroup, 'Scales')
deleteIfPresent(topGroup, 'FeatureIds')
deleteIfPresent(topGroup, 'SelectionMatrix')
deleteIfPresent(topGroup, 'FeatureListFilename')
deleteIfPresent(topGroup, 'ComputeIn2d')
# Store the new values (as numpy arrays)
topGroup.create_dataset('Scales',
data=self.topLevelOperator.Scales.value)
feature_ids = list(
map(lambda s: s.encode('utf-8'),
self.topLevelOperator.FeatureIds.value))
topGroup.create_dataset('FeatureIds', data=feature_ids)
if self.topLevelOperator.SelectionMatrix.ready():
topGroup.create_dataset(
'SelectionMatrix',
data=self.topLevelOperator.SelectionMatrix.value)
if self.topLevelOperator.FeatureListFilename.ready():
fnames = []
for slot in self.topLevelOperator.FeatureListFilename:
fnames.append(slot.value)
if fnames:
fnames = map(lambda s: s.encode('utf-8'), fnames)
topGroup.create_dataset('FeatureListFilename', data=fnames)
if self.topLevelOperator.ComputeIn2d.ready():
topGroup.create_dataset(
'ComputeIn2d', data=self.topLevelOperator.ComputeIn2d.value)
self._dirty = False
def serialize(self, group):
if not self.shouldSerialize(group):
return
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
for i in range(len(mainOperator)):
subgroup = getOrCreateGroup(group, str(i))
cleanBlockRois = self.blockslot[i].value
for roi in cleanBlockRois:
region_features_arr = self.outslot[i]( *roi ).wait()
assert region_features_arr.shape == (1,)
region_features = region_features_arr[0]
roi_grp = subgroup.create_group(name=str(roi))
logger.debug('Saving region features into group: "{}"'.format( roi_grp.name ))
for key, val in region_features.iteritems():
roi_grp.create_dataset(name=key, data=val)
self.dirty = False
def _serializeClassifiers(self, topGroup):
with Tracer(traceLogger):
deleteIfPresent(topGroup, 'Classifiers')
self._dirtyFlags[Section.Classifiers] = False
if not self.mainOperator.Classifiers.ready():
return
classifiers = self.mainOperator.Classifiers
topGroup.require_group("Classifiers")
for i in range(len(classifiers)):
classifier_forests = classifiers[i].value
# Classifier can be None if there isn't any training data yet.
if classifier_forests is None:
return
for forest in classifier_forests:
if forest is None:
return
# Due to non-shared hdf5 dlls, vigra can't write directly to our open hdf5 group.
# Instead, we'll use vigra to write the classifier to a temporary file.
tmpDir = tempfile.mkdtemp()
cachePath = os.path.join(tmpDir,
'tmp_classifier_cache.h5').replace(
'\\', '/')
for j, forest in enumerate(classifier_forests):
forest.writeHDF5(
cachePath, 'ClassifierForests/Forest{:04d}'.format(j))
# Open the temp file and copy to our project group
with h5py.File(cachePath, 'r') as cacheFile:
grouppath = "Classifiers/Classifier%d" % i
topGroup.copy(cacheFile['ClassifierForests'], grouppath)
os.remove(cachePath)
os.removedirs(tmpDir)
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
preproc = topGroup
for opPre in self._o.innerOperators:
mst = opPre._prepData[0]
if mst is not None:
# The values to be saved for sigma and filter are the
# values of the last valid preprocess
#!These may differ from the current settings!
deleteIfPresent(preproc, "sigma")
deleteIfPresent(preproc, "filter")
deleteIfPresent(preproc, "graph")
preproc.create_dataset("sigma", data=opPre.initialSigma)
preproc.create_dataset("filter", data=opPre.initialFilter)
preprocgraph = getOrCreateGroup(preproc, "graph")
mst.saveH5G(preprocgraph)
opPre._unsavedData = False
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
preproc = topGroup
for opPre in self._o.innerOperators:
mst = opPre._prepData[0]
if mst is not None:
# The values to be saved for sigma and filter are the
# values of the last valid preprocess
#!These may differ from the current settings!
deleteIfPresent(preproc, "sigma")
deleteIfPresent(preproc, "filter")
deleteIfPresent(preproc, "graph")
preproc.create_dataset("sigma", data=opPre.initialSigma)
preproc.create_dataset("filter", data=opPre.initialFilter)
preprocgraph = getOrCreateGroup(preproc, "graph")
mst.saveH5G(preprocgraph)
opPre._unsavedData = False
def _serializePredictions(self, topGroup, startProgress, endProgress):
"""
Called when the currently stored predictions are dirty.
If prediction storage is currently enabled, store them to the file.
Otherwise, just delete them/
(Avoid inconsistent project states, e.g. don't allow old predictions to be stored with a new classifier.)
"""
with Tracer(traceLogger):
# If the predictions are missing, then maybe the user wants them stored (even if they aren't dirty)
if self._dirtyFlags[Section.Predictions] or 'Pdigital signal processing bookredictions' not in topGroup.keys():
deleteIfPresent(topGroup, 'Predictions')
# Disconnect the precomputed prediction inputs.
for i,slot in enumerate( self.mainOperator.PredictionsFromDisk ):
slot.disconnect()
if self.predictionStorageEnabled:
predictionDir = topGroup.create_group('Predictions')
failedToSave = False
try:
numImages = len(self.mainOperator.PredictionProbabilities)
if numImages > 0:
increment = (endProgress - startProgress) / float(numImages)
for imageIndex in range(numImages):
# Have we been cancelled?
if not self.predictionStorageEnabled:
break
datasetName = 'predictions{:04d}'.format(imageIndex)
progress = [startProgress]
# Use a big dataset writer to do this in chunks
opWriter = OpH5WriterBigDataset(graph=self.mainOperator.graph)
opWriter.hdf5File.setValue( predictionDir )
opWriter.hdf5Path.setValue( datasetName )
opWriter.Image.connect( self.mainOperator.PredictionProbabilities[imageIndex] )
# Create the request
self._predictionStorageRequest = opWriter.WriteImage[...]
def handleProgress(percent):
# Stop sending progress if we were cancelled
if self.predictionStorageEnabled:
progress[0] = startProgress + percent * (increment / 100.0)
self.progressSignal.emit( progress[0] )
opWriter.progressSignal.subscribe( handleProgress )
finishedEvent = threading.Event()
def handleFinish(request):
finishedEvent.set()
def handleCancel(request):
self._predictionStorageRequest = None
finishedEvent.set()
# Trigger the write and wait for it to complete or cancel.
self._predictionStorageRequest.notify(handleFinish)
self._predictionStorageRequest.onCancel(handleCancel)
finishedEvent.wait()
except:
failedToSave = True
raise
finally:
# If we were cancelled, delete the predictions we just started
if not self.predictionStorageEnabled or failedToSave:
deleteIfPresent(predictionDir, datasetName)
self._predictionsPresent = False
startProgress = progress[0]
else:
# Re-load the operator with the prediction groups we just saved
self._deserializePredictions(topGroup)
shell.openProjectFile(projectFilename)
workflowKwargs={'hintoverlayFile' : options.hintoverlayFile,
'pmapoverlayFile' : options.pmapoverlayFile }
startShellGui( functools.partial(CarvingWorkflow, **workflowKwargs), loadProject)
elif len(args) == 2:
carvingGraphFilename = os.path.abspath(args[0]).replace("\\","/")
projectFilename = args[1]
projectFile = h5py.File(projectFilename)
preproc = getOrCreateGroup(projectFile,"preprocessing")
deleteIfPresent(preproc, "sigma")
deleteIfPresent(preproc, "filter")
deleteIfPresent(preproc, "StorageVersion")
deleteIfPresent(preproc, "graph")
deleteIfPresent(preproc, "graphfile")
preproc.create_dataset("sigma",data= 1.6)
preproc.create_dataset("filter",data= 0)
preproc.create_dataset("graphfile",data = carvingGraphFilename)
preproc.create_dataset("StorageVersion",data = 0.1)
preproc = getOrCreateGroup(projectFile,"preprocessing")
dataSelect = getOrCreateGroup(projectFile,"Input Data")
dataInfo = getOrCreateGroup(dataSelect,"infos")
dataThisInfo = getOrCreateGroup(dataInfo,"info0000")
deleteIfPresent(dataThisInfo,"filePath")
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
obj = getOrCreateGroup(topGroup, "objects")
for imageIndex, opCarving in enumerate(self._o.innerOperators):
mst = opCarving._mst
for name in opCarving._dirtyObjects:
print "[CarvingSerializer] serializing %s" % name
if name in obj and name in mst.object_seeds_fg_voxels:
# group already exists
print " -> changed"
elif name not in mst.object_seeds_fg_voxels:
print " -> deleted"
else:
print " -> added"
g = getOrCreateGroup(obj, name)
deleteIfPresent(g, "fg_voxels")
deleteIfPresent(g, "bg_voxels")
deleteIfPresent(g, "sv")
deleteIfPresent(g, "bg_prio")
deleteIfPresent(g, "no_bias_below")
if not name in mst.object_seeds_fg_voxels:
# this object was deleted
deleteIfPresent(obj, name)
continue
v = mst.object_seeds_fg_voxels[name]
v = [v[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
g.create_dataset("fg_voxels", data=v)
v = mst.object_seeds_bg_voxels[name]
v = [v[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
g.create_dataset("bg_voxels", data=v)
g.create_dataset("sv", data=mst.object_lut[name])
d1 = numpy.asarray(mst.bg_priority[name], dtype=numpy.float32)
d2 = numpy.asarray(mst.no_bias_below[name], dtype=numpy.int32)
g.create_dataset("bg_prio", data=d1)
g.create_dataset("no_bias_below", data=d2)
opCarving._dirtyObjects = set()
# save current seeds
deleteIfPresent(topGroup, "fg_voxels")
deleteIfPresent(topGroup, "bg_voxels")
fg_voxels, bg_voxels = opCarving.get_label_voxels()
if fg_voxels is None:
return
if fg_voxels[0].shape[0] > 0:
v = [fg_voxels[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
topGroup.create_dataset("fg_voxels", data=v)
if bg_voxels[0].shape[0] > 0:
v = [bg_voxels[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
topGroup.create_dataset("bg_voxels", data=v)
print "saved seeds"
def _serialize(self, group, name, slot):
"""Called when the currently stored predictions are dirty. If
prediction storage is currently enabled, store them to the
file. Otherwise, just delete them/
(Avoid inconsistent project states, e.g. don't allow old
predictions to be stored with a new classifier.)
"""
predictionDir = group.create_group(self.name)
# Disconnect the operators that might be using the old data.
self.deserialize(group)
failedToSave = False
opWriter = None
try:
num = len(slot)
if num > 0:
increment = 100 / float(num)
progress = 0
for imageIndex in range(num):
# Have we been cancelled?
if not self.predictionStorageEnabled:
break
datasetName = self.subname.format(imageIndex)
# Use a big dataset writer to do this in chunks
opWriter = OpH5WriterBigDataset(graph=self.operator.graph, parent = self.operator.parent)
opWriter.hdf5File.setValue(predictionDir)
opWriter.hdf5Path.setValue(datasetName)
opWriter.Image.connect(slot[imageIndex])
def handleProgress(percent):
# Stop sending progress if we were cancelled
if self.predictionStorageEnabled:
curprogress = progress + percent * (increment / 100.0)
self.progressSignal.emit(curprogress)
opWriter.progressSignal.subscribe(handleProgress)
# Create the request
self._predictionStorageRequest = opWriter.WriteImage[...]
# Must use a threading event here because if we wait on the
# request from within a "real" thread, it refuses to be cancelled.
finishedEvent = threading.Event()
def handleFinish(result):
finishedEvent.set()
def handleCancel():
logger.info("Full volume prediction save CANCELLED.")
self._predictionStorageRequest = None
finishedEvent.set()
# Trigger the write and wait for it to complete or cancel.
self._predictionStorageRequest.notify_finished(handleFinish)
self._predictionStorageRequest.notify_cancelled(handleCancel)
self._predictionStorageRequest.submit() # Can't call wait(). See note above.
finishedEvent.wait()
progress += increment
opWriter.cleanUp()
opWriter = None
except:
failedToSave = True
raise
finally:
if opWriter is not None:
opWriter.cleanUp()
# If we were cancelled, delete the predictions we just started
if not self.predictionStorageEnabled or failedToSave:
deleteIfPresent(group, name)
def _serialize(self, group, name, slot):
"""Called when the currently stored predictions are dirty. If
prediction storage is currently enabled, store them to the
file. Otherwise, just delete them/
(Avoid inconsistent project states, e.g. don't allow old
predictions to be stored with a new classifier.)
"""
predictionDir = group.create_group(self.name)
# Disconnect the operators that might be using the old data.
self.deserialize(group)
failedToSave = False
opWriter = None
try:
num = len(slot)
if num > 0:
increment = 100 / float(num)
progress = 0
for imageIndex in range(num):
# Have we been cancelled?
if not self.predictionStorageEnabled:
break
datasetName = self.subname.format(imageIndex)
# Use a big dataset writer to do this in chunks
opWriter = OpH5WriterBigDataset(graph=self.operator.graph,
parent=self.operator.parent)
opWriter.hdf5File.setValue(predictionDir)
opWriter.hdf5Path.setValue(datasetName)
opWriter.Image.connect(slot[imageIndex])
def handleProgress(percent):
# Stop sending progress if we were cancelled
if self.predictionStorageEnabled:
curprogress = progress + percent * (increment / 100.0)
self.progressSignal(curprogress)
opWriter.progressSignal.subscribe(handleProgress)
# Create the request
self._predictionStorageRequest = opWriter.WriteImage[...]
# Must use a threading event here because if we wait on the
# request from within a "real" thread, it refuses to be cancelled.
finishedEvent = threading.Event()
def handleFinish(result):
finishedEvent.set()
def handleCancel():
logger.info("Full volume prediction save CANCELLED.")
self._predictionStorageRequest = None
finishedEvent.set()
# Trigger the write and wait for it to complete or cancel.
self._predictionStorageRequest.notify_finished(handleFinish)
self._predictionStorageRequest.notify_cancelled(handleCancel)
self._predictionStorageRequest.submit(
) # Can't call wait(). See note above.
finishedEvent.wait()
progress += increment
opWriter.cleanUp()
opWriter = None
except:
failedToSave = True
raise
finally:
if opWriter is not None:
opWriter.cleanUp()
# If we were cancelled, delete the predictions we just started
if not self.predictionStorageEnabled or failedToSave:
deleteIfPresent(group, name)
def test_deleteIfPresent_2(self):
self.assertTrue("b" not in self.tmpFile)
deleteIfPresent(self.tmpFile, "b")
self.assertTrue("b" not in self.tmpFile)
shell.openProjectFile(projectFilename)
workflowKwargs={'hintoverlayFile' : options.hintoverlayFile,
'pmapoverlayFile' : options.pmapoverlayFile }
startShellGui( functools.partial(CarvingWorkflow, **workflowKwargs), loadProject)
elif len(args) == 2:
carvingGraphFilename = os.path.abspath(args[0]).replace("\\","/")
projectFilename = args[1]
projectFile = h5py.File(projectFilename)
preproc = getOrCreateGroup(projectFile,"preprocessing")
deleteIfPresent(preproc, "sigma")
deleteIfPresent(preproc, "filter")
deleteIfPresent(preproc, "StorageVersion")
deleteIfPresent(preproc, "graph")
deleteIfPresent(preproc, "graphfile")
preproc.create_dataset("sigma",data= 1.6)
preproc.create_dataset("filter",data= 0)
preproc.create_dataset("graphfile",data = carvingGraphFilename)
preproc.create_dataset("StorageVersion",data = 0.1)
preproc = getOrCreateGroup(projectFile,"preprocessing")
dataSelect = getOrCreateGroup(projectFile,"Input Data")
dataInfo = getOrCreateGroup(dataSelect,"infos")
dataThisInfo = getOrCreateGroup(dataInfo,"info0000")
deleteIfPresent(dataThisInfo,"filePath")
def test_deleteIfPresent_1(self):
self.assertTrue("a" in self.tmpFile)
deleteIfPresent(self.tmpFile, "a")
self.assertTrue("a" not in self.tmpFile)
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
obj = getOrCreateGroup(topGroup, "objects")
for imageIndex, opCarving in enumerate(self._o.innerOperators):
mst = opCarving._mst
if mst is None:
# Nothing to save
return
# Populate a list of objects to save:
objects_to_save = set(list(mst.object_names.keys()))
objects_already_saved = set(list(topGroup["objects"]))
# 1.) all objects that are in mst.object_names that are not in saved
objects_to_save = objects_to_save.difference(objects_already_saved)
# 2.) add opCarving._dirtyObjects:
objects_to_save = objects_to_save.union(opCarving._dirtyObjects)
for name in objects_to_save:
logger.info("[CarvingSerializer] serializing %s" % name)
if name in obj and name in mst.object_seeds_fg_voxels:
# group already exists
logger.info(" -> changed")
elif name not in mst.object_seeds_fg_voxels:
logger.info(" -> deleted")
else:
logger.info(" -> added")
if name not in mst.object_seeds_fg_voxels:
# this object was deleted
deleteIfPresent(obj, name)
continue
g = getOrCreateGroup(obj, name)
deleteIfPresent(g, "fg_voxels")
deleteIfPresent(g, "bg_voxels")
deleteIfPresent(g, "sv")
deleteIfPresent(g, "bg_prio")
deleteIfPresent(g, "no_bias_below")
v = mst.object_seeds_fg_voxels[name]
v = [v[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
g.create_dataset("fg_voxels", data=v)
v = mst.object_seeds_bg_voxels[name]
v = [v[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
g.create_dataset("bg_voxels", data=v)
g.create_dataset("sv", data=mst.object_lut[name])
d1 = numpy.asarray(mst.bg_priority[name], dtype=numpy.float32)
d2 = numpy.asarray(mst.no_bias_below[name], dtype=numpy.int32)
g.create_dataset("bg_prio", data=d1)
g.create_dataset("no_bias_below", data=d2)
opCarving._dirtyObjects = set()
# save current seeds
deleteIfPresent(topGroup, "fg_voxels")
deleteIfPresent(topGroup, "bg_voxels")
fg_voxels, bg_voxels = opCarving.get_label_voxels()
if fg_voxels is None:
return
if fg_voxels[0].shape[0] > 0:
v = [fg_voxels[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
topGroup.create_dataset("fg_voxels", data=v)
if bg_voxels[0].shape[0] > 0:
v = [bg_voxels[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
topGroup.create_dataset("bg_voxels", data=v)
logger.info("saved seeds")
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
obj = getOrCreateGroup(topGroup, "objects")
for imageIndex, opCarving in enumerate( self._o.innerOperators ):
mst = opCarving._mst
# Populate a list of objects to save:
objects_to_save = set(list(mst.object_names.keys()))
objects_already_saved = set(list(topGroup["objects"]))
# 1.) all objects that are in mst.object_names that are not in saved
objects_to_save = objects_to_save.difference(objects_already_saved)
# 2.) add opCarving._dirtyObjects:
objects_to_save = objects_to_save.union(opCarving._dirtyObjects)
for name in objects_to_save:
logger.info( "[CarvingSerializer] serializing %s" % name )
if name in obj and name in mst.object_seeds_fg_voxels:
#group already exists
logger.info( " -> changed" )
elif name not in mst.object_seeds_fg_voxels:
logger.info( " -> deleted" )
else:
logger.info( " -> added" )
g = getOrCreateGroup(obj, name)
deleteIfPresent(g, "fg_voxels")
deleteIfPresent(g, "bg_voxels")
deleteIfPresent(g, "sv")
deleteIfPresent(g, "bg_prio")
deleteIfPresent(g, "no_bias_below")
if not name in mst.object_seeds_fg_voxels:
#this object was deleted
deleteIfPresent(obj, name)
continue
v = mst.object_seeds_fg_voxels[name]
v = [v[i][:,numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
g.create_dataset("fg_voxels", data=v)
v = mst.object_seeds_bg_voxels[name]
v = [v[i][:,numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
g.create_dataset("bg_voxels", data=v)
g.create_dataset("sv", data=mst.object_lut[name])
d1 = numpy.asarray(mst.bg_priority[name], dtype=numpy.float32)
d2 = numpy.asarray(mst.no_bias_below[name], dtype=numpy.int32)
g.create_dataset("bg_prio", data=d1)
g.create_dataset("no_bias_below", data=d2)
opCarving._dirtyObjects = set()
# save current seeds
deleteIfPresent(topGroup, "fg_voxels")
deleteIfPresent(topGroup, "bg_voxels")
fg_voxels, bg_voxels = opCarving.get_label_voxels()
if fg_voxels is None:
return
if fg_voxels[0].shape[0] > 0:
v = [fg_voxels[i][:,numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
topGroup.create_dataset("fg_voxels", data = v)
if bg_voxels[0].shape[0] > 0:
v = [bg_voxels[i][:,numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
topGroup.create_dataset("bg_voxels", data = v)
logger.info( "saved seeds" )
def test_deleteIfPresent_1(self):
self.assertTrue("a" in self.tmpFile)
deleteIfPresent(self.tmpFile, "a")
self.assertTrue("a" not in self.tmpFile)
def _serializePredictions(self, topGroup, startProgress, endProgress):
"""
Called when the currently stored predictions are dirty.
If prediction storage is currently enabled, store them to the file.
Otherwise, just delete them/
(Avoid inconsistent project states, e.g. don't allow old predictions to be stored with a new classifier.)
"""
with Tracer(traceLogger):
# If the predictions are missing, then maybe the user wants them stored (even if they aren't dirty)
if self._dirtyFlags[
Section.
Predictions] or 'Pdigital signal processing bookredictions' not in topGroup.keys(
):
deleteIfPresent(topGroup, 'Predictions')
# Disconnect the precomputed prediction inputs.
for i, slot in enumerate(
self.mainOperator.PredictionsFromDisk):
slot.disconnect()
if self.predictionStorageEnabled:
predictionDir = topGroup.create_group('Predictions')
failedToSave = False
try:
numImages = len(
self.mainOperator.PredictionProbabilities)
if numImages > 0:
increment = (endProgress -
startProgress) / float(numImages)
for imageIndex in range(numImages):
# Have we been cancelled?
if not self.predictionStorageEnabled:
break
datasetName = 'predictions{:04d}'.format(
imageIndex)
progress = [startProgress]
# Use a big dataset writer to do this in chunks
opWriter = OpH5WriterBigDataset(
graph=self.mainOperator.graph)
opWriter.hdf5File.setValue(predictionDir)
opWriter.hdf5Path.setValue(datasetName)
opWriter.Image.connect(
self.mainOperator.
PredictionProbabilities[imageIndex])
# Create the request
self._predictionStorageRequest = opWriter.WriteImage[
...]
def handleProgress(percent):
# Stop sending progress if we were cancelled
if self.predictionStorageEnabled:
progress[0] = startProgress + percent * (
increment / 100.0)
self.progressSignal.emit(progress[0])
opWriter.progressSignal.subscribe(handleProgress)
finishedEvent = threading.Event()
def handleFinish(request):
finishedEvent.set()
def handleCancel(request):
self._predictionStorageRequest = None
finishedEvent.set()
# Trigger the write and wait for it to complete or cancel.
self._predictionStorageRequest.notify(handleFinish)
self._predictionStorageRequest.onCancel(
handleCancel)
finishedEvent.wait()
except:
failedToSave = True
raise
finally:
# If we were cancelled, delete the predictions we just started
if not self.predictionStorageEnabled or failedToSave:
deleteIfPresent(predictionDir, datasetName)
self._predictionsPresent = False
startProgress = progress[0]
else:
# Re-load the operator with the prediction groups we just saved
self._deserializePredictions(topGroup)
def test_deleteIfPresent_2(self):
self.assertTrue("b" not in self.tmpFile)
deleteIfPresent(self.tmpFile, "b")
self.assertTrue("b" not in self.tmpFile)
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
# Write any missing local datasets to the local_data group
localDataGroup = getOrCreateGroup(topGroup, 'local_data')
wroteInternalData = False
for laneIndex, multislot in enumerate(self.topLevelOperator.DatasetGroup):
for roleIndex, slot in enumerate( multislot ):
if not slot.ready():
continue
info = slot.value
# If this dataset should be stored in the project, but it isn't there yet
if info.location == DatasetInfo.Location.ProjectInternal \
and info.datasetId not in localDataGroup.keys():
# Obtain the data from the corresponding output and store it to the project.
dataSlot = self.topLevelOperator._NonTransposedImageGroup[laneIndex][roleIndex]
try:
opWriter = OpH5WriterBigDataset(parent=self.topLevelOperator.parent, graph=self.topLevelOperator.graph)
opWriter.CompressionEnabled.setValue(False) # Compression slows down browsing a lot, and raw data tends to be noisy and doesn't compress very well, anyway.
opWriter.hdf5File.setValue( localDataGroup )
opWriter.hdf5Path.setValue( info.datasetId )
opWriter.Image.connect(dataSlot)
# Trigger the copy
success = opWriter.WriteImage.value
assert success
finally:
opWriter.cleanUp()
# Add axistags and drange attributes, in case someone uses this dataset outside ilastik
localDataGroup[info.datasetId].attrs['axistags'] = dataSlot.meta.axistags.toJSON()
if dataSlot.meta.drange is not None:
localDataGroup[info.datasetId].attrs['drange'] = dataSlot.meta.drange
# Make sure the dataSlot's axistags are updated with the dataset as we just wrote it
# (The top-level operator may use an OpReorderAxes, which changed the axisorder)
info.axistags = dataSlot.meta.axistags
wroteInternalData = True
# Construct a list of all the local dataset ids we want to keep
localDatasetIds = set()
for laneIndex, multislot in enumerate(self.topLevelOperator.DatasetGroup):
for roleIndex, slot in enumerate(multislot):
if slot.ready() and slot.value.location == DatasetInfo.Location.ProjectInternal:
localDatasetIds.add( slot.value.datasetId )
# Delete any datasets in the project that aren't needed any more
for datasetName in localDataGroup.keys():
if datasetName not in localDatasetIds:
del localDataGroup[datasetName]
if wroteInternalData:
# We can only re-configure the operator if we're not saving a snapshot
# We know we're saving a snapshot if the project file isn't the one we deserialized with.
if self._projectFilePath is None or self._projectFilePath == projectFilePath:
# Force the operator to setupOutputs() again so it gets data from the project, not external files
firstInfo = self.topLevelOperator.DatasetGroup[0][0].value
self.topLevelOperator.DatasetGroup[0][0].setValue(firstInfo, check_changed=False)
deleteIfPresent(topGroup, 'Role Names')
topGroup.create_dataset('Role Names', data=self.topLevelOperator.DatasetRoles.value)
# Access the info group
infoDir = getOrCreateGroup(topGroup, 'infos')
# Delete all infos
for infoName in infoDir.keys():
del infoDir[infoName]
# Rebuild the list of infos
roleNames = self.topLevelOperator.DatasetRoles.value
for laneIndex, multislot in enumerate(self.topLevelOperator.DatasetGroup):
laneGroupName = 'lane{:04d}'.format(laneIndex)
laneGroup = infoDir.create_group( laneGroupName )
for roleIndex, slot in enumerate(multislot):
infoGroup = laneGroup.create_group( roleNames[roleIndex] )
if slot.ready():
datasetInfo = slot.value
locationString = self.LocationStrings[datasetInfo.location]
infoGroup.create_dataset('location', data=locationString)
infoGroup.create_dataset('filePath', data=datasetInfo.filePath)
infoGroup.create_dataset('datasetId', data=datasetInfo.datasetId)
infoGroup.create_dataset('allowLabels', data=datasetInfo.allowLabels)
infoGroup.create_dataset('nickname', data=datasetInfo.nickname)
infoGroup.create_dataset('fromstack', data=datasetInfo.fromstack)
if datasetInfo.drange is not None:
infoGroup.create_dataset('drange', data=datasetInfo.drange)
# Pull the axistags from the NonTransposedImage,
# which is what the image looks like before 'forceAxisOrder' is applied,
# and before 'c' is automatically appended
axistags = self.topLevelOperator._NonTransposedImageGroup[laneIndex][roleIndex].meta.axistags
infoGroup.create_dataset('axistags', data=axistags.toJSON())
axisorder = "".join(tag.key for tag in axistags)
infoGroup.create_dataset('axisorder', data=axisorder)
if datasetInfo.subvolume_roi is not None:
infoGroup.create_dataset('subvolume_roi', data=datasetInfo.subvolume_roi)
self._dirty = False
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
# Write any missing local datasets to the local_data group
localDataGroup = getOrCreateGroup(topGroup, 'local_data')
wroteInternalData = False
for laneIndex, multislot in enumerate(self.topLevelOperator.DatasetGroup):
for roleIndex, slot in enumerate( multislot ):
if not slot.ready():
continue
info = slot.value
# If this dataset should be stored in the project, but it isn't there yet
if info.location == DatasetInfo.Location.ProjectInternal \
and info.datasetId not in localDataGroup.keys():
# Obtain the data from the corresponding output and store it to the project.
dataSlot = self.topLevelOperator._NonTransposedImageGroup[laneIndex][roleIndex]
try:
opWriter = OpH5WriterBigDataset(parent=self.topLevelOperator.parent, graph=self.topLevelOperator.graph)
opWriter.CompressionEnabled.setValue(False) # Compression slows down browsing a lot, and raw data tends to be noisy and doesn't compress very well, anyway.
opWriter.hdf5File.setValue( localDataGroup )
opWriter.hdf5Path.setValue( info.datasetId )
opWriter.Image.connect(dataSlot)
# Trigger the copy
success = opWriter.WriteImage.value
assert success
finally:
opWriter.cleanUp()
# Add axistags and drange attributes, in case someone uses this dataset outside ilastik
localDataGroup[info.datasetId].attrs['axistags'] = dataSlot.meta.axistags.toJSON()
if dataSlot.meta.drange is not None:
localDataGroup[info.datasetId].attrs['drange'] = dataSlot.meta.drange
# Make sure the dataSlot's axistags are updated with the dataset as we just wrote it
# (The top-level operator may use an OpReorderAxes, which changed the axisorder)
info.axistags = dataSlot.meta.axistags
wroteInternalData = True
# Construct a list of all the local dataset ids we want to keep
localDatasetIds = set()
for laneIndex, multislot in enumerate(self.topLevelOperator.DatasetGroup):
for roleIndex, slot in enumerate(multislot):
if slot.ready() and slot.value.location == DatasetInfo.Location.ProjectInternal:
localDatasetIds.add( slot.value.datasetId )
# Delete any datasets in the project that aren't needed any more
for datasetName in localDataGroup.keys():
if datasetName not in localDatasetIds:
del localDataGroup[datasetName]
if wroteInternalData:
# We can only re-configure the operator if we're not saving a snapshot
# We know we're saving a snapshot if the project file isn't the one we deserialized with.
if self._projectFilePath is None or self._projectFilePath == projectFilePath:
# Force the operator to setupOutputs() again so it gets data from the project, not external files
firstInfo = self.topLevelOperator.DatasetGroup[0][0].value
self.topLevelOperator.DatasetGroup[0][0].setValue(firstInfo, check_changed=False)
deleteIfPresent(topGroup, 'Role Names')
topGroup.create_dataset('Role Names', data=self.topLevelOperator.DatasetRoles.value)
# Access the info group
infoDir = getOrCreateGroup(topGroup, 'infos')
# Delete all infos
for infoName in infoDir.keys():
del infoDir[infoName]
# Rebuild the list of infos
roleNames = self.topLevelOperator.DatasetRoles.value
for laneIndex, multislot in enumerate(self.topLevelOperator.DatasetGroup):
laneGroupName = 'lane{:04d}'.format(laneIndex)
laneGroup = infoDir.create_group( laneGroupName )
for roleIndex, slot in enumerate(multislot):
infoGroup = laneGroup.create_group( roleNames[roleIndex] )
if slot.ready():
datasetInfo = slot.value
locationString = self.LocationStrings[datasetInfo.location]
infoGroup.create_dataset('location', data=locationString)
infoGroup.create_dataset('filePath', data=datasetInfo.filePath)
infoGroup.create_dataset('datasetId', data=datasetInfo.datasetId)
infoGroup.create_dataset('allowLabels', data=datasetInfo.allowLabels)
infoGroup.create_dataset('nickname', data=datasetInfo.nickname)
if datasetInfo.drange is not None:
infoGroup.create_dataset('drange', data=datasetInfo.drange)
if datasetInfo.axistags is not None:
infoGroup.create_dataset('axistags', data=datasetInfo.axistags.toJSON())
axisorder = "".join(tag.key for tag in datasetInfo.axistags)
infoGroup.create_dataset('axisorder', data=axisorder)
self._dirty = False
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
obj = getOrCreateGroup(topGroup, "objects")
for imageIndex, opCarving in enumerate(self._o.innerOperators):
mst = opCarving._mst
for name in opCarving._dirtyObjects:
print "[CarvingSerializer] serializing %s" % name
if name in obj and name in mst.object_seeds_fg_voxels:
#group already exists
print " -> changed"
elif name not in mst.object_seeds_fg_voxels:
print " -> deleted"
else:
print " -> added"
g = getOrCreateGroup(obj, name)
deleteIfPresent(g, "fg_voxels")
deleteIfPresent(g, "bg_voxels")
deleteIfPresent(g, "sv")
deleteIfPresent(g, "bg_prio")
deleteIfPresent(g, "no_bias_below")
if not name in mst.object_seeds_fg_voxels:
#this object was deleted
deleteIfPresent(obj, name)
continue
v = mst.object_seeds_fg_voxels[name]
v = [v[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
g.create_dataset("fg_voxels", data=v)
v = mst.object_seeds_bg_voxels[name]
v = [v[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
g.create_dataset("bg_voxels", data=v)
g.create_dataset("sv", data=mst.object_lut[name])
d1 = numpy.asarray(mst.bg_priority[name], dtype=numpy.float32)
d2 = numpy.asarray(mst.no_bias_below[name], dtype=numpy.int32)
g.create_dataset("bg_prio", data=d1)
g.create_dataset("no_bias_below", data=d2)
opCarving._dirtyObjects = set()
# save current seeds
deleteIfPresent(topGroup, "fg_voxels")
deleteIfPresent(topGroup, "bg_voxels")
fg_voxels, bg_voxels = opCarving.get_label_voxels()
if fg_voxels is None:
return
if fg_voxels[0].shape[0] > 0:
v = [fg_voxels[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
topGroup.create_dataset("fg_voxels", data=v)
if bg_voxels[0].shape[0] > 0:
v = [bg_voxels[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
topGroup.create_dataset("bg_voxels", data=v)
print "saved seeds"
