def serialize(self, group):
if not self.shouldSerialize(group):
return
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
for i in range(len(mainOperator)):
subgroup = getOrCreateGroup(group, "{:04}".format(i))
cleanBlockRois = self.blockslot[i].value
for roi in cleanBlockRois:
region_features_arr = self.slot[i](*roi).wait()
assert region_features_arr.shape == (1, )
region_features = region_features_arr[0]
roi_grp = subgroup.create_group(name=str(roi))
logger.debug('Saving region features into group: "{}"'.format(
roi_grp.name))
for key, val in region_features.iteritems():
plugin_group = getOrCreateGroup(roi_grp, key)
for featname, featval in val.iteritems():
plugin_group.create_dataset(name=featname,
data=featval)
self.dirty = False
def importStackAsLocalDataset(self, info):
"""
Add the given stack data to the project file as a local dataset.
Create a datainfo and append it to our operator.
"""
try:
self.progressSignal.emit(0)
projectFileHdf5 = self.topLevelOperator.ProjectFile.value
topGroup = getOrCreateGroup(projectFileHdf5, self.topGroupName)
localDataGroup = getOrCreateGroup(topGroup, 'local_data')
globstring = info.filePath
info.location = DatasetInfo.Location.ProjectInternal
opWriter = OpStackToH5Writer(graph=self.topLevelOperator.graph)
opWriter.hdf5Group.setValue(localDataGroup)
opWriter.hdf5Path.setValue(info.datasetId)
opWriter.GlobString.setValue(globstring)
# Forward progress from the writer directly to our applet
opWriter.progressSignal.subscribe( self.progressSignal.emit )
success = opWriter.WriteImage.value
numDatasets = len(self.topLevelOperator.Dataset)
self.topLevelOperator.Dataset.resize( numDatasets + 1 )
self.topLevelOperator.Dataset[numDatasets].setValue(info)
finally:
self.progressSignal.emit(100)
return success
def serialize(self, group):
if not self.shouldSerialize(group):
return
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
innerops = mainOperator.innerOperators
for i, op in enumerate(innerops):
gr = getOrCreateGroup(group, str(i))
labels_gr = getOrCreateGroup(gr, str("labels"))
if "labels" in op.Annotations.value.keys():
for t in op.Annotations.value["labels"].keys():
t_gr = getOrCreateGroup(labels_gr, str(t))
for oid in op.Annotations.value["labels"][t].keys():
l = op.Annotations.value["labels"][t][oid]
dset = list(l)
if len(dset) > 0:
t_gr.create_dataset(name=str(oid), data=dset)
divisions_gr = getOrCreateGroup(gr, str("divisions"))
dset = []
if "divisions" in op.Annotations.value.keys():
for trackid in op.Annotations.value["divisions"].keys():
(children,
t_parent) = op.Annotations.value["divisions"][trackid]
dset.append([trackid, children[0], children[1], t_parent])
if len(dset) > 0:
divisions_gr.create_dataset(name=str(i), data=dset)
self.dirty = False
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
getOrCreateGroup(topGroup, "local_data")
deleteIfPresent(topGroup, "Role Names")
role_names = [name.encode("utf-8") for name in self.topLevelOperator.DatasetRoles.value]
topGroup.create_dataset("Role Names", data=role_names)
# Access the info group
infoDir = getOrCreateGroup(topGroup, "infos")
# Delete all infos
infoDir.clear()
# Rebuild the list of infos
roleNames = self.topLevelOperator.DatasetRoles.value
internal_datasets_to_keep = set()
for laneIndex, multislot in enumerate(self.topLevelOperator.DatasetGroup):
laneGroupName = "lane{:04d}".format(laneIndex)
laneGroup = infoDir.create_group(laneGroupName)
for roleIndex, slot in enumerate(multislot):
infoGroup = laneGroup.create_group(roleNames[roleIndex])
if slot.ready():
datasetInfo = slot.value
if isinstance(datasetInfo, ProjectInternalDatasetInfo):
internal_datasets_to_keep.add(hdf5File[datasetInfo.inner_path])
for k, v in datasetInfo.to_json_data().items():
if v is not None:
infoGroup.create_dataset(k, data=v)
if self.local_data_path.as_posix() in hdf5File:
for dataset in hdf5File[self.local_data_path.as_posix()].values():
if dataset not in internal_datasets_to_keep:
del hdf5File[dataset.name]
self._dirty = False
def serialize(self, group):
if not self.shouldSerialize(group):
return
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
innerops = mainOperator.innerOperators
for i, op in enumerate(innerops):
gr = getOrCreateGroup(group, str(i))
labels_gr = getOrCreateGroup(gr, str("labels"))
if "labels" in op.Annotations.value.keys():
for t in op.Annotations.value["labels"].keys():
t_gr = getOrCreateGroup(labels_gr, str(t))
for oid in op.Annotations.value["labels"][t].keys():
l = op.Annotations.value["labels"][t][oid]
dset = list(l)
if len(dset) > 0:
t_gr.create_dataset(name=str(oid), data=dset)
divisions_gr = getOrCreateGroup(gr, str("divisions"))
dset = []
if "divisions" in op.Annotations.value.keys():
for trackid in op.Annotations.value["divisions"].keys():
(children, t_parent) = op.Annotations.value["divisions"][trackid]
dset.append([trackid, children[0], children[1], t_parent])
if len(dset) > 0:
divisions_gr.create_dataset(name=str(i), data=dset)
self.dirty = False
def importStackAsLocalDataset(self, info):
"""
Add the given stack data to the project file as a local dataset.
Does not update the topLevelOperator
"""
try:
self.progressSignal.emit(0)
projectFileHdf5 = self.topLevelOperator.ProjectFile.value
topGroup = getOrCreateGroup(projectFileHdf5, self.topGroupName)
localDataGroup = getOrCreateGroup(topGroup, 'local_data')
globstring = info.filePath
info.location = DatasetInfo.Location.ProjectInternal
firstPathParts = PathComponents(info.filePath.split('//')[0])
info.filePath = firstPathParts.externalDirectory + '/??' + firstPathParts.extension
opWriter = OpStackToH5Writer(graph=self.topLevelOperator.graph)
opWriter.hdf5Group.setValue(localDataGroup)
opWriter.hdf5Path.setValue(info.datasetId)
opWriter.GlobString.setValue(globstring)
# Forward progress from the writer directly to our applet
opWriter.progressSignal.subscribe( self.progressSignal.emit )
success = opWriter.WriteImage.value
finally:
opWriter.cleanUp()
self.progressSignal.emit(100)
return success
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
split_settings_grp = getOrCreateGroup(topGroup, "split_settings")
for laneIndex in range(len( self._topLevelOperator )):
lane_grp = getOrCreateGroup(split_settings_grp, "{}".format( laneIndex ))
opLaneView = self._topLevelOperator.getLane(laneIndex)
if opLaneView.AnnotationFilepath.ready():
annotation_filepath = opLaneView.AnnotationFilepath.value
deleteIfPresent( lane_grp, "annotation_filepath" )
lane_grp.create_dataset("annotation_filepath", data=annotation_filepath)
# Now save the regular the carving data.
super( SplitBodyCarvingSerializer, self )._serializeToHdf5( topGroup, hdf5File, projectFilePath )
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
split_settings_grp = getOrCreateGroup(topGroup, "split_settings")
for laneIndex in range(len( self._topLevelOperator )):
lane_grp = getOrCreateGroup(split_settings_grp, "{}".format( laneIndex ))
opLaneView = self._topLevelOperator.getLane(laneIndex)
if opLaneView.AnnotationFilepath.ready():
annotation_filepath = opLaneView.AnnotationFilepath.value
deleteIfPresent( lane_grp, "annotation_filepath" )
lane_grp.create_dataset("annotation_filepath", data=annotation_filepath)
# Now save the regular the carving data.
super( SplitBodyCarvingSerializer, self )._serializeToHdf5( topGroup, hdf5File, projectFilePath )
self.__dirty = False
def serialize(self, group):
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
innerops = mainOperator.innerOperators
for i, op in enumerate(innerops):
gr = getOrCreateGroup(group, str(i))
for t in op.labels.keys():
t_gr = getOrCreateGroup(gr, str(t))
for oid in op.labels[t].keys():
l = op.labels[t][oid]
dset = list(l)
if len(dset) > 0:
t_gr.create_dataset(name=str(oid), data=dset)
self.dirty = False
def serialize(self, group):
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
innerops = mainOperator.innerOperators
for i, op in enumerate(innerops):
gr = getOrCreateGroup(group, str(i))
for t in op.labels.keys():
t_gr = getOrCreateGroup(gr, str(t))
for oid in op.labels[t].keys():
l = op.labels[t][oid]
dset = list(l)
if len(dset) > 0:
t_gr.create_dataset(name=str(oid), data=dset)
self.dirty = False
def serialize(self, group):
if not self.shouldSerialize(group):
return
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
for i, op in enumerate(mainOperator.innerOperators):
ts = op._processedTimeSteps
if len(ts) > 0:
subgroup = getOrCreateGroup(group, str(i))
subgroup.create_dataset(name='timesteps', data=list(ts))
src = op._mem_h5
subgroup.copy(src['/TranslationVectors'], subgroup, name='data')
self.dirty = False
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
preproc = topGroup
for opPre in self._o.innerOperators:
mst = opPre._prepData[0]
if mst is not None:
#The values to be saved for sigma and filter are the
#values of the last valid preprocess
#!These may differ from the current settings!
deleteIfPresent(preproc, "sigma")
deleteIfPresent(preproc, "filter")
deleteIfPresent(preproc, "watershed_source")
deleteIfPresent(preproc, "invert_watershed_source")
deleteIfPresent(preproc, "graph")
preproc.create_dataset("sigma",data= opPre.initialSigma)
preproc.create_dataset("filter",data= opPre.initialFilter)
ws_source = str(opPre.WatershedSource.value)
assert isinstance( ws_source, str ), "WatershedSource was {}, but it should be a string.".format( ws_source )
preproc.create_dataset("watershed_source", data=ws_source)
preproc.create_dataset("invert_watershed_source", data=opPre.InvertWatershedSource.value)
preprocgraph = getOrCreateGroup(preproc, "graph")
mst.saveH5G(preprocgraph)
opPre._unsavedData = False
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
preproc = topGroup
for opPre in self._o.innerOperators:
mst = opPre._prepData[0]
if mst is not None:
#The values to be saved for sigma and filter are the
#values of the last valid preprocess
#!These may differ from the current settings!
deleteIfPresent(preproc, "sigma")
deleteIfPresent(preproc, "filter")
deleteIfPresent(preproc, "watershed_source")
deleteIfPresent(preproc, "invert_watershed_source")
deleteIfPresent(preproc, "graph")
preproc.create_dataset("sigma", data=opPre.initialSigma)
preproc.create_dataset("filter", data=opPre.initialFilter)
ws_source = str(opPre.WatershedSource.value)
assert isinstance(
ws_source, str
), "WatershedSource was {}, but it should be a string.".format(
ws_source)
preproc.create_dataset("watershed_source", data=ws_source)
preproc.create_dataset("invert_watershed_source",
data=opPre.InvertWatershedSource.value)
preprocgraph = getOrCreateGroup(preproc, "graph")
mst.saveH5G(preprocgraph)
opPre._unsavedData = False
def test_getOrCreateGroup_2(self):
self.assertTrue("b" not in self.tmpFile)
group = getOrCreateGroup(self.tmpFile, "b")
self.assertEqual(group.name, "/b")
self.assertTrue("b" in self.tmpFile)
self.assertTrue(isinstance(self.tmpFile["b"], h5py.Group))
def test_getOrCreateGroup_2(self):
self.assertTrue("b" not in self.tmpFile)
group = getOrCreateGroup(self.tmpFile, "b")
self.assertEqual(group.name, "/b")
self.assertTrue("b" in self.tmpFile)
self.assertTrue(isinstance(self.tmpFile["b"], h5py.Group))
def test_getOrCreateGroup_3(self):
self.assertTrue("c" in self.tmpFile)
self.assertTrue(isinstance(self.tmpFile["c"], h5py.Dataset))
self.assertRaises(TypeError, lambda : getOrCreateGroup(self.tmpFile, "c"))
self.assertTrue("c" in self.tmpFile)
self.assertTrue(isinstance(self.tmpFile["c"], h5py.Dataset))
def importStackAsLocalDataset(self, info):
"""
Add the given stack data to the project file as a local dataset.
Does not update the topLevelOperator.
:param info: A DatasetInfo object.
Note: info.filePath must be a str which lists the stack files, delimited with os.path.pathsep
Note: info will be MODIFIED by this function. Use the modified info when assigning it to a dataset.
"""
try:
self.progressSignal.emit(0)
projectFileHdf5 = self.topLevelOperator.ProjectFile.value
topGroup = getOrCreateGroup(projectFileHdf5, self.topGroupName)
localDataGroup = getOrCreateGroup(topGroup, 'local_data')
globstring = info.filePath
info.location = DatasetInfo.Location.ProjectInternal
firstPathParts = PathComponents(
info.filePath.split(os.path.pathsep)[0])
info.filePath = firstPathParts.externalDirectory + '/??' + firstPathParts.extension
info.fromstack = True
# Use absolute path
cwd = self.topLevelOperator.WorkingDirectory
if os.path.pathsep not in globstring and not os.path.isabs(
globstring):
globstring = os.path.normpath(os.path.join(cwd, globstring))
opWriter = OpStackToH5Writer(parent=self.topLevelOperator.parent,
graph=self.topLevelOperator.graph)
opWriter.hdf5Group.setValue(localDataGroup)
opWriter.hdf5Path.setValue(info.datasetId)
opWriter.GlobString.setValue(globstring)
# Forward progress from the writer directly to our applet
opWriter.progressSignal.subscribe(self.progressSignal.emit)
success = opWriter.WriteImage.value
finally:
opWriter.cleanUp()
self.progressSignal.emit(100)
return success
def test_getOrCreateGroup_1(self):
self.assertTrue("a" in self.tmpFile)
self.assertTrue(isinstance(self.tmpFile["a"], h5py.Group))
group = getOrCreateGroup(self.tmpFile, "a")
self.assertEqual(group.name, "/a")
self.assertTrue("a" in self.tmpFile)
self.assertTrue(isinstance(self.tmpFile["a"], h5py.Group))
def serialize(self, group):
if not self.shouldSerialize(group):
return
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
innerops = mainOperator.innerOperators
for i, op in enumerate(innerops):
gr = getOrCreateGroup(group, str(i))
for t in list(op.disappearances.keys()):
t_gr = getOrCreateGroup(gr, str(t))
for oid in list(op.disappearances[t].keys()):
oid_gr = getOrCreateGroup(t_gr, str(oid))
for track in list(op.disappearances[t][oid].keys()):
app = op.disappearances[t][oid][track]
if app:
oid_gr.create_dataset(name=str(track), data=app)
self.dirty = False
def test_getOrCreateGroup_1(self):
self.assertTrue("a" in self.tmpFile)
self.assertTrue(isinstance(self.tmpFile["a"], h5py.Group))
group = getOrCreateGroup(self.tmpFile, "a")
self.assertEqual(group.name, "/a")
self.assertTrue("a" in self.tmpFile)
self.assertTrue(isinstance(self.tmpFile["a"], h5py.Group))
def serialize(self, group):
if not self.shouldSerialize(group):
return
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
innerops = mainOperator.innerOperators
for i, op in enumerate(innerops):
gr = getOrCreateGroup(group, str(i))
for t in list(op.disappearances.keys()):
t_gr = getOrCreateGroup(gr, str(t))
for oid in list(op.disappearances[t].keys()):
oid_gr = getOrCreateGroup(t_gr, str(oid))
for track in list(op.disappearances[t][oid].keys()):
app = op.disappearances[t][oid][track]
if app:
oid_gr.create_dataset(name=str(track), data=app)
self.dirty = False
def test_getOrCreateGroup_3(self):
self.assertTrue("c" in self.tmpFile)
self.assertTrue(isinstance(self.tmpFile["c"], h5py.Dataset))
self.assertRaises(TypeError,
lambda: getOrCreateGroup(self.tmpFile, "c"))
self.assertTrue("c" in self.tmpFile)
self.assertTrue(isinstance(self.tmpFile["c"], h5py.Dataset))
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
obj = getOrCreateGroup(topGroup, "objects")
for imageIndex, opCarving in enumerate( self._o.opCarving.innerOperators ):
mst = opCarving._mst
for name in opCarving._dirtyObjects:
print "[CarvingSerializer] serializing %s" % name
if name in obj and name in mst.object_seeds_fg_voxels:
#group already exists
print " -> changed"
elif name not in mst.object_seeds_fg_voxels:
print " -> deleted"
else:
print " -> added"
g = getOrCreateGroup(obj, name)
deleteIfPresent(g, "fg_voxels")
deleteIfPresent(g, "bg_voxels")
deleteIfPresent(g, "sv")
deleteIfPresent(g, "bg_prio")
deleteIfPresent(g, "no_bias_below")
if not name in mst.object_seeds_fg_voxels:
#this object was deleted
deleteIfPresent(obj, name)
continue
v = mst.object_seeds_fg_voxels[name]
v = [v[i][:,numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
g.create_dataset("fg_voxels", data=v)
v = mst.object_seeds_bg_voxels[name]
v = [v[i][:,numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
g.create_dataset("bg_voxels", data=v)
g.create_dataset("sv", data=mst.object_lut[name])
d1 = numpy.asarray(mst.bg_priority[name], dtype=numpy.float32)
d2 = numpy.asarray(mst.no_bias_below[name], dtype=numpy.int32)
g.create_dataset("bg_prio", data=d1)
g.create_dataset("no_bias_below", data=d2)
opCarving._dirtyObjects = set()
def importStackAsLocalDataset(self, info):
"""
Add the given stack data to the project file as a local dataset.
Does not update the topLevelOperator.
:param info: A DatasetInfo object.
Note: info.filePath must be a str which lists the stack files, delimited with os.path.pathsep
Note: info will be MODIFIED by this function. Use the modified info when assigning it to a dataset.
"""
try:
self.progressSignal.emit(0)
projectFileHdf5 = self.topLevelOperator.ProjectFile.value
topGroup = getOrCreateGroup(projectFileHdf5, self.topGroupName)
localDataGroup = getOrCreateGroup(topGroup, 'local_data')
globstring = info.filePath
info.location = DatasetInfo.Location.ProjectInternal
firstPathParts = PathComponents(info.filePath.split(os.path.pathsep)[0])
info.filePath = firstPathParts.externalDirectory + '/??' + firstPathParts.extension
info.fromstack = True
# Use absolute path
cwd = self.topLevelOperator.WorkingDirectory
if os.path.pathsep not in globstring and not os.path.isabs(globstring):
globstring = os.path.normpath( os.path.join(cwd, globstring) )
opWriter = OpStackToH5Writer(parent=self.topLevelOperator.parent, graph=self.topLevelOperator.graph)
opWriter.hdf5Group.setValue(localDataGroup)
opWriter.hdf5Path.setValue(info.datasetId)
opWriter.GlobString.setValue(globstring)
# Forward progress from the writer directly to our applet
opWriter.progressSignal.subscribe( self.progressSignal.emit )
success = opWriter.WriteImage.value
finally:
opWriter.cleanUp()
self.progressSignal.emit(100)
return success
def serialize(self, group):
if not self.shouldSerialize(group):
return
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
for i in range(len(mainOperator)):
subgroup = getOrCreateGroup(group, str(i))
cleanBlockRois = self.blockslot[i].value
for roi in cleanBlockRois:
region_features_arr = self.outslot[i]( *roi ).wait()
assert region_features_arr.shape == (1,)
region_features = region_features_arr[0]
roi_grp = subgroup.create_group(name=str(roi))
logger.debug('Saving region features into group: "{}"'.format( roi_grp.name ))
for key, val in region_features.iteritems():
roi_grp.create_dataset(name=key, data=val)
self.dirty = False
def serialize(self, group):
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
innerops = mainOperator.innerOperators
for i, op in enumerate(innerops):
dset = []
for trackid in op.divisions.keys():
(children, t_parent) = op.divisions[trackid]
dset.append([trackid, children[0], children[1], t_parent])
if len(dset) > 0:
group.create_dataset(name=str(i), data=dset)
self.dirty = False
def serialize(self, group):
deleteIfPresent(group, self.name)
group = getOrCreateGroup(group, self.name)
mainOperator = self.slot.getRealOperator()
innerops = mainOperator.innerOperators
for i, op in enumerate(innerops):
dset = []
for trackid in op.divisions.keys():
(children, t_parent) = op.divisions[trackid]
dset.append([trackid, children[0], children[1], t_parent])
if len(dset) > 0:
group.create_dataset(name=str(i), data=dset)
self.dirty = False
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
preproc = topGroup
for opPre in self._o.innerOperators:
mst = opPre._prepData[0]
if mst is not None:
# The values to be saved for sigma and filter are the
# values of the last valid preprocess
#!These may differ from the current settings!
deleteIfPresent(preproc, "sigma")
deleteIfPresent(preproc, "filter")
deleteIfPresent(preproc, "graph")
preproc.create_dataset("sigma", data=opPre.initialSigma)
preproc.create_dataset("filter", data=opPre.initialFilter)
preprocgraph = getOrCreateGroup(preproc, "graph")
mst.saveH5G(preprocgraph)
opPre._unsavedData = False
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
preproc = topGroup
for opPre in self._o.innerOperators:
mst = opPre._prepData[0]
if mst is not None:
# The values to be saved for sigma and filter are the
# values of the last valid preprocess
#!These may differ from the current settings!
deleteIfPresent(preproc, "sigma")
deleteIfPresent(preproc, "filter")
deleteIfPresent(preproc, "graph")
preproc.create_dataset("sigma", data=opPre.initialSigma)
preproc.create_dataset("filter", data=opPre.initialFilter)
preprocgraph = getOrCreateGroup(preproc, "graph")
mst.saveH5G(preprocgraph)
opPre._unsavedData = False
def loadProject(shell):
shell.openProjectFile(projectFilename)
workflowKwargs={'hintoverlayFile' : options.hintoverlayFile,
'pmapoverlayFile' : options.pmapoverlayFile }
startShellGui( functools.partial(CarvingWorkflow, **workflowKwargs), loadProject)
elif len(args) == 2:
carvingGraphFilename = os.path.abspath(args[0]).replace("\\","/")
projectFilename = args[1]
projectFile = h5py.File(projectFilename)
preproc = getOrCreateGroup(projectFile,"preprocessing")
deleteIfPresent(preproc, "sigma")
deleteIfPresent(preproc, "filter")
deleteIfPresent(preproc, "StorageVersion")
deleteIfPresent(preproc, "graph")
deleteIfPresent(preproc, "graphfile")
preproc.create_dataset("sigma",data= 1.6)
preproc.create_dataset("filter",data= 0)
preproc.create_dataset("graphfile",data = carvingGraphFilename)
preproc.create_dataset("StorageVersion",data = 0.1)
preproc = getOrCreateGroup(projectFile,"preprocessing")
dataSelect = getOrCreateGroup(projectFile,"Input Data")
dataInfo = getOrCreateGroup(dataSelect,"infos")
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
obj = getOrCreateGroup(topGroup, "objects")
for imageIndex, opCarving in enumerate( self._o.innerOperators ):
mst = opCarving._mst
# Populate a list of objects to save:
objects_to_save = set(list(mst.object_names.keys()))
objects_already_saved = set(list(topGroup["objects"]))
# 1.) all objects that are in mst.object_names that are not in saved
objects_to_save = objects_to_save.difference(objects_already_saved)
# 2.) add opCarving._dirtyObjects:
objects_to_save = objects_to_save.union(opCarving._dirtyObjects)
for name in objects_to_save:
logger.info( "[CarvingSerializer] serializing %s" % name )
if name in obj and name in mst.object_seeds_fg_voxels:
#group already exists
logger.info( " -> changed" )
elif name not in mst.object_seeds_fg_voxels:
logger.info( " -> deleted" )
else:
logger.info( " -> added" )
g = getOrCreateGroup(obj, name)
deleteIfPresent(g, "fg_voxels")
deleteIfPresent(g, "bg_voxels")
deleteIfPresent(g, "sv")
deleteIfPresent(g, "bg_prio")
deleteIfPresent(g, "no_bias_below")
if not name in mst.object_seeds_fg_voxels:
#this object was deleted
deleteIfPresent(obj, name)
continue
v = mst.object_seeds_fg_voxels[name]
v = [v[i][:,numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
g.create_dataset("fg_voxels", data=v)
v = mst.object_seeds_bg_voxels[name]
v = [v[i][:,numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
g.create_dataset("bg_voxels", data=v)
g.create_dataset("sv", data=mst.object_lut[name])
d1 = numpy.asarray(mst.bg_priority[name], dtype=numpy.float32)
d2 = numpy.asarray(mst.no_bias_below[name], dtype=numpy.int32)
g.create_dataset("bg_prio", data=d1)
g.create_dataset("no_bias_below", data=d2)
opCarving._dirtyObjects = set()
# save current seeds
deleteIfPresent(topGroup, "fg_voxels")
deleteIfPresent(topGroup, "bg_voxels")
fg_voxels, bg_voxels = opCarving.get_label_voxels()
if fg_voxels is None:
return
if fg_voxels[0].shape[0] > 0:
v = [fg_voxels[i][:,numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
topGroup.create_dataset("fg_voxels", data = v)
if bg_voxels[0].shape[0] > 0:
v = [bg_voxels[i][:,numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
topGroup.create_dataset("bg_voxels", data = v)
logger.info( "saved seeds" )
def loadProject(shell):
shell.openProjectFile(projectFilename)
workflowKwargs={'hintoverlayFile' : options.hintoverlayFile,
'pmapoverlayFile' : options.pmapoverlayFile }
startShellGui( functools.partial(CarvingWorkflow, **workflowKwargs), loadProject)
elif len(args) == 2:
carvingGraphFilename = os.path.abspath(args[0]).replace("\\","/")
projectFilename = args[1]
projectFile = h5py.File(projectFilename)
preproc = getOrCreateGroup(projectFile,"preprocessing")
deleteIfPresent(preproc, "sigma")
deleteIfPresent(preproc, "filter")
deleteIfPresent(preproc, "StorageVersion")
deleteIfPresent(preproc, "graph")
deleteIfPresent(preproc, "graphfile")
preproc.create_dataset("sigma",data= 1.6)
preproc.create_dataset("filter",data= 0)
preproc.create_dataset("graphfile",data = carvingGraphFilename)
preproc.create_dataset("StorageVersion",data = 0.1)
preproc = getOrCreateGroup(projectFile,"preprocessing")
dataSelect = getOrCreateGroup(projectFile,"Input Data")
dataInfo = getOrCreateGroup(dataSelect,"infos")
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
# Write any missing local datasets to the local_data group
localDataGroup = getOrCreateGroup(topGroup, 'local_data')
wroteInternalData = False
for laneIndex, multislot in enumerate(self.topLevelOperator.DatasetGroup):
for roleIndex, slot in enumerate( multislot ):
if not slot.ready():
continue
info = slot.value
# If this dataset should be stored in the project, but it isn't there yet
if info.location == DatasetInfo.Location.ProjectInternal \
and info.datasetId not in localDataGroup.keys():
# Obtain the data from the corresponding output and store it to the project.
dataSlot = self.topLevelOperator._NonTransposedImageGroup[laneIndex][roleIndex]
try:
opWriter = OpH5WriterBigDataset(parent=self.topLevelOperator.parent, graph=self.topLevelOperator.graph)
opWriter.CompressionEnabled.setValue(False) # Compression slows down browsing a lot, and raw data tends to be noisy and doesn't compress very well, anyway.
opWriter.hdf5File.setValue( localDataGroup )
opWriter.hdf5Path.setValue( info.datasetId )
opWriter.Image.connect(dataSlot)
# Trigger the copy
success = opWriter.WriteImage.value
assert success
finally:
opWriter.cleanUp()
# Add axistags and drange attributes, in case someone uses this dataset outside ilastik
localDataGroup[info.datasetId].attrs['axistags'] = dataSlot.meta.axistags.toJSON()
if dataSlot.meta.drange is not None:
localDataGroup[info.datasetId].attrs['drange'] = dataSlot.meta.drange
# Make sure the dataSlot's axistags are updated with the dataset as we just wrote it
# (The top-level operator may use an OpReorderAxes, which changed the axisorder)
info.axistags = dataSlot.meta.axistags
wroteInternalData = True
# Construct a list of all the local dataset ids we want to keep
localDatasetIds = set()
for laneIndex, multislot in enumerate(self.topLevelOperator.DatasetGroup):
for roleIndex, slot in enumerate(multislot):
if slot.ready() and slot.value.location == DatasetInfo.Location.ProjectInternal:
localDatasetIds.add( slot.value.datasetId )
# Delete any datasets in the project that aren't needed any more
for datasetName in localDataGroup.keys():
if datasetName not in localDatasetIds:
del localDataGroup[datasetName]
if wroteInternalData:
# We can only re-configure the operator if we're not saving a snapshot
# We know we're saving a snapshot if the project file isn't the one we deserialized with.
if self._projectFilePath is None or self._projectFilePath == projectFilePath:
# Force the operator to setupOutputs() again so it gets data from the project, not external files
firstInfo = self.topLevelOperator.DatasetGroup[0][0].value
self.topLevelOperator.DatasetGroup[0][0].setValue(firstInfo, check_changed=False)
deleteIfPresent(topGroup, 'Role Names')
topGroup.create_dataset('Role Names', data=self.topLevelOperator.DatasetRoles.value)
# Access the info group
infoDir = getOrCreateGroup(topGroup, 'infos')
# Delete all infos
for infoName in infoDir.keys():
del infoDir[infoName]
# Rebuild the list of infos
roleNames = self.topLevelOperator.DatasetRoles.value
for laneIndex, multislot in enumerate(self.topLevelOperator.DatasetGroup):
laneGroupName = 'lane{:04d}'.format(laneIndex)
laneGroup = infoDir.create_group( laneGroupName )
for roleIndex, slot in enumerate(multislot):
infoGroup = laneGroup.create_group( roleNames[roleIndex] )
if slot.ready():
datasetInfo = slot.value
locationString = self.LocationStrings[datasetInfo.location]
infoGroup.create_dataset('location', data=locationString)
infoGroup.create_dataset('filePath', data=datasetInfo.filePath)
infoGroup.create_dataset('datasetId', data=datasetInfo.datasetId)
infoGroup.create_dataset('allowLabels', data=datasetInfo.allowLabels)
infoGroup.create_dataset('nickname', data=datasetInfo.nickname)
infoGroup.create_dataset('fromstack', data=datasetInfo.fromstack)
if datasetInfo.drange is not None:
infoGroup.create_dataset('drange', data=datasetInfo.drange)
# Pull the axistags from the NonTransposedImage,
# which is what the image looks like before 'forceAxisOrder' is applied,
# and before 'c' is automatically appended
axistags = self.topLevelOperator._NonTransposedImageGroup[laneIndex][roleIndex].meta.axistags
infoGroup.create_dataset('axistags', data=axistags.toJSON())
axisorder = "".join(tag.key for tag in axistags)
infoGroup.create_dataset('axisorder', data=axisorder)
if datasetInfo.subvolume_roi is not None:
infoGroup.create_dataset('subvolume_roi', data=datasetInfo.subvolume_roi)
self._dirty = False
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
# Write any missing local datasets to the local_data group
localDataGroup = getOrCreateGroup(topGroup, 'local_data')
wroteInternalData = False
for index, slot in enumerate(self.topLevelOperator.Dataset):
info = slot.value
# If this dataset should be stored in the project, but it isn't there yet
if info.location == DatasetInfo.Location.ProjectInternal \
and info.datasetId not in localDataGroup.keys():
# Obtain the data from the corresponding output and store it to the project.
dataSlot = self.topLevelOperator.Image[index]
opWriter = OpH5WriterBigDataset(graph=self.topLevelOperator.graph)
opWriter.hdf5File.setValue( localDataGroup )
opWriter.hdf5Path.setValue( info.datasetId )
opWriter.Image.connect(dataSlot)
# Trigger the copy
success = opWriter.WriteImage.value
assert success
# Add the axistags attribute to the dataset we just created
localDataGroup[info.datasetId].attrs['axistags'] = dataSlot.meta.axistags.toJSON()
# Update the dataset info with no path, just filename base to remind us what this data is
# (operator will react to the change when we call setValue(), below)
# Directly set the private member to avoid getting a new datasetid
info._filePath = PathComponents(info.filePath).filenameBase
wroteInternalData = True
# Construct a list of all the local dataset ids we want to keep
localDatasetIds = [ slot.value.datasetId
for index, slot
in enumerate(self.topLevelOperator.Dataset)
if slot.value.location == DatasetInfo.Location.ProjectInternal ]
# Delete any datasets in the project that aren't needed any more
for datasetName in localDataGroup.keys():
if datasetName not in localDatasetIds:
del localDataGroup[datasetName]
if wroteInternalData:
# We can only re-configure the operator if we're not saving a snapshot
# We know we're saving a snapshot if the project file isn't the one we deserialized with.
if self._projectFilePath is None or self._projectFilePath == projectFilePath:
# Force the operator to setupOutputs() again so it gets data from the project, not external files
firstInfo = self.topLevelOperator.Dataset[0].value
self.topLevelOperator.Dataset[0].setValue(firstInfo, False)
# Access the info group
infoDir = getOrCreateGroup(topGroup, 'infos')
# Delete all infos
for infoName in infoDir.keys():
del infoDir[infoName]
# Rebuild the list of infos
for index, slot in enumerate(self.topLevelOperator.Dataset):
infoGroup = infoDir.create_group('info{:04d}'.format(index))
datasetInfo = slot.value
locationString = self.LocationStrings[datasetInfo.location]
infoGroup.create_dataset('location', data=locationString)
infoGroup.create_dataset('filePath', data=datasetInfo.filePath)
infoGroup.create_dataset('datasetId', data=datasetInfo.datasetId)
infoGroup.create_dataset('allowLabels', data=datasetInfo.allowLabels)
if datasetInfo.axisorder is not None:
infoGroup.create_dataset('axisorder', data=datasetInfo.axisorder)
self._dirty = False
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
obj = getOrCreateGroup(topGroup, "objects")
for imageIndex, opCarving in enumerate(self._o.innerOperators):
mst = opCarving._mst
if mst is None:
# Nothing to save
return
# Populate a list of objects to save:
objects_to_save = set(list(mst.object_names.keys()))
objects_already_saved = set(list(topGroup["objects"]))
# 1.) all objects that are in mst.object_names that are not in saved
objects_to_save = objects_to_save.difference(objects_already_saved)
# 2.) add opCarving._dirtyObjects:
objects_to_save = objects_to_save.union(opCarving._dirtyObjects)
for name in objects_to_save:
logger.info("[CarvingSerializer] serializing %s" % name)
if name in obj and name in mst.object_seeds_fg_voxels:
# group already exists
logger.info(" -> changed")
elif name not in mst.object_seeds_fg_voxels:
logger.info(" -> deleted")
else:
logger.info(" -> added")
if name not in mst.object_seeds_fg_voxels:
# this object was deleted
deleteIfPresent(obj, name)
continue
g = getOrCreateGroup(obj, name)
deleteIfPresent(g, "fg_voxels")
deleteIfPresent(g, "bg_voxels")
deleteIfPresent(g, "sv")
deleteIfPresent(g, "bg_prio")
deleteIfPresent(g, "no_bias_below")
v = mst.object_seeds_fg_voxels[name]
v = [v[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
g.create_dataset("fg_voxels", data=v)
v = mst.object_seeds_bg_voxels[name]
v = [v[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
g.create_dataset("bg_voxels", data=v)
g.create_dataset("sv", data=mst.object_lut[name])
d1 = numpy.asarray(mst.bg_priority[name], dtype=numpy.float32)
d2 = numpy.asarray(mst.no_bias_below[name], dtype=numpy.int32)
g.create_dataset("bg_prio", data=d1)
g.create_dataset("no_bias_below", data=d2)
opCarving._dirtyObjects = set()
# save current seeds
deleteIfPresent(topGroup, "fg_voxels")
deleteIfPresent(topGroup, "bg_voxels")
fg_voxels, bg_voxels = opCarving.get_label_voxels()
if fg_voxels is None:
return
if fg_voxels[0].shape[0] > 0:
v = [fg_voxels[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
topGroup.create_dataset("fg_voxels", data=v)
if bg_voxels[0].shape[0] > 0:
v = [bg_voxels[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
topGroup.create_dataset("bg_voxels", data=v)
logger.info("saved seeds")
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
# Write any missing local datasets to the local_data group
localDataGroup = getOrCreateGroup(topGroup, 'local_data')
wroteInternalData = False
for laneIndex, multislot in enumerate(self.topLevelOperator.DatasetGroup):
for roleIndex, slot in enumerate( multislot ):
if not slot.ready():
continue
info = slot.value
# If this dataset should be stored in the project, but it isn't there yet
if info.location == DatasetInfo.Location.ProjectInternal \
and info.datasetId not in localDataGroup.keys():
# Obtain the data from the corresponding output and store it to the project.
dataSlot = self.topLevelOperator._NonTransposedImageGroup[laneIndex][roleIndex]
try:
opWriter = OpH5WriterBigDataset(parent=self.topLevelOperator.parent, graph=self.topLevelOperator.graph)
opWriter.CompressionEnabled.setValue(False) # Compression slows down browsing a lot, and raw data tends to be noisy and doesn't compress very well, anyway.
opWriter.hdf5File.setValue( localDataGroup )
opWriter.hdf5Path.setValue( info.datasetId )
opWriter.Image.connect(dataSlot)
# Trigger the copy
success = opWriter.WriteImage.value
assert success
finally:
opWriter.cleanUp()
# Add axistags and drange attributes, in case someone uses this dataset outside ilastik
localDataGroup[info.datasetId].attrs['axistags'] = dataSlot.meta.axistags.toJSON()
if dataSlot.meta.drange is not None:
localDataGroup[info.datasetId].attrs['drange'] = dataSlot.meta.drange
# Make sure the dataSlot's axistags are updated with the dataset as we just wrote it
# (The top-level operator may use an OpReorderAxes, which changed the axisorder)
info.axistags = dataSlot.meta.axistags
wroteInternalData = True
# Construct a list of all the local dataset ids we want to keep
localDatasetIds = set()
for laneIndex, multislot in enumerate(self.topLevelOperator.DatasetGroup):
for roleIndex, slot in enumerate(multislot):
if slot.ready() and slot.value.location == DatasetInfo.Location.ProjectInternal:
localDatasetIds.add( slot.value.datasetId )
# Delete any datasets in the project that aren't needed any more
for datasetName in localDataGroup.keys():
if datasetName not in localDatasetIds:
del localDataGroup[datasetName]
if wroteInternalData:
# We can only re-configure the operator if we're not saving a snapshot
# We know we're saving a snapshot if the project file isn't the one we deserialized with.
if self._projectFilePath is None or self._projectFilePath == projectFilePath:
# Force the operator to setupOutputs() again so it gets data from the project, not external files
firstInfo = self.topLevelOperator.DatasetGroup[0][0].value
self.topLevelOperator.DatasetGroup[0][0].setValue(firstInfo, check_changed=False)
deleteIfPresent(topGroup, 'Role Names')
topGroup.create_dataset('Role Names', data=self.topLevelOperator.DatasetRoles.value)
# Access the info group
infoDir = getOrCreateGroup(topGroup, 'infos')
# Delete all infos
for infoName in infoDir.keys():
del infoDir[infoName]
# Rebuild the list of infos
roleNames = self.topLevelOperator.DatasetRoles.value
for laneIndex, multislot in enumerate(self.topLevelOperator.DatasetGroup):
laneGroupName = 'lane{:04d}'.format(laneIndex)
laneGroup = infoDir.create_group( laneGroupName )
for roleIndex, slot in enumerate(multislot):
infoGroup = laneGroup.create_group( roleNames[roleIndex] )
if slot.ready():
datasetInfo = slot.value
locationString = self.LocationStrings[datasetInfo.location]
infoGroup.create_dataset('location', data=locationString)
infoGroup.create_dataset('filePath', data=datasetInfo.filePath)
infoGroup.create_dataset('datasetId', data=datasetInfo.datasetId)
infoGroup.create_dataset('allowLabels', data=datasetInfo.allowLabels)
infoGroup.create_dataset('nickname', data=datasetInfo.nickname)
if datasetInfo.drange is not None:
infoGroup.create_dataset('drange', data=datasetInfo.drange)
if datasetInfo.axistags is not None:
infoGroup.create_dataset('axistags', data=datasetInfo.axistags.toJSON())
axisorder = "".join(tag.key for tag in datasetInfo.axistags)
infoGroup.create_dataset('axisorder', data=axisorder)
self._dirty = False
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath):
obj = getOrCreateGroup(topGroup, "objects")
for imageIndex, opCarving in enumerate(self._o.innerOperators):
mst = opCarving._mst
for name in opCarving._dirtyObjects:
print "[CarvingSerializer] serializing %s" % name
if name in obj and name in mst.object_seeds_fg_voxels:
#group already exists
print " -> changed"
elif name not in mst.object_seeds_fg_voxels:
print " -> deleted"
else:
print " -> added"
g = getOrCreateGroup(obj, name)
deleteIfPresent(g, "fg_voxels")
deleteIfPresent(g, "bg_voxels")
deleteIfPresent(g, "sv")
deleteIfPresent(g, "bg_prio")
deleteIfPresent(g, "no_bias_below")
if not name in mst.object_seeds_fg_voxels:
#this object was deleted
deleteIfPresent(obj, name)
continue
v = mst.object_seeds_fg_voxels[name]
v = [v[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
g.create_dataset("fg_voxels", data=v)
v = mst.object_seeds_bg_voxels[name]
v = [v[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
g.create_dataset("bg_voxels", data=v)
g.create_dataset("sv", data=mst.object_lut[name])
d1 = numpy.asarray(mst.bg_priority[name], dtype=numpy.float32)
d2 = numpy.asarray(mst.no_bias_below[name], dtype=numpy.int32)
g.create_dataset("bg_prio", data=d1)
g.create_dataset("no_bias_below", data=d2)
opCarving._dirtyObjects = set()
# save current seeds
deleteIfPresent(topGroup, "fg_voxels")
deleteIfPresent(topGroup, "bg_voxels")
fg_voxels, bg_voxels = opCarving.get_label_voxels()
if fg_voxels is None:
return
if fg_voxels[0].shape[0] > 0:
v = [fg_voxels[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
topGroup.create_dataset("fg_voxels", data=v)
if bg_voxels[0].shape[0] > 0:
v = [bg_voxels[i][:, numpy.newaxis] for i in range(3)]
v = numpy.concatenate(v, axis=1)
topGroup.create_dataset("bg_voxels", data=v)
print "saved seeds"
