def __init__(self,
model=None,
weights=None,
result=None,
model_filename=None,
weights_filename=None,
result_filename=None,
progressVisitor=DefaultProgressVisitor()):
assert (weights is None or weights_filename is None)
assert (model is None or model_filename is None)
assert (result is None or result_filename is None)
# default values
self.traxelIdPerTimestepToUniqueIdMap = {}
if model is None:
self.model = {
'segmentationHypotheses': [],
'linkingHypotheses': [],
'exclusions': [],
'divisionHypotheses': [],
'traxelToUniqueId': self.traxelIdPerTimestepToUniqueIdMap,
'settings': {
'statesShareWeights': True,
'allowPartialMergerAppearance': False,
'requireSeparateChildrenOfDivision': True,
'optimizerEpGap': 0.01,
'optimizerVerbose': True,
'optimizerNumThreads': 1
}
}
else:
assert ('segmentationHypotheses' in model)
assert ('linkingHypotheses' in model)
self.model = model
self.weights = weights
self.result = result
self.uuidToTraxelMap = {}
# load from file if specified
if model_filename is not None:
getLogger().debug("Loading model file: " + model_filename)
self.model = readFromJSON(model_filename)
if weights_filename is not None:
getLogger().debug("Loading weights file: " + weights_filename)
self.weights = readFromJSON(weights_filename)
if result_filename is not None:
getLogger().debug("Loading result file: " + result_filename)
self.result = readFromJSON(result_filename)
# further initializations
if model is not None or model_filename is not None:
self.traxelIdPerTimestepToUniqueIdMap, self.uuidToTraxelMap = \
getMappingsBetweenUUIDsAndTraxels(self.model)
self._nextUuid = 0
self.progressVisitor = progressVisitor
def __setstate__(self, state):
"""Restore state from the unpickled state values."""
try:
self._graph, \
self.withTracklets, \
self.allowLengthOneTracks, \
self._nextNodeUuid, \
self.maxNumObjects, \
self.skipLinksBias, \
self.transitionClassifier, \
self.transitionParameter, \
self.withDivisions, \
self.fieldOfView, \
self.probabilityGenerator, \
self.timeRange, \
self.numNearestNeighbors, \
self.divisionThreshold, \
self.borderAwareWidth, \
self.maxNeighborDistance, \
self.skipLinks \
= state
except:
pass
self.progressVisitor = DefaultProgressVisitor()
def __init__(self, parent=None, graph=None):
super(OpConservationTracking, self).__init__(parent=parent,
graph=graph)
self._opCache = OpBlockedArrayCache(parent=self)
self._opCache.name = "OpConservationTracking._opCache"
self._opCache.Input.connect(self.Output)
self.CleanBlocks.connect(self._opCache.CleanBlocks)
self.CachedOutput.connect(self._opCache.Output)
self.zeroProvider = OpZeroDefault(parent=self)
self.zeroProvider.MetaInput.connect(self.LabelImage)
# As soon as input data is available, check its constraints
self.RawImage.notifyReady(self._checkConstraints)
self.LabelImage.notifyReady(self._checkConstraints)
self.ExportSettings.setValue((None, None))
self._mergerOpCache = OpBlockedArrayCache(parent=self)
self._mergerOpCache.name = "OpConservationTracking._mergerOpCache"
self._mergerOpCache.Input.connect(self.MergerOutput)
self.MergerCleanBlocks.connect(self._mergerOpCache.CleanBlocks)
self.MergerCachedOutput.connect(self._mergerOpCache.Output)
self._relabeledOpCache = OpBlockedArrayCache(parent=self)
self._relabeledOpCache.name = "OpConservationTracking._mergerOpCache"
self._relabeledOpCache.Input.connect(self.RelabeledImage)
self.RelabeledCleanBlocks.connect(self._relabeledOpCache.CleanBlocks)
self.RelabeledCachedOutput.connect(self._relabeledOpCache.Output)
# Merger resolver plugin manager (contains GMM fit routine)
self.pluginPaths = [
os.path.join(os.path.dirname(os.path.abspath(hytra.__file__)),
'plugins')
]
pluginManager = TrackingPluginManager(verbose=False,
pluginPaths=self.pluginPaths)
self.mergerResolverPlugin = pluginManager.getMergerResolver()
self.result = None
# gui progress
self.progressWindow = None
self.progressVisitor = DefaultProgressVisitor()
def __init__(self,
pluginPaths=[os.path.abspath('../hytra/plugins')],
numSplits=None,
verbose=False,
progressVisitor=DefaultProgressVisitor()):
self.unresolvedGraph = None
self.resolvedGraph = None
self.mergersPerTimestep = None
self.detectionsPerTimestep = None
self.pluginManager = TrackingPluginManager(verbose=verbose,
pluginPaths=pluginPaths)
self.mergerResolverPlugin = self.pluginManager.getMergerResolver()
self.numSplits = numSplits
# should be filled by constructors of derived classes!
self.model = None
self.result = None
self.progressVisitor = progressVisitor
def __init__(self,
probabilityGenerator,
timeRange,
maxNumObjects,
numNearestNeighbors,
fieldOfView,
divisionThreshold=0.1,
withDivisions=True,
borderAwareWidth=10,
maxNeighborDistance=200,
transitionParameter=5.0,
transitionClassifier=None,
skipLinks=1,
skipLinksBias=20,
progressVisitor=DefaultProgressVisitor()):
'''
Constructor
'''
super(IlastikHypothesesGraph, self).__init__()
# store values
self.probabilityGenerator = probabilityGenerator
self.timeRange = timeRange
self.maxNumObjects = maxNumObjects
self.numNearestNeighbors = numNearestNeighbors
self.fieldOfView = fieldOfView
self.divisionThreshold = divisionThreshold
self.withDivisions = withDivisions
self.borderAwareWidth = borderAwareWidth
self.maxNeighborDistance = maxNeighborDistance
self.transitionClassifier = transitionClassifier
self.transitionParameter = transitionParameter
self.skipLinks = skipLinks
self.skipLinksBias = skipLinksBias
self.progressVisitor = progressVisitor
# build hypotheses graph
self.buildFromProbabilityGenerator(
probabilityGenerator,
numNearestNeighbors=numNearestNeighbors,
maxNeighborDist=maxNeighborDistance,
withDivisions=withDivisions,
divisionThreshold=divisionThreshold,
skipLinks=skipLinks)
def __init__(self):
self._graph = nx.DiGraph()
self.withTracklets = False
self.allowLengthOneTracks = True
self._nextNodeUuid = 0
self.progressVisitor=DefaultProgressVisitor()
class HypothesesGraph(object):
"""
Replacement for pgmlink's hypotheses graph,
with a similar API so it can be used as drop-in replacement.
Internally it uses [networkx](http://networkx.github.io/) to construct the graph.
Use the insertEnergies() method to populate the nodes and arcs with the energies for different
configurations (according to DPCT's JSON style'), derived from given probability generation functions.
**Notes:** `self._graph.node`'s are indexed by tuples (int(timestep), int(id)), and contain either a
single `'traxel'` attribute, or a list of traxels in `'tracklet'`.
Nodes also get a unique ID assigned once they are added to the graph.
"""
def __init__(self):
self._graph = nx.DiGraph()
self.withTracklets = False
self.allowLengthOneTracks = True
self._nextNodeUuid = 0
self.progressVisitor=DefaultProgressVisitor()
def nodeIterator(self):
return self._graph.nodes_iter()
def arcIterator(self):
return self._graph.edges_iter()
def countNodes(self):
return self._graph.number_of_nodes()
def countArcs(self):
return self._graph.number_of_edges()
def hasNode(self, node):
return self._graph.has_node(node)
def hasEdge(self, u, v):
return self._graph.has_edge(u, v)
@staticmethod
def source(edge):
return edge[0]
@staticmethod
def target(edge):
return edge[1]
def _findNearestNeighbors(self, kdtreeObjectPair, traxel, numNeighbors, maxNeighborDist):
"""
Return a list of object IDs which are the 'numNeighbors' closest elements
in the kdtree less than maxNeighborDist away of the traxel.
"""
kdtree, objectIdList = kdtreeObjectPair
if len(objectIdList) <= numNeighbors:
return objectIdList
distances, neighbors = kdtree.query([self._extractCenter(
traxel)], k=numNeighbors, return_distance=True)
return [objectIdList[index] for distance, index in zip(distances[0], neighbors[0]) if
distance < maxNeighborDist]
def _extractCenter(self, traxel):
try:
# python probabilityGenerator
if 'com' in traxel.Features:
return traxel.Features['com']
else:
return traxel.Features['RegionCenter']
except:
# C++ pgmlink probabilityGenerator
try:
return getTraxelFeatureVector(traxel, 'com')
except:
try:
return getTraxelFeatureVector(traxel, 'RegionCenter')
except:
raise ValueError('given traxel (t={},id={}) does not have '
'"com" or "RegionCenter"'.format(traxel.Timestep, traxel.Id))
def _traxelMightDivide(self, traxel, divisionThreshold):
assert 'divProb' in traxel.Features
return traxel.Features['divProb'][0] > divisionThreshold
def _buildFrameKdTree(self, traxelDict):
"""
Collect the centers of all traxels and their ids of this frame's traxels.
Then build a kdtree and return (kdtree, listOfObjectIdsInFrame), where the second argument
is needed to decode the object id of the nearest neighbors in _findNearestNeighbors().
"""
objectIdList = []
features = []
for obj, traxel in traxelDict.items():
if obj == 0:
continue
objectIdList.append(obj)
features.append(list(self._extractCenter(traxel)))
return (KDTree(features, metric='euclidean'), objectIdList)
def _addNodesForFrame(self, frame, traxelDict):
"""
Insert nodes for all objects in this frame, with the attribute "traxel"
"""
for obj, traxel in traxelDict.items():
if obj == 0:
continue
self._graph.add_node((frame, obj), traxel=traxel, id=self._nextNodeUuid)
self._nextNodeUuid += 1
def addNodeFromTraxel(self, traxel, **kwargs):
"""
Insert a single node specified by a traxel.
All keyword arguments are passed to the node as well.
"""
assert(traxel is not None)
assert(not self.withTracklets)
self._graph.add_node((traxel.Timestep, traxel.Id), traxel=traxel, id=self._nextNodeUuid, **kwargs)
self._nextNodeUuid += 1
def buildFromProbabilityGenerator(self, probabilityGenerator, maxNeighborDist=200, numNearestNeighbors=1,
forwardBackwardCheck=True, withDivisions=True, divisionThreshold=0.1, skipLinks=1):
"""
Takes a python probabilityGenerator containing traxel features and finds probable links between frames.
Builds a kdTree with the 'numNearestneighbors' for each frame and adds the nodes. In the same iteration, it adds
a number of 'skipLinks' between the nodes separated by 'skipLinks' frames.
"""
assert (probabilityGenerator is not None)
assert (len(probabilityGenerator.TraxelsPerFrame) > 0)
assert (skipLinks > 0)
def checkNodeWhileAddingLinks(frame, obj):
if (frame, obj) not in self._graph:
getLogger().warning("Adding node ({}, {}) when setting up links".format(frame, obj))
kdTreeFrames = [None]*(skipLinks + 1)
# len(probabilityGenerator.TraxelsPerFrame.keys()) is NOT an indicator for the total number of frames,
# because an empty frame does not create a key in the dictionary. E.g. for one frame in the middle of the
# dataset, we won't access the last one.
# Idea: take the max key in the dict. Remember, frame numbering starts with 0.
frameMax = max(probabilityGenerator.TraxelsPerFrame.keys())
frameMin = min(probabilityGenerator.TraxelsPerFrame.keys())
numFrames = frameMax - frameMin + 1
self.progressVisitor.showState("Probability Generator")
countFrames = 0
for frame in range(numFrames):
countFrames += 1
self.progressVisitor.showProgress(countFrames/float(numFrames))
if frame > 0:
del kdTreeFrames[0] # this is the current frame
if frame + skipLinks < numFrames and frameMin + frame + skipLinks in probabilityGenerator.TraxelsPerFrame.keys():
kdTreeFrames.append(self._buildFrameKdTree(probabilityGenerator.TraxelsPerFrame[frameMin + frame + skipLinks]))
self._addNodesForFrame(frameMin + frame + skipLinks, probabilityGenerator.TraxelsPerFrame[frameMin + frame + skipLinks])
else:
for i in range(0, skipLinks+1):
if frameMin + frame + i in probabilityGenerator.TraxelsPerFrame.keys(): # empty frame
kdTreeFrames[i] = self._buildFrameKdTree(probabilityGenerator.TraxelsPerFrame[frameMin + frame + i])
self._addNodesForFrame(frameMin + frame + i, probabilityGenerator.TraxelsPerFrame[frameMin + frame + i])
# find forward links
if frameMin + frame in probabilityGenerator.TraxelsPerFrame.keys(): # 'frame' could be empty
for obj, traxel in probabilityGenerator.TraxelsPerFrame[frameMin + frame].items():
divisionPreservingNumNearestNeighbors = numNearestNeighbors
if divisionPreservingNumNearestNeighbors < 2 \
and withDivisions \
and self._traxelMightDivide(traxel, divisionThreshold):
divisionPreservingNumNearestNeighbors = 2
for i in range(1, skipLinks+1):
if frame + i < numFrames and frameMin + frame + i in probabilityGenerator.TraxelsPerFrame.keys():
neighbors = (self._findNearestNeighbors(kdTreeFrames[i],
traxel,
divisionPreservingNumNearestNeighbors,
maxNeighborDist))
# type(neighbors) is list
for n in neighbors:
checkNodeWhileAddingLinks(frameMin + frame, obj)
checkNodeWhileAddingLinks(frameMin + frame + i, n)
self._graph.add_edge((frameMin + frame, obj), (frameMin + frame + i, n))
self._graph.edge[frameMin + frame, obj][frameMin + frame + i, n]['src'] = self._graph.node[(frameMin + frame, obj)]['id']
self._graph.edge[frameMin + frame, obj][frameMin + frame + i, n]['dest'] = self._graph.node[(frameMin + frame + i, n)]['id']
# find backward links
if forwardBackwardCheck:
for i in range(1, skipLinks+1):
if frame + i < numFrames:
if frameMin + frame + i in probabilityGenerator.TraxelsPerFrame.keys(): # empty frame
for obj, traxel in probabilityGenerator.TraxelsPerFrame[frameMin + frame + i].items():
if kdTreeFrames[0] is not None:
neighbors = (self._findNearestNeighbors(kdTreeFrames[0],
traxel,
numNearestNeighbors,
maxNeighborDist))
for n in neighbors:
checkNodeWhileAddingLinks(frameMin + frame, n)
checkNodeWhileAddingLinks(frameMin + frame + i, obj)
self._graph.add_edge((frameMin + frame, n), (frameMin + frame + i, obj))
self._graph.edge[frameMin + frame, n][frameMin + frame + i, obj]['src'] = self._graph.node[(frameMin + frame, n)]['id']
self._graph.edge[frameMin + frame, n][frameMin + frame + i, obj]['dest'] = self._graph.node[(frameMin + frame + i, obj)]['id']
def generateTrackletGraph(self):
'''
**Return** a new hypotheses graph where chains of detections with only one possible
incoming/outgoing transition are contracted into one node in the graph.
The returned graph will have `withTracklets` set to `True`!
The `'tracklet'` node map contains a list of traxels that each node represents.
'''
getLogger().info("generating tracklet graph...")
tracklet_graph = copy.copy(self)
tracklet_graph._graph = tracklet_graph._graph.copy()
tracklet_graph.withTracklets = True
tracklet_graph.referenceTraxelGraph = self
tracklet_graph.progressVisitor = self.progressVisitor
self.progressVisitor.showState("Initializing Tracklet Graph")
# initialize tracklet map to contain a list of only one traxel per node
countNodes = 0
numNodes = tracklet_graph.countNodes()
for node in tracklet_graph._graph.nodes_iter():
countNodes += 1
self.progressVisitor.showProgress(countNodes/float(numNodes))
tracklet_graph._graph.node[node]['tracklet'] = [tracklet_graph._graph.node[node]['traxel']]
del tracklet_graph._graph.node[node]['traxel']
# set up a list of links that indicates whether the target's in- and source's out-degree
# are one, meaning the edge can be contracted
links_to_be_contracted = []
node_remapping = {}
self.progressVisitor.showState("Finding Tracklets in Graph")
countEdges = 0
numEdges = tracklet_graph.countArcs()
for edge in tracklet_graph._graph.edges_iter():
countEdges += 1
self.progressVisitor.showProgress(countEdges/float(numEdges))
if tracklet_graph._graph.out_degree(edge[0]) == 1 and tracklet_graph._graph.in_degree(edge[1]) == 1:
links_to_be_contracted.append(edge)
for i in [0, 1]:
node_remapping[edge[i]] = edge[i]
# apply edge contraction
self.progressVisitor.showState("Contracting Edges in Tracklet Graph")
countLinks = 0
numLinks = len(links_to_be_contracted)
for edge in links_to_be_contracted:
countLinks += 1
self.progressVisitor.showProgress(countLinks/float(numLinks))
src = node_remapping[edge[0]]
dest = node_remapping[edge[1]]
if tracklet_graph._graph.in_degree(src) == 0 and tracklet_graph._graph.out_degree(dest) == 0:
# if this tracklet would contract to a single node without incoming or outgoing edges,
# then do NOT contract, as our tracking cannot handle length-one-tracks
continue
tracklet_graph._graph.node[src]['tracklet'].extend(tracklet_graph._graph.node[dest]['tracklet'])
# duplicate out arcs with new source
for out_edge in tracklet_graph._graph.out_edges(dest):
tracklet_graph._graph.add_edge(src, out_edge[1])
# adjust node remapping to point to new source for all contracted traxels
for t in tracklet_graph._graph.node[dest]['tracklet']:
node_remapping[(t.Timestep, t.Id)] = src
tracklet_graph._graph.remove_node(dest)
getLogger().info("tracklet graph has {} nodes and {} edges (before {},{})".format(
tracklet_graph.countNodes(), tracklet_graph.countArcs(), self.countNodes(), self.countArcs()))
return tracklet_graph
def getNodeTraxelMap(self):
return NodeMap(self._graph, 'traxel')
def getNodeTrackletMap(self):
return NodeMap(self._graph, 'tracklet')
def insertEnergies(self,
maxNumObjects,
detectionProbabilityFunc,
transitionProbabilityFunc,
boundaryCostMultiplierFunc,
divisionProbabilityFunc,
skipLinksBias):
'''
Insert energies for detections, divisions and links into the hypotheses graph,
by transforming the probabilities for certain
events (given by the `*ProbabilityFunc`-functions per traxel) into energies. If the given graph
contained tracklets (`self.withTracklets is True`), then also the probabilities over all contained traxels will be
accumulated for those nodes in the graph.
The energies are stored in the networkx graph under the following attribute names (to match the format for solvers):
* detection energies: `self._graph.node[n]['features']`
* division energies: `self._graph.node[n]['divisionFeatures']`
* appearance energies: `self._graph.node[n]['appearanceFeatures']`
* disappearance energies: `self._graph.node[n]['disappearanceFeatures']`
* transition energies: `self._graph.edge[src][dest]['features']`
* additionally we also store the timestep (range for traxels) per node as `timestep` attribute
** Parameters: **
* `maxNumObjects`: the max number of objects per detections
* `detectionProbabilityFunc`: should take a traxel and return its detection probabilities
([prob0objects, prob1object,...])
* `transitionProbabilityFunc`: should take two traxels and return this link's probabilities
([prob0objectsInTransition, prob1objectsInTransition,...])
* `boundaryCostMultiplierFunc`: should take a traxel and a boolean that is true if we are seeking for an appearance cost multiplier,
false for disappearance, and return a scalar multiplier between 0 and 1 for the
appearance/disappearance cost that depends on the traxel's distance to the spacial and time boundary
* `divisionProbabilityFunc`: should take a traxel and return its division probabilities ([probNoDiv, probDiv])
'''
numElements = self._graph.number_of_nodes() + self._graph.number_of_edges()
self.progressVisitor.showState("Inserting energies")
# insert detection probabilities for all detections (and some also get a div probability)
countElements = 0
for n in self._graph.nodes_iter():
countElements += 1
if not self.withTracklets:
# only one traxel, but make it a list so everything below works the same
traxels = [self._graph.node[n]['traxel']]
else:
traxels = self._graph.node[n]['tracklet']
# accumulate features over all contained traxels
previousTraxel = None
detectionFeatures = np.zeros(maxNumObjects + 1)
for t in traxels:
detectionFeatures += np.array(negLog(detectionProbabilityFunc(t)))
if previousTraxel is not None:
detectionFeatures += np.array(negLog(transitionProbabilityFunc(previousTraxel, t)))
previousTraxel = t
detectionFeatures = listify(list(detectionFeatures))
# division only if probability is big enough
divisionFeatures = divisionProbabilityFunc(traxels[-1])
if divisionFeatures is not None:
divisionFeatures = listify(negLog(divisionFeatures))
# appearance/disappearance
appearanceFeatures = listify([0.0] + [boundaryCostMultiplierFunc(traxels[0], True)] * maxNumObjects)
disappearanceFeatures = listify([0.0] + [boundaryCostMultiplierFunc(traxels[-1], False)] * maxNumObjects)
self._graph.node[n]['features'] = detectionFeatures
if divisionFeatures is not None:
self._graph.node[n]['divisionFeatures'] = divisionFeatures
self._graph.node[n]['appearanceFeatures'] = appearanceFeatures
self._graph.node[n]['disappearanceFeatures'] = disappearanceFeatures
self._graph.node[n]['timestep'] = [traxels[0].Timestep, traxels[-1].Timestep]
self.progressVisitor.showProgress(countElements/float(numElements))
# insert transition probabilities for all links
for a in self._graph.edges_iter():
countElements += 1
self.progressVisitor.showProgress(countElements/float(numElements))
if not self.withTracklets:
srcTraxel = self._graph.node[self.source(a)]['traxel']
destTraxel = self._graph.node[self.target(a)]['traxel']
else:
srcTraxel = self._graph.node[self.source(a)]['tracklet'][-1] # src is last of the traxels in source tracklet
destTraxel = self._graph.node[self.target(a)]['tracklet'][0] # dest is first of traxels in destination tracklet
features = listify(negLog(transitionProbabilityFunc(srcTraxel, destTraxel)))
# add feature for additional Frames. Since we do not want these edges to be primarily taken, we add a bias to the edge. Now: hard coded, future: parameter
frame_gap = destTraxel.Timestep - srcTraxel.Timestep
# 1. method
if frame_gap > 1:
features[1][0] = features[1][0] + skipLinksBias*frame_gap
# # 2. method
# # introduce a new energies like: [[6], [15]] -> [[6, 23], [15, 23]] for first links and
# # [[6], [15]] -> [[23, 6], [23, 15]] for second links, and so on for 3rd order links
# # !!! this will introduce a new weight in the weight.json file. For the 2nd link, comes in 2nd row and so on.
# # drawback: did not manage to adjust parameter to get sensible results.
# for feat in features:
# for i in range(frame_gap):
# feat.append(23)
# if frame_gap > 1:
# feat[frame_gap-1], feat[0] = feat[0], feat[frame_gap-1]
self._graph.edge[a[0]][a[1]]['src'] = self._graph.node[a[0]]['id']
self._graph.edge[a[0]][a[1]]['dest'] = self._graph.node[a[1]]['id']
self._graph.edge[a[0]][a[1]]['features'] = features
def getMappingsBetweenUUIDsAndTraxels(self):
'''
Extract the mapping from UUID to traxel and vice versa from the networkx graph.
** Returns: a tuple of **
* `traxelIdPerTimestepToUniqueIdMap`: a dictionary of the structure `{str(timestep):{str(labelimageId):int(uuid),
str(labelimageId):int(uuid), ...}, str(nextTimestep):{}, ...}`
* `uuidToTraxelMap`: a dictionary with keys = int(uuid), values = list(of timestep-Id-tuples (int(Timestep), int(Id)))
'''
uuidToTraxelMap = {}
traxelIdPerTimestepToUniqueIdMap = {}
for n in self._graph.nodes_iter():
uuid = self._graph.node[n]['id']
traxels = []
if self.withTracklets:
traxels = self._graph.node[n]['tracklet']
else:
traxels = [self._graph.node[n]['traxel']]
uuidToTraxelMap[uuid] = [(t.Timestep, t.Id) for t in traxels]
for t in uuidToTraxelMap[uuid]:
traxelIdPerTimestepToUniqueIdMap.setdefault(str(t[0]), {})[str(t[1])] = uuid
# sort the list of traxels per UUID by their timesteps
for v in uuidToTraxelMap.values():
v.sort(key=lambda timestepIdTuple: timestepIdTuple[0])
return traxelIdPerTimestepToUniqueIdMap, uuidToTraxelMap
def toTrackingGraph(self, noFeatures=False):
'''
Create a dictionary representation of this graph which can be passed to the solvers directly.
The resulting graph (=model) is wrapped within a `hytra.jsongraph.JsonTrackingGraph` structure for convenience.
If `noFeatures` is `True`, then only the structure of the graph will be exported.
'''
requiredNodeAttribs = ['id']
requiredLinkAttribs = ['src', 'dest']
if not noFeatures:
requiredNodeAttribs.append('features')
requiredLinkAttribs.append('features')
def translateNodeToDict(n):
result = {}
attrs = self._graph.node[n]
for k in ['id', 'features', 'appearanceFeatures', 'disappearanceFeatures', 'divisionFeatures', 'timestep']:
if k in attrs:
result[k] = attrs[k]
elif k in requiredNodeAttribs:
raise ValueError('Cannot use graph nodes without assigned ID and features, run insertEnergies() first')
return result
def translateLinkToDict(l):
result = {}
attrs = self._graph.edge[l[0]][l[1]]
for k in ['src', 'dest', 'features']:
if k in attrs:
result[k] = attrs[k]
elif k in requiredLinkAttribs:
raise ValueError('Cannot use graph links without source, target, and features, run insertEnergies() first')
return result
traxelIdPerTimestepToUniqueIdMap, _ = self.getMappingsBetweenUUIDsAndTraxels()
model = {
'segmentationHypotheses':[translateNodeToDict(n) for n in self._graph.nodes_iter()],
'linkingHypotheses':[translateLinkToDict(e) for e in self._graph.edges_iter()],
'divisionHypotheses':[],
'traxelToUniqueId':traxelIdPerTimestepToUniqueIdMap,
'settings':{'statesShareWeights':True,
'allowPartialMergerAppearance':False,
'requireSeparateChildrenOfDivision':True,
'optimizerEpGap':0.01,
'optimizerVerbose':True,
'optimizerNumThreads':1
}
}
# extract exclusion sets:
exclusions = set([])
for n in self._graph.nodes_iter():
if self.withTracklets:
traxel = self._graph.node[n]['tracklet'][0]
else:
traxel = self._graph.node[n]['traxel']
if traxel.conflictingTraxelIds is not None:
if self.withTracklets:
getLogger().error("Exclusion constraints do not work with tracklets yet!")
conflictingIds = [traxelIdPerTimestepToUniqueIdMap[str(traxel.Timestep)][str(i)] for i in traxel.conflictingTraxelIds]
myId = traxelIdPerTimestepToUniqueIdMap[str(traxel.Timestep)][str(traxel.Id)]
for ci in conflictingIds:
# insert pairwise exclusion constraints only, and always put the lower id first
if ci < myId:
exclusions.add((ci, myId))
else:
exclusions.add((myId, ci))
model['exclusions'] = [list(t) for t in exclusions]
# TODO: this recomputes the uuidToTraxelMap even though we have it already...
trackingGraph = hytra.core.jsongraph.JsonTrackingGraph(
model=model,
progressVisitor=self.progressVisitor
)
return trackingGraph
def insertSolution(self, resultDictionary):
'''
Add solution values to nodes and arcs from dictionary representation of solution.
The resulting graph (=model) gets an additional property "value" that represents the number of objects inside a detection/arc
Additionally a division indicator is saved in the node property "divisionValue".
The link also gets a new attribute: the gap that is covered. E.g. 1, if consecutive timeframes, 2 if link skipping one timeframe.
'''
_, uuidToTraxelMap = self.getMappingsBetweenUUIDsAndTraxels()
if self.withTracklets:
traxelgraph = self.referenceTraxelGraph
else:
traxelgraph = self
# reset all values
for n in traxelgraph._graph.nodes_iter():
traxelgraph._graph.node[n]['value'] = 0
traxelgraph._graph.node[n]['divisionValue'] = False
for e in traxelgraph._graph.edges_iter():
traxelgraph._graph.edge[e[0]][e[1]]['value'] = 0
# store values from dict
for detection in resultDictionary["detectionResults"]:
traxels = uuidToTraxelMap[detection["id"]]
for traxel in traxels:
traxelgraph._graph.node[traxel]['value'] = detection["value"]
for internal_edge in zip(traxels,traxels[1:]):
traxelgraph._graph.edge[internal_edge[0]][internal_edge[1]]['value'] = detection["value"]
if "linkingResults" in resultDictionary and resultDictionary["linkingResults"] is not None:
for link in resultDictionary["linkingResults"]:
source, dest = uuidToTraxelMap[link["src"]][-1], uuidToTraxelMap[link["dest"]][0]
if source in list(traxelgraph._graph.edge.keys()) and dest in list(traxelgraph._graph.edge[source].keys()):
traxelgraph._graph.edge[source][dest]['value'] = link["value"]
traxelgraph._graph.edge[source][dest]['gap'] = dest[0] - source[0]
if "divisionResults" in resultDictionary and resultDictionary["divisionResults"] is not None:
for division in resultDictionary["divisionResults"]:
traxelgraph._graph.node[uuidToTraxelMap[division["id"]][-1]]['divisionValue'] = division["value"]
def getSolutionDictionary(self):
'''
Return the solution encoded in the `value` and `divisionValue` attributes of nodes and edges
as a python dictionary in the style that can be saved to JSON or sent to our solvers as ground truths.
'''
resultDictionary = {}
if self.withTracklets:
traxelgraph = self.referenceTraxelGraph
else:
traxelgraph = self
detectionList = []
divisionList = []
linkList = []
def checkAttributeValue(element, attribName, default):
if attribName in element:
return element[attribName]
else:
return default
for n in traxelgraph._graph.nodes_iter():
newDetection = {}
newDetection['id'] = traxelgraph._graph.node[n]['id']
newDetection['value'] = checkAttributeValue(traxelgraph._graph.node[n], 'value', 0)
detectionList.append(newDetection)
if 'divisionValue' in traxelgraph._graph.node[n]:
newDivsion = {}
newDivsion['id'] = traxelgraph._graph.node[n]['id']
newDivsion['value'] = checkAttributeValue(traxelgraph._graph.node[n], 'divisionValue', False)
divisionList.append(newDivsion)
for a in traxelgraph.arcIterator():
newLink = {}
src = self.source(a)
dest = self.target(a)
newLink['src'] = traxelgraph._graph.node[src]['id']
newLink['dest'] = traxelgraph._graph.node[dest]['id']
newLink['value'] = checkAttributeValue(traxelgraph._graph.edge[src][dest], 'value', 0)
newLink['gap'] = checkAttributeValue(traxelgraph._graph.edge[src][dest], 'gap', 1)
linkList.append(newLink)
resultDictionary["detectionResults"] = detectionList
resultDictionary["linkingResults"] = linkList
resultDictionary["divisionResults"] = divisionList
return resultDictionary
def countIncomingObjects(self, node):
'''
Once a solution was written to the graph, this returns the number of
incoming objects of a node, and the number of active incoming edges.
If the latter is greater than 1, this shows that we have a merger.
'''
numberOfIncomingObject = 0
numberOfIncomingEdges = 0
for in_edge in self._graph.in_edges(node):
if 'value' in self._graph.edge[in_edge[0]][node]:
numberOfIncomingObject += self._graph.edge[in_edge[0]][node]['value']
numberOfIncomingEdges += 1
return numberOfIncomingObject, numberOfIncomingEdges
def countOutgoingObjects(self, node):
'''
Once a solution was written to the graph, this returns the number of
outgoing objects of a node, and the number of active outgoing edges.
If the latter is greater than 1, this shows that we have a merger splitting up, or a division.
'''
numberOfOutgoingObject = 0
numberOfOutgoingEdges = 0
for out_edge in self._graph.out_edges(node):
if 'value' in self._graph.edge[node][out_edge[1]] and self._graph.edge[node][out_edge[1]]['value'] > 0:
numberOfOutgoingObject += self._graph.edge[node][out_edge[1]]['value']
numberOfOutgoingEdges += 1
return numberOfOutgoingObject, numberOfOutgoingEdges
def computeLineage(self, firstTrackId=2, firstLineageId=2, skipLinks=1):
"""
computes lineage and track id for every node in the graph
"""
update_queue = []
# start lineages / tracks at 2, because 0 means background=black, 1 means misdetection in ilastik
max_lineage_id = firstLineageId
max_track_id = firstTrackId
if self.withTracklets:
traxelgraph = self.referenceTraxelGraph
else:
traxelgraph = self
self.progressVisitor.showState("Compute lineage")
# find start of lineages
numElements = 2*traxelgraph.countNodes()
countElements = 0
for n in traxelgraph.nodeIterator():
countElements += 1
self.progressVisitor.showProgress(countElements/float(numElements))
if traxelgraph.countIncomingObjects(n)[0] == 0 \
and 'value' in traxelgraph._graph.node[n] \
and traxelgraph._graph.node[n]['value'] > 0 \
and (self.allowLengthOneTracks or traxelgraph.countOutgoingObjects(n)[0] > 0):
# found start of a track
update_queue.append((n,max_lineage_id,max_track_id))
max_lineage_id += 1
max_track_id += 1
else:
traxelgraph._graph.node[n]["lineageId"] = None
traxelgraph._graph.node[n]["trackId"] = None
while len(update_queue) > 0:
countElements += 1
current_node,lineage_id,track_id = update_queue.pop()
self.progressVisitor.showProgress(countElements/float(numElements))
# if we did not run merger resolving, it can happen that we reach a node several times,
# and would propagate the new lineage+track IDs to all descendants again! We simply
# stop propagating in that case and just use the lineageID that reached the node first.
if traxelgraph._graph.node[current_node].get("lineageId", None) is not None and \
traxelgraph._graph.node[current_node].get("trackId", None) is not None:
getLogger().debug("Several tracks are merging here, stopping a later one")
continue
# set a new trackID
traxelgraph._graph.node[current_node]["lineageId"] = lineage_id
traxelgraph._graph.node[current_node]["trackId"] = track_id
numberOfOutgoingObject, numberOfOutgoingEdges = traxelgraph.countOutgoingObjects(current_node)
if (numberOfOutgoingObject != numberOfOutgoingEdges):
getLogger().warning("running lineage computation on unresolved graphs depends on a race condition")
if 'divisionValue' in traxelgraph._graph.node[current_node] and traxelgraph._graph.node[current_node]['divisionValue']:
assert(traxelgraph.countOutgoingObjects(current_node)[1] == 2)
traxelgraph._graph.node[current_node]['children'] = []
for a in traxelgraph._graph.out_edges(current_node):
if 'value' in traxelgraph._graph.edge[current_node][a[1]] and traxelgraph._graph.edge[current_node][a[1]]['value'] > 0:
traxelgraph._graph.node[a[1]]['gap'] = skipLinks
traxelgraph._graph.node[current_node]['children'].append(a[1])
traxelgraph._graph.node[a[1]]['parent'] = current_node
update_queue.append((traxelgraph.target(a),
lineage_id,
max_track_id))
max_track_id += 1
else:
if traxelgraph.countOutgoingObjects(current_node)[1] > 1:
getLogger().debug('Found merger splitting into several objects, propagating lineage and track to all descendants!')
for a in traxelgraph._graph.out_edges(current_node):
if 'value' in traxelgraph._graph.edge[current_node][a[1]] and traxelgraph._graph.edge[current_node][a[1]]['value'] > 0:
if ('gap' in traxelgraph._graph.edge[current_node][a[1]] and traxelgraph._graph.edge[current_node][a[1]]['gap'] == 1) or 'gap' not in traxelgraph._graph.edge[current_node][a[1]]:
traxelgraph._graph.node[a[1]]['gap'] = 1
update_queue.append((traxelgraph.target(a),
lineage_id,
track_id))
if 'gap' in traxelgraph._graph.edge[current_node][a[1]] and traxelgraph._graph.edge[current_node][a[1]]['gap'] > 1:
traxelgraph._graph.node[a[1]]['gap'] = skipLinks
traxelgraph._graph.node[a[1]]['gap_parent'] = current_node
update_queue.append((traxelgraph.target(a),
lineage_id,
max_track_id))
max_track_id += 1
def pruneGraphToSolution(self, distanceToSolution=0):
'''
creates a new pruned HypothesesGraph that around the result. Assumes that value==0 corresponds
to unlabeled parts of the graph.
distanceToSolution determines how many negative examples are included
distanceToSolution = 0: only include negative edges that connect used objects
distanceToSolution = 1: additionally include edges that connect used objects with unlabeled objects
'''
prunedGraph = HypothesesGraph()
for n in self.nodeIterator():
if 'value' in self._graph.node[n] and self._graph.node[n]['value'] > 0:
prunedGraph._graph.add_node(n,**self._graph.node[n])
for e in self.arcIterator():
src = self.source(e)
dest = self.target(e)
if distanceToSolution == 0:
if src in prunedGraph._graph and dest in prunedGraph._graph:
prunedGraph._graph.add_edge(src,dest,**self._graph.edge[src][dest])
# TODO: can be optimized by looping over the pruned graph nodes(might sacrifice readability)
for distance in range(1,distanceToSolution+1):
for e in self.arcIterator():
src = self.source(e)
dest = self.target(e)
if src in prunedGraph._graph or dest in prunedGraph._graph:
prunedGraph._graph.add_node(src,**self._graph.node[src])
prunedGraph._graph.add_node(dest,**self._graph.node[dest])
prunedGraph._graph.add_edge(src,dest,**self._graph.edge[src][dest])
# in case a node is NOT an appearance and
# has all the incoming edges with value 0, we remove all these incoming edges
#
# in case a node is NOT a disappearance and
# has all the outgoing edges with value 0, we remove all these outgoing edges
withAppearanceFeatures = True
withDisappearanceFeatures = True
withFeatures = True
correctAppearanceFeatureLength = True
correctDisappearanceFeatureLength = True
correctFeatureLength = True
maxNumObjects = None
maxNumObjectsAppearance = None
maxNumObjectsDisappearance = None
for n in self.nodeIterator():
try:
maxNumObjectsApp = len(self._graph.node[n]['appearanceFeatures'])-1
if maxNumObjectsAppearance is None:
maxNumObjectsAppearance = maxNumObjectsApp
elif not maxNumObjectsApp == maxNumObjectsAppearance:
correctAppearanceFeatureLength = False
getLogger().info('Appearance/disappearance features have different lengths!')
except:
withAppearanceFeatures = False
getLogger().info('There are no appearance features in node properties!')
break
try:
maxNumObjectsDis = len(self._graph.node[n]['disappearanceFeatures'])-1
if maxNumObjectsDisappearance is None:
maxNumObjectsDisappearance = maxNumObjectsDis
elif not maxNumObjectsDis == maxNumObjectsDisappearance:
correctDisappearanceFeatureLength = False
getLogger().info('Disappearance features have different lengths!')
except:
withDisappearanceFeatures = False
getLogger().info('There are no disappearance features in node properties!')
break
if withAppearanceFeatures and withDisappearanceFeatures:
if correctAppearanceFeatureLength and correctDisappearanceFeatureLength and maxNumObjectsAppearance == maxNumObjectsDisappearance:
maxNumObjects = maxNumObjectsAppearance
else:
correctFeatureLength = False
getLogger().info('Appearance and disappearance features have different lengths!')
else:
withFeatures = False
if withFeatures and correctFeatureLength:
for n in self.nodeIterator():
if not ('appearance' in self._graph.node[n].keys() and self._graph.node[n]['appearance']):
allArcsWithValueZero = True
in_edges = self._graph.in_edges(n)
for edge in list(in_edges):
if 'value' in self._graph.edge[edge[0]][edge[1]].keys() and not self._graph.edge[edge[0]][edge[1]]['value'] == 0:
allArcsWithValueZero = False
break
self._graph.node[n]['appearanceFeatures'] = listify([0.0] + [0.0] * maxNumObjects)
if allArcsWithValueZero:
if not in_edges == []:
self._graph.remove_edges_from(in_edges)
if not('disappearance' in self._graph.node[n].keys() and self._graph.node[n]['disappearance']):
allArcsWithValueZero = True
out_edges = self._graph.out_edges(n)
for edge in list(out_edges):
if 'value' in self._graph.edge[edge[0]][edge[1]].keys() and not self._graph.edge[edge[0]][edge[1]]['value'] == 0:
allArcsWithValueZero = False
break
self._graph.node[n]['disappearanceFeatures'] = listify([0.0] + [0.0] * maxNumObjects)
if allArcsWithValueZero:
if not out_edges == []:
self._graph.remove_edges_from(out_edges)
return prunedGraph
def _getNodeAttribute(self, timestep, objectId, attribute):
'''
return some attribute of a certain node specified by timestep and objectId
'''
try:
return self._graph.node[(int(timestep), int(objectId))][attribute]
except KeyError:
getLogger().error(attribute + ' not found in graph node properties, call computeLineage() first!')
raise
def getLineageId(self, timestep, objectId):
'''
return the lineage Id of a certain node specified by timestep and objectId
'''
if self.withTracklets:
traxelgraph = self.referenceTraxelGraph
else:
traxelgraph = self
return traxelgraph._getNodeAttribute(timestep, objectId, 'lineageId')
def getTrackId(self, timestep, objectId):
'''
return the track Id of a certain node specified by timestep and objectId
'''
if self.withTracklets:
traxelgraph = self.referenceTraxelGraph
else:
traxelgraph = self
return traxelgraph._getNodeAttribute(timestep, objectId, 'trackId')
def track(self,
time_range,
x_range,
y_range,
z_range,
size_range=(0, 100000),
x_scale=1.0,
y_scale=1.0,
z_scale=1.0,
maxDist=30,
maxObj=2,
divThreshold=0.5,
avgSize=[0],
withTracklets=False,
sizeDependent=True,
detWeight=10.0,
divWeight=10.0,
transWeight=10.0,
withDivisions=True,
withOpticalCorrection=True,
withClassifierPrior=False,
ndim=3,
cplex_timeout=None,
withMergerResolution=True,
borderAwareWidth=0.0,
withArmaCoordinates=True,
appearance_cost=500,
disappearance_cost=500,
motionModelWeight=10.0,
force_build_hypotheses_graph=False,
max_nearest_neighbors=1,
numFramesPerSplit=0,
withBatchProcessing=False,
solverName="Flow-based",
progressWindow=None,
progressVisitor=DefaultProgressVisitor()):
"""
Main conservation tracking function. Runs tracking solver, generates hypotheses graph, and resolves mergers.
"""
self.progressWindow = progressWindow
self.progressVisitor = progressVisitor
if self.parent.parent._with_progress_bar and progressVisitor == DefaultProgressVisitor(
):
self.progressVisitor = CommandLineProgressVisitor()
if not self.Parameters.ready():
raise Exception("Parameter slot is not ready")
# it is assumed that the self.Parameters object is changed only at this
# place (ugly assumption). Therefore we can track any changes in the
# parameters as done in the following lines: If the same value for the
# key is already written in the parameters dictionary, the
# paramters_changed dictionary will get a "False" entry for this key,
# otherwise it is set to "True"
parameters = self.Parameters.value
parameters['maxDist'] = maxDist
parameters['maxObj'] = maxObj
parameters['divThreshold'] = divThreshold
parameters['avgSize'] = avgSize
parameters['withTracklets'] = withTracklets
parameters['sizeDependent'] = sizeDependent
parameters['detWeight'] = detWeight
parameters['divWeight'] = divWeight
parameters['transWeight'] = transWeight
parameters['withDivisions'] = withDivisions
parameters['withOpticalCorrection'] = withOpticalCorrection
parameters['withClassifierPrior'] = withClassifierPrior
parameters['withMergerResolution'] = withMergerResolution
parameters['borderAwareWidth'] = borderAwareWidth
parameters['withArmaCoordinates'] = withArmaCoordinates
parameters['appearanceCost'] = appearance_cost
parameters['disappearanceCost'] = disappearance_cost
parameters['scales'] = [x_scale, y_scale, z_scale]
parameters['time_range'] = [min(time_range), max(time_range)]
parameters['x_range'] = x_range
parameters['y_range'] = y_range
parameters['z_range'] = z_range
parameters['max_nearest_neighbors'] = max_nearest_neighbors
parameters['numFramesPerSplit'] = numFramesPerSplit
parameters['solver'] = str(solverName)
# Set a size range with a minimum area equal to the max number of objects (since the GMM throws an error if we try to fit more gaussians than the number of pixels in the object)
size_range = (max(maxObj, size_range[0]), size_range[1])
parameters['size_range'] = size_range
if cplex_timeout:
parameters['cplex_timeout'] = cplex_timeout
else:
parameters['cplex_timeout'] = ''
cplex_timeout = float(1e75)
self.Parameters.setValue(parameters, check_changed=False)
if withClassifierPrior:
if not self.DetectionProbabilities.ready() or len(
self.DetectionProbabilities([0]).wait()[0]) == 0:
raise DatasetConstraintError(
'Tracking',
'Classifier not ready yet. Did you forget to train the Object Count Classifier?'
)
if not self.NumLabels.ready() or self.NumLabels.value < (maxObj +
1):
raise DatasetConstraintError('Tracking', 'The max. number of objects must be consistent with the number of labels given in Object Count Classification.\n' +\
'Check whether you have (i) the correct number of label names specified in Object Count Classification, and (ii) provided at least ' +\
'one training example for each class.')
if len(self.DetectionProbabilities(
[0]).wait()[0][0]) < (maxObj + 1):
raise DatasetConstraintError('Tracking', 'The max. number of objects must be consistent with the number of labels given in Object Count Classification.\n' +\
'Check whether you have (i) the correct number of label names specified in Object Count Classification, and (ii) provided at least ' +\
'one training example for each class.')
hypothesesGraph = self._createHypothesesGraph()
hypothesesGraph.allowLengthOneTracks = True
if withTracklets:
hypothesesGraph = hypothesesGraph.generateTrackletGraph()
hypothesesGraph.insertEnergies()
trackingGraph = hypothesesGraph.toTrackingGraph()
trackingGraph.convexifyCosts()
model = trackingGraph.model
model['settings']['allowLengthOneTracks'] = True
detWeight = 10.0 # FIXME: Should we store this weight in the parameters slot?
weights = trackingGraph.weightsListToDict([
transWeight, detWeight, divWeight, appearance_cost,
disappearance_cost
])
stepStr = "Tracking solver"
self.progressVisitor.showState(stepStr)
self.progressVisitor.showProgress(0)
if solverName == 'Flow-based' and dpct:
if numFramesPerSplit:
# Run solver with frame splits (split, solve, and stitch video to improve running-time)
from hytra.core.splittracking import SplitTracking
result = SplitTracking.trackFlowBasedWithSplits(
model, weights, numFramesPerSplit=numFramesPerSplit)
else:
result = dpct.trackFlowBased(model, weights)
elif solverName == 'ILP' and mht:
result = mht.track(model, weights)
else:
raise ValueError("Invalid tracking solver selected")
self.progressVisitor.showProgress(1.0)
# Insert the solution into the hypotheses graph and from that deduce the lineages
if hypothesesGraph:
hypothesesGraph.insertSolution(result)
# Merger resolution
resolvedMergersDict = {}
if withMergerResolution:
stepStr = "Merger resolution"
self.progressVisitor.showState(stepStr)
resolvedMergersDict = self._resolveMergers(hypothesesGraph, model)
# Set value of resolved mergers slot (Should be empty if mergers are disabled)
self.ResolvedMergers.setValue(resolvedMergersDict, check_changed=False)
# Computing tracking lineage IDs from within Hytra
hypothesesGraph.computeLineage()
# Uncomment to export a hypothese graph diagram
#logger.info("Exporting hypotheses graph diagram")
#from hytra.util.hypothesesgraphdiagram import HypothesesGraphDiagram
#hgv = HypothesesGraphDiagram(hypothesesGraph._graph, timeRange=(0, 10), fileName='HypothesesGraph.png' )
# Set value of hypotheses grap slot (use referenceTraxelGraph if using tracklets)
hypothesesGraph = hypothesesGraph.referenceTraxelGraph if withTracklets else hypothesesGraph
self.HypothesesGraph.setValue(hypothesesGraph, check_changed=False)
# Set all the output slots dirty (See execute() function)
self.Output.setDirty()
self.MergerOutput.setDirty()
self.RelabeledImage.setDirty()
def _onTrackButtonPressed(self):
if not self.mainOperator.ObjectFeatures.ready():
self._criticalMessage("You have to compute object features first.")
return
withMergerResolution = self._drawer.mergerResolutionBox.isChecked()
withTracklets = True
numStages = 6
# creating traxel store
# generating probabilities
# insert energies
# convexify costs
# solver
# compute lineages
if withMergerResolution:
numStages += 1 # merger resolution
if withTracklets:
numStages += 3 # initializing tracklet graph, finding tracklets, contracting edges in tracklet graph
self.mainOperator.parent.parent._with_progress_bar = self._drawer.progressBarCheckBox.isChecked(
)
if self.mainOperator.parent.parent._with_progress_bar:
self.progressWindow = TrackProgressDialog(parent=self,
numStages=numStages)
self.progressWindow.run()
self.progressWindow.show()
self.progressVisitor = GuiProgressVisitor(
progressWindow=self.progressWindow)
else:
self.progressVisitor = DefaultProgressVisitor()
def _track():
self.applet.busy = True
self.applet.appletStateUpdateRequested.emit()
maxDist = self._drawer.maxDistBox.value()
maxObj = self._drawer.maxObjectsBox.value()
divThreshold = self._drawer.divThreshBox.value()
from_t = self._drawer.from_time.value()
to_t = self._drawer.to_time.value()
from_x = self._drawer.from_x.value()
to_x = self._drawer.to_x.value()
from_y = self._drawer.from_y.value()
to_y = self._drawer.to_y.value()
from_z = self._drawer.from_z.value()
to_z = self._drawer.to_z.value()
from_size = self._drawer.from_size.value()
to_size = self._drawer.to_size.value()
self.time_range = range(from_t, to_t + 1)
avgSize = [self._drawer.avgSizeBox.value()]
cplex_timeout = None
if len(str(self._drawer.timeoutBox.text())):
cplex_timeout = int(self._drawer.timeoutBox.text())
withTracklets = True
sizeDependent = self._drawer.sizeDepBox.isChecked()
hardPrior = self._drawer.hardPriorBox.isChecked()
classifierPrior = self._drawer.classifierPriorBox.isChecked()
detWeight = self._drawer.detWeightBox.value()
divWeight = self._drawer.divWeightBox.value()
transWeight = self._drawer.transWeightBox.value()
withDivisions = self._drawer.divisionsBox.isChecked()
withOpticalCorrection = self._drawer.opticalBox.isChecked()
withMergerResolution = self._drawer.mergerResolutionBox.isChecked()
borderAwareWidth = self._drawer.bordWidthBox.value()
withArmaCoordinates = True
appearanceCost = self._drawer.appearanceBox.value()
disappearanceCost = self._drawer.disappearanceBox.value()
solverName = self.topLevelOperatorView._solver
ndim = 3
if (to_z - from_z == 0):
ndim = 2
try:
if True:
self.mainOperator.track(
time_range=self.time_range,
x_range=(from_x, to_x + 1),
y_range=(from_y, to_y + 1),
z_range=(from_z, to_z + 1),
size_range=(from_size, to_size + 1),
x_scale=self._drawer.x_scale.value(),
y_scale=self._drawer.y_scale.value(),
z_scale=self._drawer.z_scale.value(),
maxDist=maxDist,
maxObj=maxObj,
divThreshold=divThreshold,
avgSize=avgSize,
withTracklets=withTracklets,
sizeDependent=sizeDependent,
detWeight=detWeight,
divWeight=divWeight,
transWeight=transWeight,
withDivisions=withDivisions,
withOpticalCorrection=withOpticalCorrection,
withClassifierPrior=classifierPrior,
ndim=ndim,
withMergerResolution=withMergerResolution,
borderAwareWidth=borderAwareWidth,
withArmaCoordinates=withArmaCoordinates,
cplex_timeout=cplex_timeout,
appearance_cost=appearanceCost,
disappearance_cost=disappearanceCost,
#graph_building_parameter_changed = True,
#trainingToHardConstraints = self._drawer.trainingToHardConstraints.isChecked(),
max_nearest_neighbors=self._maxNearestNeighbors,
solverName=solverName,
progressWindow=self.progressWindow,
progressVisitor=self.progressVisitor)
except Exception:
ex_type, ex, tb = sys.exc_info()
traceback.print_tb(tb)
self._criticalMessage("Exception(" + str(ex_type) + "): " +
str(ex))
return
def _handle_finished(*args):
self.applet.busy = False
self.applet.appletStateUpdateRequested.emit()
self._drawer.TrackButton.setEnabled(True)
self._drawer.exportButton.setEnabled(True)
self._drawer.exportTifButton.setEnabled(True)
self._setLayerVisible("Objects", False)
def _handle_failure(exc, exc_info):
self.applet.busy = False
self.applet.appletStateUpdateRequested.emit()
traceback.print_exception(*exc_info)
sys.stderr.write(
"Exception raised during tracking. See traceback above.\n")
self._drawer.TrackButton.setEnabled(True)
req = Request(_track)
req.notify_failed(_handle_failure)
req.notify_finished(_handle_finished)
req.submit()
def _onRunStructuredLearningButtonPressed(self, withBatchProcessing=False):
numStages = 4
# creating traxel store
# generating probabilities
# insert energies
# structured learning
self.mainOperator.parent.parent._with_progress_bar = self._drawer.progressBarCheckBox.isChecked(
)
if self.mainOperator.parent.parent._with_progress_bar:
self.progressWindow = TrackProgressDialog(parent=self,
numStages=numStages)
self.progressWindow.run()
self.progressWindow.show()
self.progressVisitor = GuiProgressVisitor(
progressWindow=self.progressWindow)
else:
self.progressVisitor = DefaultProgressVisitor()
def _learn():
self.applet.busy = True
self.applet.appletStateUpdateRequested.emit()
try:
self.topLevelOperatorView._runStructuredLearning(
(self._drawer.from_z.value(), self._drawer.to_z.value()),
self._maxNumObj,
self._maxNearestNeighbors,
self._drawer.maxDistBox.value(),
self._drawer.divThreshBox.value(),
(self._drawer.x_scale.value(),
self._drawer.y_scale.value(),
self._drawer.z_scale.value()),
(self._drawer.from_size.value(),
self._drawer.to_size.value()),
self._drawer.divisionsBox.isChecked(),
self._drawer.bordWidthBox.value(),
self._drawer.classifierPriorBox.isChecked(),
withBatchProcessing,
progressWindow=self.progressWindow,
progressVisitor=self.progressVisitor)
except Exception:
ex_type, ex, tb = sys.exc_info()
traceback.print_tb(tb)
self._criticalMessage("Exception(" + str(ex_type) + "): " +
str(ex))
return
def _handle_finished(*args):
self.applet.busy = False
self.applet.appletStateUpdateRequested.emit()
def _handle_failure(exc, exc_info):
self.applet.busy = False
self.applet.appletStateUpdateRequested.emit()
traceback.print_exception(*exc_info)
sys.stderr.write(
"Exception raised during learning. See traceback above.\n")
req = Request(_learn)
req.notify_failed(_handle_failure)
req.notify_finished(_handle_finished)
req.submit()
def _runStructuredLearning(self,
z_range,
maxObj,
maxNearestNeighbors,
maxDist,
divThreshold,
scales,
size_range,
withDivisions,
borderAwareWidth,
withClassifierPrior,
withBatchProcessing=False,
progressWindow=None,
progressVisitor=DefaultProgressVisitor()
):
if not withBatchProcessing:
gui = self.parent.parent.trackingApplet._gui.currentGui()
self.progressWindow = progressWindow
self.progressVisitor=progressVisitor
if self.parent.parent._with_progress_bar and progressVisitor==DefaultProgressVisitor():
self.progressVisitor = CommandLineProgressVisitor()
emptyAnnotations = False
for crop in self.Annotations.value.keys():
emptyCrop = self.Annotations.value[crop]["divisions"]=={} and self.Annotations.value[crop]["labels"]=={}
if emptyCrop and not withBatchProcessing:
gui._criticalMessage("Error: Weights can not be calculated because training annotations for crop {} are missing. ".format(crop) +\
"Go back to Training applet and Save your training for each crop.")
emptyAnnotations = emptyAnnotations or emptyCrop
if emptyAnnotations:
return [self.DetectionWeight.value, self.DivisionWeight.value, self.TransitionWeight.value, self.AppearanceWeight.value, self.DisappearanceWeight.value]
self._updateCropsFromOperator()
median_obj_size = [0]
from_z = z_range[0]
to_z = z_range[1]
ndim=3
if (to_z - from_z == 0):
ndim=2
time_range = [0, self.LabelImage.meta.shape[0]-1]
x_range = [0, self.LabelImage.meta.shape[1]]
y_range = [0, self.LabelImage.meta.shape[2]]
z_range = [0, self.LabelImage.meta.shape[3]]
parameters = self.Parameters.value
parameters['maxDist'] = maxDist
parameters['maxObj'] = maxObj
parameters['divThreshold'] = divThreshold
parameters['withDivisions'] = withDivisions
parameters['withClassifierPrior'] = withClassifierPrior
parameters['borderAwareWidth'] = borderAwareWidth
parameters['scales'] = scales
parameters['time_range'] = [min(time_range), max(time_range)]
parameters['x_range'] = x_range
parameters['y_range'] = y_range
parameters['z_range'] = z_range
parameters['max_nearest_neighbors'] = maxNearestNeighbors
parameters['withTracklets'] = False
# Set a size range with a minimum area equal to the max number of objects (since the GMM throws an error if we try to fit more gaussians than the number of pixels in the object)
size_range = (max(maxObj, size_range[0]), size_range[1])
parameters['size_range'] = size_range
self.Parameters.setValue(parameters, check_changed=False)
foundAllArcs = False
new_max_nearest_neighbors = max ([maxNearestNeighbors-1,1])
maxObjOK = True
parameters['max_nearest_neighbors'] = maxNearestNeighbors
while not foundAllArcs and maxObjOK and new_max_nearest_neighbors<10:
new_max_nearest_neighbors += 1
logger.info("new_max_nearest_neighbors: {}".format(new_max_nearest_neighbors))
time_range = range (0,self.LabelImage.meta.shape[0])
parameters['max_nearest_neighbors'] = new_max_nearest_neighbors
self.Parameters.setValue(parameters, check_changed=False)
hypothesesGraph = self._createHypothesesGraph()
if hypothesesGraph.countNodes() == 0:
raise DatasetConstraintError('Structured Learning', 'Can not track frames with 0 objects, abort.')
logger.info("Structured Learning: Adding Training Annotations to Hypotheses Graph")
mergeMsgStr = "Your tracking annotations contradict this model assumptions! All tracks must be continuous, tracks of length one are not allowed, and mergers may merge or split but all tracks in a merger appear/disappear together."
foundAllArcs = True;
numAllAnnotatedDivisions = 0
self.features = self.ObjectFeatures(range(0,self.LabelImage.meta.shape[0])).wait()
for cropKey in self.Crops.value.keys():
if foundAllArcs:
if not cropKey in self.Annotations.value.keys():
if not withBatchProcessing:
gui._criticalMessage("You have not trained or saved your training for " + str(cropKey) + \
". \nGo back to the Training applet and save all your training!")
return [self.DetectionWeight.value, self.DivisionWeight.value, self.TransitionWeight.value, self.AppearanceWeight.value, self.DisappearanceWeight.value]
crop = self.Annotations.value[cropKey]
timeRange = self.Crops.value[cropKey]['time']
if "labels" in crop.keys():
labels = crop["labels"]
for time in labels.keys():
if time in range(timeRange[0],timeRange[1]+1):
if not foundAllArcs:
break
for label in labels[time].keys():
if not foundAllArcs:
break
trackSet = labels[time][label]
center = self.features[time][default_features_key]['RegionCenter'][label]
trackCount = len(trackSet)
if trackCount > maxObj:
logger.info("Your track count for object {} in time frame {} is {} =| {} |, which is greater than maximum object number {} defined by object count classifier!".format(label,time,trackCount,trackSet,maxObj))
logger.info("Either remove track(s) from this object or train the object count classifier with more labels!")
maxObjOK = False
raise DatasetConstraintError('Structured Learning', "Your track count for object "+str(label)+" in time frame " +str(time)+ " equals "+str(trackCount)+"=|"+str(trackSet)+"|," + \
" which is greater than the maximum object number "+str(maxObj)+" defined by object count classifier! " + \
"Either remove track(s) from this object or train the object count classifier with more labels!")
for track in trackSet:
if not foundAllArcs:
logger.info("[structuredTrackingGui] Increasing max nearest neighbors!")
break
# is this a FIRST, INTERMEDIATE, LAST, SINGLETON(FIRST_LAST) object of a track (or FALSE_DETECTION)
type = self._type(cropKey, time, track) # returns [type, previous_label] if type=="LAST" or "INTERMEDIATE" (else [type])
if type == None:
raise DatasetConstraintError('Structured Learning', mergeMsgStr)
elif type[0] in ["LAST", "INTERMEDIATE"]:
previous_label = int(type[1])
previousTrackSet = labels[time-1][previous_label]
intersectionSet = trackSet.intersection(previousTrackSet)
trackCountIntersection = len(intersectionSet)
if trackCountIntersection > maxObj:
logger.info("Your track count for transition ( {},{} ) ---> ( {},{} ) is {} =| {} |, which is greater than maximum object number {} defined by object count classifier!".format(previous_label,time-1,label,time,trackCountIntersection,intersectionSet,maxObj))
logger.info("Either remove track(s) from these objects or train the object count classifier with more labels!")
maxObjOK = False
raise DatasetConstraintError('Structured Learning', "Your track count for transition ("+str(previous_label)+","+str(time-1)+") ---> ("+str(label)+","+str(time)+") is "+str(trackCountIntersection)+"=|"+str(intersectionSet)+"|, " + \
"which is greater than maximum object number "+str(maxObj)+" defined by object count classifier!" + \
"Either remove track(s) from these objects or train the object count classifier with more labels!")
sink = (time, int(label))
foundAllArcs = False
for edge in hypothesesGraph._graph.in_edges(sink): # an edge is a tuple of source and target nodes
logger.info("Looking at in edge {} of node {}, searching for ({},{})".format(edge, sink, time-1, previous_label))
# print "Looking at in edge {} of node {}, searching for ({},{})".format(edge, sink, time-1, previous_label)
if edge[0][0] == time-1 and edge[0][1] == int(previous_label): # every node 'id' is a tuple (timestep, label), so we need the in-edge coming from previous_label
foundAllArcs = True
hypothesesGraph._graph.edge[edge[0]][edge[1]]['value'] = int(trackCountIntersection)
break
if not foundAllArcs:
logger.info("[structuredTrackingGui] Increasing max nearest neighbors! LABELS/MERGERS t:{} id:{}".format(time-1, int(previous_label)))
# print "[structuredTrackingGui] Increasing max nearest neighbors! LABELS/MERGERS t:{} id:{}".format(time-1, int(previous_label))
break
if type == None:
raise DatasetConstraintError('Structured Learning', mergeMsgStr)
elif type[0] in ["FIRST", "LAST", "INTERMEDIATE", "SINGLETON(FIRST_LAST)"]:
if (time, int(label)) in hypothesesGraph._graph.node.keys():
hypothesesGraph._graph.node[(time, int(label))]['value'] = trackCount
logger.info("[structuredTrackingGui] NODE: {} {}".format(time, int(label)))
# print "[structuredTrackingGui] NODE: {} {} {}".format(time, int(label), int(trackCount))
else:
logger.info("[structuredTrackingGui] NODE: {} {} NOT found".format(time, int(label)))
# print "[structuredTrackingGui] NODE: {} {} NOT found".format(time, int(label))
foundAllArcs = False
break
if foundAllArcs and "divisions" in crop.keys():
divisions = crop["divisions"]
numAllAnnotatedDivisions = numAllAnnotatedDivisions + len(divisions)
for track in divisions.keys():
if not foundAllArcs:
break
division = divisions[track]
time = int(division[1])
parent = int(self.getLabelInCrop(cropKey, time, track))
if parent >=0:
children = [int(self.getLabelInCrop(cropKey, time+1, division[0][i])) for i in [0, 1]]
parentNode = (time, parent)
hypothesesGraph._graph.node[parentNode]['divisionValue'] = 1
foundAllArcs = False
for child in children:
for edge in hypothesesGraph._graph.out_edges(parentNode): # an edge is a tuple of source and target nodes
if edge[1][0] == time+1 and edge[1][1] == int(child): # every node 'id' is a tuple (timestep, label), so we need the in-edge coming from previous_label
foundAllArcs = True
hypothesesGraph._graph.edge[edge[0]][edge[1]]['value'] = 1
break
if not foundAllArcs:
break
if not foundAllArcs:
logger.info("[structuredTrackingGui] Increasing max nearest neighbors! DIVISION {} {}".format(time, parent))
# print "[structuredTrackingGui] Increasing max nearest neighbors! DIVISION {} {}".format(time, parent)
break
logger.info("max nearest neighbors= {}".format(new_max_nearest_neighbors))
if new_max_nearest_neighbors > maxNearestNeighbors:
maxNearestNeighbors = new_max_nearest_neighbors
parameters['maxNearestNeighbors'] = maxNearestNeighbors
if not withBatchProcessing:
gui._drawer.maxNearestNeighborsSpinBox.setValue(maxNearestNeighbors)
detectionWeight = self.DetectionWeight.value
divisionWeight = self.DivisionWeight.value
transitionWeight = self.TransitionWeight.value
disappearanceWeight = self.DisappearanceWeight.value
appearanceWeight = self.AppearanceWeight.value
if not foundAllArcs:
logger.info("[structuredTracking] Increasing max nearest neighbors did not result in finding all training arcs!")
return [transitionWeight, detectionWeight, divisionWeight, appearanceWeight, disappearanceWeight]
hypothesesGraph.insertEnergies()
self.progressVisitor.showState("Structured learning")
self.progressVisitor.showProgress(0)
# crops away everything (arcs and nodes) that doesn't have 'value' set
prunedGraph = hypothesesGraph.pruneGraphToSolution(distanceToSolution=0) # width of non-annotated border needed for negative training examples
trackingGraph = prunedGraph.toTrackingGraph()
# trackingGraph.convexifyCosts()
model = trackingGraph.model
model['settings']['optimizerEpGap'] = 0.005
model['settings']['allowLengthOneTracks'] = False
gt = prunedGraph.getSolutionDictionary()
initialWeights = trackingGraph.weightsListToDict([transitionWeight, detectionWeight, divisionWeight, appearanceWeight, disappearanceWeight])
mht.trainWithWeightInitialization(model,gt, initialWeights)
weightsDict = mht.train(model, gt)
weights = trackingGraph.weightsDictToList(weightsDict)
self.progressVisitor.showProgress(1)
if not withBatchProcessing and withDivisions and numAllAnnotatedDivisions == 0 and not weights[2] == 0.0:
gui._informationMessage("Divisible objects are checked, but you did not annotate any divisions in your tracking training. " + \
"The resulting division weight might be arbitrarily and if there are divisions present in the dataset, " +\
"they might not be present in the tracking solution.")
norm = 0
for i in range(len(weights)):
norm += weights[i]*weights[i]
norm = math.sqrt(norm)
if norm > 0.0000001:
self.TransitionWeight.setValue(weights[0]/norm)
self.DetectionWeight.setValue(weights[1]/norm)
self.DivisionWeight.setValue(weights[2]/norm)
self.AppearanceWeight.setValue(weights[3]/norm)
self.DisappearanceWeight.setValue(weights[4]/norm)
if not withBatchProcessing:
gui._drawer.detWeightBox.setValue(self.DetectionWeight.value)
gui._drawer.divWeightBox.setValue(self.DivisionWeight.value)
gui._drawer.transWeightBox.setValue(self.TransitionWeight.value)
gui._drawer.appearanceBox.setValue(self.AppearanceWeight.value)
gui._drawer.disappearanceBox.setValue(self.DisappearanceWeight.value)
if not withBatchProcessing:
if self.DetectionWeight.value < 0.0:
gui._informationMessage ("Detection weight calculated was negative. Tracking solution will be re-calculated with non-negativity constraints for learning weights. " + \
"Furthermore, you should add more training and recalculate the learning weights in order to improve your tracking solution.")
elif self.DivisionWeight.value < 0.0:
gui._informationMessage ("Division weight calculated was negative. Tracking solution will be re-calculated with non-negativity constraints for learning weights. " + \
"Furthermore, you should add more division cells to your training and recalculate the learning weights in order to improve your tracking solution.")
elif self.TransitionWeight.value < 0.0:
gui._informationMessage ("Transition weight calculated was negative. Tracking solution will be re-calculated with non-negativity constraints for learning weights. " + \
"Furthermore, you should add more transitions to your training and recalculate the learning weights in order to improve your tracking solution.")
elif self.AppearanceWeight.value < 0.0:
gui._informationMessage ("Appearance weight calculated was negative. Tracking solution will be re-calculated with non-negativity constraints for learning weights. " + \
"Furthermore, you should add more appearances to your training and recalculate the learning weights in order to improve your tracking solution.")
elif self.DisappearanceWeight.value < 0.0:
gui._informationMessage ("Disappearance weight calculated was negative. Tracking solution will be re-calculated with non-negativity constraints for learning weights. " + \
"Furthermore, you should add more disappearances to your training and recalculate the learning weights in order to improve your tracking solution.")
if self.DetectionWeight.value < 0.0 or self.DivisionWeight.value < 0.0 or self.TransitionWeight.value < 0.0 or \
self.AppearanceWeight.value < 0.0 or self.DisappearanceWeight.value < 0.0:
model['settings']['nonNegativeWeightsOnly'] = True
weightsDict = mht.train(model, gt)
weights = trackingGraph.weightsDictToList(weightsDict)
norm = 0
for i in range(len(weights)):
norm += weights[i]*weights[i]
norm = math.sqrt(norm)
if norm > 0.0000001:
self.TransitionWeight.setValue(weights[0]/norm)
self.DetectionWeight.setValue(weights[1]/norm)
self.DivisionWeight.setValue(weights[2]/norm)
self.AppearanceWeight.setValue(weights[3]/norm)
self.DisappearanceWeight.setValue(weights[4]/norm)
if not withBatchProcessing:
gui._drawer.detWeightBox.setValue(self.DetectionWeight.value)
gui._drawer.divWeightBox.setValue(self.DivisionWeight.value)
gui._drawer.transWeightBox.setValue(self.TransitionWeight.value)
gui._drawer.appearanceBox.setValue(self.AppearanceWeight.value)
gui._drawer.disappearanceBox.setValue(self.DisappearanceWeight.value)
logger.info("Structured Learning Tracking Weights (normalized):")
logger.info(" detection weight = {}".format(self.DetectionWeight.value))
logger.info(" division weight = {}".format(self.DivisionWeight.value))
logger.info(" transition weight = {}".format(self.TransitionWeight.value))
logger.info(" appearance weight = {}".format(self.AppearanceWeight.value))
logger.info(" disappearance weight = {}".format(self.DisappearanceWeight.value))
parameters['detWeight'] = self.DetectionWeight.value
parameters['divWeight'] = self.DivisionWeight.value
parameters['transWeight'] = self.TransitionWeight.value
parameters['appearanceCost'] = self.AppearanceWeight.value
parameters['disappearanceCost'] = self.DisappearanceWeight.value
self.Parameters.setValue(parameters)
if self.progressWindow is not None:
self.progressWindow.onTrackDone()
return [self.DetectionWeight.value, self.DivisionWeight.value, self.TransitionWeight.value, self.AppearanceWeight.value, self.DisappearanceWeight.value]