def errorFunction(X, n_cameras, n_points, x2d_splits, x2ds_labels, x2ds): camera_params = X[:n_cameras * 11].reshape((n_cameras, 11)) x3ds = X[n_cameras * 11:].reshape((n_points, 3)) projected_x2ds = np.zeros_like(x2ds) for camVec, c0, c1 in zip(camera_params, x2d_splits[:-1], x2d_splits[1:]): P, distortion = vecToMat(camVec) x3d_labels = np.int32([x2ds_labels[i] for i in xrange(c0, c1)]) proj_x2ds, proj_splits, proj_labels = ISCV.project(np.float32(x3ds[x3d_labels]), x3d_labels, np.float32([P])) assert np.all(x3d_labels == proj_labels) ISCV.distort_points(proj_x2ds, float(camVec[9]), float(camVec[10]), float(distortion[0]), float(distortion[1]), proj_x2ds) projected_x2ds[c0:c1, :] = proj_x2ds return (projected_x2ds - x2ds).ravel()
def cook(self, location, interface, attrs):
if not self.useFrame(interface.frame(), attrs['frameRange']): return
if not attrs['calibration'] or not attrs['x3ds']: return
calibrationLocation = attrs['calibration']
Ps = interface.attr('Ps', atLocation=calibrationLocation)
mats = interface.attr('mats', atLocation=calibrationLocation)
if Ps is None:
if mats is None:
self.logger.warning('Could not find calibration data at: %s' %
calibrationLocation)
return
Ps = interface.getPsFromMats(mats)
if Ps is None: return
# Get the x3ds (and labels if available) from the cooked location
x3ds = interface.attr('x3ds', atLocation=attrs['x3ds'])
if x3ds is None:
self.logger.error('No 3D points found at: %s' % attrs['x3ds'])
return
which_labels = interface.attr('which_labels')
if which_labels is None:
which_labels = np.arange(len(x3ds))
x3ds = np.ascontiguousarray(x3ds, dtype=np.float32)
normals = interface.attr('normals', atLocation=attrs['x3ds'])
if 'x3dIndex' in attrs and attrs['x3dIndex'] >= 0:
idx = attrs['x3dIndex']
x3ds = x3ds[idx].reshape(1, -1)
which_labels = [idx]
# Check if we've got visibility lods
visibilityLod = None
if 'skeleton' in attrs and attrs['skeleton']:
skeletonLoc = attrs['skeleton']
skelDict = interface.attr('skelDict', atLocation=skeletonLoc)
visibilityLod = interface.getChild('visibilityLod',
parent=skeletonLoc)
if attrs['useVisibility'] and visibilityLod is None:
self.logger.error('No visibility LODs found at skeleton: %s' %
attrs['skeleton'])
return
mats = interface.attr('mats', atLocation=calibrationLocation)
cameraPositions = np.array([m[4] for m in mats], dtype=np.float32)
if self.visibility is None:
self.visibility = ISCV.ProjectVisibility.create()
# if attrs['useVisibility'] and normals is not None and visibilityLod is not None:
if attrs['useVisibility'] and visibilityLod is not None:
lodNames = visibilityLod['names']
lodTris = visibilityLod['tris']
lodVerts = visibilityLod['verts']
lodNormals = visibilityLod['faceNormals']
tris = lodVerts[lodTris]
if attrs['useNormals'] and normals is not None:
self.visibility.setNormalsAndLods(normals, tris,
cameraPositions,
np.concatenate((lodNormals)),
attrs['intersect_threshold'],
attrs['generateNormals'])
else:
self.visibility.setLods(tris, cameraPositions,
np.concatenate((lodNormals)),
attrs['intersect_threshold'],
attrs['generateNormals'])
x2ds, x2ds_splits, x2d_labels = ISCV.project_visibility(
x3ds, which_labels, Ps, self.visibility)
elif attrs['useNormals'] and normals is not None:
self.visibility.setNormals(normals)
x2ds, x2ds_splits, x2d_labels = ISCV.project_visibility(
x3ds, which_labels, Ps, self.visibility)
else:
x2ds, x2ds_splits, x2d_labels = ISCV.project(
x3ds, which_labels, Ps)
# Distort if needed
if 'distort' in attrs and attrs['distort']:
for ci, (s, e) in enumerate(zip(x2ds_splits[:-1],
x2ds_splits[1:])):
K, RT, P, ks, T, wh = mats[ci]
dets = x2ds[s:e]
ISCV.distort_points(dets, float(-K[0, 2]), float(-K[1, 2]),
float(ks[0]), float(ks[1]), dets)
x2ds[s:e] = dets
detsAttrs = {
'x2ds': x2ds,
'x2ds_splits': x2ds_splits,
'labels': x2d_labels,
'x2ds_colour': eval(attrs['colour']),
'x2ds_pointSize': attrs['pointSize']
}
if 'cameraOffset' in attrs and attrs['cameraOffset'] > 0:
x2ds_splits_render = np.insert(
x2ds_splits, np.zeros(attrs['cameraOffset'], dtype=np.int32),
0)
detsAttrs['x2ds_splits_render'] = x2ds_splits_render
interface.createChild(interface.name(),
'detections',
atLocation=interface.parentPath(),
attrs=detsAttrs)
def cook(self, location, interface, attrs):
# Get x3ds and 3D labels from the cooked location
x3ds = interface.attr('x3ds')
if x3ds is None:
self.logger.error('Could not find attribute: x3ds')
return
x3ds_labels = interface.attr('x3ds_labels')
if x3ds_labels is None:
self.logger.error('Could not find attribute: x3ds_labels')
return
normals = interface.attr('normals')
x2d_threshold = attrs['x2d_threshold']
# Set the detections and calibration locations as the cook location if not defined
x2ds_location = attrs['x2ds']
if not x2ds_location: x2ds_location = location
calibrationLocation = attrs['calibration']
if not calibrationLocation: calibrationLocation = interface.root()
# Fetch 2D and calibration data
x2ds = interface.attr('x2ds', atLocation=x2ds_location)
x2ds_splits = interface.attr('x2ds_splits', atLocation=x2ds_location)
Ps = interface.attr('Ps', atLocation=interface.root() + '/cameras')
if x2ds is None or x2ds_splits is None:
self.logger.error('2D detection data at %s is not valid' % x2ds_location)
return
if Ps is None:
mats = interface.attr('mats', atLocation=interface.root())
if mats:
Ps = np.array([m[2] / (np.sum(m[2][0, :3] ** 2) ** 0.5) for m in mats], dtype=np.float32)
else:
self.logger.error('Attribute mats not found at %s' % calibrationLocation)
self.logger.error('Attribute Ps not found at %s' % calibrationLocation)
return
# Check if we've got visibility lods
if 'skeleton' in attrs and attrs['skeleton']:
skeletonLoc = attrs['skeleton']
skelDict = interface.attr('skelDict', atLocation=skeletonLoc)
visibilityLod = interface.getChild('visibilityLod', parent=skeletonLoc)
if visibilityLod is None:
self.logger.error('No visibility LODs found at skeleton: %s' % attrs['skeleton'])
return
lodNames = visibilityLod['names']
lodTris = visibilityLod['tris']
lodVerts = visibilityLod['verts']
lodNormals = visibilityLod['faceNormals']
tris = lodVerts[lodTris]
mats = interface.attr('mats', atLocation=attrs['calibration'])
cameraPositions = np.array([m[4] for m in mats], dtype=np.float32)
clouds = ISCV.HashCloud2DList(x2ds, x2ds_splits, x2d_threshold)
if self.visibility is None: self.visibility = ISCV.ProjectVisibility.create()
proj_x2ds = None
if attrs['useVisibility'] and normals is not None:
self.visibility.setNormalsAndLods(normals, tris, cameraPositions, np.concatenate((lodNormals)), attrs['intersect_threshold'], attrs['generateNormals'])
# proj_x2ds, proj_splits, proj_labels = ISCV.project_visibility(x3ds, x3ds_labels, Ps, self.visibility)
# score, x2d_labels, residuals = clouds.assign(proj_x2ds, proj_splits, proj_labels, x2d_threshold)
score, x2d_labels, residuals = clouds.project_assign_visibility(x3ds, x3ds_labels, Ps, x2d_threshold, self.visibility)
elif attrs['useNormals'] and normals is not None:
self.visibility.setNormals(normals)
proj_x2ds, proj_splits, proj_labels = ISCV.project_visibility(x3ds, x3ds_labels, Ps, self.visibility)
score, x2d_labels, residuals = clouds.assign(proj_x2ds, proj_splits, proj_labels, x2d_threshold)
else:
proj_x2ds, proj_splits, proj_labels = ISCV.project(x3ds, x3ds_labels, Ps)
score, x2d_labels, vels = clouds.assign(proj_x2ds, proj_splits, proj_labels, x2d_threshold)
if proj_x2ds is not None:
projectedLocsAttrs = {
'x2ds': proj_x2ds, 'x2ds_splits': proj_splits, 'labels': proj_labels,
'x2ds_colour': (1.0, 0.0, 0.0, 0.7), 'x2ds_pointSize': 10,
'score': score
}
if 'showProjected' in attrs and attrs['showProjected']:
interface.createChild('projected', 'points2d', attrs=projectedLocsAttrs)
else:
interface.createChild('projected', 'group', attrs=projectedLocsAttrs)
if attrs['newLocation']:
locAttrs = {
'x2ds': x2ds, 'x2ds_splits': x2ds_splits, 'labels': x2d_labels,
'x2ds_colour': eval(attrs['colour']), 'x2ds_pointSize': attrs['pointSize'],
'score': score
}
labelColours = interface.getLabelColours(x2d_labels, eval(attrs['colour']))
if labelColours.any():
locAttrs['x2ds_colours'] = labelColours
interface.createChild('assigned', 'points2d', attrs=locAttrs)
else:
interface.setAttr('labels', x2d_labels, atLocation=x2ds_location)
interface.setAttr('score', score)
labelColours = interface.getLabelColours(x2d_labels, interface.attr('x2ds_colour', atLocation=x2ds_location))
if labelColours.any():
interface.setAttr('x2ds_colours', labelColours, atLocation=x2ds_location)