def cook(self, location, interface, attrs):
imgs = interface.attr('jpegimgs')
if imgs is None: return
img_count = len(imgs)
vwidth, vheight = attrs['vwidth'], attrs['vheight']
self.mats = interface.attr('mats', atLocation=attrs['cameraLocations'])
updateMats = interface.attr('updateMats',
atLocation=attrs['cameraLocations'],
default=False)
if self.mats is None or updateMats:
self.mats = []
for i in xrange(img_count):
self.mats.append(
Calibrate.makeUninitialisedMat(i, (vheight, vwidth)))
interface.setAttr('updateMats',
False,
atLocation=attrs['cameraLocations'])
self.camAttrs = {
'vheight': [vheight] * img_count,
'vwidth': [vwidth] * img_count,
'camera_ids': range(img_count),
'mats': self.mats,
'updateImage': True,
'jpegimgs': imgs,
'updateMats': updateMats
}
for k, v in self.camAttrs.iteritems():
interface.setAttr(k, v)
def cook(self, location, interface, attrs):
if not self.listeners: return
import StringIO
imgs, vwidths, vheights, camera_ids, camera_names, mats = [], [], [], [], [], []
ci = 0
for r, listener in self.listeners.iteritems():
data = listener.poll()
if data is None:
self.logger.error('No data on %d' % r)
continue
print data
timeCode, imgStr = data
sio = StringIO.StringIO(imgStr)
img = PIL.Image.open(sio)
if not self.initialised:
vwidth, vheight = attrs['vwidth'], attrs['vheight']
mat = Calibrate.makeUninitialisedMat(0, (vheight, vwidth))
vwidths.append(vwidth)
vheights.append(vheight)
camera_ids.append('Camera %d' % ci)
camera_names.append(str(r))
mats.append(mat)
imgs.append(img.tobytes())
ci += 1
if not self.initialised:
self.camAttrs['vheight'] = vheights
self.camAttrs['vwidth'] = vwidths
# self.camAttrs['camera_ids'] = camera_ids
# self.camAttrs['camera_names'] = camera_names
self.camAttrs['camera_ids'] = interface.attr('camera_ids')
self.camAttrs['camera_names'] = camera_names
self.camAttrs['mats'] = interface.attr('mats')
self.initialised = True
if imgs:
self.camAttrs['imgs'] = imgs
# self.camAttrs['updateImage'] = True
interface.createChild(interface.name(),
'cameras',
atLocation=interface.parentPath(),
attrs=self.camAttrs)
tcAttrs = {
'x3ds': np.array([[0, 0, 0]], dtype=np.float32),
'x3ds_labels': [timeCode]
}
interface.createChild(interface.name() + '/tc',
'points',
atLocation=interface.parentPath(),
attrs=tcAttrs)
def cook(self, location, interface, attrs):
vwidth, vheight = attrs['vwidth'], attrs['vheight']
if self.cam is None:
self.cam = CamVideoStream(src=0).start()
self.mats = [Calibrate.makeUninitialisedMat(0, (vheight, vwidth))]
self.camAttrs = {
'vheight': [vheight],
'vwidth': [vwidth],
'camera_ids': [0],
'mats': self.mats,
'updateImage': True
}
md = {'frame': self.cam.read()}
self.camAttrs['imgs'] = [md['frame']]
# self.attrs['imgs'] = np.array(frame, dtype=np.uint8)
interface.createChild(interface.name(),
'cameras',
atLocation=interface.parentPath(),
attrs=self.camAttrs)
def cook(self, location, interface, attrs):
if not self.useFrame(interface.frame(), attrs['frameRange']):
interface.setAttr('updateMats', False)
return
# We need 2D data e.g. wand detections from a wand op
# We need 3D wand data from e.g. c3d or a 3D wand detector
dets_location = attrs['detections']
x3ds_location = attrs['x3ds']
if not dets_location or not x3ds_location: return
# Get the 2D and 3D data
x2ds = interface.attr('rx2ds', atLocation=dets_location)
x2d_splits = interface.attr('x2ds_splits', atLocation=dets_location)
x3ds = interface.attr('x3ds', atLocation=x3ds_location)
if x2ds is None or x2d_splits is None or x3ds is None: return
numCameras = len(x2d_splits) - 1
error_threshold = attrs['error_threshold']
# Get the data we've collected already so we can add to it
frame = interface.frame()
dets_colours = interface.attr('x2ds_colours', atLocation=dets_location)
collectedDets = interface.attr('collect_rx2ds')
collectedX3ds = interface.attr('collect_x3ds')
lastFrame = interface.attr('lastFrame', [frame] * numCameras)
emptyFrame3d = np.array([[]], dtype=np.float32).reshape(-1, 3)
# This is potentially used by other ops so we only set it when we have some confidence
# (and we might reset or tweak the values to indicate confidence levels at some point)
cameraErrors = interface.attr('cameraErrors', [-1] * numCameras)
# This is never modified to allow checking the camera rms values regardless of what we make of them
rmsValues = interface.attr('rms', [-1] * numCameras)
# Get the width and height for the videos
vwidth = interface.attr('vwidth', [1920] * numCameras)
vheight = interface.attr('vheight', [1080] * numCameras)
# Get the frame mapping for x3ds
x3ds_frames = interface.attr('x3ds_frames', {})
x2ds_frames = interface.attr('x2ds_frames', [[] for i in xrange(numCameras)])
# Get the camera matrices. We initialise them with default settings if we don't find any
mats = interface.attr('mats', atLocation=location)
if mats is None:
mats = []
for ci in range(numCameras):
mats.append(Calibrate.makeUninitialisedMat(ci, (vheight[ci], vwidth[ci])))
# Allow overriding the error threshold using an attribute (on the cooked location)
error_threshold_attr = interface.attr('error_threshold')
if error_threshold_attr is not None:
error_threshold = error_threshold_attr
Ps = interface.attr('Ps')
if Ps is None: Ps = [np.array([], dtype=np.float32) for n in range(numCameras)]
# Get the minimum number of samples we need to start solving distortion etc. as specified by the user
minSamples = attrs['min_samples']
# Prepare the collected data for further processing (or initialise if nothing has been collected)
if collectedDets is not None:
c_x2ds, c_splits = collectedDets
cams_collected = [c_x2ds[c0:c1] for ci, (c0, c1) in enumerate(zip(c_splits[:-1], c_splits[1:]))]
else:
cams_collected = [[] for ci, (c0, c1) in enumerate(zip(x2d_splits[:-1], x2d_splits[1:]))]
collectedX3ds = []
for ci, (c0, c1) in enumerate(zip(x2d_splits[:-1], x2d_splits[1:])):
collectedX3ds.append(emptyFrame3d)
# Process each camera by looking for a wand and attempt a calibration. If we're happy with the results we'll
# add it to our collection
for ci, (c0, c1) in enumerate(zip(x2d_splits[:-1], x2d_splits[1:])):
elapsed = frame - lastFrame[ci]
if 0 < elapsed < attrs['jumpFrames']: continue
# Get the 2Ds and 3Ds for the wand in this camera (if any)
cameraDetections = x2ds[c0:c1]
cameraX3ds = x3ds
if not cameraDetections.any() or not cameraX3ds.any(): continue
# Add the new detection to the existing collection as a candidate for a new collection
if cams_collected[ci] is None or len(cams_collected[ci]) == 0:
proposalDets, proposalX3ds = cameraDetections, cameraX3ds
else:
proposalDets = np.concatenate((cams_collected[ci], cameraDetections))
proposalX3ds = np.concatenate((collectedX3ds[ci], cameraX3ds))
# Check if we want to solve for distortion and focal length by looking at the number of samples
# we've got already compared to our minimum number of samples required
numSamples = len(proposalDets) / 5
# if numSamples == minSamples: self.logger.info('Camera %d reached min samples of %d' % (ci, minSamples))
solveTrigger = True if numSamples > minSamples else False
solve_focal_length = attrs['solve_focal_length'] if solveTrigger else False
solve_distortion = attrs['solve_distortion'] if solveTrigger else False
# The wand is assumed to have 5 points so we make sure we've got at least one wand before attempting
# to calibrate
if len(proposalDets) >= 5 and len(proposalX3ds) >= 5:
P, ks, rms = Calibrate.cv2_solve_camera_from_3d(proposalX3ds, proposalDets,
solve_focal_length=solve_focal_length,
solve_distortion=solve_distortion)
if ks[0] < -3. or ks[0] > 3.: ks[0] = 0.
if ks[1] < -3. or ks[1] > 3.: ks[1] = 0.
# This shouldn't' happen but if we lose confidence in the camera we can visualise it
# by resetting the camera error (this will change the colour in the UI)
if rms > error_threshold:
cameraErrors[ci] = -1
continue
# See how the rms for the calibration compares to the last recorded value for this camera
prevRms = rms if rmsValues[ci] == -1 else rmsValues[ci]
rmsDelta = rms - prevRms
# If the rms is lower than the last recorded error for this camera then
# we want to keep this data
if rmsDelta <= 0 or not solveTrigger:
cams_collected[ci] = proposalDets
collectedX3ds[ci] = proposalX3ds
if frame not in x3ds_frames:
x3ds_frames[frame] = proposalX3ds[-5:]
x2ds_frames[ci] += ([frame] * 5)
else:
continue
# Record the rms value for the camera
rmsValues[ci] = rms
# Once we've solved for distortion etc. we are more confident with the accuracy of our
# error so we start reporting it, where the value can be used for visualiation etc.
if solveTrigger: cameraErrors[ci] = rms
lastFrame[ci] = frame
# Everything has gone well so far so we create and add the new camera matrix
mat = Calibrate.makeMat(P, ks, (vheight[ci], vwidth[ci]))
mats[ci] = mat
Ps[ci] = P
# Concatenate the results from all the cameras
cams = [np.concatenate((cc)) for cc in cams_collected if len(cc)]
if not cams:
# We haven't found a wand in any camera so we just keep calm and return
return
# Build our collections and write to the interface
collectedDets = np.array(np.concatenate(cams), dtype=np.float32).reshape(-1, 2), \
Interface.makeSplitBoundaries(map(len, cams_collected))
interface.setAttr('collect_rx2ds', collectedDets)
interface.setAttr('collect_x3ds', collectedX3ds)
interface.setAttr('x2ds_frames', x2ds_frames)
interface.setAttr('x3ds_frames', x3ds_frames)
interface.setAttr('lastFrame', lastFrame)
# Write the calibration data to the interface and request an update at render time
interface.setAttr('mats', mats)
interface.setAttr('Ps', Ps)
interface.setAttr('rms', rmsValues)
interface.setAttr('cameraErrors', cameraErrors)
interface.setAttr('updateMats', True)
# Optionally display all the collected wand detections
if 'showDetections' in attrs and attrs['showDetections']:
colours = np.tile(dets_colours, (len(collectedDets[0]) / 5, 1))
allAttrs = {'x2ds': collectedDets[0], 'x2ds_splits': collectedDets[1],
'x2ds_colours': colours}
interface.createChild('collected', 'detections', attrs=allAttrs)
def load_xcp_and_x2d(xcp_filename, x2d_filename, raw=False):
'''Load an x2d, xcp pair and make a valid data structure.
The returned cameras are in the order of the x2d file -- as it happens, this is in order of deviceid.
If any particular camera is not in the xcp then it's initialised on the positive x-axis.
If a camera is only in the xcp then it is discarded.'''
from GCore import Calibrate
print ('loading xcp')
vicon_mats,xcp_data = loadXCP(xcp_filename)
xcp_camera_ids = np.array([int(x['DEVICEID']) for x in xcp_data],dtype=np.int32)
camera_names = ['%s:%s'%(x['LABEL'],x['DEVICEID']) for x in xcp_data]
camera_vicon_errors = np.array([x['IMAGE_ERROR'] for x in xcp_data],dtype=np.float32)
#print (camera_names)
#print ('vicon_errors',camera_vicon_errors,np.min(camera_vicon_errors),np.max(camera_vicon_errors),np.mean(camera_vicon_errors))
print ('loading x2d',x2d_filename)
x2d_dict = loadX2D(x2d_filename)
cameras_info = extractCameraInfo(x2d_dict) # W,H,ID per camera
x2d_cids = cameras_info[:,2]
x2ds = [(x[0][:,:2].copy(),x[1]) if raw else frameCentroidsToDets(x, vicon_mats) for x in x2d_dict['frames']]
if not(np.all(xcp_camera_ids == x2d_cids)):
print ('WARNING not all the cameras from the x2d were in the xcp file?') # TODO, we should report this
print (xcp_camera_ids, x2d_cids)
vicon_mats = [vicon_mats[list(xcp_camera_ids).index(ci)] if ci in list(xcp_camera_ids) else Calibrate.makeUninitialisedMat(ci,(w,h)) for w,h,ci in cameras_info]
camera_names = ['CAM_%s'%x for x in x2d_cids]
xcp_camera_ids = [f for f in x2d_cids]
Ps = np.array([m[2]/(np.sum(m[2][0,:3]**2)**0.5) for m in vicon_mats],dtype=np.float32)
headerMetadata = readX2DMetadata(x2d_dict)
return Ps, vicon_mats, x2d_cids, camera_names, x2ds, x2d_dict['header']
def initialise(self, interface, attrs):
directory = self.resolvePath(attrs['directory'])
if not directory: return False
prefix = attrs['prefix']
prefixFilename = self.resolvePath(attrs['prefixFilename'])
if prefix and not prefixFilename: return
calibration = attrs['calibration']
calibrationFilename = self.resolvePath(attrs['calibrationFilename'])
calibrationLocation = self.resolvePath(attrs['calibrationLocation'])
if calibration and (not calibrationFilename
and not calibrationLocation):
return False
movieFilenames = []
try:
for file in os.listdir(directory):
if prefixFilename and not file.startswith(prefixFilename):
continue
if file.endswith('.avi') or file.endswith(
'.mov') or file.endswith('mp4'):
movieFilenames.append(os.path.join(directory, file))
except WindowsError as e:
self.logger.error('Could not find videos: % s' % str(e))
if not movieFilenames:
# TODO: Here we'll have to clear the cameras etc.
return False
# Windows will produce a wonky order, i.e. 1, 10, 11, .., 2, 3, ..
# Use natural sorting to rectify
movieFilenames.sort(key=self.alphaNumKey)
self.camera_ids = []
self.camera_names = []
self.movies = []
self.mats = []
vheights = []
vwidths = []
timecodes = []
hasTimecode = False
useTimecode = attrs['useTimecode'] if 'useTimecode' in attrs else True
offset = attrs['offset']
if 'offsets' in attrs and attrs['offsets']:
offsets = eval(attrs['offsets'])
else:
offsets = [offset] * len(movieFilenames)
for ci, mf in enumerate(movieFilenames):
self.logger.info('Loading MovieReader: %s' % mf)
movieData = MovieReader.open_file(mf,
audio=False,
frame_offset=offsets[ci])
if movieData['vbuffer'] is not None:
self.movies.append(movieData)
self.timecodeOffsets.append(0)
if 'timecode' in movieData and movieData['timecode']:
hasTimecode = True
timecodes.append(movieData['timecode'])
# Make sure we have all the cameras before continuing
if len(self.movies) != len(movieFilenames):
self.logger.error('Could not load all movies in sequence')
return
# Make sure we have as many time codes as movies (if we have any)
if hasTimecode and len(self.movies) != len(timecodes):
self.logger.error('Not all movie files have a time code')
return
# See if we can get the offsets using the time codes
if hasTimecode and useTimecode:
print 'Video timecodes:', timecodes
fps_all = [round(m['fps']) for m in self.movies]
print 'FPS:', fps_all
timecodeValues = [
Timecode.TCFtoInt(tc, fps)
for tc, fps in zip(timecodes, fps_all)
]
tcOrderDesc = [
timecodes.index(tc) for tc in sorted(timecodes, reverse=True)
]
# Set the first offset to 0
firstTcIndex = tcOrderDesc[0]
self.timecodeOffsets[firstTcIndex] = 0
largestTc = timecodes[firstTcIndex]
offsetStartIndex = 1
# We can also get the timecode destination from an incoming location, e.g. 2D detections
if 'timecodeLocation' in attrs and attrs['timecodeLocation']:
tcSyncTime = interface.attr(
'timecode', atLocation=attrs['timecodeLocation'])
if tcSyncTime is not None:
tcSyncValue = Timecode.TCFtoInt(tcSyncTime, fps_all[0])
if tcSyncValue < timecodeValues[firstTcIndex]:
self.logger.error(
'Sync timecode %s is smaller than video timecodes (%s).'
% (tcSyncTime, largestTc))
return
largestTc = tcSyncTime
offsetStartIndex = 0
self.timecode = largestTc
self.logger.info('Setting timecode to: %s' % (largestTc))
# Calculate the offset for each camera to get it up to speed with the target timecode
# TODO: Replace hard coded timecode fps and multiplier
timecodeFps, timecodeMultiplier = 25., 2.
for tcInd in tcOrderDesc[offsetStartIndex:]:
diff = Timecode.TCSub(largestTc, timecodes[tcInd], timecodeFps)
self.timecodeOffsets[tcInd] = Timecode.TCFtoInt(
diff, timecodeFps) * timecodeMultiplier
if self.timecodeOffsets:
print 'Video timecode offsets:', self.timecodeOffsets
self.camera_ids = [
'Camera %d' % ci for ci in xrange(len(movieFilenames))
]
self.movies = self.movies
if not calibrationLocation: calibrationLocation = interface.root()
if calibrationFilename or interface.hasAttr(
'mats', atLocation=calibrationLocation):
if calibrationFilename:
# TODO: Detect filetype, e.g. .cal and .xcp and handle accordingly
try:
self.mats, rawCalData = OptitrackReader.load_CAL(
calibrationFilename)
if not self.mats: return False
except IOError as e:
self.logger.error('Could not load calibration file: %s' %
str(e))
return False
else:
self.mats = interface.attr('mats',
atLocation=calibrationLocation)
if not self.mats:
self.logger.error('Could not find calibration mats: %s' %
calibrationLocation)
return False
else:
from GCore import Calibrate
for ci, (cid, md) in enumerate(zip(self.camera_ids, self.movies)):
if md is not None:
self.mats.append(
Calibrate.makeUninitialisedMat(
ci, (md['vheight'], md['vwidth'])))
for md in self.movies:
vheights.append(md['vheight'])
vwidths.append(md['vwidth'])
Ps = interface.getPsFromMats(self.mats)
self.attrs = {
'vheight': vheights,
'vwidth': vwidths,
'camera_ids': self.camera_ids,
'Ps': Ps,
'mats': self.mats,
'colour': eval(attrs['colour'])
}
if self.camera_names:
self.attrs['camera_names'] = self.camera_names
self.initialised = True
return True