def cook(self, location, interface, attrs): imgs = interface.attr('jpegimgs') if imgs is None: return img_count = len(imgs) vwidth, vheight = attrs['vwidth'], attrs['vheight'] self.mats = interface.attr('mats', atLocation=attrs['cameraLocations']) updateMats = interface.attr('updateMats', atLocation=attrs['cameraLocations'], default=False) if self.mats is None or updateMats: self.mats = [] for i in xrange(img_count): self.mats.append( Calibrate.makeUninitialisedMat(i, (vheight, vwidth))) interface.setAttr('updateMats', False, atLocation=attrs['cameraLocations']) self.camAttrs = { 'vheight': [vheight] * img_count, 'vwidth': [vwidth] * img_count, 'camera_ids': range(img_count), 'mats': self.mats, 'updateImage': True, 'jpegimgs': imgs, 'updateMats': updateMats } for k, v in self.camAttrs.iteritems(): interface.setAttr(k, v)
def cook(self, location, interface, attrs): if not self.listeners: return import StringIO imgs, vwidths, vheights, camera_ids, camera_names, mats = [], [], [], [], [], [] ci = 0 for r, listener in self.listeners.iteritems(): data = listener.poll() if data is None: self.logger.error('No data on %d' % r) continue print data timeCode, imgStr = data sio = StringIO.StringIO(imgStr) img = PIL.Image.open(sio) if not self.initialised: vwidth, vheight = attrs['vwidth'], attrs['vheight'] mat = Calibrate.makeUninitialisedMat(0, (vheight, vwidth)) vwidths.append(vwidth) vheights.append(vheight) camera_ids.append('Camera %d' % ci) camera_names.append(str(r)) mats.append(mat) imgs.append(img.tobytes()) ci += 1 if not self.initialised: self.camAttrs['vheight'] = vheights self.camAttrs['vwidth'] = vwidths # self.camAttrs['camera_ids'] = camera_ids # self.camAttrs['camera_names'] = camera_names self.camAttrs['camera_ids'] = interface.attr('camera_ids') self.camAttrs['camera_names'] = camera_names self.camAttrs['mats'] = interface.attr('mats') self.initialised = True if imgs: self.camAttrs['imgs'] = imgs # self.camAttrs['updateImage'] = True interface.createChild(interface.name(), 'cameras', atLocation=interface.parentPath(), attrs=self.camAttrs) tcAttrs = { 'x3ds': np.array([[0, 0, 0]], dtype=np.float32), 'x3ds_labels': [timeCode] } interface.createChild(interface.name() + '/tc', 'points', atLocation=interface.parentPath(), attrs=tcAttrs)
def cook(self, location, interface, attrs): vwidth, vheight = attrs['vwidth'], attrs['vheight'] if self.cam is None: self.cam = CamVideoStream(src=0).start() self.mats = [Calibrate.makeUninitialisedMat(0, (vheight, vwidth))] self.camAttrs = { 'vheight': [vheight], 'vwidth': [vwidth], 'camera_ids': [0], 'mats': self.mats, 'updateImage': True } md = {'frame': self.cam.read()} self.camAttrs['imgs'] = [md['frame']] # self.attrs['imgs'] = np.array(frame, dtype=np.uint8) interface.createChild(interface.name(), 'cameras', atLocation=interface.parentPath(), attrs=self.camAttrs)
def cook(self, location, interface, attrs): if not self.useFrame(interface.frame(), attrs['frameRange']): interface.setAttr('updateMats', False) return # We need 2D data e.g. wand detections from a wand op # We need 3D wand data from e.g. c3d or a 3D wand detector dets_location = attrs['detections'] x3ds_location = attrs['x3ds'] if not dets_location or not x3ds_location: return # Get the 2D and 3D data x2ds = interface.attr('rx2ds', atLocation=dets_location) x2d_splits = interface.attr('x2ds_splits', atLocation=dets_location) x3ds = interface.attr('x3ds', atLocation=x3ds_location) if x2ds is None or x2d_splits is None or x3ds is None: return numCameras = len(x2d_splits) - 1 error_threshold = attrs['error_threshold'] # Get the data we've collected already so we can add to it frame = interface.frame() dets_colours = interface.attr('x2ds_colours', atLocation=dets_location) collectedDets = interface.attr('collect_rx2ds') collectedX3ds = interface.attr('collect_x3ds') lastFrame = interface.attr('lastFrame', [frame] * numCameras) emptyFrame3d = np.array([[]], dtype=np.float32).reshape(-1, 3) # This is potentially used by other ops so we only set it when we have some confidence # (and we might reset or tweak the values to indicate confidence levels at some point) cameraErrors = interface.attr('cameraErrors', [-1] * numCameras) # This is never modified to allow checking the camera rms values regardless of what we make of them rmsValues = interface.attr('rms', [-1] * numCameras) # Get the width and height for the videos vwidth = interface.attr('vwidth', [1920] * numCameras) vheight = interface.attr('vheight', [1080] * numCameras) # Get the frame mapping for x3ds x3ds_frames = interface.attr('x3ds_frames', {}) x2ds_frames = interface.attr('x2ds_frames', [[] for i in xrange(numCameras)]) # Get the camera matrices. We initialise them with default settings if we don't find any mats = interface.attr('mats', atLocation=location) if mats is None: mats = [] for ci in range(numCameras): mats.append(Calibrate.makeUninitialisedMat(ci, (vheight[ci], vwidth[ci]))) # Allow overriding the error threshold using an attribute (on the cooked location) error_threshold_attr = interface.attr('error_threshold') if error_threshold_attr is not None: error_threshold = error_threshold_attr Ps = interface.attr('Ps') if Ps is None: Ps = [np.array([], dtype=np.float32) for n in range(numCameras)] # Get the minimum number of samples we need to start solving distortion etc. as specified by the user minSamples = attrs['min_samples'] # Prepare the collected data for further processing (or initialise if nothing has been collected) if collectedDets is not None: c_x2ds, c_splits = collectedDets cams_collected = [c_x2ds[c0:c1] for ci, (c0, c1) in enumerate(zip(c_splits[:-1], c_splits[1:]))] else: cams_collected = [[] for ci, (c0, c1) in enumerate(zip(x2d_splits[:-1], x2d_splits[1:]))] collectedX3ds = [] for ci, (c0, c1) in enumerate(zip(x2d_splits[:-1], x2d_splits[1:])): collectedX3ds.append(emptyFrame3d) # Process each camera by looking for a wand and attempt a calibration. If we're happy with the results we'll # add it to our collection for ci, (c0, c1) in enumerate(zip(x2d_splits[:-1], x2d_splits[1:])): elapsed = frame - lastFrame[ci] if 0 < elapsed < attrs['jumpFrames']: continue # Get the 2Ds and 3Ds for the wand in this camera (if any) cameraDetections = x2ds[c0:c1] cameraX3ds = x3ds if not cameraDetections.any() or not cameraX3ds.any(): continue # Add the new detection to the existing collection as a candidate for a new collection if cams_collected[ci] is None or len(cams_collected[ci]) == 0: proposalDets, proposalX3ds = cameraDetections, cameraX3ds else: proposalDets = np.concatenate((cams_collected[ci], cameraDetections)) proposalX3ds = np.concatenate((collectedX3ds[ci], cameraX3ds)) # Check if we want to solve for distortion and focal length by looking at the number of samples # we've got already compared to our minimum number of samples required numSamples = len(proposalDets) / 5 # if numSamples == minSamples: self.logger.info('Camera %d reached min samples of %d' % (ci, minSamples)) solveTrigger = True if numSamples > minSamples else False solve_focal_length = attrs['solve_focal_length'] if solveTrigger else False solve_distortion = attrs['solve_distortion'] if solveTrigger else False # The wand is assumed to have 5 points so we make sure we've got at least one wand before attempting # to calibrate if len(proposalDets) >= 5 and len(proposalX3ds) >= 5: P, ks, rms = Calibrate.cv2_solve_camera_from_3d(proposalX3ds, proposalDets, solve_focal_length=solve_focal_length, solve_distortion=solve_distortion) if ks[0] < -3. or ks[0] > 3.: ks[0] = 0. if ks[1] < -3. or ks[1] > 3.: ks[1] = 0. # This shouldn't' happen but if we lose confidence in the camera we can visualise it # by resetting the camera error (this will change the colour in the UI) if rms > error_threshold: cameraErrors[ci] = -1 continue # See how the rms for the calibration compares to the last recorded value for this camera prevRms = rms if rmsValues[ci] == -1 else rmsValues[ci] rmsDelta = rms - prevRms # If the rms is lower than the last recorded error for this camera then # we want to keep this data if rmsDelta <= 0 or not solveTrigger: cams_collected[ci] = proposalDets collectedX3ds[ci] = proposalX3ds if frame not in x3ds_frames: x3ds_frames[frame] = proposalX3ds[-5:] x2ds_frames[ci] += ([frame] * 5) else: continue # Record the rms value for the camera rmsValues[ci] = rms # Once we've solved for distortion etc. we are more confident with the accuracy of our # error so we start reporting it, where the value can be used for visualiation etc. if solveTrigger: cameraErrors[ci] = rms lastFrame[ci] = frame # Everything has gone well so far so we create and add the new camera matrix mat = Calibrate.makeMat(P, ks, (vheight[ci], vwidth[ci])) mats[ci] = mat Ps[ci] = P # Concatenate the results from all the cameras cams = [np.concatenate((cc)) for cc in cams_collected if len(cc)] if not cams: # We haven't found a wand in any camera so we just keep calm and return return # Build our collections and write to the interface collectedDets = np.array(np.concatenate(cams), dtype=np.float32).reshape(-1, 2), \ Interface.makeSplitBoundaries(map(len, cams_collected)) interface.setAttr('collect_rx2ds', collectedDets) interface.setAttr('collect_x3ds', collectedX3ds) interface.setAttr('x2ds_frames', x2ds_frames) interface.setAttr('x3ds_frames', x3ds_frames) interface.setAttr('lastFrame', lastFrame) # Write the calibration data to the interface and request an update at render time interface.setAttr('mats', mats) interface.setAttr('Ps', Ps) interface.setAttr('rms', rmsValues) interface.setAttr('cameraErrors', cameraErrors) interface.setAttr('updateMats', True) # Optionally display all the collected wand detections if 'showDetections' in attrs and attrs['showDetections']: colours = np.tile(dets_colours, (len(collectedDets[0]) / 5, 1)) allAttrs = {'x2ds': collectedDets[0], 'x2ds_splits': collectedDets[1], 'x2ds_colours': colours} interface.createChild('collected', 'detections', attrs=allAttrs)
def load_xcp_and_x2d(xcp_filename, x2d_filename, raw=False): '''Load an x2d, xcp pair and make a valid data structure. The returned cameras are in the order of the x2d file -- as it happens, this is in order of deviceid. If any particular camera is not in the xcp then it's initialised on the positive x-axis. If a camera is only in the xcp then it is discarded.''' from GCore import Calibrate print ('loading xcp') vicon_mats,xcp_data = loadXCP(xcp_filename) xcp_camera_ids = np.array([int(x['DEVICEID']) for x in xcp_data],dtype=np.int32) camera_names = ['%s:%s'%(x['LABEL'],x['DEVICEID']) for x in xcp_data] camera_vicon_errors = np.array([x['IMAGE_ERROR'] for x in xcp_data],dtype=np.float32) #print (camera_names) #print ('vicon_errors',camera_vicon_errors,np.min(camera_vicon_errors),np.max(camera_vicon_errors),np.mean(camera_vicon_errors)) print ('loading x2d',x2d_filename) x2d_dict = loadX2D(x2d_filename) cameras_info = extractCameraInfo(x2d_dict) # W,H,ID per camera x2d_cids = cameras_info[:,2] x2ds = [(x[0][:,:2].copy(),x[1]) if raw else frameCentroidsToDets(x, vicon_mats) for x in x2d_dict['frames']] if not(np.all(xcp_camera_ids == x2d_cids)): print ('WARNING not all the cameras from the x2d were in the xcp file?') # TODO, we should report this print (xcp_camera_ids, x2d_cids) vicon_mats = [vicon_mats[list(xcp_camera_ids).index(ci)] if ci in list(xcp_camera_ids) else Calibrate.makeUninitialisedMat(ci,(w,h)) for w,h,ci in cameras_info] camera_names = ['CAM_%s'%x for x in x2d_cids] xcp_camera_ids = [f for f in x2d_cids] Ps = np.array([m[2]/(np.sum(m[2][0,:3]**2)**0.5) for m in vicon_mats],dtype=np.float32) headerMetadata = readX2DMetadata(x2d_dict) return Ps, vicon_mats, x2d_cids, camera_names, x2ds, x2d_dict['header']
def initialise(self, interface, attrs): directory = self.resolvePath(attrs['directory']) if not directory: return False prefix = attrs['prefix'] prefixFilename = self.resolvePath(attrs['prefixFilename']) if prefix and not prefixFilename: return calibration = attrs['calibration'] calibrationFilename = self.resolvePath(attrs['calibrationFilename']) calibrationLocation = self.resolvePath(attrs['calibrationLocation']) if calibration and (not calibrationFilename and not calibrationLocation): return False movieFilenames = [] try: for file in os.listdir(directory): if prefixFilename and not file.startswith(prefixFilename): continue if file.endswith('.avi') or file.endswith( '.mov') or file.endswith('mp4'): movieFilenames.append(os.path.join(directory, file)) except WindowsError as e: self.logger.error('Could not find videos: % s' % str(e)) if not movieFilenames: # TODO: Here we'll have to clear the cameras etc. return False # Windows will produce a wonky order, i.e. 1, 10, 11, .., 2, 3, .. # Use natural sorting to rectify movieFilenames.sort(key=self.alphaNumKey) self.camera_ids = [] self.camera_names = [] self.movies = [] self.mats = [] vheights = [] vwidths = [] timecodes = [] hasTimecode = False useTimecode = attrs['useTimecode'] if 'useTimecode' in attrs else True offset = attrs['offset'] if 'offsets' in attrs and attrs['offsets']: offsets = eval(attrs['offsets']) else: offsets = [offset] * len(movieFilenames) for ci, mf in enumerate(movieFilenames): self.logger.info('Loading MovieReader: %s' % mf) movieData = MovieReader.open_file(mf, audio=False, frame_offset=offsets[ci]) if movieData['vbuffer'] is not None: self.movies.append(movieData) self.timecodeOffsets.append(0) if 'timecode' in movieData and movieData['timecode']: hasTimecode = True timecodes.append(movieData['timecode']) # Make sure we have all the cameras before continuing if len(self.movies) != len(movieFilenames): self.logger.error('Could not load all movies in sequence') return # Make sure we have as many time codes as movies (if we have any) if hasTimecode and len(self.movies) != len(timecodes): self.logger.error('Not all movie files have a time code') return # See if we can get the offsets using the time codes if hasTimecode and useTimecode: print 'Video timecodes:', timecodes fps_all = [round(m['fps']) for m in self.movies] print 'FPS:', fps_all timecodeValues = [ Timecode.TCFtoInt(tc, fps) for tc, fps in zip(timecodes, fps_all) ] tcOrderDesc = [ timecodes.index(tc) for tc in sorted(timecodes, reverse=True) ] # Set the first offset to 0 firstTcIndex = tcOrderDesc[0] self.timecodeOffsets[firstTcIndex] = 0 largestTc = timecodes[firstTcIndex] offsetStartIndex = 1 # We can also get the timecode destination from an incoming location, e.g. 2D detections if 'timecodeLocation' in attrs and attrs['timecodeLocation']: tcSyncTime = interface.attr( 'timecode', atLocation=attrs['timecodeLocation']) if tcSyncTime is not None: tcSyncValue = Timecode.TCFtoInt(tcSyncTime, fps_all[0]) if tcSyncValue < timecodeValues[firstTcIndex]: self.logger.error( 'Sync timecode %s is smaller than video timecodes (%s).' % (tcSyncTime, largestTc)) return largestTc = tcSyncTime offsetStartIndex = 0 self.timecode = largestTc self.logger.info('Setting timecode to: %s' % (largestTc)) # Calculate the offset for each camera to get it up to speed with the target timecode # TODO: Replace hard coded timecode fps and multiplier timecodeFps, timecodeMultiplier = 25., 2. for tcInd in tcOrderDesc[offsetStartIndex:]: diff = Timecode.TCSub(largestTc, timecodes[tcInd], timecodeFps) self.timecodeOffsets[tcInd] = Timecode.TCFtoInt( diff, timecodeFps) * timecodeMultiplier if self.timecodeOffsets: print 'Video timecode offsets:', self.timecodeOffsets self.camera_ids = [ 'Camera %d' % ci for ci in xrange(len(movieFilenames)) ] self.movies = self.movies if not calibrationLocation: calibrationLocation = interface.root() if calibrationFilename or interface.hasAttr( 'mats', atLocation=calibrationLocation): if calibrationFilename: # TODO: Detect filetype, e.g. .cal and .xcp and handle accordingly try: self.mats, rawCalData = OptitrackReader.load_CAL( calibrationFilename) if not self.mats: return False except IOError as e: self.logger.error('Could not load calibration file: %s' % str(e)) return False else: self.mats = interface.attr('mats', atLocation=calibrationLocation) if not self.mats: self.logger.error('Could not find calibration mats: %s' % calibrationLocation) return False else: from GCore import Calibrate for ci, (cid, md) in enumerate(zip(self.camera_ids, self.movies)): if md is not None: self.mats.append( Calibrate.makeUninitialisedMat( ci, (md['vheight'], md['vwidth']))) for md in self.movies: vheights.append(md['vheight']) vwidths.append(md['vwidth']) Ps = interface.getPsFromMats(self.mats) self.attrs = { 'vheight': vheights, 'vwidth': vwidths, 'camera_ids': self.camera_ids, 'Ps': Ps, 'mats': self.mats, 'colour': eval(attrs['colour']) } if self.camera_names: self.attrs['camera_names'] = self.camera_names self.initialised = True return True