def create_trajectory (self, nframes, h5filebase, start, finish): # need to make some kalman rows! # kalman rows = [obj_id, frame, time of acquisition, positions[3], velocities[3], other crap] self.n_artificial_trajecs = self.n_artificial_trajecs + 1 # get extra, pretty much just for the time model ca = core_analysis.get_global_CachingAnalyzer() (obj_ids, use_obj_ids, is_mat_file, data_file, extra) = ca.initial_file_load(h5filebase) start = np.array(start) finish = np.array(finish) mean_vel = (finish - start) delta_t = extra['time_model'].framestamp2timestamp(2) - extra['time_model'].framestamp2timestamp(1) kalman_rows = np.zeros([nframes, 9]) for n in range(nframes): kalman_rows[n,0] = 1 kalman_rows[n,1] = n kalman_rows[n,2] = 0 kalman_rows[n,6:9] = mean_vel kalman_rows[n,3:6] = start + mean_vel*delta_t traj_id = (str('artificial'+'_'+str(self.n_artificial_trajecs))) self.trajecs.setdefault(traj_id, Trajectory(kalman_rows, extra, stimulus = self.stimulus) )
def load_data(self, filename, kalman_smoothing=False, dynamic_model=None, fps=None, info={}, save_covariance=False): # use info to pass information to trajectory instances as a dictionary. # eg. info={"post_type": "black"} # save_covariance: set to True if you need access to the covariance data. Keep as False if this is not important for analysis (takes up lots of space) # set up analyzer ca = core_analysis.get_global_CachingAnalyzer() (obj_ids, use_obj_ids, is_mat_file, data_file, extra) = ca.initial_file_load(filename) data_file.flush() # data set defaults if fps is None: fps = result_utils.get_fps(data_file) if dynamic_model is None: try: dyn_model = extra['dynamic_model_name'] except: print 'cannot find dynamic model' print 'using EKF mamarama, units: mm' dyn_model = 'EKF mamarama, units: mm' if dynamic_model is not None: dyn_model = dynamic_model # if kalman smoothing is on, then we cannot use the EKF model - remove that from the model name print '** Kalman Smoothing is: ', kalman_smoothing, ' **' if kalman_smoothing is True: dyn_model = dyn_model[4:] print 'using dynamic model: ', dyn_model print 'framerate: ', fps print 'loading data.... ' self.dynamic_model = dyn_model self.fps = fps # load object id's and save as Trajectory instances for obj_id in use_obj_ids: print 'processing: ', obj_id try: print obj_id kalman_rows = ca.load_data( obj_id, data_file, dynamic_model_name = dyn_model, use_kalman_smoothing= kalman_smoothing, frames_per_second= fps) except: print 'object id failed to load (probably no data): ', obj_id continue # couple object ID dictionary with trajectory objects trajec_id = str(obj_id) # this is not necessarily redundant with the obj_id, it allows for making a unique trajectory id when merging multiple datasets tmp = Trajectory(trajec_id, kalman_rows, info=info, fps=fps, save_covariance=save_covariance, extra=extra) self.trajecs.setdefault(trajec_id, tmp) return
def consider_stimulus(h5file, verbose_problems=False, fanout_name="fanout.xml"): """ Parses the corresponding fanout XML and finds IDs to use as well as the stimulus. Returns 3 values: valid, use_objs_ids, stimulus. valid is false if something was wrong """ try: dirname = os.path.dirname(h5file) fanout_xml = os.path.join(dirname, fanout_name) if not(os.path.exists(fanout_xml)): if verbose_problems: logger.error("Stim_xml path not found '%s' for file '%s'" % (h5file, fanout_xml)) return False, None, None ca = core_analysis.get_global_CachingAnalyzer() (_, use_obj_ids, _, _, _) = ca.initial_file_load(h5file) file_timestamp = timestamp_string_from_filename(h5file) fanout = xml_stimulus.xml_fanout_from_filename(fanout_xml) include_obj_ids, exclude_obj_ids = \ fanout.get_obj_ids_for_timestamp(timestamp_string=file_timestamp) if include_obj_ids is not None: use_obj_ids = include_obj_ids if exclude_obj_ids is not None: use_obj_ids = list(set(use_obj_ids).difference(exclude_obj_ids)) episode = fanout._get_episode_for_timestamp( timestamp_string=file_timestamp) (_, _, stim_fname) = episode return True, use_obj_ids, stim_fname except xml_stimulus.WrongXMLTypeError: if verbose_problems: logger.error("Caught WrongXMLTypeError for '%s'" % file_timestamp) return False, None, None except ValueError, ex: if verbose_problems: logger.error("Caught ValueError for '%s': %s" % (file_timestamp, ex)) return False, None, None
def __init__(self,kalman_filename): import flydra.a2.core_analysis as core_analysis import flydra.analysis.result_utils as result_utils self.ca = core_analysis.get_global_CachingAnalyzer() (obj_ids, use_obj_ids, is_mat_file, data_file, extra) = self.ca.initial_file_load(kalman_filename) self.data_file = data_file self.up_dir = None if 1: dynamic_model = extra['dynamic_model_name'] print 'detected file loaded with dynamic model "%s"'%dynamic_model if dynamic_model.startswith('EKF '): dynamic_model = dynamic_model[4:] print ' for smoothing, will use dynamic model "%s"'%dynamic_model self.dynamic_model = dynamic_model self.fps = result_utils.get_fps( data_file ) self.use_kalman_smoothing = True self.up_dir = (0,0,1)
def test(self, filename, kalman_smoothing=False, dynamic_model=None, fps=None, info={}, save_covariance=False): ca = core_analysis.get_global_CachingAnalyzer() (obj_ids, use_obj_ids, is_mat_file, data_file, extra) = ca.initial_file_load(filename) return ca, obj_ids, use_obj_ids, is_mat_file, data_file, extra
def __init__(self, h5): self._ca = core_analysis.get_global_CachingAnalyzer() obj_ids, use_obj_ids, is_mat_file, data_file, extra = self._ca.initial_file_load( h5) x = [] y = [] z = [] for obj_id_enum, obj_id in enumerate(use_obj_ids): rows = self._ca.load_data(obj_id, data_file, use_kalman_smoothing=False) verts = np.array([rows['x'], rows['y'], rows['z']]).T x.append(verts[:, 0]) y.append(verts[:, 1]) z.append(verts[:, 2]) x = np.concatenate(x) y = np.concatenate(y) z = np.concatenate(z) coords = np.array([x, y, z]).T self.pubcyl = None self.pubpt = None self.p = pcl.PointCloud() self.p.from_array(coords) self.p.to_file('/tmp/fuckit.pcd') self.np = len(coords) self._cx, self._cy, self._cz, self._ax, self._ay, self._az, self._radius = ( 0, 0, 0, 0, 0, 0, 0) self.pubpts, pts = calib.visualization.create_point_cloud_message_publisher( coords, topic_name='/flydracalib/points', publish_now=True, latch=True) self.pubptsinliers = calib.visualization.create_point_cloud_message_publisher( [(0, 0, 0)], topic_name='/flydracalib/pointsinliers', publish_now=False, latch=True) self.pubcyl = calib.visualization.create_cylinder_publisher( self._cx, self._cy, self._cz, self._ax, self._ay, self._az, self._radius, topic_name='/flydracalib/cyl', publish_now=False, latch=True, length=5, color=(0, 1, 1, 0.4)) self.pubpt = calib.visualization.create_point_publisher( self._cx, self._cy, self._cz, r=0.1, topic_name='/flydracalib/cylcenter', publish_now=False, latch=True, color=(1, 0, 0, 0.5)) thisdir = os.path.dirname(os.path.abspath(__file__)) self.ui = Gtk.Builder() self.ui.add_from_file(os.path.join(thisdir, "tune.ui")) self._ks = -1.0 self.ui.get_object("ks_adjustment").props.value = self._ks self._sr = 0.5 self.ui.get_object("sr_adjustment").props.value = self._sr self._nd = 0.5 self.ui.get_object("nd_adjustment").props.value = self._nd self._dt = 0.10 self.ui.get_object("dt_adjustment").props.value = self._dt self._minr = 0.5 self.ui.get_object("minr_adjustment").props.value = self._minr self._maxr = 1.5 self.ui.get_object("maxr_adjustment").props.value = self._maxr self._exbtn = self.ui.get_object("execute_btn") self._exbtn.connect("clicked", self._calculate) self._axadj = self.ui.get_object("ax_adjustment") self._ayadj = self.ui.get_object("ay_adjustment") self._azadj = self.ui.get_object("az_adjustment") self._cxadj = self.ui.get_object("cx_adjustment") self._cyadj = self.ui.get_object("cy_adjustment") self._czadj = self.ui.get_object("cz_adjustment") self._radj = self.ui.get_object("r_adjustment") self._in = self.ui.get_object("inliersentry") self.ui.connect_signals(self) w = self.ui.get_object("window1") w.connect("delete-event", rosgobject.main_quit) w.show_all()
def load_data(self, filename, kalman_smoothing=False, dynamic_model=None, fps=None, info={}, save_covariance=False, experiment_length=19*60*60): # use info to pass information to trajectory instances as a dictionary. # eg. info={"post_type": "black"} # save_covariance: set to True if you need access to the covariance data. Keep as False if this is not important for analysis (takes up lots of space) # set up analyzer ca = core_analysis.get_global_CachingAnalyzer() (obj_ids, use_obj_ids, is_mat_file, data_file, extra) = ca.initial_file_load(filename) data_file.flush() # data set defaults if fps is None: fps = result_utils.get_fps(data_file) if dynamic_model is None: try: dyn_model = extra['dynamic_model_name'] except: print 'cannot find dynamic model' print 'using EKF mamarama, units: mm' dyn_model = 'EKF mamarama, units: mm' if dynamic_model is not None: dyn_model = dynamic_model # if kalman smoothing is on, then we cannot use the EKF model - remove that from the model name print '** Kalman Smoothing is: ', kalman_smoothing, ' **' if kalman_smoothing is True: dyn_model = dyn_model[4:] print 'using dynamic model: ', dyn_model print 'framerate: ', fps print 'loading data.... ' self.dynamic_model = dyn_model self.fps = fps time_start = None # load object id's and save as Trajectory instances for obj_id in use_obj_ids: print 'processing: ', obj_id try: print obj_id kalman_rows = ca.load_data( obj_id, data_file, dynamic_model_name = dyn_model, use_kalman_smoothing= kalman_smoothing, frames_per_second= fps) except: print 'object id failed to load (probably no data): ', obj_id continue # couple object ID dictionary with trajectory objects filenamebase = os.path.basename(filename) trajecbase = filenamebase.rstrip('5').rstrip('.h').lstrip('DATA').rstrip('.kalmanized') trajec_id = trajecbase + '_' + str(obj_id) # filename details + original object id - this is unique tmp = Trajectory(trajec_id, kalman_rows, info=info, fps=fps, save_covariance=save_covariance, extra=extra) tmp.h5 = os.path.basename(filename) if time_start is None: time_start = tmp.timestamp_epoch else: if tmp.timestamp_epoch - time_start > experiment_length: break else: self.trajecs.setdefault(trajec_id, tmp) self.h5_files_loaded.append(os.path.basename(filename)) return
def load_data (self, filename, calibration_file = None, objs = None, obj_filelist = None, kalman_smoothing = True, fps = None, dynamic_model = None, load_2d=True,): self.datasets.append(len(self.datasets)+1) # raw h5 - this is the 2d raw camera data: if load_2d is True: h5 = PT.openFile( filename, mode='r' ) camn2cam_id, cam_id2camns = result_utils.get_caminfo_dicts(h5) cam_ids = cam_id2camns.keys() cams2d = [] for cam_id in cam_ids: cams2d.append( Cam2d(cam_id, h5) ) self.cams2d.append(cams2d) # set up analyzer ca = core_analysis.get_global_CachingAnalyzer() (obj_ids, use_obj_ids, is_mat_file, data_file, extra) = ca.initial_file_load(filename) # data set defaults if fps is None: fps = result_utils.get_fps(data_file) if dynamic_model is None: try: dyn_model = extra['dynamic_model_name'] except: dyn_model = 'EKF mamarama, units: mm' if dynamic_model is not None: dyn_model = dynamic_model # if kalman smoothing is on, then we cannot use the EKF model - remove that from the model name print '**Kalman Smoothing is: ', kalman_smoothing, ' **' if kalman_smoothing is True: dyn_model = dyn_model[4:] print 'using dynamic model: ', dyn_model if objs is None and obj_filelist is None: print "running through all object id's, this might take a while..." obj_only = use_obj_ids # this is all the unique object id's if obj_filelist is not None: tmp = np.loadtxt(obj_filelist,delimiter=',') obj_only = np.array(tmp[:,0], dtype='int') elif objs is not None: obj_only = np.array(objs) print 'loading data.... ' for obj_id in obj_only: try: kalman_rows = ca.load_data( obj_id, data_file, dynamic_model_name = dyn_model, use_kalman_smoothing= kalman_smoothing, frames_per_second= fps) except: print 'object id failed to load (probably no data): ', obj_id continue # couple object ID dictionary with trajectory objects traj_id = (str(self.datasets[-1])+'_'+str(obj_id)) self.trajecs.setdefault(traj_id, ffa.Trajectory(kalman_rows, extra, stimulus = self.stimulus, fps = fps) )
def __init__(self, h5): self._ca = core_analysis.get_global_CachingAnalyzer() obj_ids, use_obj_ids, is_mat_file, data_file, extra = self._ca.initial_file_load(h5) x = [] y = [] z = [] for obj_id_enum,obj_id in enumerate(use_obj_ids): rows = self._ca.load_data(obj_id, data_file, use_kalman_smoothing=False) verts = np.array( [rows['x'], rows['y'], rows['z']] ).T x.append( verts[:,0] ) y.append( verts[:,1] ) z.append( verts[:,2] ) x = np.concatenate(x) y = np.concatenate(y) z = np.concatenate(z) coords = np.array([x,y,z]).T self.pubcyl = None self.pubpt = None self.p = pcl.PointCloud() self.p.from_array(coords) self.p.to_file('/tmp/fuckit.pcd') self.np = len(coords) self._cx,self._cy,self._cz,self._ax,self._ay,self._az,self._radius = (0,0,0,0,0,0,0) self.pubpts, pts = calib.visualization.create_point_cloud_message_publisher( coords, topic_name='/flydracalib/points', publish_now=True, latch=True) self.pubptsinliers = calib.visualization.create_point_cloud_message_publisher( [(0,0,0)], topic_name='/flydracalib/pointsinliers', publish_now=False, latch=True) self.pubcyl = calib.visualization.create_cylinder_publisher( self._cx,self._cy,self._cz,self._ax,self._ay,self._az,self._radius, topic_name='/flydracalib/cyl', publish_now=False, latch=True, length=5, color=(0,1,1,0.4)) self.pubpt = calib.visualization.create_point_publisher( self._cx,self._cy,self._cz, r=0.1, topic_name='/flydracalib/cylcenter', publish_now=False, latch=True, color=(1,0,0,0.5)) thisdir = os.path.dirname(os.path.abspath(__file__)) self.ui = Gtk.Builder() self.ui.add_from_file(os.path.join(thisdir,"tune.ui")) self._ks = -1.0 self.ui.get_object("ks_adjustment").props.value = self._ks self._sr = 0.5 self.ui.get_object("sr_adjustment").props.value = self._sr self._nd = 0.5 self.ui.get_object("nd_adjustment").props.value = self._nd self._dt = 0.10 self.ui.get_object("dt_adjustment").props.value = self._dt self._minr = 0.5 self.ui.get_object("minr_adjustment").props.value = self._minr self._maxr = 1.5 self.ui.get_object("maxr_adjustment").props.value = self._maxr self._exbtn = self.ui.get_object("execute_btn") self._exbtn.connect("clicked", self._calculate) self._axadj = self.ui.get_object("ax_adjustment") self._ayadj = self.ui.get_object("ay_adjustment") self._azadj = self.ui.get_object("az_adjustment") self._cxadj = self.ui.get_object("cx_adjustment") self._cyadj = self.ui.get_object("cy_adjustment") self._czadj = self.ui.get_object("cz_adjustment") self._radj = self.ui.get_object("r_adjustment") self._in = self.ui.get_object("inliersentry") self.ui.connect_signals(self) w = self.ui.get_object("window1") w.connect("delete-event", rosgobject.main_quit) w.show_all()
def get_good_smoothed_tracks(filename, obj_ids, min_frames_per_track, use_smoothing, dynamic_model_name): ''' Yields (obj_id, rows) for each track in obj_ids in the file that has the given minimum number of frames. ''' frames_per_second = 60.0 dt = 1 / frames_per_second ca = core_analysis.get_global_CachingAnalyzer() #warned = False #obj_ids, unique_obj_ids, is_mat_file, data_file, extra = \ # ca.initial_file_load(filename) data_file = filename for obj_id in obj_ids: try: frows = ca.load_data(obj_id, data_file, use_kalman_smoothing=False) # don't consider tracks too small if len(frows) < min_frames_per_track: continue # write timestamp entry # The 'timestamp' field returned by flydra is the time # when the computation was made, not the actual data timestamp. # For computing the actual timestamp, use the frame number # and multiply by dt global warned_fixed_dt if not warned_fixed_dt: warned_fixed_dt = True logger.info('Warning: We are assuming that the data is ' \ 'equally spaced, and fps = %s.' % frames_per_second) for i in range(len(frows)): frows['timestamp'][i] = frows['frame'][i] * dt for i in range(len(frows) - 1): if frows['obj_id'][i] == frows['obj_id'][i + 1]: if not (frows['timestamp'][i] < frows['timestamp'][i + 1]): print("fishy behavior at index %d" % i) # return raw data if smoothing is not requested if not use_smoothing: yield obj_id, frows continue # otherwise, run the smoothing srows = ca.load_data(obj_id, data_file, use_kalman_smoothing=True, frames_per_second=frames_per_second, dynamic_model_name=dynamic_model_name); # make a copy, just in case srows = srows.copy() for i in range(len(srows)): srows['timestamp'][i] = srows['frame'][i] * dt # From Andrew: # I'm pretty sure there is an inconsistency in some of this # unit stuff. Basically, I used to do the camera calibrations # all in mm (so that the 3D coords would come out in mm). Then, # I started doing analyses in meters... And I think some of # the calibration and dynamic model stuff got defaulted to meters. # And basically there are inconsistencies in there. # Anyhow, I think the extent of the issue is that you'll be off # by 1000, so hopefully you can just determine that by looking at the data. # quick fix if dynamic_model_name == "mamarama, units: mm": #and not warned: #warned = True # logger.info("Warning: Implementing simple workaround for flydra's " \ # "units inconsistencies (multiplying xvel,yvel by 1000).") #srows['xvel'] *= 1000 #srows['yvel'] *= 1000 #srows['xvel'] *= 1000 v = numpy.hypot(srows['xvel'], srows['yvel'], srows['zvel']) perc = [1, 5, 50, 95, 99] scores = map(lambda x: scipy.stats.scoreatpercentile(v, x), perc) print "scores: %s" % map(lambda x: "%f " % x, scores) score95 = scipy.stats.scoreatpercentile(v, 95) if score95 < 100.0: logger.debug(" score95 = %f, assuming m" % score95) else: logger.debug(" score95 = %f, assuming mm" % score95) srows['xvel'] *= 0.001 srows['yvel'] *= 0.001 srows['xvel'] *= 0.001 v = numpy.hypot(srows['xvel'], srows['yvel'], srows['zvel']) final_score95 = scipy.stats.scoreatpercentile(v, 95) logger.info('After much deliberation, 95%% score is %f.' % final_score95) yield obj_id, srows except core_analysis.NotEnoughDataToSmoothError: logger.warning('not enough data to smooth obj_id %d, skipping.' % (obj_id,)) continue ca.close()
def main(): np.seterr(all='raise') parser = OptionParser() parser.add_option("--output_dir", default='saccade_detect_output', help="Output directory") parser.add_option("--min_frames_per_track", default=400, help="Minimum number of frames per track [= %default]") parser.add_option("--confirm_problems", help="Stop interactively on problems with log files'\ '(e.g.: cannot find valid obj_ids) [default: %default]", default=False, action="store_true") parser.add_option("--dynamic_model_name", help="Smoothing dynamical model [default: %default]", default="mamarama, units: mm") parser.add_option("--debug_output", help="Creates debug figures.", default=False, action="store_true") parser.add_option("--nocache", help="Ignores already computed results.", default=False, action="store_true") parser.add_option("--smoothing", help="Uses Kalman-smoothed data.", default=False, action="store_true") # detection parameters dt = 1.0 / 60 parser.add_option("--deltaT_inner_sec", default=4 * dt, type='float', help="Inner interval [= %default]") parser.add_option("--deltaT_outer_sec", default=10 * dt, type='float', help="Outer interval [= %default]") parser.add_option("--min_amplitude_deg", default=25, type='float', help="Minimum saccade amplitude (deg) [= %default]") parser.add_option("--min_linear_velocity", default=0.1, type='float', help="Minimum linear velocity when saccading (m/s) " "[= %default]") parser.add_option("--max_linear_acceleration", default=20, type='float', help="Maximum linear acceleration when saccading " "(m/s^2) [= %default]") parser.add_option("--max_angular_velocity", default=8000, type='float', help="Maximum angular velocity when saccading (deg/s) " "[= %default]") parser.add_option("--max_orientation_dispersion_deg", default=15, type='float', help="Maximum dispersion (deg) [= %default]") parser.add_option("--minimum_interval_sec", default=10 * dt, type='float', help="Minimum interval between saccades. [= %default]") (options, args) = parser.parse_args() if not args: logger.error('No files or directories specified.') sys.exit(-1) # Create processed string processed = 'geometric_saccade_detector %s %s %s@%s Python %s' % \ (__version__, datetime.now().strftime("%Y%m%d_%H%M%S"), get_user(), platform.node(), platform.python_version()) if not os.path.exists(options.output_dir): os.makedirs(options.output_dir) good_files = get_good_files(where=args, pattern="*.kh5", confirm_problems=options.confirm_problems) if len(good_files) == 0: logger.error("No good files to process.") sys.exit(1) try: n = len(good_files) for i in range(n): (filename, obj_ids, stim_fname) = good_files[i] # only maintain basename stim_fname = os.path.splitext(os.path.basename(stim_fname))[0] basename = os.path.splitext(os.path.basename(filename))[0] output_basename = os.path.join(options.output_dir, basename + '-saccades') output_saccades_hdf = output_basename + '.h5' if os.path.exists(output_saccades_hdf) and not options.nocache: logger.info('File %r exists; skipping. ' '(use --nocache to ignore)' % output_saccades_hdf) continue logger.info("File %d/%d %s %s %s " % (i, n, str(filename), str(obj_ids), stim_fname)) # concatenate all in one track all_data = None for _, rows in get_good_smoothed_tracks( filename=filename, obj_ids=obj_ids, min_frames_per_track=options.min_frames_per_track, dynamic_model_name=options.dynamic_model_name, use_smoothing=options.smoothing): all_data = rows.copy() if all_data is None \ else np.concatenate((all_data, rows)) if all_data is None: logger.info('Not enough data found for %s; skipping.' % filename) continue params = { 'deltaT_inner_sec': options.deltaT_inner_sec, 'deltaT_outer_sec': options. deltaT_outer_sec, 'min_amplitude_deg': options.min_amplitude_deg, 'max_orientation_dispersion_deg': options.max_orientation_dispersion_deg, 'minimum_interval_sec': options.minimum_interval_sec, 'max_linear_acceleration': options.max_linear_acceleration, 'min_linear_velocity': options.min_linear_velocity, 'max_angular_velocity': options.max_angular_velocity, } saccades, annotated_data = geometric_saccade_detect(all_data, params) for saccade in saccades: check_saccade_is_well_formed(saccade) # other fields used for managing different samples, # used in the analysis saccades['species'] = 'Dmelanogaster' saccades['stimulus'] = stim_fname sample_name = 'DATA' + timestamp_string_from_filename(filename) saccades['sample'] = sample_name saccades['sample_num'] = -1 # will be filled in by someone else saccades['processed'] = processed logger.info("Writing to %s {h5,mat,pickle}" % output_basename) saccades_write_all(output_basename, saccades) # Write debug figures if options.debug_output: debug_output_dir = os.path.join(options.output_dir, basename) logger.info("Writing HTML+png to %s" % debug_output_dir) write_debug_output(debug_output_dir, basename, annotated_data, saccades) except Exception as e: logger.error('Error while processing. Exception and traceback follow.') logger.error(str(e)) logger.error(traceback.format_exc()) sys.exit(-2) finally: print('Closing flydra cache') ca = core_analysis.get_global_CachingAnalyzer() ca.close() sys.exit(0)
def get_good_smoothed_tracks(filename, obj_ids, min_frames_per_track, use_smoothing, dynamic_model_name): ''' Yields (obj_id, rows) for each track in obj_ids in the file that has the given minimum number of frames. ''' frames_per_second = 60.0 dt = 1 / frames_per_second ca = core_analysis.get_global_CachingAnalyzer() warned = False #obj_ids, unique_obj_ids, is_mat_file, data_file, extra = \ # ca.initial_file_load(filename) data_file = filename for obj_id in obj_ids: try: frows = ca.load_data(obj_id, data_file, use_kalman_smoothing=False) # don't consider tracks too small if len(frows) < min_frames_per_track: continue # write timestamp entry # The 'timestamp' field returned by flydra is the time # when the computation was made, not the actual data timestamp. # For computing the actual timestamp, use the frame number # and multiply by dt global warned_fixed_dt if not warned_fixed_dt: warned_fixed_dt = True logger.info('Warning: We are assuming that the data is ' \ 'equally spaced, and fps = %s.' % frames_per_second) for i in range(len(frows)): frows['timestamp'][i] = frows['frame'][i] * dt for i in range(len(frows) - 1): if frows['obj_id'][i] == frows['obj_id'][i + 1]: assert frows['timestamp'][i] < frows['timestamp'][i + 1] # return raw data if smoothing is not requested if not use_smoothing: yield (obj_id, extract_interesting_fields(frows, np.dtype(rows_dtype))) continue # otherwise, run the smoothing srows = ca.load_data(obj_id, data_file, use_kalman_smoothing=True, frames_per_second=frames_per_second, dynamic_model_name=dynamic_model_name) # make a copy, just in case srows = srows.copy() for i in range(len(srows)): srows['timestamp'][i] = srows['frame'][i] * dt # From Andrew: # I'm pretty sure there is an inconsistency in some of this # unit stuff. Basically, I used to do the camera calibrations # all in mm (so that the 3D coords would come out in mm). Then, # I started doing analyses in meters... And I think some of # the calibration and dynamic model stuff got defaulted to meters. # And basically there are inconsistencies in there. # Anyhow, I think the extent of the issue is that you'll be off # by 1000, so hopefully you can just determine that by looking # at the data. # quick fix if dynamic_model_name == "mamarama, units: mm" and not warned: warned = True logger.info("Warning: Implementing simple workaround" " for flydra's " "units inconsistencies " "(multiplying xvel,yvel by 1000).") srows['xvel'] *= 1000 srows['yvel'] *= 1000 yield obj_id, extract_interesting_fields(srows, np.dtype(rows_dtype)) except core_analysis.NotEnoughDataToSmoothError: #logger.warning('not enough data to # smooth obj_id %d, skipping.'%(obj_id,)) continue ca.close()
def load_data (self, filename, calibration_file = None, objs = None, obj_filelist = None, kalman_smoothing = True, fps = None, dynamic_model = None, gender = None, post_type=None): self.datasets.append(len(self.datasets)+1) self.filename.append(filename) if calibration_file is not None: print "Calibration Files not yet supported!!!" # set up analyzer ca = core_analysis.get_global_CachingAnalyzer() (obj_ids, use_obj_ids, is_mat_file, data_file, extra) = ca.initial_file_load(filename) # data set defaults if fps is None: fps = result_utils.get_fps(data_file) if dynamic_model is None: try: dyn_model = extra['dynamic_model_name'] except: dyn_model = 'EKF mamarama, units: mm' if dynamic_model is not None: dyn_model = dynamic_model # if kalman smoothing is on, then we cannot use the EKF model - remove that from the model name print '**Kalman Smoothing is: ', kalman_smoothing, ' **' if kalman_smoothing is True: dyn_model = dyn_model[4:] print 'using dynamic model: ', dyn_model if objs is None and obj_filelist is None: print "running through all object id's, this might take a while..." obj_only = use_obj_ids # this is all the unique object id's if obj_filelist is not None: tmp = np.loadtxt(obj_filelist,delimiter=',') obj_only = np.array(tmp[:,0], dtype='int') elif objs is not None: if type(objs) is not list: objs = [objs] obj_only = np.array(objs) print fps print 'loading data.... ' for obj_id in obj_only: print 'processing: ', obj_id try: print obj_id # kalman rows = [ kalman_rows = ca.load_data( obj_id, data_file, dynamic_model_name = dyn_model, use_kalman_smoothing= kalman_smoothing, frames_per_second= fps) except: print 'object id failed to load (probably no data): ', obj_id continue #print kalman_rows[0] # couple object ID dictionary with trajectory objects traj_id = (str(self.datasets[-1])+'_'+str(obj_id)) self.trajecs.setdefault(traj_id, Trajectory(kalman_rows, extra, stimulus = self.stimulus, fps = fps, post_type=post_type) )