def make_and_save_plot(data_path, figure_path): data = pd.read_pickle(data_path) try: core_data = foraging.data_to_change_detection_core(data) except KeyError: warnings.warn( 'attempting to load foraging2 file. CAUTION: no support for custom "time"' ) core_data = foraging2.data_to_change_detection_core(data) extended_trials = create_extended_dataframe( trials=core_data['trials'], metadata=core_data['metadata'], licks=core_data['licks'], time=core_data['time'], ) fig = make_daily_figure(extended_trials) fig.set_size_inches(11, 8.5) fig.savefig( figure_path, transparent=False, orientation='landscape', dpi=300, ) return fig
def beh_pkl_path_to_df(path): data = pd.read_pickle(path) time_arr = np.hstack((0, np.cumsum(data['vsyncintervals']) / 1000.)) core_data = foraging.data_to_change_detection_core(data, time_arr) df = create_extended_dataframe(trials=core_data['trials'], metadata=core_data['metadata'], licks=core_data['licks'], time=core_data['time']) return df
def get_trials_df(behavior_data): core_data = data_to_change_detection_core(behavior_data) trials = create_extended_dataframe( trials=core_data['trials'], metadata=core_data['metadata'], licks=core_data['licks'], time=core_data['time']) return(trials)
def rewrite_record(uuid): pkl_path = db.get_pkl_path(uuid) data = pd.read_pickle(pkl_path) core_data = data_to_change_detection_core(data) trials = create_extended_dataframe(**core_data).drop( columns=['date', 'LDT_mode']) summary = summarize.session_level_summary(trials).iloc[0].to_dict() summary.update({'error_on_load': 0}) vb = db.Database('visual_behavior_data') db.update_or_create(vb['behavior_data']['summary'], db.simplify_entry(summary), ['behavior_session_uuid'], force_write=False) vb.close()
def test_foraging2_translator_schema(foraging2_data_stage4_2018_05_10): core_data = foraging2.data_to_change_detection_core( foraging2_data_stage4_2018_05_10) core_data['metadata']['behavior_session_uuid'] = create_session_uuid( core_data['metadata']['mouseid'], core_data['metadata']['startdatetime'], ) core_data['trials']['response_time'] = [[]] * len(core_data['trials']) core_data['trials']['lick_times'] = [[]] * len(core_data['trials']) _test_core_data_schemas(core_data) generate_qc_report(core_data) change_detection_metrics(core_data) trials = create_extended_dataframe(**core_data) make_daily_figure(trials) make_summary_figure(trials, core_data['metadata']['mouseid'])
def getTrialsDF(self, pklpath): p = pd.read_pickle(pklpath) #print(pklpath) if 'behavior' in p['items']: try: core_data = data_to_change_detection_core(p) trials = create_extended_dataframe( trials=core_data['trials'], metadata=core_data['metadata'], licks=core_data['licks'], time=core_data['time']) print('Loaded data from {}'.format(os.path.basename(pklpath))) except Exception as e: print('Error loading ' + pklpath, e) trials = pd.DataFrame.from_dict({'stage':[None]}) else: print('Found non-behavior pickle file: ' + pklpath) trials = pd.DataFrame.from_dict({'stage':[None]}) return trials
def _test_core_data_schemas(core_data): # metadata # core dataframes dataframe_schemas = ( (StimulusSchema, core_data['visual_stimuli']), (RunningSchema, core_data['running']), (LickSchema, core_data['licks']), (RewardSchema, core_data['rewards']), (TrialSchema, core_data['trials']), ) for Schema, data in dataframe_schemas: errors = Schema(many=True).validate(data.to_dict(orient='records')) for row, row_errors in errors.items(): assert len(row_errors) == 0, (Schema, data, row_errors) extended_trials = create_extended_dataframe( trials=core_data['trials'], metadata=core_data['metadata'], licks=core_data['licks'], time=core_data['time'], ) errors = ExtendedTrialSchema(many=True).validate( extended_trials.to_dict(orient='records')) for row, row_errors in errors.items(): assert len(row_errors) == 0, row_errors errors = MetadataSchema().validate(core_data['metadata']) assert len(errors) == 0, errors.keys() errors = ImageSetSchema().validate(core_data['image_set']) assert len(errors) == 0, errors.keys()
trainingStage.append([]) changeTriggeredRunning.append([]) flashTriggeredRunning.append([]) rewardsEarned.append([]) dprimeOverall.append([]) dprimeEngaged.append([]) probEngaged.append([]) imageHitRate.append([]) imageHitRateEngaged.append([]) unloadablePklFiles.append([]) for pklFile in glob.glob(os.path.join(pickleDir, mouseID, '*.pkl')): try: core_data = data_to_change_detection_core(pd.read_pickle(pklFile)) trials = create_extended_dataframe(trials=core_data['trials'], metadata=core_data['metadata'], licks=core_data['licks'], time=core_data['time']) except: unloadablePkl.append(pklFile) continue data['trainingDate'][mouseInd].append( datetime.datetime.strptime( os.path.basename(pklFile)[:6], '%y%m%d')) data['trainingStage'][mouseInd].append(core_data['metadata']['stage']) data['rigID'][mouseInd].append(core_data['metadata']['rig_id']) autoRewarded = np.array(trials['auto_rewarded']).astype(bool) earlyResponse = np.array(trials['response_type'] == 'EARLY_RESPONSE') ignore = earlyResponse | autoRewarded miss = np.array(trials['response_type'] == 'MISS')
def get_trials(core_data): trials = create_extended_dataframe(trials=core_data['trials'], metadata=core_data['metadata'], licks=core_data['licks'], time=core_data['time']) return trials
def load_data(self): self.data = pd.read_pickle(self.pkl_path) self.core_data = data_to_change_detection_core(self.data) self.trials = create_extended_dataframe(**self.core_data)
def getBehaviorData(self): # get behavior data if not hasattr(self, 'vsyncTimes'): self.getFrameTimes() self.pkl_file = glob.glob(os.path.join(self.dataDir, '*[0-9].pkl'))[0] behaviordata = pd.read_pickle(self.pkl_file) self.core_data = data_to_change_detection_core(behaviordata) self.trials = create_extended_dataframe( trials=self.core_data['trials'], metadata=self.core_data['metadata'], licks=self.core_data['licks'], time=self.core_data['time']) self.behaviorVsyncCount = self.core_data[ 'time'].size # same as self.trials['endframe'].values[-1] + 1 self.images = self.core_data['image_set']['images'] newSize = tuple(int(s / 10) for s in self.images[0].shape[::-1]) self.imagesDownsampled = [ cv2.resize(img, newSize, interpolation=cv2.INTER_AREA) for img in self.images ] self.imageNames = [ i['image_name'] for i in self.core_data['image_set']['image_attributes'] ] # get running data self.behaviorRunTime = self.vsyncTimes[self.core_data['running'].frame] self.behaviorRunSpeed = self.core_data['running'].speed # get run start times self.behaviorRunStartTimes = find_run_transitions( self.behaviorRunSpeed, self.behaviorRunTime) #make_daily_figure(trials) # align trials to sync self.trial_start_frames = np.array(self.trials['startframe']) self.trial_end_frames = np.array(self.trials['endframe']) self.trial_start_times = self.frameAppearTimes[self.trial_start_frames] self.trial_end_times = self.frameAppearTimes[self.trial_end_frames] # trial info self.autoRewarded = np.array(self.trials['auto_rewarded']).astype(bool) self.earlyResponse = np.array( self.trials['response_type'] == 'EARLY_RESPONSE') self.ignore = self.earlyResponse | self.autoRewarded self.miss = np.array(self.trials['response_type'] == 'MISS') self.hit = np.array(self.trials['response_type'] == 'HIT') self.falseAlarm = np.array(self.trials['response_type'] == 'FA') self.correctReject = np.array(self.trials['response_type'] == 'CR') self.initialImage = np.array(self.trials['initial_image_name']) self.changeImage = np.array(self.trials['change_image_name']) candidateOmittedFlashFrames = behaviordata['items']['behavior'][ 'stimuli']['images']['flashes_omitted'] drawlog = behaviordata['items']['behavior']['stimuli']['images'][ 'draw_log'] self.omittedFlashFrames = np.array( [c for c in candidateOmittedFlashFrames if not drawlog[c]]) imageFrameIndexBeforeOmitted = np.searchsorted( self.core_data['visual_stimuli']['frame'], self.omittedFlashFrames) - 1 self.omittedFlashImage = np.array( self.core_data['visual_stimuli'] ['image_name'])[imageFrameIndexBeforeOmitted] self.behaviorStimDur = np.array( self.core_data['visual_stimuli']['duration']) self.preGrayDur = np.stack( self.trials['blank_duration_range']) # where is actual gray dur self.lastBehaviorTime = self.frameAppearTimes[ self.trials['endframe'].values[-1]]
def getBehaviorData(self): # get behavior data if not hasattr(self, 'vsyncTimes'): self.getFrameTimes() pkl = glob.glob( os.path.join(self.dataDir, '*replay-session*behavior*.pkl')) if len(pkl) > 0: self.pkl_file = pkl[0] else: self.pkl_file = glob.glob(os.path.join(self.dataDir, '*[0-9].pkl'))[0] behaviordata = pd.read_pickle(self.pkl_file) self.core_data = data_to_change_detection_core(behaviordata) self.trials = create_extended_dataframe( trials=self.core_data['trials'], metadata=self.core_data['metadata'], licks=self.core_data['licks'], time=self.core_data['time']) self.behaviorVsyncCount = self.core_data[ 'time'].size # same as self.trials['endframe'].values[-1] + 1 self.flashFrames = np.array(self.core_data['visual_stimuli']['frame']) self.flashImage = self.core_data['visual_stimuli']['image_name'] self.changeFrames = np.array(self.trials['change_frame']).astype( int) + 1 #add one to correct for change frame indexing problem self.initialImage = np.array(self.trials['initial_image_name']) self.changeImage = np.array(self.trials['change_image_name']) self.images = self.core_data['image_set']['images'] newSize = tuple(int(s / 10) for s in self.images[0].shape[::-1]) self.imagesDownsampled = [ cv2.resize(img, newSize, interpolation=cv2.INTER_AREA) for img in self.images ] self.imageNames = [ str(i['image_name'], 'utf-8') for i in self.core_data['image_set']['image_attributes'] ] candidateOmittedFlashFrames = behaviordata['items']['behavior'][ 'stimuli']['images']['flashes_omitted'] drawlog = behaviordata['items']['behavior']['stimuli']['images'][ 'draw_log'] self.omittedFlashFrames = np.array( [c for c in candidateOmittedFlashFrames if not drawlog[c]]) imageFrameIndexBeforeOmitted = np.searchsorted( self.core_data['visual_stimuli']['frame'], self.omittedFlashFrames) - 1 self.omittedFlashImage = np.array( self.core_data['visual_stimuli'] ['image_name'])[imageFrameIndexBeforeOmitted] self.behaviorStimDur = np.array( self.core_data['visual_stimuli']['duration']) self.preGrayDur = np.stack( self.trials['blank_duration_range']) # where is actual gray dur self.lastBehaviorTime = self.frameAppearTimes[ self.trials['endframe'].values[-1]] # align trials to sync self.trial_start_frames = np.array(self.trials['startframe']) self.trial_end_frames = np.array(self.trials['endframe']) self.trial_start_times = self.frameAppearTimes[self.trial_start_frames] self.trial_end_times = self.frameAppearTimes[self.trial_end_frames] # trial info self.autoRewarded = np.array(self.trials['auto_rewarded']).astype(bool) self.earlyResponse = np.array( self.trials['response_type'] == 'EARLY_RESPONSE') self.ignore = self.earlyResponse | self.autoRewarded self.miss = np.array(self.trials['response_type'] == 'MISS') self.hit = np.array(self.trials['response_type'] == 'HIT') self.falseAlarm = np.array(self.trials['response_type'] == 'FA') self.correctReject = np.array(self.trials['response_type'] == 'CR') # get running data self.behaviorRunTime = self.vsyncTimes[self.core_data['running'].frame] self.behaviorRunSpeed = self.core_data['running'].speed self.behaviorRunDx = self.core_data['running'].dx # get run start times self.behaviorRunStartTimes = find_run_transitions( self.behaviorRunSpeed, self.behaviorRunTime) #get lick times self.lickTimes = probeSync.get_sync_line_data(self.syncDataset, channel=31)[0] if len(self.lickTimes) == 0: self.lickTimes = self.vsyncTimes[np.concatenate( [f for f in self.trials['lick_frames']]).astype(int)] # get reward times for all trials (nan for non-rewarded trials) rewardTimes = [] for f in self.trials['reward_frames']: if len(f) > 0: rewardTimes.append(self.vsyncTimes[f[0]]) else: rewardTimes.append(np.nan) self.rewardTimes = np.array(rewardTimes)