Ejemplo n.º 1
0
def make_and_save_plot(data_path, figure_path):

    data = pd.read_pickle(data_path)

    try:
        core_data = foraging.data_to_change_detection_core(data)
    except KeyError:
        warnings.warn(
            'attempting to load foraging2 file. CAUTION: no support for custom "time"'
        )
        core_data = foraging2.data_to_change_detection_core(data)

    extended_trials = create_extended_dataframe(
        trials=core_data['trials'],
        metadata=core_data['metadata'],
        licks=core_data['licks'],
        time=core_data['time'],
    )

    fig = make_daily_figure(extended_trials)
    fig.set_size_inches(11, 8.5)
    fig.savefig(
        figure_path,
        transparent=False,
        orientation='landscape',
        dpi=300,
    )
    return fig
Ejemplo n.º 2
0
def get_core_data(pkl, stimulus_timestamps):
    try:
        core_data = foraging.data_to_change_detection_core(
            pkl, time=stimulus_timestamps)
    except KeyError:
        core_data = foraging2.data_to_change_detection_core(
            pkl, time=stimulus_timestamps)
    return core_data
Ejemplo n.º 3
0
def get_trials_df(behavior_data):
    
    core_data = data_to_change_detection_core(behavior_data)
    trials = create_extended_dataframe(
        trials=core_data['trials'],
        metadata=core_data['metadata'],
        licks=core_data['licks'],
        time=core_data['time'])
    
    return(trials)
Ejemplo n.º 4
0
 def getRunning(self, pklpath):
     p = pd.read_pickle(pklpath)
     if 'behavior' in p['items']:
         core_data = data_to_change_detection_core(p)
         rtime = core_data['running']['time']
         rspeed = core_data['running']['speed']
     else:
         rtime = np.zeros(5).astype(float)
         rspeed = np.zeros(5).astype(float)
     
     return [rtime, rspeed]
def load_running_df(bsid=None,pkl_path=None,camstim_type='foraging2'):
    '''
    loads running data from pkl file using VBA
    input is either the behavior session ID (bsid) or the pkl path (not both!)
    '''
    if bsid:
        pkl_path = db.get_pkl_path(int(bsid))
    
    data = pd.read_pickle(pkl_path)
    if camstim_type == 'foraging2':
        core_data = data_to_change_detection_core(data)
    else:
        core_data = data_to_change_detection_core_legacy(data)
    return core_data['running']
def rewrite_record(uuid):
    pkl_path = db.get_pkl_path(uuid)
    data = pd.read_pickle(pkl_path)

    core_data = data_to_change_detection_core(data)
    trials = create_extended_dataframe(**core_data).drop(
        columns=['date', 'LDT_mode'])
    summary = summarize.session_level_summary(trials).iloc[0].to_dict()
    summary.update({'error_on_load': 0})

    vb = db.Database('visual_behavior_data')
    db.update_or_create(vb['behavior_data']['summary'],
                        db.simplify_entry(summary), ['behavior_session_uuid'],
                        force_write=False)
    vb.close()
def test_foraging2_translator_schema(foraging2_data_stage4_2018_05_10):
    core_data = foraging2.data_to_change_detection_core(
        foraging2_data_stage4_2018_05_10)
    core_data['metadata']['behavior_session_uuid'] = create_session_uuid(
        core_data['metadata']['mouseid'],
        core_data['metadata']['startdatetime'],
    )

    core_data['trials']['response_time'] = [[]] * len(core_data['trials'])
    core_data['trials']['lick_times'] = [[]] * len(core_data['trials'])

    _test_core_data_schemas(core_data)
    generate_qc_report(core_data)
    change_detection_metrics(core_data)
    trials = create_extended_dataframe(**core_data)
    make_daily_figure(trials)
    make_summary_figure(trials, core_data['metadata']['mouseid'])
def run_qc(pkls):
    all_results = []
    for mouse, mouse_pkls in pkls.iteritems():
        for stage, stage_pkl in mouse_pkls.iteritems():

            print('mouse = {}, stage = {}'.format(mouse, stage))
            try:
                data = pd.read_pickle(stage_pkl)

                core_data = data_to_change_detection_core(data)
                results = generate_qc_report(core_data)
                results.update({'mouse': mouse, 'stage': stage})

            except Exception as e:
                print('error before validation: {}'.format(e))
                results = {'mouse': mouse, 'stage': stage}

            all_results.append(results)

    return all_results
Ejemplo n.º 9
0
    def getTrialsDF(self, pklpath):
        p = pd.read_pickle(pklpath)
        #print(pklpath)

        if 'behavior' in p['items']:
            try:
                core_data = data_to_change_detection_core(p)
                trials = create_extended_dataframe(
                        trials=core_data['trials'],
                        metadata=core_data['metadata'],
                        licks=core_data['licks'],
                        time=core_data['time'])
                print('Loaded data from {}'.format(os.path.basename(pklpath)))
            
            except Exception as e:
                print('Error loading ' + pklpath, e)
                trials = pd.DataFrame.from_dict({'stage':[None]})
        else:
            print('Found non-behavior pickle file: ' + pklpath)
            trials = pd.DataFrame.from_dict({'stage':[None]})
            
        return trials
Ejemplo n.º 10
0
    rigID = []
    trainingDate.append([])
    trainingStage.append([])
    changeTriggeredRunning.append([])
    flashTriggeredRunning.append([])
    rewardsEarned.append([])
    dprimeOverall.append([])
    dprimeEngaged.append([])
    probEngaged.append([])
    imageHitRate.append([])
    imageHitRateEngaged.append([])
    unloadablePklFiles.append([])

    for pklFile in glob.glob(os.path.join(pickleDir, mouseID, '*.pkl')):
        try:
            core_data = data_to_change_detection_core(pd.read_pickle(pklFile))
            trials = create_extended_dataframe(trials=core_data['trials'],
                                               metadata=core_data['metadata'],
                                               licks=core_data['licks'],
                                               time=core_data['time'])
        except:
            unloadablePkl.append(pklFile)
            continue

        data['trainingDate'][mouseInd].append(
            datetime.datetime.strptime(
                os.path.basename(pklFile)[:6], '%y%m%d'))
        data['trainingStage'][mouseInd].append(core_data['metadata']['stage'])
        data['rigID'][mouseInd].append(core_data['metadata']['rig_id'])

        autoRewarded = np.array(trials['auto_rewarded']).astype(bool)
daysToInclude = 20
data = [[], [], []]
for mouse in mouseIDs:
    try:
        mousedir = os.path.join(pickleFileDir, mouse)
        pklfiles = np.sort(os.listdir(mousedir))[-daysToInclude:]
    except:
        print('could not find mouse ' + mouse + ' in ' + pickleFileDir)
        continue
    
    allcounts = np.zeros((8,8))
    allhits = np.zeros((8,8))
    for pkl in pklfiles:
        try:
            core_data = data_to_change_detection_core(pd.read_pickle(os.path.join(mousedir,pkl)))
            trials = create_extended_dataframe(
                    trials=core_data['trials'],
                    metadata=core_data['metadata'],
                    licks=core_data['licks'],
                    time=core_data['time'])
            if 'im061' in trials.change_image_name.to_list():
            
                lastRewardTime = 0
                interval = 0
                for it, tr in enumerate(trials.reward_times.values[0:]):
                    if len(tr)>0:
                        interval = tr[0] - lastRewardTime
                        lastRewardTime = tr[0]
                    if interval>120 and it>0:
                        break
Ejemplo n.º 12
0
    def load_data(self):

        self.data = pd.read_pickle(self.pkl_path)
        self.core_data = data_to_change_detection_core(self.data)
        self.trials = create_extended_dataframe(**self.core_data)
Ejemplo n.º 13
0
    def getBehaviorData(self):
        # get behavior data
        if not hasattr(self, 'vsyncTimes'):
            self.getFrameTimes()

        self.pkl_file = glob.glob(os.path.join(self.dataDir, '*[0-9].pkl'))[0]
        behaviordata = pd.read_pickle(self.pkl_file)
        self.core_data = data_to_change_detection_core(behaviordata)

        self.trials = create_extended_dataframe(
            trials=self.core_data['trials'],
            metadata=self.core_data['metadata'],
            licks=self.core_data['licks'],
            time=self.core_data['time'])

        self.behaviorVsyncCount = self.core_data[
            'time'].size  # same as self.trials['endframe'].values[-1] + 1

        self.images = self.core_data['image_set']['images']
        newSize = tuple(int(s / 10) for s in self.images[0].shape[::-1])
        self.imagesDownsampled = [
            cv2.resize(img, newSize, interpolation=cv2.INTER_AREA)
            for img in self.images
        ]
        self.imageNames = [
            i['image_name']
            for i in self.core_data['image_set']['image_attributes']
        ]

        # get running data
        self.behaviorRunTime = self.vsyncTimes[self.core_data['running'].frame]
        self.behaviorRunSpeed = self.core_data['running'].speed

        # get run start times
        self.behaviorRunStartTimes = find_run_transitions(
            self.behaviorRunSpeed, self.behaviorRunTime)

        #make_daily_figure(trials)

        # align trials to sync
        self.trial_start_frames = np.array(self.trials['startframe'])
        self.trial_end_frames = np.array(self.trials['endframe'])
        self.trial_start_times = self.frameAppearTimes[self.trial_start_frames]
        self.trial_end_times = self.frameAppearTimes[self.trial_end_frames]

        # trial info
        self.autoRewarded = np.array(self.trials['auto_rewarded']).astype(bool)
        self.earlyResponse = np.array(
            self.trials['response_type'] == 'EARLY_RESPONSE')
        self.ignore = self.earlyResponse | self.autoRewarded
        self.miss = np.array(self.trials['response_type'] == 'MISS')
        self.hit = np.array(self.trials['response_type'] == 'HIT')
        self.falseAlarm = np.array(self.trials['response_type'] == 'FA')
        self.correctReject = np.array(self.trials['response_type'] == 'CR')
        self.initialImage = np.array(self.trials['initial_image_name'])
        self.changeImage = np.array(self.trials['change_image_name'])
        candidateOmittedFlashFrames = behaviordata['items']['behavior'][
            'stimuli']['images']['flashes_omitted']
        drawlog = behaviordata['items']['behavior']['stimuli']['images'][
            'draw_log']
        self.omittedFlashFrames = np.array(
            [c for c in candidateOmittedFlashFrames if not drawlog[c]])
        imageFrameIndexBeforeOmitted = np.searchsorted(
            self.core_data['visual_stimuli']['frame'],
            self.omittedFlashFrames) - 1
        self.omittedFlashImage = np.array(
            self.core_data['visual_stimuli']
            ['image_name'])[imageFrameIndexBeforeOmitted]

        self.behaviorStimDur = np.array(
            self.core_data['visual_stimuli']['duration'])
        self.preGrayDur = np.stack(
            self.trials['blank_duration_range'])  # where is actual gray dur
        self.lastBehaviorTime = self.frameAppearTimes[
            self.trials['endframe'].values[-1]]
Ejemplo n.º 14
0
    def getBehaviorData(self):
        # get behavior data
        if not hasattr(self, 'vsyncTimes'):
            self.getFrameTimes()

        pkl = glob.glob(
            os.path.join(self.dataDir, '*replay-session*behavior*.pkl'))
        if len(pkl) > 0:
            self.pkl_file = pkl[0]
        else:
            self.pkl_file = glob.glob(os.path.join(self.dataDir,
                                                   '*[0-9].pkl'))[0]
        behaviordata = pd.read_pickle(self.pkl_file)
        self.core_data = data_to_change_detection_core(behaviordata)

        self.trials = create_extended_dataframe(
            trials=self.core_data['trials'],
            metadata=self.core_data['metadata'],
            licks=self.core_data['licks'],
            time=self.core_data['time'])

        self.behaviorVsyncCount = self.core_data[
            'time'].size  # same as self.trials['endframe'].values[-1] + 1

        self.flashFrames = np.array(self.core_data['visual_stimuli']['frame'])
        self.flashImage = self.core_data['visual_stimuli']['image_name']
        self.changeFrames = np.array(self.trials['change_frame']).astype(
            int) + 1  #add one to correct for change frame indexing problem
        self.initialImage = np.array(self.trials['initial_image_name'])
        self.changeImage = np.array(self.trials['change_image_name'])

        self.images = self.core_data['image_set']['images']
        newSize = tuple(int(s / 10) for s in self.images[0].shape[::-1])
        self.imagesDownsampled = [
            cv2.resize(img, newSize, interpolation=cv2.INTER_AREA)
            for img in self.images
        ]
        self.imageNames = [
            str(i['image_name'], 'utf-8')
            for i in self.core_data['image_set']['image_attributes']
        ]

        candidateOmittedFlashFrames = behaviordata['items']['behavior'][
            'stimuli']['images']['flashes_omitted']
        drawlog = behaviordata['items']['behavior']['stimuli']['images'][
            'draw_log']
        self.omittedFlashFrames = np.array(
            [c for c in candidateOmittedFlashFrames if not drawlog[c]])
        imageFrameIndexBeforeOmitted = np.searchsorted(
            self.core_data['visual_stimuli']['frame'],
            self.omittedFlashFrames) - 1
        self.omittedFlashImage = np.array(
            self.core_data['visual_stimuli']
            ['image_name'])[imageFrameIndexBeforeOmitted]

        self.behaviorStimDur = np.array(
            self.core_data['visual_stimuli']['duration'])
        self.preGrayDur = np.stack(
            self.trials['blank_duration_range'])  # where is actual gray dur
        self.lastBehaviorTime = self.frameAppearTimes[
            self.trials['endframe'].values[-1]]

        # align trials to sync
        self.trial_start_frames = np.array(self.trials['startframe'])
        self.trial_end_frames = np.array(self.trials['endframe'])
        self.trial_start_times = self.frameAppearTimes[self.trial_start_frames]
        self.trial_end_times = self.frameAppearTimes[self.trial_end_frames]

        # trial info
        self.autoRewarded = np.array(self.trials['auto_rewarded']).astype(bool)
        self.earlyResponse = np.array(
            self.trials['response_type'] == 'EARLY_RESPONSE')
        self.ignore = self.earlyResponse | self.autoRewarded
        self.miss = np.array(self.trials['response_type'] == 'MISS')
        self.hit = np.array(self.trials['response_type'] == 'HIT')
        self.falseAlarm = np.array(self.trials['response_type'] == 'FA')
        self.correctReject = np.array(self.trials['response_type'] == 'CR')

        # get running data
        self.behaviorRunTime = self.vsyncTimes[self.core_data['running'].frame]
        self.behaviorRunSpeed = self.core_data['running'].speed
        self.behaviorRunDx = self.core_data['running'].dx

        # get run start times
        self.behaviorRunStartTimes = find_run_transitions(
            self.behaviorRunSpeed, self.behaviorRunTime)

        #get lick times
        self.lickTimes = probeSync.get_sync_line_data(self.syncDataset,
                                                      channel=31)[0]
        if len(self.lickTimes) == 0:
            self.lickTimes = self.vsyncTimes[np.concatenate(
                [f for f in self.trials['lick_frames']]).astype(int)]

        # get reward times for all trials (nan for non-rewarded trials)
        rewardTimes = []
        for f in self.trials['reward_frames']:
            if len(f) > 0:
                rewardTimes.append(self.vsyncTimes[f[0]])
            else:
                rewardTimes.append(np.nan)
        self.rewardTimes = np.array(rewardTimes)