def make_and_save_plot(data_path, figure_path):
data = pd.read_pickle(data_path)
try:
core_data = foraging.data_to_change_detection_core(data)
except KeyError:
warnings.warn(
'attempting to load foraging2 file. CAUTION: no support for custom "time"'
)
core_data = foraging2.data_to_change_detection_core(data)
extended_trials = create_extended_dataframe(
trials=core_data['trials'],
metadata=core_data['metadata'],
licks=core_data['licks'],
time=core_data['time'],
)
fig = make_daily_figure(extended_trials)
fig.set_size_inches(11, 8.5)
fig.savefig(
figure_path,
transparent=False,
orientation='landscape',
dpi=300,
)
return fig
def beh_pkl_path_to_df(path):
data = pd.read_pickle(path)
time_arr = np.hstack((0, np.cumsum(data['vsyncintervals']) / 1000.))
core_data = foraging.data_to_change_detection_core(data, time_arr)
df = create_extended_dataframe(trials=core_data['trials'],
metadata=core_data['metadata'],
licks=core_data['licks'],
time=core_data['time'])
return df
def get_trials_df(behavior_data):
core_data = data_to_change_detection_core(behavior_data)
trials = create_extended_dataframe(
trials=core_data['trials'],
metadata=core_data['metadata'],
licks=core_data['licks'],
time=core_data['time'])
return(trials)
def rewrite_record(uuid):
pkl_path = db.get_pkl_path(uuid)
data = pd.read_pickle(pkl_path)
core_data = data_to_change_detection_core(data)
trials = create_extended_dataframe(**core_data).drop(
columns=['date', 'LDT_mode'])
summary = summarize.session_level_summary(trials).iloc[0].to_dict()
summary.update({'error_on_load': 0})
vb = db.Database('visual_behavior_data')
db.update_or_create(vb['behavior_data']['summary'],
db.simplify_entry(summary), ['behavior_session_uuid'],
force_write=False)
vb.close()
def test_foraging2_translator_schema(foraging2_data_stage4_2018_05_10):
core_data = foraging2.data_to_change_detection_core(
foraging2_data_stage4_2018_05_10)
core_data['metadata']['behavior_session_uuid'] = create_session_uuid(
core_data['metadata']['mouseid'],
core_data['metadata']['startdatetime'],
)
core_data['trials']['response_time'] = [[]] * len(core_data['trials'])
core_data['trials']['lick_times'] = [[]] * len(core_data['trials'])
_test_core_data_schemas(core_data)
generate_qc_report(core_data)
change_detection_metrics(core_data)
trials = create_extended_dataframe(**core_data)
make_daily_figure(trials)
make_summary_figure(trials, core_data['metadata']['mouseid'])
def getTrialsDF(self, pklpath):
p = pd.read_pickle(pklpath)
#print(pklpath)
if 'behavior' in p['items']:
try:
core_data = data_to_change_detection_core(p)
trials = create_extended_dataframe(
trials=core_data['trials'],
metadata=core_data['metadata'],
licks=core_data['licks'],
time=core_data['time'])
print('Loaded data from {}'.format(os.path.basename(pklpath)))
except Exception as e:
print('Error loading ' + pklpath, e)
trials = pd.DataFrame.from_dict({'stage':[None]})
else:
print('Found non-behavior pickle file: ' + pklpath)
trials = pd.DataFrame.from_dict({'stage':[None]})
return trials
def _test_core_data_schemas(core_data):
# metadata
# core dataframes
dataframe_schemas = (
(StimulusSchema, core_data['visual_stimuli']),
(RunningSchema, core_data['running']),
(LickSchema, core_data['licks']),
(RewardSchema, core_data['rewards']),
(TrialSchema, core_data['trials']),
)
for Schema, data in dataframe_schemas:
errors = Schema(many=True).validate(data.to_dict(orient='records'))
for row, row_errors in errors.items():
assert len(row_errors) == 0, (Schema, data, row_errors)
extended_trials = create_extended_dataframe(
trials=core_data['trials'],
metadata=core_data['metadata'],
licks=core_data['licks'],
time=core_data['time'],
)
errors = ExtendedTrialSchema(many=True).validate(
extended_trials.to_dict(orient='records'))
for row, row_errors in errors.items():
assert len(row_errors) == 0, row_errors
errors = MetadataSchema().validate(core_data['metadata'])
assert len(errors) == 0, errors.keys()
errors = ImageSetSchema().validate(core_data['image_set'])
assert len(errors) == 0, errors.keys()
trainingStage.append([])
changeTriggeredRunning.append([])
flashTriggeredRunning.append([])
rewardsEarned.append([])
dprimeOverall.append([])
dprimeEngaged.append([])
probEngaged.append([])
imageHitRate.append([])
imageHitRateEngaged.append([])
unloadablePklFiles.append([])
for pklFile in glob.glob(os.path.join(pickleDir, mouseID, '*.pkl')):
try:
core_data = data_to_change_detection_core(pd.read_pickle(pklFile))
trials = create_extended_dataframe(trials=core_data['trials'],
metadata=core_data['metadata'],
licks=core_data['licks'],
time=core_data['time'])
except:
unloadablePkl.append(pklFile)
continue
data['trainingDate'][mouseInd].append(
datetime.datetime.strptime(
os.path.basename(pklFile)[:6], '%y%m%d'))
data['trainingStage'][mouseInd].append(core_data['metadata']['stage'])
data['rigID'][mouseInd].append(core_data['metadata']['rig_id'])
autoRewarded = np.array(trials['auto_rewarded']).astype(bool)
earlyResponse = np.array(trials['response_type'] == 'EARLY_RESPONSE')
ignore = earlyResponse | autoRewarded
miss = np.array(trials['response_type'] == 'MISS')
def get_trials(core_data):
trials = create_extended_dataframe(trials=core_data['trials'],
metadata=core_data['metadata'],
licks=core_data['licks'],
time=core_data['time'])
return trials
def load_data(self):
self.data = pd.read_pickle(self.pkl_path)
self.core_data = data_to_change_detection_core(self.data)
self.trials = create_extended_dataframe(**self.core_data)
def getBehaviorData(self):
# get behavior data
if not hasattr(self, 'vsyncTimes'):
self.getFrameTimes()
self.pkl_file = glob.glob(os.path.join(self.dataDir, '*[0-9].pkl'))[0]
behaviordata = pd.read_pickle(self.pkl_file)
self.core_data = data_to_change_detection_core(behaviordata)
self.trials = create_extended_dataframe(
trials=self.core_data['trials'],
metadata=self.core_data['metadata'],
licks=self.core_data['licks'],
time=self.core_data['time'])
self.behaviorVsyncCount = self.core_data[
'time'].size # same as self.trials['endframe'].values[-1] + 1
self.images = self.core_data['image_set']['images']
newSize = tuple(int(s / 10) for s in self.images[0].shape[::-1])
self.imagesDownsampled = [
cv2.resize(img, newSize, interpolation=cv2.INTER_AREA)
for img in self.images
]
self.imageNames = [
i['image_name']
for i in self.core_data['image_set']['image_attributes']
]
# get running data
self.behaviorRunTime = self.vsyncTimes[self.core_data['running'].frame]
self.behaviorRunSpeed = self.core_data['running'].speed
# get run start times
self.behaviorRunStartTimes = find_run_transitions(
self.behaviorRunSpeed, self.behaviorRunTime)
#make_daily_figure(trials)
# align trials to sync
self.trial_start_frames = np.array(self.trials['startframe'])
self.trial_end_frames = np.array(self.trials['endframe'])
self.trial_start_times = self.frameAppearTimes[self.trial_start_frames]
self.trial_end_times = self.frameAppearTimes[self.trial_end_frames]
# trial info
self.autoRewarded = np.array(self.trials['auto_rewarded']).astype(bool)
self.earlyResponse = np.array(
self.trials['response_type'] == 'EARLY_RESPONSE')
self.ignore = self.earlyResponse | self.autoRewarded
self.miss = np.array(self.trials['response_type'] == 'MISS')
self.hit = np.array(self.trials['response_type'] == 'HIT')
self.falseAlarm = np.array(self.trials['response_type'] == 'FA')
self.correctReject = np.array(self.trials['response_type'] == 'CR')
self.initialImage = np.array(self.trials['initial_image_name'])
self.changeImage = np.array(self.trials['change_image_name'])
candidateOmittedFlashFrames = behaviordata['items']['behavior'][
'stimuli']['images']['flashes_omitted']
drawlog = behaviordata['items']['behavior']['stimuli']['images'][
'draw_log']
self.omittedFlashFrames = np.array(
[c for c in candidateOmittedFlashFrames if not drawlog[c]])
imageFrameIndexBeforeOmitted = np.searchsorted(
self.core_data['visual_stimuli']['frame'],
self.omittedFlashFrames) - 1
self.omittedFlashImage = np.array(
self.core_data['visual_stimuli']
['image_name'])[imageFrameIndexBeforeOmitted]
self.behaviorStimDur = np.array(
self.core_data['visual_stimuli']['duration'])
self.preGrayDur = np.stack(
self.trials['blank_duration_range']) # where is actual gray dur
self.lastBehaviorTime = self.frameAppearTimes[
self.trials['endframe'].values[-1]]
def getBehaviorData(self):
# get behavior data
if not hasattr(self, 'vsyncTimes'):
self.getFrameTimes()
pkl = glob.glob(
os.path.join(self.dataDir, '*replay-session*behavior*.pkl'))
if len(pkl) > 0:
self.pkl_file = pkl[0]
else:
self.pkl_file = glob.glob(os.path.join(self.dataDir,
'*[0-9].pkl'))[0]
behaviordata = pd.read_pickle(self.pkl_file)
self.core_data = data_to_change_detection_core(behaviordata)
self.trials = create_extended_dataframe(
trials=self.core_data['trials'],
metadata=self.core_data['metadata'],
licks=self.core_data['licks'],
time=self.core_data['time'])
self.behaviorVsyncCount = self.core_data[
'time'].size # same as self.trials['endframe'].values[-1] + 1
self.flashFrames = np.array(self.core_data['visual_stimuli']['frame'])
self.flashImage = self.core_data['visual_stimuli']['image_name']
self.changeFrames = np.array(self.trials['change_frame']).astype(
int) + 1 #add one to correct for change frame indexing problem
self.initialImage = np.array(self.trials['initial_image_name'])
self.changeImage = np.array(self.trials['change_image_name'])
self.images = self.core_data['image_set']['images']
newSize = tuple(int(s / 10) for s in self.images[0].shape[::-1])
self.imagesDownsampled = [
cv2.resize(img, newSize, interpolation=cv2.INTER_AREA)
for img in self.images
]
self.imageNames = [
str(i['image_name'], 'utf-8')
for i in self.core_data['image_set']['image_attributes']
]
candidateOmittedFlashFrames = behaviordata['items']['behavior'][
'stimuli']['images']['flashes_omitted']
drawlog = behaviordata['items']['behavior']['stimuli']['images'][
'draw_log']
self.omittedFlashFrames = np.array(
[c for c in candidateOmittedFlashFrames if not drawlog[c]])
imageFrameIndexBeforeOmitted = np.searchsorted(
self.core_data['visual_stimuli']['frame'],
self.omittedFlashFrames) - 1
self.omittedFlashImage = np.array(
self.core_data['visual_stimuli']
['image_name'])[imageFrameIndexBeforeOmitted]
self.behaviorStimDur = np.array(
self.core_data['visual_stimuli']['duration'])
self.preGrayDur = np.stack(
self.trials['blank_duration_range']) # where is actual gray dur
self.lastBehaviorTime = self.frameAppearTimes[
self.trials['endframe'].values[-1]]
# align trials to sync
self.trial_start_frames = np.array(self.trials['startframe'])
self.trial_end_frames = np.array(self.trials['endframe'])
self.trial_start_times = self.frameAppearTimes[self.trial_start_frames]
self.trial_end_times = self.frameAppearTimes[self.trial_end_frames]
# trial info
self.autoRewarded = np.array(self.trials['auto_rewarded']).astype(bool)
self.earlyResponse = np.array(
self.trials['response_type'] == 'EARLY_RESPONSE')
self.ignore = self.earlyResponse | self.autoRewarded
self.miss = np.array(self.trials['response_type'] == 'MISS')
self.hit = np.array(self.trials['response_type'] == 'HIT')
self.falseAlarm = np.array(self.trials['response_type'] == 'FA')
self.correctReject = np.array(self.trials['response_type'] == 'CR')
# get running data
self.behaviorRunTime = self.vsyncTimes[self.core_data['running'].frame]
self.behaviorRunSpeed = self.core_data['running'].speed
self.behaviorRunDx = self.core_data['running'].dx
# get run start times
self.behaviorRunStartTimes = find_run_transitions(
self.behaviorRunSpeed, self.behaviorRunTime)
#get lick times
self.lickTimes = probeSync.get_sync_line_data(self.syncDataset,
channel=31)[0]
if len(self.lickTimes) == 0:
self.lickTimes = self.vsyncTimes[np.concatenate(
[f for f in self.trials['lick_frames']]).astype(int)]
# get reward times for all trials (nan for non-rewarded trials)
rewardTimes = []
for f in self.trials['reward_frames']:
if len(f) > 0:
rewardTimes.append(self.vsyncTimes[f[0]])
else:
rewardTimes.append(np.nan)
self.rewardTimes = np.array(rewardTimes)
