def save_TrialDict_pkl(self,f_name=None):
if not f_name:
f_name = "{0}.pkl".format(self.getName())
print "file name: {0}".format(os.path.join(os.getcwd(),f_name))
else:
print "file name: {0}".format(f_name)
FileTools.saveFile(f_name,self.generateTrialDict())
def load_experiment_img_jcamf(raw_exp_img_path,
exp_img_kwargs={
"row": 1024,
"column": 1024
}):
exp_img, _, _ = ft.importRawJCamF(raw_exp_img_path, **exp_img_kwargs)
return exp_img[0]
def get_channel_geometry(kilosort_folder, channel_names=None):
"""
get channel geometry from kilosort folder
:param kilosort_folder: path to kilosort output folder
:param channel_names: list of strings, name of each channel, ideally should match channel names and order saved in
the .nwb file
:return: dictionary, {channel name : xy position of the channel}
"""
positions = np.load(os.path.join(kilosort_folder, 'channel_positions.npy'))
channel_num = positions.shape[0]
if channel_names is None:
channel_names = ['ch_' + ft.int2str(i, 4) for i in range(channel_num)]
else:
if len(channel_names) != channel_num:
raise ValueError(
'number of kilosort output channels does not equal length of channel_names.'
)
channel_pos = {}
for i, ch_n in enumerate(channel_names):
channel_pos.update({ch_n: positions[i]})
return channel_pos
def test_getSparseNoiseOnsetIndex():
allOnsetInd, onsetIndWithLocationSign = sca.get_sparse_noise_onset_index(
ft.loadFile(sparseNoiseDisplayLogPath))
# print list(allOnsetInd[0:10])
# print onsetIndWithLocationSign[2][0]
assert (list(allOnsetInd[0:10]) == [0, 6, 12, 18, 24, 30, 36, 42, 48, 54])
assert (np.array_equal(onsetIndWithLocationSign[2][0], np.array([0.,
70.])))
def run():
curr_folder = os.path.dirname(os.path.realpath(__file__))
os.chdir(curr_folder)
data_f = h5py.File('rois_and_traces.hdf5')
traces_raw = data_f['traces_center_raw'].value
traces_srround = data_f['traces_surround_raw'].value
traces_subtracted = np.zeros(traces_raw.shape, np.float32)
ratio = np.zeros(traces_raw.shape[0], np.float32)
err = np.zeros(traces_raw.shape[0], np.float32)
for i in range(traces_raw.shape[0]):
curr_trace_c = traces_raw[i]
curr_trace_s = traces_srround[i]
curr_r, curr_err, curr_trace_sub = hl.neural_pil_subtraction(
curr_trace_c, curr_trace_s, lam=lam)
print "roi_%s \tr = %.4f; error = %.4f." % (ft.int2str(
i, 5), curr_r, curr_err)
traces_subtracted[i] = curr_trace_sub
ratio[i] = curr_r
err[i] = curr_err
print('\nplotting neuropil subtraction results ...')
figures_folder = 'figures/neuropil_subtraction_lam_{}'.format(lam)
if not os.path.isdir(figures_folder):
os.makedirs(figures_folder)
params = []
for roi_ind in range(traces_raw.shape[0]):
curr_traces = np.array([
traces_raw[roi_ind], traces_srround[roi_ind],
traces_subtracted[roi_ind]
])
params.append((curr_traces, plot_chunk_size, roi_ind, figures_folder))
p = Pool(process_num)
p.map(plot_traces_for_multi_process, params)
# wait for keyboard abortion
# msg = raw_input('Do you want to save? (y/n)\n')
# while True:
# if msg == 'y':
# break
# elif msg == 'n':
# sys.exit('Stop process without saving.')
# else:
# msg = raw_input('Do you want to save? (y/n)\n')
data_f['traces_center_subtracted'] = traces_subtracted
data_f['neuropil_r'] = ratio
data_f['neuropil_err'] = err
data_f.close()
def load_events(file_path, channels=None):
"""
return time stamps in seconds of each digital channel
:param file_path:
:param channels: name of channels
:return: dictionary, {channel: {'rise':[timestamps of rising events in seconds],
'fall':[timestamps of falling events in seconds}}
"""
print('\n')
if file_path[-7:] != '.events':
raise LookupError('The input file: ' + file_path + ' is not a .events file!')
with open(file_path) as f:
header = oe.readHeader(f)
fs = float(header['sampleRate'])
events = oe.loadEvents(file_path)
detected_channel_number = int(max(events['channel']) + 1)
real_channels = ['ch_' + ft.int2str(c, 3) for c in range(detected_channel_number)]
if channels is not None:
if detected_channel_number != len(channels):
warning_msg = '\nThe number of digital channels detected: ' + str(detected_channel_number) + \
' does not match input channel number: ' + str(len(channels))
warnings.warn(warning_msg)
if len(channels) <= detected_channel_number:
real_channels[0:len(channels)] = channels
else:
real_channels = channels[0:detected_channel_number]
output = {}
for i, ch in enumerate(real_channels):
output.update({ch : {'rise' : [],
'fall' : []}
})
rise = events['timestamps'][np.logical_and(events['channel'] == i, events['eventId'] == 1)]
output[ch]['rise'] = np.array(rise.astype(np.float32) / fs).astype(np.float32)
fall = events['timestamps'][np.logical_and(events['channel'] == i, events['eventId'] == 0)]
output[ch]['fall'] = np.array(fall.astype(np.float32) / fs).astype(np.float32)
print('events loaded.\n')
return output
def get_traces(params):
t0 = time.time()
chunk_ind, chunk_start, chunk_end, nwb_path, data_path, curr_folder, center_array, surround_array = params
nwb_f = h5py.File(nwb_path, 'r')
print('\nstart analyzing chunk: {}'.format(chunk_ind))
curr_mov = nwb_f[data_path][chunk_start:chunk_end]
nwb_f.close()
# print 'extracting traces'
curr_traces_center = np.empty((center_array.shape[0], curr_mov.shape[0]),
dtype=np.float32)
curr_traces_surround = np.empty((center_array.shape[0], curr_mov.shape[0]),
dtype=np.float32)
for i in range(center_array.shape[0]):
curr_center = ia.WeightedROI(center_array[i])
curr_surround = ia.ROI(surround_array[i])
curr_traces_center[i, :] = curr_center.get_weighted_trace_pixelwise(
curr_mov)
# scale surround trace to be similar as center trace
mean_center_weight = curr_center.get_mean_weight()
curr_traces_surround[i, :] = curr_surround.get_binary_trace_pixelwise(
curr_mov) * mean_center_weight
# print 'saveing chunk {} ...'.format(chunk_ind)
chunk_folder = os.path.join(curr_folder, 'chunks')
if not os.path.isdir(chunk_folder):
os.mkdir(chunk_folder)
chunk_f = h5py.File(
os.path.join(chunk_folder,
'chunk_temp_' + ft.int2str(chunk_ind, 4) + '.hdf5'))
chunk_f['traces_center'] = curr_traces_center
chunk_f['traces_surround'] = curr_traces_surround
chunk_f.close()
print('\n\t{:06d} seconds: chunk: {}; demixing finished.'.format(
int(time.time() - t0), chunk_ind))
return None
def get_clusters(csv_output):
"""
get cluster for each single unit and one cluster for all multi-unit activity
:param csv_output: output of read_csv function. list of tuples (cluster id, type)
:return: dictionary {unit_name : cluster_id, 'mua' : list of mua cluster_ids}
"""
output = {'unit_mua': []}
for cluster in csv_output:
try:
cluster_id = int(cluster[0])
except ValueError:
print(cluster[0], 'can not be converted into integer.')
continue
cluster_type = cluster[1]
if cluster_type == 'good':
output.update({'unit_' + ft.int2str(cluster_id, 5): cluster_id})
if cluster_type == 'mua':
output['unit_mua'].append(cluster_id)
return output
visualStimType='KSstim'
visualStimBackground='gray'
analysisParams ={}
if vasMapPaths:
vasMap = hl.getVasMap(vasMapPaths,dtype=vasMapDtype,headerLength=vasMapHeaderLength,tailerLength=vasMapTailerLength,
column=vasMapColumn,row=vasMapRow,frame=vasMapFrame,crop=vasMapCrop,mergeMethod=vasMapMergeMethod)
else:
print 'No vasculature map find. Taking first frame of movie as vasculature map.'
vasMap = BinarySlicer(movPath)[0,:,:]
vasMap = ia.array_nor(vasMap).astype(np.float32)
tf.imsave(os.path.join(saveFolder,dateRecorded+'_M'+mouseID+'_Trial'+trialNum+'_vasMap.tif'),vasMap)
_, jphys = ft.importRawNewJPhys(jphysPath,dtype=jphysDtype,headerLength=jphysHeaderLength,channels=jphysChannels,sf=jphysFs)
pd = jphys['photodiode']
displayOnsets = hl.segmentPhotodiodeSignal(pd, digitizeThr=pdDigitizeThr, filterSize=pdFilterSize, segmentThr=pdSegmentThr, Fs=jphysFs)
imgFrameTS = ta.get_onset_timeStamps(jphys['read'], Fs=jphysFs, threshold=readThreshold, onsetType=readOnsetType)
logPath = hl.findLogPath(date=dateRecorded,mouseID=mouseID,stimulus='KSstimAllDir',userID=userID,fileNumber=str(fileNum),displayFolder=dataFolder)
displayInfo = hl.analysisMappingDisplayLog(logPath)
sweepNum = len(displayInfo['B2U']['ind']+displayInfo['U2B']['ind']+displayInfo['L2R']['ind']+displayInfo['R2L']['ind'])
if len(displayOnsets) != sweepNum:
warningMessage = '\nNumber of detected photodiode onsets ('+str(len(displayOnsets))+') is not equal to display sweep number ('+str(sweepNum)+')!\n'
warnings.warn(warningMessage)
curr_mov = ia.rigid_transform_cv2_2d(curr_mov[:, ::-1, :], rotation=135)
curr_frame_num = curr_mov.shape[0] / len(ch_ns)
if curr_frame_num % frames_per_step != 0:
raise ValueError(
'{}: total frame number is not divisible by frames per step.'.
format(file_n))
curr_mov_chs = []
for ch_i in range(len(ch_ns)):
curr_mov_chs.append(curr_mov[ch_i::len(ch_ns)])
steps = curr_frame_num // frames_per_step
for step_ind in range(steps):
print('current step: {}'.format(curr_step))
for ch_i in range(len(ch_ns)):
curr_step_mov_ch = curr_mov_chs[ch_i][step_ind *
frames_per_step:(step_ind +
1) *
frames_per_step, :, :]
curr_step_n = 'step_' + ft.int2str(curr_step, 4)
curr_step_folder = os.path.join(save_folders[ch_i], curr_step_n)
os.mkdir(curr_step_folder)
tf.imsave(os.path.join(curr_step_folder, curr_step_n + '.tif'),
curr_step_mov_ch)
curr_step += 1
vas_map_paths = [
r"\\allen\programs\braintv\workgroups\nc-ophys\Jun\raw_data_rabies_project"
r"\180404-M360495-2p\vasmap_wf\180404JCamF100",
r"\\allen\programs\braintv\workgroups\nc-ophys\Jun\raw_data_rabies_project"
r"\180404-M360495-2p\vasmap_wf\180404JCamF101",
r"\\allen\programs\braintv\workgroups\nc-ophys\Jun\raw_data_rabies_project"
r"\180404-M360495-2p\vasmap_wf\180404JCamF102",
]
saveFolder = os.path.dirname(os.path.realpath(__file__))
os.chdir(saveFolder)
vas_maps = []
for vas_map_path in vas_map_paths:
vas_map_focused, _, _ = ft.importRawJCamF(vas_map_path,
column=1024,
row=1024,
headerLength=116,
tailerLength=452)
vas_map_focused = vas_map_focused[2:]
vas_map_focused = vas_map_focused[:, ::-1, :]
vas_map_focused[vas_map_focused > 50000] = 400
vas_map_focused = np.mean(vas_map_focused, axis=0)
vas_maps.append(ia.array_nor(vas_map_focused))
vas_map = ia.array_nor(np.mean(vas_maps, axis=0))
tf.imsave('vas_map_focused_wf.tif', vas_map.astype(np.float32))
data_f = h5py.File('rois_and_traces.hdf5')
traces_raw = data_f['traces_center_raw'].value
traces_srround = data_f['traces_surround_raw'].value
traces_subtracted = np.zeros(traces_raw.shape, np.float32)
ratio = np.zeros(traces_raw.shape[0], np.float32)
err = np.zeros(traces_raw.shape[0], np.float32)
for i in range(traces_raw.shape[0]):
curr_trace_c = traces_raw[i]
curr_trace_s = traces_srround[i]
curr_r, curr_err, curr_trace_sub = hl.neural_pil_subtraction(curr_trace_c,
curr_trace_s,
lam=lam)
print "roi_%s \tr = %.4f; error = %.4f." % (ft.int2str(
i, 5), curr_r, curr_err)
traces_subtracted[i] = curr_trace_sub
ratio[i] = curr_r
err[i] = curr_err
print('\nplotting neuropil subtraction results ...')
figures_folder = 'figures/neuropil_subtraction_lam_{}'.format(lam)
if not os.path.isdir(figures_folder):
os.makedirs(figures_folder)
for roi_ind in range(traces_raw.shape[0]):
print('roi_{:04d}'.format(roi_ind))
curr_traces = np.array([
traces_raw[roi_ind], traces_srround[roi_ind],
traces_subtracted[roi_ind]
])
curr_fig = plot_traces_chunks(traces=curr_traces,
['patch07', 'MMA'],
['patch08', 'MMP'],
['patch09', 'LLA'],
# ['patch10', 'AM'],
# ['patch11', 'LLA'],
# ['patch12', 'MMP'],
# ['patch13', 'MMP']
# ['patch14', 'MMP']
]
currFolder = os.path.dirname(os.path.realpath(__file__))
os.chdir(currFolder)
trialPath = os.path.join(currFolder, trialName)
trialDict = ft.loadFile(trialPath)
finalPatches = dict(trialDict['finalPatches'])
for i, namePair in enumerate(names):
currPatch = finalPatches.pop(namePair[0])
newPatchDict = {namePair[1]: currPatch}
finalPatches.update(newPatchDict)
trialDict.update({'finalPatchesMarked': finalPatches})
ft.saveFile(trialPath, trialDict)
trial, _ = rm.loadTrial(trialPath)
f = plt.figure(figsize=(10, 10))
ax = f.add_subplot(111)
def add_rois_and_traces(
data_folder,
nwb_f,
plane_n,
imaging_depth,
mov_path='/processing/motion_correction/MotionCorrection'):
mov_grp = nwb_f.file_pointer[mov_path + '/' + plane_n + '/corrected']
data_f = h5py.File(os.path.join(data_folder, 'rois_and_traces.hdf5'), 'r')
mask_arr_c = data_f['masks_center'].value
mask_arr_s = data_f['masks_surround'].value
traces_center_raw = data_f['traces_center_raw'].value
# traces_center_demixed = data_f['traces_center_demixed'].value
traces_center_subtracted = data_f['traces_center_subtracted'].value
# traces_center_dff = data_f['traces_center_dff'].value
traces_surround_raw = data_f['traces_surround_raw'].value
neuropil_r = data_f['neuropil_r'].value
neuropil_err = data_f['neuropil_err'].value
data_f.close()
if traces_center_raw.shape[1] != mov_grp['num_samples'].value:
raise ValueError(
'number of trace time points ({}) does not match frame number of '
'corresponding movie ({}).'.format(traces_center_raw.shape[1],
mov_grp['num_samples'].value))
# traces_center_raw = traces_center_raw[:, :mov_grp['num_samples'].value]
# traces_center_subtracted = traces_center_subtracted[:, :mov_grp['num_samples'].value]
# traces_surround_raw = traces_surround_raw[:, :mov_grp['num_samples'].value]
rf_img_max = tf.imread(
os.path.join(data_folder, 'corrected_max_projection.tif'))
rf_img_mean = tf.imread(
os.path.join(data_folder, 'corrected_mean_projection.tif'))
print 'adding segmentation results ...'
rt_mo = nwb_f.create_module('rois_and_traces_' + plane_n)
rt_mo.set_value('imaging_depth_micron', imaging_depth)
is_if = rt_mo.create_interface('ImageSegmentation')
is_if.create_imaging_plane('imaging_plane', description='')
is_if.add_reference_image('imaging_plane', 'max_projection', rf_img_max)
is_if.add_reference_image('imaging_plane', 'mean_projection', rf_img_mean)
for i in range(mask_arr_c.shape[0]):
curr_cen = mask_arr_c[i]
curr_cen_n = 'roi_' + ft.int2str(i, 4)
curr_cen_roi = ia.WeightedROI(curr_cen)
curr_cen_pixels_yx = curr_cen_roi.get_pixel_array()
curr_cen_pixels_xy = np.array(
[curr_cen_pixels_yx[:, 1], curr_cen_pixels_yx[:, 0]]).transpose()
is_if.add_roi_mask_pixels(image_plane='imaging_plane',
roi_name=curr_cen_n,
desc='',
pixel_list=curr_cen_pixels_xy,
weights=curr_cen_roi.weights,
width=512,
height=512)
curr_sur = mask_arr_s[i]
curr_sur_n = 'surround_' + ft.int2str(i, 4)
curr_sur_roi = ia.ROI(curr_sur)
curr_sur_pixels_yx = curr_sur_roi.get_pixel_array()
curr_sur_pixels_xy = np.array(
[curr_sur_pixels_yx[:, 1], curr_sur_pixels_yx[:, 0]]).transpose()
is_if.add_roi_mask_pixels(image_plane='imaging_plane',
roi_name=curr_sur_n,
desc='',
pixel_list=curr_sur_pixels_xy,
weights=None,
width=512,
height=512)
is_if.finalize()
trace_f_if = rt_mo.create_interface('Fluorescence')
seg_if_path = '/processing/rois_and_traces_' + plane_n + '/ImageSegmentation/imaging_plane'
# print seg_if_path
ts_path = mov_path + '/' + plane_n + '/corrected'
print 'adding center fluorescence raw'
trace_raw_ts = nwb_f.create_timeseries('RoiResponseSeries', 'f_center_raw')
trace_raw_ts.set_data(traces_center_raw,
unit='au',
conversion=np.nan,
resolution=np.nan)
trace_raw_ts.set_value('data_format', 'roi (row) x time (column)')
trace_raw_ts.set_value('data_range', '[-8192, 8191]')
trace_raw_ts.set_description(
'fluorescence traces extracted from the center region of each roi')
trace_raw_ts.set_time_as_link(ts_path)
trace_raw_ts.set_value_as_link('segmentation_interface', seg_if_path)
roi_names = [
'roi_' + ft.int2str(ind, 4)
for ind in range(traces_center_raw.shape[0])
]
trace_raw_ts.set_value('roi_names', roi_names)
trace_raw_ts.set_value('num_samples', traces_center_raw.shape[1])
trace_f_if.add_timeseries(trace_raw_ts)
trace_raw_ts.finalize()
print 'adding neuropil fluorescence raw'
trace_sur_ts = nwb_f.create_timeseries('RoiResponseSeries',
'f_surround_raw')
trace_sur_ts.set_data(traces_surround_raw,
unit='au',
conversion=np.nan,
resolution=np.nan)
trace_sur_ts.set_value('data_format', 'roi (row) x time (column)')
trace_sur_ts.set_value('data_range', '[-8192, 8191]')
trace_sur_ts.set_description(
'neuropil traces extracted from the surroud region of each roi')
trace_sur_ts.set_time_as_link(ts_path)
trace_sur_ts.set_value_as_link('segmentation_interface', seg_if_path)
sur_names = [
'surround_' + ft.int2str(ind, 4)
for ind in range(traces_center_raw.shape[0])
]
trace_sur_ts.set_value('roi_names', sur_names)
trace_sur_ts.set_value('num_samples', traces_surround_raw.shape[1])
trace_f_if.add_timeseries(trace_sur_ts)
trace_sur_ts.finalize()
roi_center_n_path = '/processing/rois_and_traces_' + plane_n + '/Fluorescence/f_center_raw/roi_names'
# print 'adding center fluorescence demixed'
# trace_demix_ts = nwb_f.create_timeseries('RoiResponseSeries', 'f_center_demixed')
# trace_demix_ts.set_data(traces_center_demixed, unit='au', conversion=np.nan, resolution=np.nan)
# trace_demix_ts.set_value('data_format', 'roi (row) x time (column)')
# trace_demix_ts.set_description('center traces after overlapping demixing for each roi')
# trace_demix_ts.set_time_as_link(mov_path + '/' + plane_n + '/corrected')
# trace_demix_ts.set_value_as_link('segmentation_interface', seg_if_path)
# trace_demix_ts.set_value('roi_names', roi_names)
# trace_demix_ts.set_value('num_samples', traces_center_demixed.shape[1])
# trace_f_if.add_timeseries(trace_demix_ts)
# trace_demix_ts.finalize()
print 'adding center fluorescence after neuropil subtraction'
trace_sub_ts = nwb_f.create_timeseries('RoiResponseSeries',
'f_center_subtracted')
trace_sub_ts.set_data(traces_center_subtracted,
unit='au',
conversion=np.nan,
resolution=np.nan)
trace_sub_ts.set_value('data_format', 'roi (row) x time (column)')
trace_sub_ts.set_description(
'center traces after overlap demixing and neuropil subtraction for each roi'
)
trace_sub_ts.set_time_as_link(mov_path + '/' + plane_n + '/corrected')
trace_sub_ts.set_value_as_link('segmentation_interface', seg_if_path)
trace_sub_ts.set_value_as_link('roi_names', roi_center_n_path)
trace_sub_ts.set_value('num_samples', traces_center_subtracted.shape[1])
trace_sub_ts.set_value('r', neuropil_r, dtype='float32')
trace_sub_ts.set_value('rmse', neuropil_err, dtype='float32')
trace_sub_ts.set_comments(
'value "r": neuropil contribution ratio for each roi. '
'value "rmse": RMS error of neuropil subtraction for each roi')
trace_f_if.add_timeseries(trace_sub_ts)
trace_sub_ts.finalize()
trace_f_if.finalize()
# print 'adding global dF/F traces for each roi'
# trace_dff_if = rt_mo.create_interface('DfOverF')
#
# trace_dff_ts = nwb_f.create_timeseries('RoiResponseSeries', 'dff_center')
# trace_dff_ts.set_data(traces_center_dff, unit='au', conversion=np.nan, resolution=np.nan)
# trace_dff_ts.set_value('data_format', 'roi (row) x time (column)')
# trace_dff_ts.set_description('global df/f traces for each roi center, input fluorescence is the trace after demixing'
# ' and neuropil subtraction. global df/f is calculated by '
# 'allensdk.brain_observatory.dff.compute_dff() function.')
# trace_dff_ts.set_time_as_link(ts_path)
# trace_dff_ts.set_value_as_link('segmentation_interface', seg_if_path)
# trace_dff_ts.set_value('roi_names', roi_names)
# trace_dff_ts.set_value('num_samples', traces_center_dff.shape[1])
# trace_dff_if.add_timeseries(trace_dff_ts)
# trace_dff_ts.finalize()
# trace_dff_if.finalize()
rt_mo.finalize()
def get_stim_dict_list(pkl_path, lsn_npy_path=None):
pkl_dict = ft.loadFile(pkl_path)
stimuli = pkl_dict['stimuli']
pre_blank_sec = pkl_dict['pre_blank_sec']
post_blank_sec = pkl_dict['post_blank_sec']
total_fps = pkl_dict['fps']
start_frame_num = int(total_fps * pre_blank_sec)
assert (pkl_dict['vsynccount'] == pkl_dict['total_frames'] +
(pre_blank_sec + post_blank_sec) * total_fps)
# print('\n'.join(pkl_dict.keys()))
stim_dict_lst = []
for stim_ind, stim in enumerate(stimuli):
# print('\n'.join(stim.keys()))
# print stim['stim_path']
# get stim_type
stim_str = stim['stim']
if '(' in stim_str:
if stim_str[0:stim_str.index('(')] == 'GratingStim':
if 'Phase' in stim['sweep_params'].keys():
stim_type = 'static_grating_camstim'
elif 'TF' in stim['sweep_params'].keys():
stim_type = 'drifting_grating_camstim'
else:
print('\n\nunknow stimulus type:')
print(stim['stim_path'])
print(stim['stim_text'])
stim_type = None
elif stim_str[0:stim_str.index('(')] == 'ImageStimNumpyuByte':
if 'locally_sparse_noise' in stim['stim_path']:
stim_type = 'locally_sparse_noise'
else:
print('\n\nunknow stimulus type:')
print(stim['stim_path'])
print(stim['stim_text'])
stim_type = None
else:
print('\n\nunknow stimulus type:')
print(stim['stim_path'])
print(stim['stim_text'])
stim_type = None
else:
print('\n\nunknow stimulus type:')
print(stim['stim_path'])
print(stim['stim_text'])
stim_type = None
if stim_type == 'drifting_grating_camstim':
stim_name = '{:03d}_DriftingGratingCamStim'.format(stim_ind)
print('\n\nextracting stimulus: ' + stim_name)
stim_dict = get_stim_dict_drifting_grating(input_dict=stim,
stim_name=stim_name)
stim_dict['sweep_onset_frames'] = stim_dict[
'sweep_onset_frames'] + start_frame_num
stim_dict.update({'stim_type': 'drifting_grating_camstim'})
start_frame_num = stim_dict['total_frame_num']
elif stim_type == 'locally_sparse_noise':
stim_name = '{:03d}_LocallySparseNoiseCamStim'.format(stim_ind)
print('\n\nextracting stimulus: ' + stim_name)
stim_dict = get_stim_dict_locally_sparse_noise(
input_dict=stim, stim_name=stim_name, npy_path=lsn_npy_path)
stim_dict['global_frame_ind'] = stim_dict[
'local_frame_ind'] + start_frame_num
stim_dict.update({'stim_type': 'locally_sparse_noise_camstim'})
start_frame_num = stim_dict['total_frame_num']
elif stim_type == 'static_gratings':
print('\n\nskip static_gratings stimulus. stim index: {}.'.format(
stim_ind))
# needs to fill in
stim_dict = {
'stim_name': '{:03d}_StaticGratingCamStim'.format(stim_ind),
'stim_type': 'static_grating_camstim',
'total_frame_num': stim['total_frames']
}
start_frame_num = stim['total_frame_num']
else:
print(
'\nskip unknow stimstimulus. stim index: {}.'.format(stim_ind))
# place holder
stim_dict = {
'stim_name': '{:03d}_UnknownCamStim'.format(stim_ind),
'stim_type': 'unknow_camstim',
'total_frame_num': stim['total_frames']
}
start_frame_num = stim['total_frame_num']
stim_dict_lst.append(stim_dict)
return stim_dict_lst
'signMapFilterSigma': 9.,
'signMapThr': 0.3,
'eccMapFilterSigma': 15.0,
'splitLocalMinCutStep': 10.,
'closeIter': 3,
'openIter': 3,
'dilationIter': 15,
'borderWidth': 1,
'smallPatchThr': 100,
'visualSpacePixelSize': 0.5,
'visualSpaceCloseIter': 15,
'splitOverlapThr': 1.1,
'mergeOverlapThr': 0.1
}
currFolder = os.path.dirname(os.path.realpath(__file__))
os.chdir(currFolder)
trial, _ = rm.loadTrial(trialName)
trial.params = params
_ = trial.processTrial(isPlot=True)
trialDict = trial.generateTrialDict()
trial.plotTrial(isSave=isSave, saveFolder=currFolder)
plt.show()
if isSave:
ft.saveFile(trial.getName() + '.pkl', trialDict)
def get_MPath(self):
self.axes.clear()
self.canvas.draw()
self.button_MPath.setStyleSheet('QPushButton {color: #888888}')
self.button_MPath.setEnabled(False)
fnames = QFileDialog.getOpenFileNames(
self,
'Choose Retinotopic Mapping Dictionary of TIFF/JCam file(s):',
self.currMatchingFolder)
fnames = list(fnames)
fnames = [str(x) for x in fnames]
try:
if len(fnames) == 0: # no file is chosen
print("no file is chosen! Setting matching map as None...")
self.textbrowser_MPath.clear()
self.MatchingVasMap = None
self.MatchingVasMapRaw = None
self.MatchingVasMapAfterChange = None
elif len(fnames) == 1: # only one file is chosen
filePath = fnames[0]
if filePath[-3:] == 'pkl': # mapping dictionary pkl file
self.trialDict = ft.loadFile(filePath)
self.MatchingVasMap = pt.merge_normalized_images(
[self.trialDict['vasculatureMap']])
self.MatchingVasMapRaw = self.trialDict['vasculatureMap']
self.textbrowser_MPath.setText(filePath)
self.MatchingVasMapAfterChange = None
elif filePath[-3:] == 'tif': # tiff file
self.MatchingVasMap = pt.merge_normalized_images(
[tf.imread(filePath)])
self.MatchingVasMapRaw = tf.imread(filePath)
self.textbrowser_MPath.setText(filePath)
self.MatchingVasMapAfterChange = None
else: # raw binary file
fileFolder, fileName = os.path.split(filePath)
if 'JCamF' in fileName:
currMap, _, _ = ft.importRawJCamF(filePath,
column=1024,
row=1024)
self.MatchingVasMap = pt.merge_normalized_images(
[currMap[0]])
self.MatchingVasMapRaw = currMap[0]
self.textbrowser_MPath.setText(filePath)
elif 'JCam' in fileName:
currMap, _ = ft.importRawJCam(filePath)
self.MatchingVasMap = pt.merge_normalized_images(
[currMap[0]])
self.MatchingVasMapRaw = currMap[0]
self.textbrowser_MPath.setText(filePath)
else:
print('Can not read matching map ' + filePath)
self.textbrowser_MPath.clear()
self.MatchingVasMap = None
self.MatchingVasMapAfterChange = None
else: # more than one file is chosen
displayText = ';'.join(fnames)
mapList = []
for i, filePath in enumerate(fnames):
if filePath[-3:] == 'tif': # tiff file
mapList.append(tf.imread(filePath))
else: # raw binary file
fileFolder, fileName = os.path.split(filePath)
if 'JCamF' in fileName:
currMap, _, _ = ft.importRawJCamF(filePath,
column=1024,
row=1024)
elif 'JCam' in fileName:
currMap, _ = ft.importRawJCam(filePath)
else:
print('Can not read ' + filePath)
mapList.append(currMap[0].astype(np.float32))
if len(mapList) == 0:
print(
"no file can be read! Setting matching map as None...")
self.textbrowser_MPath.clear()
self.MatchingVasMap = None
self.MatchingVasMapRaw = None
self.MatchingVasMapAfterChange = None
else:
self.MatchingVasMap = pt.merge_normalized_images(
mapList, dtype=np.float32)
self.MatchingVasMapRaw = pt.merge_normalized_images(
mapList, dtype=np.float32, isFilter=False)
self.textbrowser_MPath.setText(displayText)
self.MatchingVasMapAfterChange = None
except Exception as e:
print(e, '\n\n')
print('Can not load matching Map! Setting it as None...')
self.textbrowser_MPath.clear()
self.MatchingVasMap = None
self.MatchingVasMapRaw = None
self.MatchingVasMapAfterChange = None
self.button_MPath.setEnabled(True)
self.button_MPath.setStyleSheet('QPushButton {color: #000000}')
self.setZero()
self.currMatchingFolder = os.path.split(fnames[0])[0]
save_name = 'vasmap_wf'
data_folder = r"\\allen\programs\braintv\workgroups\nc-ophys\Jun\raw_data\190228-M426525-2p\vasmap_wf"
saveFolder = os.path.dirname(os.path.realpath(__file__))
os.chdir(saveFolder)
vasmap_fns = [f for f in os.listdir(data_folder) if 'JCam' in f]
vasmap_fns.sort()
print('\n'.join(vasmap_fns))
vasmaps = []
for vasmap_fn in vasmap_fns:
vasmap_focused, _, _ = ft.importRawJCamF(
os.path.join(data_folder, vasmap_fn),
column=1024,
row=1024,
headerLength=116,
tailerLength=452) # try 452 if 218 does not work
vasmap_focused = vasmap_focused[2:]
vasmap_focused[vasmap_focused > 50000] = 400
vasmap_focused = np.mean(vasmap_focused, axis=0)
vasmaps.append(ia.array_nor(vasmap_focused))
vasmap = ia.array_nor(np.mean(vasmaps, axis=0))
vasmap_r = vasmap[::-1, :]
tf.imsave('{}.tif'.format(save_name), vasmap.astype(np.float32))
tf.imsave('{}_rotated.tif'.format(save_name), vasmap_r.astype(np.float32))
# save figures
pt.save_figure_without_borders(f,
os.path.join(
save_folder,
'2P_refined_ROIs_with_background.png'),
dpi=300)
pt.save_figure_without_borders(f2,
os.path.join(
save_folder,
'2P_refined_ROIs_without_background.png'),
dpi=300)
# save h5 file
save_file = h5py.File(save_file_name, 'w')
i = 0
for retain_cell in retain_cells:
print retain_cell, ':', cells[retain_cell].get_binary_area()
currGroup = save_file.create_group('cell' + ft.int2str(i, 4))
currGroup.attrs['name'] = retain_cell
roiGroup = currGroup.create_group('roi')
cells[retain_cell].to_h5_group(roiGroup)
i += 1
for attr, value in dfile.attrs.iteritems():
save_file.attrs[attr] = value
save_file.close()
dfile.close()
tf.imsave(dateRecorded + '_M' + mouseID + '_vasMap.tif',
vasMap.astype(np.float32))
for fileNum in fileNumList:
movPath = os.path.join(dataFolder, [
f for f in fileList
if (dateRecorded + 'JCamF' + str(fileNum) in f) and ('.npy' in f)
][0])
jphysPath = os.path.join(dataFolder, [
f for f in fileList if dateRecorded + 'JPhys' + str(fileNum) in f
][0])
_, jphys = ft.importRawNewJPhys(jphysPath,
dtype=jphysDtype,
headerLength=jphysHeaderLength,
channels=jphysChannels,
sf=jphysFs)
pd = jphys['photodiode']
displayOnsets = hl.segmentPhotodiodeSignal(
pd,
digitizeThr=pdDigitizeThr,
filterSize=pdFilterSize,
segmentThr=pdSegmentThr,
Fs=jphysFs,
smallestInterval=smallestInterval)
imgFrameTS = ta.get_onset_timeStamps(jphys['read'],
Fs=jphysFs,
def pack_folders(folder_list, output_folder, output_filename, continous_channels, prefix, digital_channels):
"""
:param folder_list:
:param output_folder:
:param output_filename:
:param continous_channels:
:param digital_channels:
:param prefix:
:return:
"""
output_path_dat = os.path.join(output_folder, output_filename + '.dat')
output_path_h5 = os.path.join(output_folder, output_filename + '.hdf5')
if os.path.isfile(output_path_dat) or os.path.isfile(output_path_h5):
raise IOError('Output path already exists!')
h5_file = h5py.File(output_path_h5)
h5_file.attrs['device'] = 'tetrode'
_ = h5_file.create_dataset('channels', data=continous_channels)
curr_folder_start_ind = 0
data_all = []
sampling_rate = None
for i, folder in enumerate(folder_list):
curr_group = h5_file.create_group('folder' + ft.int2str(i, 4))
curr_group.attrs['path'] = folder
curr_con_group = curr_group.create_group('continuous')
curr_dig_group = curr_group.create_group('digital')
curr_ts_group = curr_group.create_group('timestamps')
curr_trace_dict, curr_sample_num, fs = pack_folder(folder, prefix, digital_channels=digital_channels)
all_channels = list(curr_trace_dict.keys())
print('\nall channels in folder ', folder, ':')
print(all_channels)
print()
if sampling_rate is None:
sampling_rate = fs
else:
if fs != sampling_rate:
err = 'The sampling rate (' + str(fs) + 'Hz) of folder: (' + folder + ') does not match the sampling' +\
' rate (' + str(sampling_rate) + ') of other folders.'
raise ValueError(err)
curr_data_array = []
# add electrode channels
for channel in continous_channels:
curr_prefix = prefix + '_CH' + str(channel)
curr_key = [k for k in all_channels if k[:len(curr_prefix)] == curr_prefix]
if len(curr_key) == 0:
raise LookupError('no file is found in ' + folder +' for channel ' + str(channel) + '!')
elif len(curr_key) > 1:
raise LookupError('more than one files are found in ' + folder +' for channel ' + str(channel) + '!')
curr_key = curr_key[0]
curr_dset = curr_con_group.create_dataset('channel_' + ft.int2str(int(channel), 4),
data=curr_trace_dict[curr_key])
curr_dset.attrs['unit'] = 'arbitrary_unit'
curr_data_array.append(curr_trace_dict[curr_key])
curr_data_array = np.array(curr_data_array, dtype=np.int16)
data_all.append(curr_data_array.flatten(order='F'))
# add continuous channels
for ch, trace in curr_trace_dict.items():
if '_CH' not in ch and ch != 'events':
curr_dset = curr_con_group.create_dataset(ch[len(prefix) + 1:], data=trace)
curr_dset.attrs['unit'] = 'volt'
# add digital events
events = curr_trace_dict['events']
for dch, dch_dict in events.items():
curr_dch_group = curr_dig_group.create_group(dch)
curr_dch_group.create_dataset('rise', data=dch_dict['rise'])
curr_dch_group.create_dataset('fall', data=dch_dict['fall'])
curr_group.attrs['start_index'] = curr_folder_start_ind
curr_group.attrs['end_index'] = curr_folder_start_ind + curr_sample_num
curr_folder_start_ind += curr_sample_num
h5_file.create_dataset('fs_hz', data=float(sampling_rate))
h5_file.close()
data_all = np.concatenate(data_all)
data_all.tofile(output_path_dat)
def run():
# pixels, masks with center location within this pixel region at the image border will be discarded
center_margin = [10, 20, 25, 10] # [top margin, bottom margin, left margin, right margin]
# area range, range of number of pixels of a valid roi
area_range = [150, 1000]
# for the two masks that are overlapping, if the ratio between overlap and the area of the smaller mask is larger than
# this value, the smaller mask will be discarded.
overlap_thr = 0.2
save_folder = 'figures'
data_file_name = 'cells.hdf5'
save_file_name = 'cells_refined.hdf5'
background_file_name = "corrected_mean_projections.tif"
curr_folder = os.path.dirname(os.path.realpath(__file__))
os.chdir(curr_folder)
if not os.path.isdir(save_folder):
os.makedirs(save_folder)
# read cells
dfile = h5py.File(data_file_name)
cells = {}
for cellname in dfile.iterkeys():
cells.update({cellname:ia.WeightedROI.from_h5_group(dfile[cellname])})
print 'total number of cells:', len(cells)
# get the names of cells which are on the edge
edge_cells = []
for cellname, cellmask in cells.iteritems():
dimension = cellmask.dimension
center = cellmask.get_center()
if center[0] < center_margin[0] or \
center[0] > dimension[0] - center_margin[1] or \
center[1] < center_margin[2] or \
center[1] > dimension[1] - center_margin[3]:
# cellmask.plot_binary_mask_border(color='#ff0000', borderWidth=1)
# plt.title(cellname)
# plt.show()
edge_cells.append(cellname)
print '\ncells to be removed because they are on the edges:'
print '\n'.join(edge_cells)
# remove edge cells
for edge_cell in edge_cells:
_ = cells.pop(edge_cell)
# get dictionary of cell areas
cell_areas = {}
for cellname, cellmask in cells.iteritems():
cell_areas.update({cellname: cellmask.get_binary_area()})
# remove cellnames that have area outside of the area_range
invalid_cell_ns = []
for cellname, cellarea in cell_areas.items():
if cellarea < area_range[0] or cellarea > area_range[1]:
invalid_cell_ns.append(cellname)
print "cells to be removed because they do not meet area criterion:"
print "\n".join(invalid_cell_ns)
for invalid_cell_n in invalid_cell_ns:
cell_areas.pop(invalid_cell_n)
# sort cells with their binary area
cell_areas_sorted = sorted(cell_areas.items(), key=operator.itemgetter(1))
cell_areas_sorted.reverse()
cell_names_sorted = [c[0] for c in cell_areas_sorted]
# print '\n'.join([str(c) for c in cell_areas_sorted])
# get the name of cells that needs to be removed because of overlapping
retain_cells = []
remove_cells = []
for cell1_name in cell_names_sorted:
cell1_mask = cells[cell1_name]
is_remove = 0
cell1_area = cell1_mask.get_binary_area()
for cell2_name in retain_cells:
cell2_mask = cells[cell2_name]
cell2_area = cell2_mask.get_binary_area()
curr_overlap = cell1_mask.binary_overlap(cell2_mask)
if float(curr_overlap) / cell1_area > overlap_thr:
remove_cells.append(cell1_name)
is_remove = 1
print cell1_name, ':', cell1_mask.get_binary_area(), ': removed'
# f = plt.figure(figsize=(10,10))
# ax = f.add_subplot(111)
# cell1_mask.plot_binary_mask_border(plotAxis=ax, color='#ff0000', borderWidth=1)
# cell2_mask.plot_binary_mask_border(plotAxis=ax, color='#0000ff', borderWidth=1)
# ax.set_title('red:'+cell1_name+'; blue:'+cell2_name)
# plt.show()
break
if is_remove == 0:
retain_cells.append(cell1_name)
print cell1_name, ':', cell1_mask.get_binary_area(), ': retained'
print '\ncells to be removed because of overlapping:'
print '\n'.join(remove_cells)
print '\ntotal number of reatined cells:', len(retain_cells)
# plotting
colors = pt.random_color(len(cells.keys()))
bgImg = ia.array_nor(np.max(tf.imread(background_file_name), axis=0))
f = plt.figure(figsize=(10, 10))
ax = f.add_subplot(111)
ax.imshow(ia.array_nor(bgImg), cmap='gray', vmin=0, vmax=0.5, interpolation='nearest')
f2 = plt.figure(figsize=(10, 10))
ax2 = f2.add_subplot(111)
ax2.imshow(np.zeros(bgImg.shape, dtype=np.uint8), vmin=0, vmax=1, cmap='gray', interpolation='nearest')
i = 0
for retain_cell in retain_cells:
cells[retain_cell].plot_binary_mask_border(plotAxis=ax, color=colors[i], borderWidth=1)
cells[retain_cell].plot_binary_mask_border(plotAxis=ax2, color=colors[i], borderWidth=1)
i += 1
# plt.show()
# save figures
pt.save_figure_without_borders(f, os.path.join(save_folder, '2P_refined_ROIs_with_background.png'), dpi=300)
pt.save_figure_without_borders(f2, os.path.join(save_folder, '2P_refined_ROIs_without_background.png'), dpi=300)
# save h5 file
save_file = h5py.File(save_file_name, 'w')
i = 0
for retain_cell in retain_cells:
print retain_cell, ':', cells[retain_cell].get_binary_area()
currGroup = save_file.create_group('cell' + ft.int2str(i, 4))
currGroup.attrs['name'] = retain_cell
roiGroup = currGroup.create_group('roi')
cells[retain_cell].to_h5_group(roiGroup)
i += 1
for attr, value in dfile.attrs.iteritems():
save_file.attrs[attr] = value
save_file.close()
dfile.close()