def run_single_file_for_multi_process(params):
file_path, file_ind, total_file_num, t0 = params
print('{:6.1f} min; {} / {}; {}: regenerating strf ...'.format(
(time.time() - t0) / 60., file_ind + 1, total_file_num, file_path))
nwb_f = nt.RecordedFile(file_path)
lsn_name = '001_LocallySparseNoiseRetinotopicMapping'
if dt.get_strf_grp_key(nwb_f.file_pointer) is None:
if lsn_name in nwb_f.file_pointer['analysis/photodiode_onsets']:
nwb_f.get_spatial_temporal_receptive_field_retinotopic_mapping(
stim_name=lsn_name, time_window=strf_t_win, verbose=False)
dgc_name = '003_DriftingGratingCircleRetinotopicMapping'
if dt.get_dgcrm_grp_key(nwb_f.file_pointer) is None:
if dgc_name in nwb_f.file_pointer['analysis/photodiode_onsets']:
nwb_f.get_drifting_grating_response_table_retinotopic_mapping(
stim_name=dgc_name, time_window=dgc_t_win)
dgc_name = '001_DriftingGratingCircleRetinotopicMapping'
if dt.get_dgcrm_grp_key(nwb_f.file_pointer) is None:
if dgc_name in nwb_f.file_pointer['analysis/photodiode_onsets']:
nwb_f.get_drifting_grating_response_table_retinotopic_mapping(
stim_name=dgc_name, time_window=dgc_t_win)
nwb_f.close()
def process_one_nwb_for_multi_thread(inputs):
nwb_path, params, columns, save_folder, t0 = inputs
nwb_fn = os.path.splitext(os.path.split(nwb_path)[1])[0]
save_path = os.path.join(save_folder, nwb_fn + '.xlsx')
if os.path.isfile(save_path):
print('\tt: {:5.0f} minutes, {} , excel table already exists, skip.'.
format((time.time() - t0) / 60., nwb_fn))
return None
else:
nwb_f = h5py.File(nwb_path, 'r')
plane_ns = [
k for k in nwb_f['processing'].keys()
if k[0:16] == 'rois_and_traces_'
]
plane_ns = [k[16:] for k in plane_ns]
plane_ns.sort()
# print('total plane number: {}'.format(len(plane_ns)))
total_roi_num = 10000
big_array = np.zeros((total_roi_num, len(columns)), dtype=np.float64)
big_array[:] = np.nan
df = pd.DataFrame(data=big_array, columns=columns)
curr_row_ind = 0
for plane_n in plane_ns:
print('\tt: {:5.0f} minutes, {} / {}, processing ...'.format(
(time.time() - t0) / 60., nwb_fn, plane_n))
roi_ns = nwb_f[
'processing/rois_and_traces_{}/ImageSegmentation/imaging_plane/roi_list'
.format(plane_n)].value
roi_ns = [r.encode('utf-8') for r in roi_ns if r[0:4] == 'roi_']
roi_ns.sort()
for roi_i, roi_n in enumerate(roi_ns):
# print('\t\t\troi: {} / {}'.format(roi_i+1, len(roi_ns)))
roi_properties, _, _, _, _, _, _, _, _, _, _, _, _, _ = \
dt.get_everything_from_roi(nwb_f=nwb_f, plane_n=plane_n, roi_n=roi_n, params=params)
for rp_name, rp_value in roi_properties.items():
df.loc[curr_row_ind, rp_name] = rp_value
curr_row_ind += 1
df = df[0:curr_row_ind]
save_path = os.path.join(save_folder, nwb_fn + '.xlsx')
if os.path.isfile(save_path):
os.remove(save_path)
with pd.ExcelWriter(save_path, mode='w') as writer:
df.to_excel(writer, sheet_name='sheet1')
def process_one_nwb_for_multi_thread(inputs):
nwb_path, params, columns, save_folder, t0, nwb_i, nwb_f_num, is_overwrite = inputs
nwb_fn = os.path.splitext(os.path.split(nwb_path)[1])[0]
nwb_f = h5py.File(nwb_path, 'r')
plane_ns = [
k for k in nwb_f['processing'].keys() if k[0:16] == 'rois_and_traces_'
]
plane_ns = [k[16:] for k in plane_ns]
plane_ns.sort()
# print('total plane number: {}'.format(len(plane_ns)))
for plane_n in plane_ns:
print('\tt: {:5.0f} minutes, processing {}, {} / {}, {} ...'.format(
(time.time() - t0) / 60., nwb_fn, nwb_i + 1, nwb_f_num, plane_n))
save_fn = '_'.join(nwb_fn.split('_')[0:2]) + '_' + plane_n + '.xlsx'
save_path = os.path.join(save_folder, save_fn)
if os.path.isfile(save_path):
if is_overwrite: # overwrite existing xlsx files
print('\t{}, file already exists. Overwirite.'.format(
os.path.split(save_path)[1]))
os.remove(save_path)
else: # do not overwrite existing xlsx files
print('\t{}, file already exists. Skip.'.format(
os.path.split(save_path)[1]))
return
roi_ns = nwb_f[
'processing/rois_and_traces_{}/ImageSegmentation/imaging_plane/roi_list'
.format(plane_n)].value
roi_ns = [r.encode('utf-8') for r in roi_ns if r[0:4] == 'roi_']
roi_ns.sort()
df = pd.DataFrame(np.nan, index=range(len(roi_ns)), columns=columns)
for roi_i, roi_n in enumerate(roi_ns):
# print('\t\t\troi: {} / {}'.format(roi_i+1, len(roi_ns)))
roi_properties, _, _, _, _, _, _, _, _, _, _, _, _, _ = \
dt.get_everything_from_roi(nwb_f=nwb_f, plane_n=plane_n, roi_n=roi_n, params=params)
for rp_name, rp_value in roi_properties.items():
df.loc[roi_i, rp_name] = rp_value
with pd.ExcelWriter(save_path, mode='w') as writer:
df.to_excel(writer, sheet_name='sheet1')
print('plotting rois in {} ...'.format(plane_n))
plane_grp = strf_grp[plane_n]
pdff = PdfPages(os.path.join(save_folder,
'zscore_RFs_' + plane_n + '.pdf'))
roi_lst = nwb_f['processing/rois_and_traces_' + plane_n +
'/ImageSegmentation/imaging_plane/roi_list'].value
roi_lst = [r for r in roi_lst if r[:4] == 'roi_']
roi_lst.sort()
for roi_ind, roi_n in enumerate(roi_lst):
print('roi: {} / {}'.format(roi_ind + 1, len(roi_lst)))
curr_trace, _ = dt.get_single_trace(nwb_f=nwb_f,
plane_n=plane_n,
roi_n=roi_n)
if np.min(curr_trace) < bias:
add_to_trace = -np.min(curr_trace) + bias
else:
add_to_trace = 0.
curr_strf = sca.get_strf_from_nwb(h5_grp=strf_grp[plane_n],
roi_ind=roi_ind,
trace_type=trace_type)
curr_strf_dff = curr_strf.get_local_dff_strf(
is_collaps_before_normalize=True, add_to_trace=add_to_trace)
# v_min, v_max = curr_strf_dff.get_data_range()
rf_on, rf_off = curr_strf_dff.get_zscore_receptive_field(
timeWindow=t_window)
nwb_f = h5py.File(
os.path.join(nwb_folder, '{}_{}_110_repacked.nwb'.format(date, mid)),
'r')
clu_f = h5py.File(
os.path.join(clu_folder,
'{}_{}_{}_axon_grouping.hdf5'.format(date, mid, plane_n)),
'r')
curr_dfa = dfa[(dfa['date'] == date) & (dfa['mouse_id'] == mid) &
(dfa['plane_n'] == plane_n)].reset_index()
for axon_i, axon_row in curr_dfa.iterrows():
axon_morph = dt.get_axon_morphology(clu_f=clu_f,
nwb_f=nwb_f,
plane_n=plane_n,
axon_n=axon_row['roi_n'])
axon_morph.update({
'date': date,
'mouse_id': mid,
'plane_n': plane_n,
'roi_n': axon_row['roi_n']
})
axon_morph_dict[ind] = axon_morph
ind = ind + 1
dfa_morph = pd.DataFrame.from_dict(axon_morph_dict, orient='index')
print(dfa_morph)
strs = os.path.splitext(dfa_fn)[0].split('_')
alpha=0.5,
)
ax_and_pm.get_yaxis().set_ticks([])
# =============================azi alt spatial distribution=============================================
ax_alt_or = f.add_subplot(4, 5, 4)
ax_alt_or.set_title('altitude')
ax_azi_or = f.add_subplot(4, 5, 5)
ax_azi_or.set_title('azimuth')
if len(df_or) > 0:
dt.plot_roi_retinotopy(
coords_rf=df_or[[
'rf_{}_onoff_center_alt'.format(response_dir),
'rf_{}_onoff_center_azi'.format(response_dir)
]].values,
coords_roi=df_or[['roi_center_row', 'roi_center_col']].values,
ax_alt=ax_alt_or,
ax_azi=ax_azi_or,
cmap='viridis',
canvas_shape=(512, 512),
edgecolors='#000000',
linewidth=0.5)
else:
ax_alt_or.set_xticks([])
ax_alt_or.set_yticks([])
ax_azi_or.set_xticks([])
ax_azi_or.set_yticks([])
ax_alt_on = f.add_subplot(4, 5, 9)
ax_azi_on = f.add_subplot(4, 5, 10)
if len(df_on) > 0:
dt.plot_roi_retinotopy(
save_folder = os.path.join(
save_folder, '{}_DistanceThr_{:.2f}'.format(trace_window, distance_thr))
if not os.path.isdir(save_folder):
os.makedirs(save_folder)
nwb_fns = [f for f in os.listdir(nwb_folder) if f[-4:] == '.nwb']
nwb_fns.sort()
bc = dt.BoutonClassifier(skew_filter_sigma=skew_filter_sigma,
skew_thr=skew_thr,
lowpass_sigma=lowpass_sigma,
detrend_sigma=detrend_sigma,
event_std_thr=event_std_thr,
peri_event_dur=peri_event_dur,
corr_len_thr=corr_len_thr,
corr_abs_thr=corr_abs_thr,
corr_std_thr=corr_std_thr,
is_cosine_similarity=is_cosine_similarity,
distance_metric=distance_metric,
linkage_method=linkage_method,
distance_thr=distance_thr)
for nwb_fi, nwb_fn in enumerate(nwb_fns):
print('processing {}, {}/{}'.format(nwb_fn, nwb_fi + 1, len(nwb_fns)))
nwb_f = h5py.File(os.path.join(nwb_folder, nwb_fn), 'r')
plane_ns = dt.get_plane_ns(nwb_f=nwb_f)
plane_ns.sort()
import matplotlib.pyplot as plt
nwb_folder = 'nwbs'
nwb_fn = "190326_M441626_110.nwb"
plane_n = 'plane2'
roi_n = 'roi_0001'
analysis_params = dt.ANALYSIS_PARAMS
plot_params = dt.PLOTTING_PARAMS
analysis_params['trace_type'] = 'f_center_raw'
analysis_params['response_window_dgc'] = [0.5, 1.5]
analysis_params['baseline_window_dgc'] = [-0.5, 0.5]
plot_params['sftf_vmax'] = 6
plot_params['sftf_vmin'] = -6
curr_folder = os.path.dirname(os.path.realpath(__file__))
os.chdir(curr_folder)
nwb_f = h5py.File(os.path.join(nwb_folder, nwb_fn), 'r')
f = dt.roi_page_report(nwb_f=nwb_f,
plane_n=plane_n,
roi_n=roi_n,
params=analysis_params,
plot_params=plot_params)
nwb_f.close()
plt.show()
f.savefig('{}_{}_{}.pdf'.format(os.path.splitext(nwb_fn)[0], plane_n, roi_n))
import os
import h5py
import corticalmapping.DatabaseTools as dt
import corticalmapping.HighLevel as hl
from matplotlib.backends.backend_pdf import PdfPages
import matplotlib.pyplot as plt
overlap_ratio = 0.5
trace_type = 'f_center_raw'
save_folder = 'figures'
size_thr = 20. # um^2
curr_folder = os.path.dirname(os.path.realpath(__file__))
os.chdir(curr_folder)
save_folder = os.path.join(curr_folder, save_folder)
if not os.path.isdir(save_folder):
os.makedirs(save_folder)
nwb_fn = [fn for fn in os.listdir(curr_folder) if fn[-4:] == '.nwb']
if len(nwb_fn) == 0:
raise LookupError('cannot find .nwb file.')
elif len(nwb_fn) > 1:
raise LookupError('more than one .nwb files found.')
nwb_fn = nwb_fn[0]
nwb_f = h5py.File(os.path.join(curr_folder, nwb_fn), 'r')
pdf_path = os.path.join(
save_folder,
r'overlapping_rois_{}.pdf'.format(os.path.splitext(nwb_fn)[0]))
pdff = PdfPages(pdf_path)
roi_triplets = dt.get_roi_triplets(nwb_f=nwb_f,
overlap_ratio=overlap_ratio,
size_thr=size_thr)
for i, roi_triplet in enumerate(roi_triplets):
(subdf['rf_{}_off_peak_z'.format(response_dir)] >= analysis_params['rf_z_thr_abs'])].reset_index()
if len(s2_df) > 0:
s2_grp = save_f.create_group('{}_{}_{}_{}_ONOFF'.format(date, mid, plane_n, response_dir))
s1_on_grp = save_f.create_group('{}_{}_{}_{}_ON'.format(date, mid, plane_n, response_dir))
s1_off_grp = save_f.create_group('{}_{}_{}_{}_OFF'.format(date, mid, plane_n, response_dir))
for roi_i, roi_row in s2_df.iterrows():
print('\t s2 receptive fields, {}, {} / {} ...'.format(roi_row['roi_n'], roi_i + 1, len(s2_df)))
if response_dir == 'pos':
_, _, _, srf_on, srf_off, _, _, _, _, _, _, _, _, \
_ = dt.get_everything_from_axon(nwb_f=nwb_f,
clu_f=clu_f,
plane_n=plane_n,
axon_n=roi_row['roi_n'],
params=analysis_params)
_, rf_on_new = dt.get_rf_properties(srf=srf_on,
polarity='positive',
sigma=analysis_params['gaussian_filter_sigma_rf'],
interpolate_rate=analysis_params['interpolate_rate_rf'],
z_thr_abs=analysis_params['rf_z_thr_abs'],
z_thr_rel=analysis_params['rf_z_thr_rel'])
_, rf_off_new = dt.get_rf_properties(srf=srf_off,
polarity='positive',
sigma=analysis_params['gaussian_filter_sigma_rf'],
interpolate_rate=analysis_params['interpolate_rate_rf'],
z_thr_abs=analysis_params['rf_z_thr_abs'],
local_grp.attrs['trace_type'] = sheet_name
local_grp.attrs['response_dir'] = response_dir
local_grp.attrs['skew_thr'] = 0.6
local_grp.attrs['trace_bias'] = trace_bias
local_grp.attrs['rf_peak_z_thr'] = rf_peak_z_thr
local_grp.attrs['gaussian_filter_sigma'] = gaussian_filter_sigma
local_grp.attrs['interpolation_rate'] = interpolate_rate
local_grp.attrs['response_window'] = response_window
print('\t\tprocessing roi {}/{} ...'.format(roi_i + 1, len(localdfon)))
if roi_row['roi_n'] not in local_grp.keys():
# get constant to add to trace
trace, _ = dt.get_single_trace(nwb_f=nwb_f,
plane_n=roi_row['plane_n'],
roi_n=roi_row['roi_n'],
trace_type=sheet_name)
if np.min(trace) < trace_bias:
add_to_trace = -np.min(trace) + trace_bias
else:
add_to_trace = 0.
# get strf
curr_strf = dt.get_strf(nwb_f=nwb_f,
plane_n=roi_row['plane_n'],
roi_ind=int(roi_row['roi_n'][-4:]),
trace_type='sta_'+sheet_name)
curr_strf_dff = curr_strf.get_local_dff_strf(is_collaps_before_normalize=True,
add_to_trace=add_to_trace)
curr_srf, _ = curr_strf_dff.get_zscore_receptive_field(timeWindow=response_window)
if response_dir == 'pos':
curr_folder = os.path.dirname(os.path.realpath(__file__))
os.chdir(curr_folder)
db_df = pd.read_excel(summary_fn, sheet_name=sheet_n)
print(db_df.head())
pdf_fn = 'page_report_{}.pdf'.format(
datetime.datetime.now().strftime('%y%m%d%H%M%S'))
pdff = PdfPages(pdf_fn)
for row_i, row in db_df.iterrows():
# if row['roi_area'] >= area_lim and row['skew_fil'] >= 0.5:
if row['roi_area'] >= area_lim:
print('{}_{}; {}; {}'.format(row['date'], row['mouse_id'],
row['plane_n'], row['roi_n']))
nwb_fn = '{}_{}_110.nwb'.format(row['date'], row['mouse_id'])
nwb_f = h5py.File(os.path.join(nwb_folder, nwb_fn), 'r')
f = dt.roi_page_report(nwb_f=nwb_f,
plane_n=row['plane_n'],
roi_n=row['roi_n'],
params=params,
plot_params=plot_params)
nwb_f.close()
pdff.savefig(f)
f.clear()
plt.close()
pdff.close()
def process_one_nwb_for_multi_thread(inputs):
nwb_path, df_folder, clu_folder, params, columns, save_folder, t0, nwb_i, nwb_f_num, is_overwrite = inputs
nwb_fn = os.path.splitext(os.path.split(nwb_path)[1])[0]
date, mid, _, _ = nwb_fn.split('_')
nwb_f = h5py.File(nwb_path, 'r')
plane_ns = dt.get_plane_ns(nwb_f=nwb_f)
plane_ns.sort()
for plane_n in plane_ns:
print('\tt: {:5.0f} minutes, processing {}, {} / {}, {} ...'.format(
(time.time() - t0) / 60., nwb_fn, nwb_i + 1, nwb_f_num, plane_n))
roi_df_fn = '{}_{}_{}.csv'.format(date, mid, plane_n)
roi_df = pd.read_csv(os.path.join(df_folder, roi_df_fn))
clu_fn = '{}_{}_{}_axon_grouping.hdf5'.format(date, mid, plane_n)
clu_f = h5py.File(os.path.join(clu_folder, clu_fn), 'r')
axon_ns = clu_f['axons'].keys()
axon_ns.sort()
axon_df = pd.DataFrame(np.nan,
index=range(len(axon_ns)),
columns=columns)
for axon_i, axon_n in enumerate(axon_ns):
roi_lst = clu_f['axons/{}'.format(axon_n)].value
if len(roi_lst) == 1:
curr_roi_df = roi_df[roi_df['roi_n'] ==
roi_lst[0]].reset_index()
for col in columns:
axon_df.loc[axon_i, col] = curr_roi_df.loc[0, col]
axon_df.loc[axon_i, 'roi_n'] = axon_n
else:
axon_properties, _, _, _, _, _, _, _, _, _, _, _, _, _ = \
dt.get_everything_from_axon(nwb_f=nwb_f,
clu_f=clu_f,
plane_n=plane_n,
axon_n=axon_n,
params=params,
verbose=False)
for rp_name, rp_value in axon_properties.items():
axon_df.loc[axon_i, rp_name] = rp_value
save_path = os.path.join(save_folder,
'{}_{}_{}.csv'.format(date, mid, plane_n))
if os.path.isfile(save_path):
if is_overwrite:
os.remove(save_path)
axon_df.to_csv(save_path)
else:
raise IOError(
'Axon dataframe file already exists. \npath: {}'.format(
save_path))
else:
axon_df.to_csv(save_path)
import os
import h5py
import corticalmapping.DatabaseTools as dt
nwb_folder = "nwbs"
clu_folder = r"intermediate_results\bouton_clustering\AllStimuli_DistanceThr_1.30"
strf_t_win = [-0.5, 2.]
curr_folder = os.path.dirname(os.path.realpath(__file__))
os.chdir(curr_folder)
clu_fns = [f for f in os.listdir(clu_folder) if f[-5:] == '.hdf5']
clu_fns.sort()
print('total number of planes: {}'.format(len(clu_fns)))
for clu_fi, clu_fn in enumerate(clu_fns):
date, mid, plane_n, _, _ = clu_fn.split('_')
print('processing {}_{}_{}, {} / {}'.format(date, mid, plane_n, clu_fi + 1, len(clu_fns)))
nwb_fn = '{}_{}_110_repacked.nwb'.format(date, mid)
nwb_f = h5py.File(os.path.join(nwb_folder, nwb_fn), 'r')
clu_f = h5py.File(os.path.join(clu_folder, clu_fn))
bc = dt.BoutonClassifier()
bc.add_axon_strf(nwb_f=nwb_f, clu_f=clu_f, plane_n=plane_n, t_win=strf_t_win, verbose=False)
nwb_f.close()
clu_f.close()
# positive spatial receptive fields
srf_on, srf_off = strf_dff.get_zscore_receptive_field(timeWindow=[0., 0.5])
srf = srf_on.gaussian_filter(sigma=1.)
srf = srf.interpolate(ratio=10.)
print(np.max(srf.weights))
f_o = h5py.File(fn_original, 'r')
strf_o = sca.SpatialTemporalReceptiveField.from_h5_group(
f_o['analysis/STRFs/plane0/strf_roi_{:04d}'.format(roi_ind)])
print_peak_z(strf_o)
# print(strf_o.data['traces'][0][5])
# print(strf_o.time)
f_o.close()
f_r = h5py.File(os.path.join('repacked', fn_repacked), 'r')
strf_r = dt.get_strf(f_r,
plane_n='plane0',
roi_ind=0,
trace_type='sta_f_center_subtracted')
print_peak_z(strf_r)
# print(strf_r.data['traces'][0][5])
# print(strf_r.time)
roi_properties, _, _, _, _, _, _, _, _, _, _, _, _, _ = \
dt.get_everything_from_roi(nwb_f=f_r, plane_n='plane0', roi_n=roi_n, params=params)
print(roi_properties['rf_pos_on_peak_z'])
f_r.close()
nwb_fn = [fn for fn in os.listdir(curr_folder) if fn[-4:] == '.nwb']
if len(nwb_fn) == 0:
raise LookupError('cannot find .nwb file.')
elif len(nwb_fn) > 1:
raise LookupError('more than one .nwb files found.')
nwb_fn = nwb_fn[0]
pdf_path = os.path.join(
save_folder, r'page_report_{}.pdf'.format(os.path.splitext(nwb_fn)[0]))
pdff = PdfPages(pdf_path)
nwb_f = h5py.File(nwb_fn, 'r')
plane_ns = dt.get_plane_ns(nwb_f)
for plane_n in plane_ns:
roi_ns = dt.get_roi_ns(nwb_f, plane_n)
for roi_n in roi_ns:
roi = dt.get_roi(nwb_f, plane_n, roi_n)
if roi.get_binary_area() >= area_lim:
print('plotting {}; {}; {}'.format(nwb_fn, plane_n, roi_n))
f = dt.roi_page_report(nwb_f=nwb_f,
plane_n=plane_n,