def analyze_pair_connectivity(amps, sign=None):
"""Given response strength records for a single pair, generate summary
statistics characterizing strength, latency, and connectivity.
Parameters
----------
amps : dict
Contains foreground and background strength analysis records
(see input format below)
sign : None, -1, or +1
If None, then automatically determine whether to treat this connection as
inhibitory or excitatory.
Input must have the following structure::
amps = {
('ic', 'fg'): recs,
('ic', 'bg'): recs,
('vc', 'fg'): recs,
('vc', 'bg'): recs,
}
Where each *recs* must be a structured array containing fields as returned
by get_amps() and get_baseline_amps().
The overall strategy here is:
1. Make an initial decision on whether to treat this pair as excitatory or
inhibitory, based on differences between foreground and background amplitude
measurements
2. Generate mean and stdev for amplitudes, deconvolved amplitudes, and deconvolved
latencies
3. Generate KS test p values describing the differences between foreground
and background distributions for amplitude, deconvolved amplitude, and
deconvolved latency
"""
requested_sign = sign
fields = {} # used to fill the new DB record
# Use KS p value to check for differences between foreground and background
qc_amps = {}
ks_pvals = {}
amp_means = {}
amp_diffs = {}
for clamp_mode in ('ic', 'vc'):
clamp_mode_fg = amps[clamp_mode, 'fg']
clamp_mode_bg = amps[clamp_mode, 'bg']
if (len(clamp_mode_fg) == 0 or len(clamp_mode_bg) == 0):
continue
for sign in ('pos', 'neg'):
# Separate into positive/negative tests and filter out responses that failed qc
qc_field = {'vc': {'pos': 'in_qc_pass', 'neg': 'ex_qc_pass'}, 'ic': {'pos': 'ex_qc_pass', 'neg': 'in_qc_pass'}}[clamp_mode][sign]
fg = clamp_mode_fg[clamp_mode_fg[qc_field]]
bg = clamp_mode_bg[clamp_mode_bg[qc_field]]
qc_amps[sign, clamp_mode, 'fg'] = fg
qc_amps[sign, clamp_mode, 'bg'] = bg
if (len(fg) == 0 or len(bg) == 0):
continue
# Measure some statistics from these records
fg = fg[sign + '_dec_amp']
bg = bg[sign + '_dec_amp']
pval = scipy.stats.ks_2samp(fg, bg).pvalue
ks_pvals[(sign, clamp_mode)] = pval
# we could ensure that the average amplitude is in the right direction:
fg_mean = np.mean(fg)
bg_mean = np.mean(bg)
amp_means[sign, clamp_mode] = {'fg': fg_mean, 'bg': bg_mean}
amp_diffs[sign, clamp_mode] = fg_mean - bg_mean
if requested_sign is None:
# Decide whether to treat this connection as excitatory or inhibitory.
# strategy: accumulate evidence for either possibility by checking
# the ks p-values for each sign/clamp mode and the direction of the deflection
is_exc = 0
# print(expt.acq_timestamp, pair.pre_cell.ext_id, pair.post_cell.ext_id)
for sign in ('pos', 'neg'):
for mode in ('ic', 'vc'):
ks = ks_pvals.get((sign, mode), None)
if ks is None:
continue
# turn p value into a reasonable scale factor
ks = norm_pvalue(ks)
dif_sign = 1 if amp_diffs[sign, mode] > 0 else -1
if mode == 'vc':
dif_sign *= -1
is_exc += dif_sign * ks
# print(" ", sign, mode, is_exc, dif_sign * ks)
else:
is_exc = requested_sign
if is_exc > 0:
fields['synapse_type'] = 'ex'
signs = {'ic':'pos', 'vc':'neg'}
else:
fields['synapse_type'] = 'in'
signs = {'ic':'neg', 'vc':'pos'}
# compute the rest of statistics for only positive or negative deflections
for clamp_mode in ('ic', 'vc'):
sign = signs[clamp_mode]
fg = qc_amps.get((sign, clamp_mode, 'fg'))
bg = qc_amps.get((sign, clamp_mode, 'bg'))
if fg is None or bg is None or len(fg) == 0 or len(bg) == 0:
fields[clamp_mode + '_n_samples'] = 0
continue
fields[clamp_mode + '_n_samples'] = len(fg)
fields[clamp_mode + '_crosstalk_mean'] = np.mean(fg['crosstalk'])
fields[clamp_mode + '_base_crosstalk_mean'] = np.mean(bg['crosstalk'])
# measure mean, stdev, and statistical differences between
# fg and bg for each measurement
for val, field in [('amp', 'amp'), ('deconv_amp', 'dec_amp'), ('latency', 'dec_latency')]:
f = fg[sign + '_' + field]
b = bg[sign + '_' + field]
fields[clamp_mode + '_' + val + '_mean'] = np.mean(f)
fields[clamp_mode + '_' + val + '_stdev'] = np.std(f)
fields[clamp_mode + '_base_' + val + '_mean'] = np.mean(b)
fields[clamp_mode + '_base_' + val + '_stdev'] = np.std(b)
# statistical tests comparing fg vs bg
# Note: we use log(1-log(pval)) because it's nicer to plot and easier to
# use as a classifier input
tt_pval = scipy.stats.ttest_ind(f, b, equal_var=False).pvalue
ks_pval = scipy.stats.ks_2samp(f, b).pvalue
fields[clamp_mode + '_' + val + '_ttest'] = norm_pvalue(tt_pval)
fields[clamp_mode + '_' + val + '_ks2samp'] = norm_pvalue(ks_pval)
### generate the average response and psp fit
# collect all bg and fg traces
# bg_traces = TraceList([Trace(data, sample_rate=db.default_sample_rate) for data in amps[clamp_mode, 'bg']['data']])
fg_traces = TraceList()
for rec in fg:
t0 = rec['response_start_time'] - rec['max_dvdt_time'] # time-align to presynaptic spike
trace = Trace(rec['data'], sample_rate=db.default_sample_rate, t0=t0)
fg_traces.append(trace)
# get averages
# bg_avg = bg_traces.mean()
fg_avg = fg_traces.mean()
base_rgn = fg_avg.time_slice(-6e-3, 0)
base = float_mode(base_rgn.data)
fields[clamp_mode + '_average_response'] = fg_avg.data
fields[clamp_mode + '_average_response_t0'] = fg_avg.t0
fields[clamp_mode + '_average_base_stdev'] = base_rgn.std()
sign = {'pos':'+', 'neg':'-'}[signs[clamp_mode]]
fg_bsub = fg_avg.copy(data=fg_avg.data - base) # remove base to help fitting
try:
fit = fit_psp(fg_bsub, mode=clamp_mode, sign=sign, xoffset=(1e-3, 0, 6e-3), yoffset=(0, None, None), rise_time_mult_factor=4)
for param, val in fit.best_values.items():
fields['%s_fit_%s' % (clamp_mode, param)] = val
fields[clamp_mode + '_fit_yoffset'] = fit.best_values['yoffset'] + base
fields[clamp_mode + '_fit_nrmse'] = fit.nrmse()
except:
print("Error in PSP fit:")
sys.excepthook(*sys.exc_info())
continue
#global fit_plot
#if fit_plot is None:
#fit_plot = FitExplorer(fit)
#fit_plot.show()
#else:
#fit_plot.set_fit(fit)
#raw_input("Waiting to continue..")
return fields
y_deconv1 = []
y_deconv2 = []
y_deconv3 = []
y_raw1 = []
y_raw2 = []
y_raw3 = []
inds = np.arange(n_responses)
for n in range(1, n_responses+1):
iters = n_responses // n
np.random.shuffle(inds)
for i in range(iters):
tl = TraceList([deconv[k] for k in inds[n*i:n*(i+1)]])
x.append(n + np.random.normal(scale=0.1))
# calculate amplitude from averaged trace
avg = tl.mean()
y_deconv1.append(measure_amp(avg))
# calculate average of amplitudes measured from individual traces
amps = []
for t in tl:
amps.append(measure_amp(t))
y_deconv2.append(np.mean(amps))
# calculate a fake amplitude from the baseline region
amps = []
for t in tl:
amps.append(measure_amp(t, baseline=(0, 2e-3), response=(6e-3, 10e-3)))
y_deconv3.append(np.mean(amps))
# calculate average amplitudes from raw traces
pulse_response
join stim_pulse on pulse_response.pulse_id=stim_pulse.id
join recording post_rec on pulse_response.recording_id=post_rec.id
join patch_clamp_recording post_pcrec on post_pcrec.recording_id=post_rec.id
join multi_patch_probe on multi_patch_probe.patch_clamp_recording_id=post_pcrec.id
join recording pre_rec on stim_pulse.recording_id=pre_rec.id
join sync_rec on post_rec.sync_rec_id=sync_rec.id
join experiment on sync_rec.experiment_id=experiment.id
where
{conditions}
""".format(conditions='\n and '.join(conditions))
print(query)
rp = session.execute(query)
recs = rp.fetchall()
data = [np.load(io.BytesIO(rec[0])) for rec in recs]
print("\n\nloaded %d records" % len(data))
plt = pg.plot(labels={'left': ('Vm', 'V')})
traces = TraceList()
for i, x in enumerate(data):
trace = Trace(x - np.median(x[:100]), sample_rate=20000)
traces.append(trace)
if i < 100:
plt.plot(trace.time_values, trace.data, pen=(255, 255, 255, 100))
avg = traces.mean()
plt.plot(avg.time_values, avg.data, pen='g')
join patch_clamp_recording post_pcrec on post_pcrec.recording_id=post_rec.id
join multi_patch_probe on multi_patch_probe.patch_clamp_recording_id=post_pcrec.id
join recording pre_rec on stim_pulse.recording_id=pre_rec.id
join sync_rec on post_rec.sync_rec_id=sync_rec.id
join experiment on sync_rec.experiment_id=experiment.id
where
{conditions}
""".format(conditions='\n and '.join(conditions))
print(query)
rp = session.execute(query)
recs = rp.fetchall()
data = [np.load(io.BytesIO(rec[0])) for rec in recs]
print("\n\nloaded %d records" % len(data))
plt = pg.plot(labels={'left': ('Vm', 'V')})
traces = TraceList()
for i,x in enumerate(data):
trace = Trace(x - np.median(x[:100]), sample_rate=20000)
traces.append(trace)
if i<100:
plt.plot(trace.time_values, trace.data, pen=(255, 255, 255, 100))
avg = traces.mean()
plt.plot(avg.time_values, avg.data, pen='g')
def analyze_pair_connectivity(amps, sign=None):
"""Given response strength records for a single pair, generate summary
statistics characterizing strength, latency, and connectivity.
Parameters
----------
amps : dict
Contains foreground and background strength analysis records
(see input format below)
sign : None, -1, or +1
If None, then automatically determine whether to treat this connection as
inhibitory or excitatory.
Input must have the following structure::
amps = {
('ic', 'fg'): recs,
('ic', 'bg'): recs,
('vc', 'fg'): recs,
('vc', 'bg'): recs,
}
Where each *recs* must be a structured array containing fields as returned
by get_amps() and get_baseline_amps().
The overall strategy here is:
1. Make an initial decision on whether to treat this pair as excitatory or
inhibitory, based on differences between foreground and background amplitude
measurements
2. Generate mean and stdev for amplitudes, deconvolved amplitudes, and deconvolved
latencies
3. Generate KS test p values describing the differences between foreground
and background distributions for amplitude, deconvolved amplitude, and
deconvolved latency
"""
requested_sign = sign
fields = {} # used to fill the new DB record
# Use KS p value to check for differences between foreground and background
qc_amps = {}
ks_pvals = {}
amp_means = {}
amp_diffs = {}
for clamp_mode in ('ic', 'vc'):
clamp_mode_fg = amps[clamp_mode, 'fg']
clamp_mode_bg = amps[clamp_mode, 'bg']
if (len(clamp_mode_fg) == 0 or len(clamp_mode_bg) == 0):
continue
for sign in ('pos', 'neg'):
# Separate into positive/negative tests and filter out responses that failed qc
qc_field = {'vc': {'pos': 'in_qc_pass', 'neg': 'ex_qc_pass'}, 'ic': {'pos': 'ex_qc_pass', 'neg': 'in_qc_pass'}}[clamp_mode][sign]
fg = clamp_mode_fg[clamp_mode_fg[qc_field]]
bg = clamp_mode_bg[clamp_mode_bg[qc_field]]
qc_amps[sign, clamp_mode, 'fg'] = fg
qc_amps[sign, clamp_mode, 'bg'] = bg
if (len(fg) == 0 or len(bg) == 0):
continue
# Measure some statistics from these records
fg = fg[sign + '_dec_amp']
bg = bg[sign + '_dec_amp']
pval = scipy.stats.ks_2samp(fg, bg).pvalue
ks_pvals[(sign, clamp_mode)] = pval
# we could ensure that the average amplitude is in the right direction:
fg_mean = np.mean(fg)
bg_mean = np.mean(bg)
amp_means[sign, clamp_mode] = {'fg': fg_mean, 'bg': bg_mean}
amp_diffs[sign, clamp_mode] = fg_mean - bg_mean
if requested_sign is None:
# Decide whether to treat this connection as excitatory or inhibitory.
# strategy: accumulate evidence for either possibility by checking
# the ks p-values for each sign/clamp mode and the direction of the deflection
is_exc = 0
# print(expt.acq_timestamp, pair.pre_cell.ext_id, pair.post_cell.ext_id)
for sign in ('pos', 'neg'):
for mode in ('ic', 'vc'):
ks = ks_pvals.get((sign, mode), None)
if ks is None:
continue
# turn p value into a reasonable scale factor
ks = norm_pvalue(ks)
dif_sign = 1 if amp_diffs[sign, mode] > 0 else -1
if mode == 'vc':
dif_sign *= -1
is_exc += dif_sign * ks
# print(" ", sign, mode, is_exc, dif_sign * ks)
else:
is_exc = requested_sign
if is_exc > 0:
fields['synapse_type'] = 'ex'
signs = {'ic':'pos', 'vc':'neg'}
else:
fields['synapse_type'] = 'in'
signs = {'ic':'neg', 'vc':'pos'}
# compute the rest of statistics for only positive or negative deflections
for clamp_mode in ('ic', 'vc'):
sign = signs[clamp_mode]
fg = qc_amps.get((sign, clamp_mode, 'fg'))
bg = qc_amps.get((sign, clamp_mode, 'bg'))
if fg is None or bg is None or len(fg) == 0 or len(bg) == 0:
fields[clamp_mode + '_n_samples'] = 0
continue
fields[clamp_mode + '_n_samples'] = len(fg)
fields[clamp_mode + '_crosstalk_mean'] = np.mean(fg['crosstalk'])
fields[clamp_mode + '_base_crosstalk_mean'] = np.mean(bg['crosstalk'])
# measure mean, stdev, and statistical differences between
# fg and bg for each measurement
for val, field in [('amp', 'amp'), ('deconv_amp', 'dec_amp'), ('latency', 'dec_latency')]:
f = fg[sign + '_' + field]
b = bg[sign + '_' + field]
fields[clamp_mode + '_' + val + '_mean'] = np.mean(f)
fields[clamp_mode + '_' + val + '_stdev'] = np.std(f)
fields[clamp_mode + '_base_' + val + '_mean'] = np.mean(b)
fields[clamp_mode + '_base_' + val + '_stdev'] = np.std(b)
# statistical tests comparing fg vs bg
# Note: we use log(1-log(pval)) because it's nicer to plot and easier to
# use as a classifier input
tt_pval = scipy.stats.ttest_ind(f, b, equal_var=False).pvalue
ks_pval = scipy.stats.ks_2samp(f, b).pvalue
fields[clamp_mode + '_' + val + '_ttest'] = norm_pvalue(tt_pval)
fields[clamp_mode + '_' + val + '_ks2samp'] = norm_pvalue(ks_pval)
### generate the average response and psp fit
# collect all bg and fg traces
# bg_traces = TraceList([Trace(data, sample_rate=db.default_sample_rate) for data in amps[clamp_mode, 'bg']['data']])
fg_traces = TraceList()
for rec in fg:
t0 = rec['response_start_time'] - rec['max_dvdt_time'] # time-align to presynaptic spike
trace = Trace(rec['data'], sample_rate=db.default_sample_rate, t0=t0)
fg_traces.append(trace)
# get averages
# bg_avg = bg_traces.mean()
fg_avg = fg_traces.mean()
base_rgn = fg_avg.time_slice(-6e-3, 0)
base = float_mode(base_rgn.data)
fields[clamp_mode + '_average_response'] = fg_avg.data
fields[clamp_mode + '_average_response_t0'] = fg_avg.t0
fields[clamp_mode + '_average_base_stdev'] = base_rgn.std()
sign = {'pos':'+', 'neg':'-'}[signs[clamp_mode]]
fg_bsub = fg_avg.copy(data=fg_avg.data - base) # remove base to help fitting
try:
fit = fit_psp(fg_bsub, mode=clamp_mode, sign=sign, xoffset=(1e-3, 0, 6e-3), yoffset=(0, None, None), rise_time_mult_factor=4)
for param, val in fit.best_values.items():
fields['%s_fit_%s' % (clamp_mode, param)] = val
fields[clamp_mode + '_fit_yoffset'] = fit.best_values['yoffset'] + base
fields[clamp_mode + '_fit_nrmse'] = fit.nrmse()
except:
print("Error in PSP fit:")
sys.excepthook(*sys.exc_info())
continue
#global fit_plot
#if fit_plot is None:
#fit_plot = FitExplorer(fit)
#fit_plot.show()
#else:
#fit_plot.set_fit(fit)
#raw_input("Waiting to continue..")
return fields
y_deconv1 = []
y_deconv2 = []
y_deconv3 = []
y_raw1 = []
y_raw2 = []
y_raw3 = []
inds = np.arange(n_responses)
for n in range(1, n_responses+1):
iters = n_responses // n
np.random.shuffle(inds)
for i in range(iters):
tl = TraceList([deconv[k] for k in inds[n*i:n*(i+1)]])
x.append(n + np.random.normal(scale=0.1))
# calculate amplitude from averaged trace
avg = tl.mean()
y_deconv1.append(measure_amp(avg))
# calculate average of amplitudes measured from individual traces
amps = []
for t in tl:
amps.append(measure_amp(t))
y_deconv2.append(np.mean(amps))
# calculate a fake amplitude from the baseline region
amps = []
for t in tl:
amps.append(measure_amp(t, baseline=(0, 2e-3), response=(6e-3, 10e-3)))
y_deconv3.append(np.mean(amps))
# calculate average amplitudes from raw traces