def populate(self):
self.items_by_pair_id = {}
self.session = db.Session()
db_expts = db.list_experiments(session=self.session)
db_expts.sort(key=lambda e: e.acq_timestamp)
for expt in db_expts:
date = expt.acq_timestamp
date_str = datetime.fromtimestamp(date).strftime('%Y-%m-%d')
slice = expt.slice
expt_item = pg.TreeWidgetItem(map(str, [date_str, expt.rig_name, slice.species, expt.target_region, slice.genotype, expt.acsf]))
expt_item.expt = expt
self.addTopLevelItem(expt_item)
for pair in expt.pairs:
if pair.n_ex_test_spikes == 0 and pair.n_in_test_spikes == 0:
continue
cells = '%d => %d' % (pair.pre_cell.ext_id, pair.post_cell.ext_id)
conn = {True:"syn", False:"-", None:"?"}[pair.synapse]
types = 'L%s %s => L%s %s' % (pair.pre_cell.target_layer or "?", pair.pre_cell.cre_type, pair.post_cell.target_layer or "?", pair.post_cell.cre_type)
pair_item = pg.TreeWidgetItem([cells, conn, types])
expt_item.addChild(pair_item)
pair_item.pair = pair
pair_item.expt = expt
self.items_by_pair_id[pair.id] = pair_item
def update_DB(limit=None,
expts=None,
parallel=True,
workers=6,
raise_exceptions=False,
session=None):
"""
"""
session = db.Session()
if expts is None:
experiments = session.query(db.Experiment.acq_timestamp).all()
expts_done = session.query(db.Experiment.acq_timestamp).join(
db.Pair).join(AvgFirstPulseFit).all()
print("Skipping %d already complete experiments" % (len(expts_done)))
experiments = [e for e in experiments if e not in set(expts_done)]
if limit > 0:
np.random.shuffle(experiments)
experiments = experiments[:limit]
jobs = [(record.acq_timestamp, index, len(experiments))
for index, record in enumerate(experiments)]
else:
jobs = [(expt, i, len(expts)) for i, expt in enumerate(expts)]
# if parallel:
# pool = multiprocessing.Pool(processes=workers)
# pool.map(pair, pairs)
# else:
for job in jobs:
compute_fit(job, raise_exceptions=raise_exceptions)
def update_analysis(limit=None):
s = db.Session()
q = s.query(db.Pair,
FirstPulseFeatures).outerjoin(FirstPulseFeatures).filter(
FirstPulseFeatures.pair_id == None)
if limit is not None:
q = q.limit(limit)
print("Updating %d pairs.." % q.count())
records = q.all()
for i, record in enumerate(records):
pair = record[0]
pulse_responses, pulse_ids, pulse_response_amps = filter_pulse_responses(
pair)
if len(pulse_responses) > 0:
results = first_pulse_features(pair, pulse_responses,
pulse_response_amps)
fpf = FirstPulseFeatures(pair=pair,
n_sweeps=len(pulse_ids),
pulse_ids=pulse_ids,
**results)
s.add(fpf)
if i % 10 == 0:
s.commit()
print("%d pairs added to the DB of %d" % (i, len(records)))
s.commit()
s.close()
def build_detection_limits():
# silence warnings about fp issues
np.seterr(all='ignore')
# read all pair records from DB
classifier = strength_analysis.get_pair_classifier(seed=0,
use_vc_features=False)
conns = strength_analysis.query_all_pairs(classifier)
# filter
mask = np.isfinite(conns['ic_deconv_amp_mean'])
filtered = conns[mask]
# remove recordings with gain errors
mask = filtered['ic_deconv_amp_mean'] < 0.02
# remove recordings with high crosstalk
mask &= abs(filtered['ic_crosstalk_mean']) < 60e-6
# remove recordings with low sample count
mask &= filtered['ic_n_samples'] > 50
typs = filtered['pre_cre_type']
mask &= typs == filtered['post_cre_type']
typ_mask = ((typs == 'sim1') | (typs == 'tlx3') | (typs == 'unknown') |
(typs == 'rorb') | (typs == 'ntsr1'))
mask &= typ_mask
filtered = filtered[mask]
c_mask = filtered['synapse'] == True
u_mask = ~c_mask
signal = filtered['confidence']
background = filtered['ic_base_deconv_amp_mean']
session = db.Session()
# do selected connections first
count = 0
for i, rec in enumerate(filtered):
print("================== %d/%d ===================== " %
(i, len(filtered)))
pair = session.query(
db.Pair).filter(db.Pair.id == rec['pair_id']).all()[0]
if pair.detection_limit is not None:
print(" skip!")
continue
try:
measure_limit(pair, session, classifier)
except Exception:
sys.excepthook(*sys.exc_info())
count += 1
if count > 100:
print("Bailing out before memory fills up.")
sys.exit(0)
def query_all_pairs():
import pandas
query = ("""
select """
# ((DATE_PART('day', experiment.acq_timestamp - '1970-01-01'::timestamp) * 24 +
# DATE_PART('hour', experiment.acq_timestamp - '1970-01-01'::timestamp)) * 60 +
# DATE_PART('minute', experiment.acq_timestamp - '1970-01-01'::timestamp)) * 60 +
# DATE_PART('second', experiment.acq_timestamp - '1970-01-01'::timestamp) as acq_timestamp,
"""
connection_strength.*,
experiment.id as experiment_id,
experiment.acq_timestamp as acq_timestamp,
ABS(connection_strength.amp_med) as abs_amp_med,
ABS(connection_strength.base_amp_med) as abs_base_amp_med,
ABS(connection_strength.amp_med_minus_base) as abs_amp_med_minus_base,
ABS(connection_strength.deconv_amp_med) as abs_deconv_amp_med,
ABS(connection_strength.deconv_base_amp_med) as abs_deconv_base_amp_med,
ABS(connection_strength.deconv_amp_med_minus_base) as abs_deconv_amp_med_minus_base,
experiment.rig_name,
experiment.acsf,
slice.species,
slice.genotype,
slice.age,
slice.slice_time,
pre_cell.ext_id as pre_cell_id,
pre_cell.cre_type as pre_cre_type,
pre_cell.target_layer as pre_target_layer,
post_cell.ext_id as post_cell_id,
post_cell.cre_type as post_cre_type,
post_cell.target_layer as post_target_layer,
pair.synapse,
pair.crosstalk_artifact,
abs(post_cell.ext_id - pre_cell.ext_id) as electrode_distance
from connection_strength
join pair on connection_strength.pair_id=pair.id
join cell pre_cell on pair.pre_cell_id=pre_cell.id
join cell post_cell on pair.post_cell_id=post_cell.id
join experiment on pair.expt_id=experiment.id
join slice on experiment.slice_id=slice.id
order by acq_timestamp
""")
session = db.Session()
df = pandas.read_sql(query, session.bind)
# test out a new metric:
df['connection_signal'] = pandas.Series(df['deconv_amp_med'] / df['latency_stdev'], index=df.index)
df['connection_background'] = pandas.Series(df['deconv_base_amp_med'] / df['base_latency_stdev'], index=df.index)
ts = [datetime_to_timestamp(t) for t in df['acq_timestamp']]
df['acq_timestamp'] = ts
recs = df.to_records()
return recs
def __init__(self):
pg.QtGui.QWidget.__init__(self)
self.layout = pg.QtGui.QVBoxLayout()
self.setLayout(self.layout)
self.update_button = pg.QtGui.QPushButton("Update Matrix")
self.layout.addWidget(self.update_button)
self.project_list = pg.QtGui.QListWidget()
self.layout.addWidget(self.project_list)
s = db.Session()
projects = s.query(db.Experiment.project_name).distinct().all()
for record in projects:
project = record[0]
project_item = pg.QtGui.QListWidgetItem(project)
project_item.setFlags(project_item.flags() | pg.QtCore.Qt.ItemIsUserCheckable)
project_item.setCheckState(pg.QtCore.Qt.Unchecked)
self.project_list.addItem(project_item)
def poll(self):
self.session = database.Session()
expts = {}
print("loading site paths..")
#site_paths = glob.glob(os.path.join(config.synphys_data, '*', 'slice_*', 'site_*'))
root_dh = getDirHandle(config.synphys_data)
print(root_dh.name())
for site_dh in self.list_expts(root_dh):
expt = ExperimentMetadata(nas_path=site_dh.name())
if expt.timestamp in expts:
continue
expts[expt.timestamp] = expt
self.check(expt)
if self.limit > 0 and len(expts) > self.limit:
break
def __init__(self):
pg.QtCore.QObject.__init__(self)
# global session for querying from DB
self.session = db.Session()
win = pg.QtGui.QSplitter()
win.setOrientation(pg.QtCore.Qt.Horizontal)
win.resize(1000, 800)
win.show()
b = ExperimentBrowser()
win.addWidget(b)
rs_plots = ResponseStrengthPlots(self.session)
win.addWidget(rs_plots)
b.itemSelectionChanged.connect(self._selected)
b.doubleClicked.connect(self._dbl_clicked)
self.win = win
self.rs_plots = rs_plots
self.browser = b
self.nwb_viewer = MultipatchNwbViewer()
# strong ex with failures
expt_id = 1537820585.767
pre_cell_id = 1
post_cell_id = 2
# strong ex, depressing
expt_id = 1536781898.381
pre_cell_id = 8
post_cell_id = 2
# expt_id = float(sys.argv[1])
# pre_cell_id = int(sys.argv[2])
# post_cell_id = int(sys.argv[3])
session = db.Session()
expt = db.experiment_from_timestamp(expt_id, session=session)
pair = expt.pairs[(pre_cell_id, post_cell_id)]
syn_type = pair.connection_strength.synapse_type
# 1. Get a list of all presynaptic spike times and the amplitudes of postsynaptic responses
raw_events = get_amps(session, pair, clamp_mode='ic')
mask = event_qc(raw_events)
events = raw_events[mask]
rec_times = 1e-9 * (events['rec_start_time'].astype(float) -
float(events['rec_start_time'][0]))
spike_times = events['max_dvdt_time'] + rec_times
def query_all_pairs(classifier=None):
columns = [
"connection_strength.*",
"experiment.id as experiment_id",
"experiment.acq_timestamp as acq_timestamp",
"experiment.rig_name",
"experiment.acsf",
"slice.species as donor_species",
"slice.genotype as donor_genotype",
"slice.age as donor_age",
"slice.sex as donor_sex",
"slice.quality as slice_quality",
"slice.weight as donor_weight",
"slice.slice_time",
"pre_cell.ext_id as pre_cell_id",
"pre_cell.cre_type as pre_cre_type",
"pre_cell.target_layer as pre_target_layer",
"post_cell.ext_id as post_cell_id",
"post_cell.cre_type as post_cre_type",
"post_cell.target_layer as post_target_layer",
"pair.synapse",
"pair.distance",
"pair.crosstalk_artifact",
"abs(post_cell.ext_id - pre_cell.ext_id) as electrode_distance",
]
# columns.extend([
# "detection_limit.minimum_amplitude",
# ])
joins = [
"join pair on connection_strength.pair_id=pair.id",
"join cell pre_cell on pair.pre_cell_id=pre_cell.id",
"join cell post_cell on pair.post_cell_id=post_cell.id",
"join experiment on pair.experiment_id=experiment.id",
"join slice on experiment.slice_id=slice.id",
]
# joins.extend([
# "left join detection_limit on detection_limit.pair_id=pair.id",
# ])
query = ("""
select
{columns}
from connection_strength
{joins}
order by acq_timestamp
""").format(
columns=", ".join(columns),
joins=" ".join(joins),
)
session = db.Session()
df = pandas.read_sql(query, session.bind)
recs = df.to_records()
if classifier is None:
return recs
# Fit classifier and add results of classifier prediction in to records
classifier.fit(recs)
prediction = classifier.predict(recs)
recs = join_struct_arrays([recs, prediction])
return recs
from __future__ import division
import time, datetime
import multipatch_analysis.database.database as db
from neuroanalysis.ui.plot_grid import PlotGrid
s = db.Session()
q = """
select
pcrec.baseline_rms_noise,
substring(experiment.original_path from 36 for 1),
recording.device_key,
recording.start_time
from
patch_clamp_recording pcrec
join recording on pcrec.recording_id=recording.id
join sync_rec on recording.sync_rec_id=sync_rec.id
join experiment on sync_rec.experiment_id=experiment.id
where
pcrec.clamp_mode='ic'
and pcrec.baseline_rms_noise is not null
and recording.device_key is not null
and experiment.original_path is not null
"""
rec = s.execute(q)
rows = rec.fetchall()
import pyqtgraph as pg
import numpy as np
def in_database(self):
session = database.Session()
expts = session.query(database.Experiment).filter(
database.Experiment.acq_timestamp == self.datetime).all()
return len(expts) == 1
def compute_fit(job_info, raise_exceptions=False):
session = db.Session(readonly=False) #create session
expt_id, index, n_jobs = job_info
print("QUERYING (expt_id=%f): %d/%d" % (expt_id, index, n_jobs))
#do query
pre_cell = db.aliased(db.Cell)
post_cell = db.aliased(db.Cell)
expt_stuff = session.query(db.Pair, db.Experiment.acq_timestamp, pre_cell.ext_id, post_cell.ext_id,pre_cell.cre_type, post_cell.cre_type)\
.join(db.Experiment)\
.join(pre_cell, db.Pair.pre_cell_id==pre_cell.id)\
.join(post_cell, db.Pair.post_cell_id==post_cell.id).filter(db.Experiment.acq_timestamp==expt_id).all()
# make sure query returned something
if len(expt_stuff) <= 0:
print('No pairs found for expt_id=%f', expt_id)
return
processed_count = 0 #index for keeping track of how many cells pairs in experiemnt have been analyzed
for ii, (pair, uid, pre_cell_id, post_cell_id, pre_cell_cre,
post_cell_cre) in enumerate(expt_stuff):
print("Number %i of %i experiment pairs: %0.3f, cell ids:%s %s" %
(ii, len(expt_stuff), uid, pre_cell_id, post_cell_id))
# grab syapse from the table
try:
excitation = pair.connection_strength.synapse_type
except:
print('\tskipping: no pair.connection_strength.synapse_type')
continue
if not pair.connection_strength.ic_fit_xoffset:
print('\tskipping: no latency to do forced latency fitting')
continue
xoffset = pair.connection_strength.ic_fit_xoffset
# -----------fit current clamp data---------------------
# get pulses
(pulse_responses_i, pulse_ids_i, psp_amps_measured_i, freq, avg_psp_i,
measured_relative_amp_i,
measured_baseline_i) = get_average_pulse_response(pair,
desired_clamp='ic')
if pulse_responses_i:
# weight and fit the trace
weight_i = np.ones(len(
avg_psp_i.data)) * 10. #set everything to ten initially
weight_i[int((time_before_spike - 3e-3) / avg_psp_i.dt):int(
time_before_spike / avg_psp_i.dt
)] = 0. #area around stim artifact note that since this is spike aligned there will be some blur in where the cross talk is
weight_i[int((time_before_spike + .0001 + xoffset) /
avg_psp_i.dt):int(
(time_before_spike + .0001 + xoffset + 4e-3) /
avg_psp_i.dt)] = 30. #area around steep PSP rise
avg_fit_i = fit_trace(avg_psp_i,
excitation=excitation,
weight=weight_i,
latency=xoffset,
latency_jitter=.5e-3)
latency_i = avg_fit_i.best_values['xoffset'] - time_before_spike
amp_i = avg_fit_i.best_values['amp']
rise_time_i = avg_fit_i.best_values['rise_time']
decay_tau_i = avg_fit_i.best_values['decay_tau']
avg_data_waveform_i = avg_psp_i.data
avg_fit_waveform_i = avg_fit_i.best_fit
dt_i = avg_psp_i.dt
nrmse_i = avg_fit_i.nrmse()
else:
print(
'\tskipping: no suitable first pulses found in current clamp')
weight_i = np.array([0])
latency_i = None
amp_i = None
rise_time_i = None
decay_tau_i = None
avg_data_waveform_i = np.array([0])
avg_fit_waveform_i = np.array([0])
dt_i = None
nrmse_i = None
# --------------fit voltage clamp data---------------------
# get pulses
(pulse_responses_v, pulse_ids_v, psp_amps_measured_v, freq_v,
avg_psp_v, measured_relative_amp_v,
measured_baseline_v) = get_average_pulse_response(pair,
desired_clamp='vc')
if pulse_responses_v:
# weight and fit the trace
weight_v = np.ones(len(
avg_psp_v.data)) * 10. #set everything to ten initially
weight_v[int((time_before_spike + .0001 + xoffset) /
avg_psp_v.dt):int(
(time_before_spike + .0001 + xoffset + 4e-3) /
avg_psp_v.dt)] = 30. #area around steep PSP rise
avg_fit_v = fit_trace(avg_psp_v,
excitation=excitation,
clamp_mode='vc',
weight=weight_v,
latency=xoffset,
latency_jitter=.5e-3)
latency_v = avg_fit_v.best_values['xoffset'] - time_before_spike
amp_v = avg_fit_v.best_values['amp']
rise_time_v = avg_fit_v.best_values['rise_time']
decay_tau_v = avg_fit_v.best_values['decay_tau']
avg_data_waveform_v = avg_psp_v.data
avg_fit_waveform_v = avg_fit_v.best_fit
dt_v = avg_psp_v.dt
nrmse_v = avg_fit_v.nrmse()
else:
print(
'\tskipping: no suitable first pulses found in voltage clamp')
weight_v = np.array([0])
latency_v = None
amp_v = None
rise_time_v = None
decay_tau_v = None
avg_data_waveform_v = np.array([0])
avg_fit_waveform_v = np.array([0])
dt_v = None
nrmse_v = None
#------------ done with fitting section ------------------------------
# dictionary for ease of translation into the output table
out_dict = {
'ic_amp': amp_i,
'ic_latency': latency_i,
'ic_rise_time': rise_time_i,
'ic_decay_tau': decay_tau_i,
'ic_avg_psp_data': avg_data_waveform_i,
'ic_avg_psp_fit': avg_fit_waveform_i,
'ic_dt': dt_i,
'ic_pulse_ids': pulse_ids_i,
'ic_nrmse': nrmse_i,
'ic_measured_baseline': measured_baseline_i,
'ic_measured_amp': measured_relative_amp_i,
'ic_weight': weight_i,
'vc_amp': amp_v,
'vc_latency': latency_v,
'vc_rise_time': rise_time_v,
'vc_decay_tau': decay_tau_v,
'vc_avg_psp_data': avg_data_waveform_v,
'vc_avg_psp_fit': avg_fit_waveform_v,
'vc_dt': dt_v,
'vc_pulse_ids': pulse_ids_v,
'vc_nrmse': nrmse_v,
'vc_measured_baseline': measured_baseline_v,
'vc_measured_amp': measured_relative_amp_v,
'vc_weight': weight_v
}
# map to pair table and commit
afpf = AvgFirstPulseFit(pair=pair, **out_dict)
if commiting is True:
session.add(afpf)
session.commit()
# TODO: I guess I need to query expt before I do this...
# expt.meta = expt.meta.copy() # required by sqlalchemy to flag as modified
# expt.meta['avg_first_pulse_fit_timestamp'] = time.time()
print("COMMITED %i pairs from expt_id=%f: %d/%d" %
(processed_count, expt_id, index, n_jobs))
#---------------------------------------------------------------------------------------
processed_count = processed_count + 1
print('processed', processed_count + 1)
