def load_traces_and_frametimes(self, key):
# -- find number of recording depths
pipe = (fuse.Activity() & key).fetch('pipe')
assert len(
np.unique(pipe)) == 1, 'Selection is from different pipelines'
pipe = dj.create_virtual_module(pipe[0], 'pipeline_' + pipe[0])
k = dict(key)
k.pop('field', None)
ndepth = len(dj.U('z') & (pipe.ScanInfo.Field() & k))
frame_times = (stimulus.Sync()
& key).fetch1('frame_times').squeeze()[::ndepth]
soma = pipe.MaskClassification.Type() & dict(type='soma')
spikes = (dj.U('field', 'channel') * pipe.Activity.Trace() * StaticScan.Unit() \
* pipe.ScanSet.UnitInfo() & soma & key)
traces, ms_delay, trace_keys = spikes.fetch(
'trace',
'ms_delay',
dj.key,
order_by='animal_id, session, scan_idx, unit_id')
delay = np.fromiter(ms_delay / 1000, dtype=np.float)
frame_times = (delay[:, None] + frame_times[None, :])
traces = np.vstack(
[fill_nans(tr.astype(np.float32)).squeeze() for tr in traces])
traces, frame_times = self.adjust_trace_len(traces, frame_times)
return traces, frame_times, trace_keys
def test_issue484():
q = dj.U().aggr(S, n='max(s)')
n = q.fetch('n')
n = q.fetch1('n')
q = dj.U().aggr(S, n='avg(s)')
result = dj.U().aggr(q, m='max(n)')
result.fetch()
def get_session_history(session_key, remove_ignored=True):
# Fetch data
q_choice_outcome = (experiment.WaterPortChoice.proj(choice='water_port')
* experiment.BehaviorTrial.proj('outcome', 'early_lick')
* experiment.SessionBlock.BlockTrial) & session_key
if remove_ignored:
q_choice_outcome &= 'outcome != "ignore"'
# TODO: session QC (warm-up and decreased motivation etc.)
# -- Choice and reward --
# 0: left, 1: right, np.nan: ignored
_choice = q_choice_outcome.fetch('choice', order_by='trial')
_choice[_choice == 'left'] = 0
_choice[_choice == 'right'] = 1
_reward = q_choice_outcome.fetch('outcome', order_by='trial') == 'hit'
reward_history = np.zeros([2, len(_reward)]) # .shape = (2, N trials)
for c in (0, 1):
reward_history[c, _choice == c] = (_reward[_choice == c] == True).astype(int)
if remove_ignored: # For model fitting, turn to integer
choice_history = np.array([_choice]).astype(int) # .shape = (1, N trials)
else: # Include np.NaNs, can't be integers
choice_history = np.array([_choice]).astype(float) # .shape = (1, N trials)
# -- ITI --
# All previous models has an effective ITI of constant 1.
# For Ulises RNN model, an important prediction is that values in line attractor decay over (actual) time with time constant tau_CANN.
# This will take into consideration (1) different ITI of each trial, and (2) long effective ITI after ignored trials.
# Thus for CANN model, the ignored trials are also removed, but they contribute to model fitting in increasing the ITI.
if (len(ephys.TrialEvent & q_choice_outcome)
and len(dj.U('trial') & (ephys.TrialEvent & q_choice_outcome)) == len(dj.U('trial') & (experiment.TrialEvent & q_choice_outcome))):
# Use NI times (trial start and trial end) if (1) ephys exists (2) len(ephys) == len(behavior)
trial_start = (ephys.TrialEvent & q_choice_outcome & 'trial_event_type = "bitcodestart"'
).fetch('trial_event_time', order_by='trial').astype(float)
trial_end = (ephys.TrialEvent & q_choice_outcome & 'trial_event_type = "trialend"'
).fetch('trial_event_time', order_by='trial').astype(float)
iti = trial_start[1:] - trial_end[:-1] # ITI [t] --> ITI between trial t-1 and t
else: # If not ephys session, we can only use PC time (should be fine because this fitting is not quite time-sensitive)
bpod_start_global = (experiment.SessionTrial & q_choice_outcome
).fetch('start_time', order_by='trial').astype(float) # This is (global) PC time
bitcodestart_local = (experiment.TrialEvent & q_choice_outcome & 'trial_event_type = "bitcodestart"'
).fetch('trial_event_time', order_by='trial').astype(float)
trial_end_local = (experiment.TrialEvent & q_choice_outcome & 'trial_event_type = "trialend"'
).fetch('trial_event_time', order_by='trial').astype(float)
if len(bitcodestart_local) == len(trial_end_local) and len(bitcodestart_local) > 0:
iti = (bpod_start_global[1:] + bitcodestart_local[1:]) - (bpod_start_global[:-1] + trial_end_local[:-1])
else:
iti = bpod_start_global[1:] - bpod_start_global[:-1]
iti = np.hstack([0, iti]) # First trial iti is irrelevant
# -- p_reward --
q_p_reward = q_choice_outcome.proj() * experiment.SessionBlock.WaterPortRewardProbability & session_key
p_reward = np.vstack([(q_p_reward & 'water_port="left"').fetch('reward_probability', order_by='trial').astype(float),
(q_p_reward & 'water_port="right"').fetch('reward_probability', order_by='trial').astype(float)])
return choice_history, reward_history, iti, p_reward, q_choice_outcome
def get_extended_qc_fields_from_alyx(level='session'):
if level == 'session':
key_source = dj.U('uuid') & \
(alyxraw.AlyxRaw.Field &
(alyxraw.AlyxRaw & 'model="actions.session"') &
'fname="extended_qc"' &
'fvalue!="None"')
fname = 'extended_qc'
elif level == 'probe':
key_source = dj.U('uuid') & \
(alyxraw.AlyxRaw.Field &
(alyxraw.AlyxRaw & 'model="experiments.probeinsertion"') &
'fname="json"' &
'fvalue like "%extended_qc%"')
fname = 'json'
else:
raise ValueError('Incorrect level argument, has to be "session" or "probe"')
eqc_fields = []
for key in tqdm(key_source):
qc_extended = str_to_dict(grf(key, fname))
if qc_extended != 'None':
if level == 'probe' and 'extended_qc' in qc_extended:
qc_extended = qc_extended['extended_qc']
eqc_fields += list(qc_extended.keys())
return set(eqc_fields)
def print_current_jobs():
"""
Return a pandas.DataFrame on the status of each table currently being processed
table | reserve_count | error_count | oldest_job | newest_job
- table: {schema}.{table} name
- reserve_count: number of workers currently working on this table
- error_count: number of jobs errors for this table
- oldest_job: timestamp of the oldest job currently being worked on
- newest_job: timestamp of the most recent job currently being worked on
Provide insights into the current status of the workers
One caveat in this function is that we don't know how many workers are being deployed,
and how they're orchestrated. We can infer by taking the sum of the reserved jobs,
but this won't reflect idling workers because there's no "key_source" to work on for some
particular tables
"""
job_status = []
for pipeline_module in (experiment, tracking, ephys, histology, psth,
foraging_analysis, oralfacial_analysis, report):
reserved = dj.U('table_name').aggr(pipeline_module.schema.jobs
& 'status = "reserved"',
reserve_count='count(table_name)',
oldest_job='MIN(timestamp)',
newest_job='MAX(timestamp)')
errored = dj.U('table_name').aggr(pipeline_module.schema.jobs
& 'status = "error"',
error_count='count(table_name)')
if dj.__version__.startswith('0.13'):
jobs_summary = reserved.join(errored, left=True)
else:
jobs_summary = reserved.aggr(errored, ...,
error_count='error_count',
keep_all_rows=True)
for job in jobs_summary.fetch(as_dict=True):
job_status.append({
'table':
f'{pipeline_module.__name__.split(".")[-1]}.{dj.utils.to_camel_case(job.pop("table_name"))}',
**job
})
if not job_status:
print('No jobs under process (0 reserved jobs)')
return
job_status_df = pd.DataFrame(job_status).set_index('table')
job_status_df.fillna(0, inplace=True)
job_status_df = job_status_df.astype({
"reserve_count": int,
"error_count": int
})
with pd.option_context('display.max_rows', None, 'display.max_columns',
None, 'display.width', None, 'display.max_colwidth',
-1):
print(job_status_df)
return job_status_df
def fill_up(self, tier, frames, cond, key, m):
existing = ConditionTier().proj() & (self & dict(tier=tier)) \
& (stimulus.Trial() * stimulus.Condition() & dict(key, **cond))
n = len(existing)
if n < m:
# all hashes that are in clips but not registered for that animal and have the right tier
candidates = dj.U('condition_hash') & \
(self & (dj.U('condition_hash') & (frames - self)) & dict(tier=tier))
keys = candidates.fetch(dj.key)
d = m - n
update = min(len(keys), d)
log.info('Inserting {} more existing {} trials'.format(
update, tier))
for k in keys[:update]:
k = (frames & k).fetch1(dj.key)
k['tier'] = tier
self.insert1(k, ignore_extra_fields=True)
existing = ConditionTier().proj() & (self & dict(tier=tier)) \
& (stimulus.Trial() * stimulus.Condition() & dict(key, **cond))
n = len(existing)
if n < m:
keys = (frames - self).fetch(dj.key)
update = m - n
log.info('Inserting {} more new {} trials'.format(update, tier))
for k in keys[:update]:
k['tier'] = tier
self.insert1(k, ignore_extra_fields=True)
def test_join(self):
rel = self.experiment*dj.U('experiment_date')
assert_equal(self.experiment.primary_key, ['subject_id', 'experiment_id'])
assert_equal(rel.primary_key, self.experiment.primary_key + ['experiment_date'])
rel = dj.U('experiment_date')*self.experiment
assert_equal(self.experiment.primary_key, ['subject_id', 'experiment_id'])
assert_equal(rel.primary_key, self.experiment.primary_key + ['experiment_date'])
def test_aggr(self):
rel = schema_simple.ArgmaxTest()
amax1 = (dj.U('val') * rel) & dj.U('secondary_key').aggr(
rel, val='min(val)')
amax2 = (dj.U('val') * rel) * dj.U('secondary_key').aggr(
rel, val='min(val)')
assert_true(
len(amax1) == len(amax2) == rel.n,
'Aggregated argmax with join and restriction does not yield same length.'
)
def test_aggr(self):
rel = schema_simple.ArgmaxTest()
amax1 = (dj.U("val") * rel) & dj.U("secondary_key").aggr(
rel, val="min(val)")
amax2 = (dj.U("val") * rel) * dj.U("secondary_key").aggr(
rel, val="min(val)")
assert_true(
len(amax1) == len(amax2) == rel.n,
"Aggregated argmax with join and restriction does not yield same length.",
)
def test_aggregations(self):
lang = schema.Language()
# test total aggregation on expression object
n1 = dj.U().aggr(lang, n='count(*)').fetch1('n')
assert_equal(n1, len(lang.fetch()))
# test total aggregation on expression class
n2 = dj.U().aggr(schema.Language, n='count(*)').fetch1('n')
assert_equal(n1, n2)
rel = dj.U('language').aggr(schema.Language, number_of_speakers='count(*)')
assert_equal(len(rel), len(set(l[1] for l in schema.Language.contents)))
assert_equal((rel & 'language="English"').fetch1('number_of_speakers'), 3)
def test_restriction(self):
language_set = {s[1] for s in self.language.contents}
rel = dj.U('language') & self.language
assert_list_equal(rel.heading.names, ['language'])
assert_true(len(rel) == len(language_set))
assert_true(set(rel.fetch('language')) == language_set)
# Test for issue #342
rel = self.trial*dj.U('start_time')
assert_list_equal(rel.primary_key, self.trial.primary_key + ['start_time'])
assert_list_equal(rel.primary_key, (rel & 'trial_id>3').primary_key)
assert_list_equal((dj.U('start_time') & self.trial).primary_key, ['start_time'])
def test_join(self):
rel = self.experiment * dj.U("experiment_date")
assert_equal(self.experiment.primary_key,
["subject_id", "experiment_id"])
assert_equal(rel.primary_key,
self.experiment.primary_key + ["experiment_date"])
rel = dj.U("experiment_date") * self.experiment
assert_equal(self.experiment.primary_key,
["subject_id", "experiment_id"])
assert_equal(rel.primary_key,
self.experiment.primary_key + ["experiment_date"])
def plot_calibration_curve(self, calibration_date, rig):
import matplotlib.pyplot as plt
import seaborn as sns
session = LaserCalibration.PowerMeasurement() & dict(
calibration_date=calibration_date, rig=rig)
sns.set_context('talk')
with sns.axes_style('darkgrid'):
fig, ax = plt.subplots()
# sns.set_palette("husl")
for k in (dj.U('pockels', 'bidirectional', 'gdd', 'wavelength')
& session).fetch.keys():
pe, po, zoom = (session & k).fetch('percentage', 'power', 'zoom')
zoom = np.unique(zoom)
ax.plot(pe,
po,
'o-',
label=(u"zoom={0:.2f} ".format(zoom[0]) +
" ".join("{0}={1}".format(*v) for v in k.items())))
ax.legend(loc='best')
ax.set_xlim((0, 100))
y_min, y_max = [np.round(y / 5) * 5 for y in ax.get_ylim()]
ax.set_yticks(np.arange(0, y_max + 5, 5))
ax.set_xlabel('power [in %]')
ax.set_ylabel('power [in mW]')
return fig, ax
def check_train_test_split(self, frames, cond):
stim = getattr(stimulus, cond['stimulus_type'].split('.')[-1])
train_test = (dj.U(*UNIQUE_FRAME[cond['stimulus_type']]).aggr(frames * stim,
train='sum(1-test)',
test='sum(test)') &
'train>0 and test>0')
assert len(train_test) == 0, 'Train and test clips do overlap'
def test_aggregations(self):
rel = dj.U('language').aggr(schema.Language(),
number_of_speakers='count(*)')
assert_equal(len(rel),
len(set(l[1] for l in schema.Language.contents)))
assert_equal((rel & 'language="English"').fetch1('number_of_speakers'),
3)
def make(self, key):
master_entry = key.copy()
rt = key.copy()
# get all trial sets and trials from that date
trial_sets_proj = (behavior.TrialSet.proj(
session_date='DATE(session_start_time)')) & key
trial_sets_keys = (behavior.TrialSet * trial_sets_proj).fetch('KEY')
n_trials, n_correct_trials = \
(behavior.TrialSet & trial_sets_keys).fetch(
'n_trials', 'n_correct_trials')
trials = behavior.TrialSet.Trial & trial_sets_keys
# compute the performance for easy trials
performance_easy = utils.compute_performance_easy(trials)
if performance_easy:
master_entry['performance_easy'] = performance_easy
# compute the performance for all trials
master_entry['performance'] = np.divide(np.sum(n_correct_trials),
np.sum(n_trials))
self.insert1(master_entry)
# compute psych results for all trials
task_protocol = (acquisition.Session
& trial_sets_keys[0]).fetch1('task_protocol')
if task_protocol and 'biased' in task_protocol:
prob_lefts = dj.U('trial_stim_prob_left') & trials
for ileft, prob_left in enumerate(prob_lefts):
p_left = prob_left['trial_stim_prob_left']
trials_sub = trials & \
'ABS(trial_stim_prob_left - {})<1e-6'.format(p_left)
# compute psych results
psych_results_tmp = utils.compute_psych_pars(trials_sub)
psych_results = {**key, **psych_results_tmp}
psych_results['prob_left'] = prob_left['trial_stim_prob_left']
psych_results['prob_left_block'] = ileft
self.PsychResults.insert1(psych_results)
# compute reaction time
rt['prob_left_block'] = ileft
rt['reaction_time'] = utils.compute_reaction_time(trials_sub)
self.ReactionTime.insert1(rt)
else:
psych_results_tmp = utils.compute_psych_pars(trials)
psych_results = {**key, **psych_results_tmp}
psych_results['prob_left'] = 0.5
psych_results['prob_left_block'] = 1
self.PsychResults.insert1(psych_results)
# compute reaction time
rt['prob_left_block'] = 1
rt['reaction_time'] = utils.compute_reaction_time(trials)
self.ReactionTime.insert1(rt)
def cell_matches(animal_id, size):
key = dict(animal_id=animal_id, **SETTINGS)
sz = tuple(i * size_factor[size] for i in [.7, .7])
df = pd.DataFrame((stack.StackSet.Unit() * dj.U('stack_session')
& key).aggr(stack.StackSet.Match() & key,
matches='COUNT(*)').fetch())
with sns.plotting_context('talk' if size == 'huge' else 'paper',
font_scale=1.3):
with sns.axes_style(style="ticks", rc={"axes.facecolor":
(0, 0, 0, 0)}):
order = sorted(pd.unique(df.matches))
g = sns.factorplot('matches',
kind='count',
hue='stack_session',
data=df,
order=order)
sns.despine(trim=True, offset=5)
g.ax.spines['bottom'].set_linewidth(1)
g.ax.spines['left'].set_linewidth(1)
g.ax.tick_params(axis='both', length=3, width=1)
g.ax.set_xlabel('scans neuron was visible in')
g.ax.set_ylabel('neurons')
g.fig.set_size_inches(sz)
return savefig(g.fig)
def create1_from_processing_task(self,
key,
is_curated=False,
curation_note=""):
"""
Given a "ProcessingTask", create a new corresponding "Curation"
"""
if key not in Processing():
raise ValueError(
f"No corresponding entry in Processing available for: "
f"{key}; run `Processing.populate(key)`")
output_dir = (ProcessingTask & key).fetch1("processing_output_dir")
method, imaging_dataset = get_loader_result(key, ProcessingTask)
if method == "caiman":
caiman_dataset = imaging_dataset
curation_time = caiman_dataset.creation_time
else:
raise NotImplementedError("Unknown method: {}".format(method))
# Synthesize curation_id
curation_id = (dj.U().aggr(
self & key, n="ifnull(max(curation_id)+1,1)").fetch1("n"))
self.insert1({
**key,
"curation_id": curation_id,
"curation_time": curation_time,
"curation_output_dir": output_dir,
"manual_curation": is_curated,
"curation_note": curation_note,
})
def make(self, key):
task_protocol = (acquisition.Session & key).fetch1('task_protocol')
trials = behavior.TrialSet.Trial & key
if task_protocol and ('biased' in task_protocol):
prob_lefts = dj.U('trial_stim_prob_left') & trials
for prob_left in prob_lefts:
p_left = prob_left['trial_stim_prob_left']
trials_sub = trials & \
'ABS(trial_stim_prob_left - {})<1e-6'.format(p_left)
# compute psych results
psych_results = utils.compute_psych_pars(trials_sub)
psych_results = {**key, **psych_results}
psych_results['prob_left'] = prob_left['trial_stim_prob_left']
if abs(p_left - 0.8) < 0.001:
psych_results['prob_left_block'] = 80
elif abs(p_left - 0.2) < 0.001:
psych_results['prob_left_block'] = 20
elif abs(p_left - 0.5) < 0.001:
psych_results['prob_left_block'] = 50
self.insert1(psych_results)
else:
psych_results = utils.compute_psych_pars(trials)
psych_results = {**key, **psych_results}
psych_results['prob_left'] = 0.5
psych_results['prob_left_block'] = 50
self.insert1(psych_results)
def unique_unit_mapping(self, dynamic_scan):
if (self * dv_nn9_scan.ScanConfig.Scan3() *
dv_scan3_scan_dataset.UnitConfig.Unique()):
key = ((self * dv_nn9_scan.ScanConfig.Scan3() *
dv_scan3_scan_dataset.UnitConfig.Unique())
& dynamic_scan).fetch1() # get unique_id
unique_unit_key = (dv_scan3_scan.Unique() & dynamic_scan
& key).fetch1("KEY")
unique_unit_rel = (
dv_scan3_scan.Unique.Unit *
dv_scan3_scan.Unique.Neuron.proj(unique_unit_id="unit_id")
& unique_unit_key)
return (dj.U("animal_id", "session", "scan_idx", "unit_id",
"unique_unit_id")
& unique_unit_rel)
elif (self * dv_nn9_scan.ScanConfig.Scan3() *
dv_scan3_scan_dataset.UnitConfig.All()
): # return a mapping from all units to themselves
units = (self * dv_nn9_scan.Scan.Unit *
dv_nn9_scan.ScanConfig().Scan3())
return units.proj(unique_unit_id='unit_id * 1')
else:
raise NotImplementedError(
"`unique_unit_mapping` is not implemented for key {}!".
format(self.fetch1()))
def test_dj_u_distinct(self):
# Test developed to see if removing DISTINCT from the select statement
# generation breakes the dj.U universal set imlementation
# Contents to be inserted
contents = [(1, 2, 3), (2, 2, 3), (3, 3, 2), (4, 5, 5)]
Stimulus.insert(contents)
# Query the whole table
test_query = Stimulus()
# Use dj.U to create a list of unique contrast and brightness combinations
result = dj.U('contrast', 'brightness') & test_query
expected_result = [{
'contrast': 2,
'brightness': 3
}, {
'contrast': 3,
'brightness': 2
}, {
'contrast': 5,
'brightness': 5
}]
fetched_result = result.fetch(as_dict=True,
order_by=('contrast', 'brightness'))
Stimulus.delete_quick()
assert fetched_result == expected_result
def make(self, key):
task_protocol = (acquisition.Session & key).fetch1('task_protocol')
trials = behavior.TrialSet.Trial & key & 'trial_stim_on_time is not NULL'
if task_protocol and ('biased' in task_protocol):
prob_lefts = dj.U('trial_stim_prob_left') & trials
for prob_left in prob_lefts:
rt = key.copy()
p_left = prob_left['trial_stim_prob_left']
trials_sub = trials & \
'ABS(trial_stim_prob_left - {})<1e-6'.format(p_left)
# compute reaction_time
rt['reaction_time_contrast'], rt['reaction_time_ci_low'], \
rt['reaction_time_ci_high'] = utils.compute_reaction_time(
trials_sub, compute_ci=True)
if abs(p_left - 0.8) < 0.001:
rt['prob_left_block'] = 80
elif abs(p_left - 0.2) < 0.001:
rt['prob_left_block'] = 20
elif abs(p_left - 0.5) < 0.001:
rt['prob_left_block'] = 50
self.insert1(rt)
else:
rt = key.copy()
rt['prob_left_block'] = 50
rt['reaction_time_contrast'], rt['reaction_time_ci_low'], \
rt['reaction_time_ci_high'] = utils.compute_reaction_time(
trials, compute_ci=True)
self.insert1(rt)
def create1_from_processing_task(self,
key,
is_curated=False,
curation_note=''):
"""
A convenient function to create a new corresponding "Curation" for a particular "ProcessingTask"
"""
if key not in Processing():
raise ValueError(
f'No corresponding entry in Processing available for: {key};'
f' do `Processing.populate(key)`')
output_dir = (ProcessingTask & key).fetch1('processing_output_dir')
method, loaded_result = get_loader_result(key, ProcessingTask)
if method == 'caiman':
loaded_caiman = loaded_result
curation_time = loaded_caiman.creation_time
elif method == 'mcgill_miniscope_analysis':
loaded_miniscope_analysis = loaded_result
curation_time = loaded_miniscope_analysis.creation_time
else:
raise NotImplementedError('Unknown method: {}'.format(method))
# Synthesize curation_id
curation_id = dj.U().aggr(self & key,
n='ifnull(max(curation_id)+1,1)').fetch1('n')
self.insert1({
**key, 'curation_id': curation_id,
'curation_time': curation_time,
'curation_output_dir': output_dir,
'manual_curation': is_curated,
'curation_note': curation_note
})
def plot_example_cells(sort_lv = 'relative_action_value_ic',
sort_ep = 'iti_all',
best_n = 10, linear_model='Q_rel + Q_tot + rpe'):
#%%
q_unit = ((ephys.Unit * ephys.ClusterMetric * ephys.UnitStat * ephys.MAPClusterMetric.DriftMetric)
& 'presence_ratio > 0.95'
& 'amplitude_cutoff < 0.1'
& 'isi_violation < 0.5'
& 'unit_amp > 100'
# & 'drift_metric < 0.1'
)
q_hist = (q_unit * histology.ElectrodeCCFPosition.ElectrodePosition) * ccf.CCFAnnotation
q_unit_n = dj.U('annotation').aggr(q_hist, area_num_units='count(*)')
q_hist *= q_unit_n
lvs = (psth_foraging.LinearModel.X & {'multi_linear_model': linear_model}).fetch('var_name')
q_all = ((psth_foraging.UnitPeriodLinearFit
* psth_foraging.UnitPeriodLinearFit.Param
* q_hist)
& {'multi_linear_model': linear_model})
# Best n (absolute value)
best_models = (q_all & f'var_name = "{sort_lv}"' & f'period = "{sort_ep}"').proj(
'actual_behavior_model', abs_t='abs(t)').fetch(order_by='abs_t desc', limit=best_n, format='frame')
for unit_key in best_models.reset_index().to_dict('records'):
unit_psth.plot_unit_psth_choice_outcome(unit_key)
unit_psth.plot_unit_psth_latent_variable_quantile(unit_key,
model_id=unit_key['actual_behavior_model'])
unit_psth.plot_unit_period_tuning(unit_key)
def fetch_oracle_raster(unit_key):
"""Fetches the responses of the provided unit to the oracle trials
Args:
unit_key (dict): dictionary to uniquely identify a functional unit (must contain the keys: "session", "scan_idx", "unit_id")
Returns:
oracle_score (float):
responses (array): array of oracle responses interpolated to scan frequency: 10 repeats x 6 oracle clips x f response frames
"""
fps = (nda.Scan & unit_key).fetch1('fps') # get frame rate of scan
oracle_rel = (dj.U('condition_hash').aggr(nda.Trial & unit_key,n='count(*)',m='min(trial_idx)') & 'n=10') # get oracle clips
oracle_hashes = oracle_rel.fetch('KEY',order_by='m ASC') # get oracle clip hashes sorted temporally
frame_times_set = []
# iterate over oracle repeats (10 repeats)
for first_clip in (nda.Trial & oracle_hashes[0] & unit_key).fetch('trial_idx'):
trial_block_rel = (nda.Trial & unit_key & f'trial_idx >= {first_clip} and trial_idx < {first_clip+6}') # uses the trial_idx of the first clip to grab subsequent 5 clips (trial_block)
start_times, end_times = trial_block_rel.fetch('start_frame_time', 'end_frame_time', order_by='condition_hash DESC') # grabs start time and end time of each clip in trial_block and orders by condition_hash to maintain order across scans
frame_times = [np.linspace(s, e , np.round(fps * (e - s)).astype(int)) for s, e in zip(start_times, end_times)] # generate time vector between start and end times according to frame rate of scan
frame_times_set.append(frame_times)
trace, fts, delay = ((nda.Activity & unit_key) * nda.FrameTimes * nda.ScanUnit).fetch1('trace', 'frame_times', 'ms_delay') # fetch trace delay and frame times for interpolation
f2a = interp1d(fts + delay/1000, trace) # create trace interpolator with unit specific time delay
oracle_traces = np.array([f2a(ft) for ft in frame_times_set]) # interpolate oracle times to match the activity trace
oracle_traces -= np.min(oracle_traces,axis=(1,2),keepdims=True) # normalize the oracle traces
oracle_traces /= np.max(oracle_traces,axis=(1,2),keepdims=True) # normalize the oracle traces
oracle_score = (nda.Oracle & unit_key).fetch1('pearson') # fetch oracle score
return oracle_traces, oracle_score
def create1_from_clustering_task(self, key, curation_note=''):
"""
A convenient function to create a new corresponding "Curation"
for a particular "ClusteringTask"
"""
if key not in Clustering():
raise ValueError(f'No corresponding entry in Clustering available'
f' for: {key}; do `Clustering.populate(key)`')
task_mode, output_dir = (ClusteringTask & key).fetch1(
'task_mode', 'clustering_output_dir')
kilosort_dir = find_full_path(get_ephys_root_data_dir(), output_dir)
creation_time, is_curated, is_qc = kilosort.extract_clustering_info(
kilosort_dir)
# Synthesize curation_id
curation_id = dj.U().aggr(self & key,
n='ifnull(max(curation_id)+1,1)').fetch1('n')
self.insert1({
**key, 'curation_id': curation_id,
'curation_time': creation_time,
'curation_output_dir': output_dir,
'quality_control': is_qc,
'manual_curation': is_curated,
'curation_note': curation_note
})
def get_fields(chosen_cell, scans, stack_key):
"""
function that takes in an munit_id, a set of scans, and a stack, and returns the field key and a set of summary images for each scan_session and scan_idx the munit appears in:
The summary images are as follows:
1) scan summary image which will be one of: average, correlation, l6norm, hybrid image depending on specification in argument 'functional_image'
2) the relevant stack field after registering the imaging field inside the stack
3) the relevant 3D segmentation image after registering the imaging field
:param chosen_cell: dictionary of munit id formatted as such: {'munit_id':00000}
:param scans: the key to restrict the stack datajoint table relation to the specified functional scans
:param stack_key: the relevant stack to restrict with
:return: List of field keys For each scan_session and scan_idx returns:
"""
field_munit_relation_table = meso.ScanSet.Unit * (
stack.StackSet.Match & stack_key & chosen_cell).proj(
'munit_id', session='scan_session') & scans
field_munit_relation_table
# choose an example cell and get unit_id's (unique id's per scan)
field_munit_relation_keys = (
dj.U('animal_id', 'stack_session', 'stack_idx', 'segmentation_method',
'session', 'scan_idx', 'field', 'unit_id', 'munit_id')
& field_munit_relation_table).fetch('KEY')
return field_munit_relation_keys
def concatenated_rel(cls,
core_segment=None,
version=-1,
return_with_meshes=False):
"""
Returns all. You can restrict by a core_segment first though.
:param core_segment: The core segment(s) to restrict by. If left empty will fetch all.
:param version: The default of -1 will fetch the highest version for each core segment
and its subsegments. If you happen to explicitely pass False, it will ignore version.
:param return_with_meshes: When set to true or 'Decimation' will default
to using the Decimation table for the meshes, otherwise 'Mesh' will
choose the Mesh table with the original meshes.
"""
subsegment_rel = cls.Subsegment.proj()
if core_segment is not None:
try:
subsegment_rel &= [
dict(segment_id=segment_id) for segment_id in core_segment
]
except TypeError:
subsegment_rel &= dict(segment_id=core_segment)
if version == -1:
version_rel = dj.U('segment_id').aggr(subsegment_rel,
version='max(version)')
elif version is False:
version_rel = subsegment_rel
else:
version_rel = subsegment_rel & dict(version=version)
a_rel = dj.U('segment_id') & version_rel
b_rel = dj.U('segment_id') & (subsegment_rel & version_rel).proj(
_='segment_id', segment_id='subsegment_id')
c_rel = a_rel + b_rel
if return_with_meshes:
if isinstance(return_with_meshes,
str) and return_with_meshes.lower() == 'mesh':
c_rel = minnie.Mesh & c_rel
else:
c_rel = minnie.Decimation & c_rel
return c_rel
def load_frame_times(self, key):
pipe = (fuse.Activity() & key).fetch('pipe')
assert len(np.unique(pipe)) == 1, 'Selection is from different pipelines'
pipe = dj.create_virtual_module(pipe[0], 'pipeline_' + pipe[0])
k = dict(key)
k.pop('field', None)
ndepth = len(dj.U('z') & (pipe.ScanInfo.Field() & k))
return (stimulus.Sync() & key).fetch1('frame_times').squeeze()[::ndepth]
def key_source(self):
ret_rel = stack.Area.proj(ret_session='scan_session',
ret_scan_idx='scan_idx',
ret_channel='scan_channel')
key_source = ret_rel * StackCoordinates
heading_str = list(self.heading)
return dj.U(*heading_str) & key_source
