def _get_units(self, filter_by_validity: bool = True, **unit_filter_kwargs) -> pd.DataFrame:
path = self.get_cache_path(None, self.UNITS_KEY)
units = one_file_call_caching(path, self.fetch_api.get_units, write_csv, read_csv, num_tries=self.fetch_tries)
units = units.rename(columns={
'PT_ratio': 'waveform_PT_ratio',
'amplitude': 'waveform_amplitude',
'duration': 'waveform_duration',
'halfwidth': 'waveform_halfwidth',
'recovery_slope': 'waveform_recovery_slope',
'repolarization_slope': 'waveform_repolarization_slope',
'spread': 'waveform_spread',
'velocity_above': 'waveform_velocity_above',
'velocity_below': 'waveform_velocity_below',
'l_ratio': 'L_ratio',
})
units = units[
(units["amplitude_cutoff"] <= get_unit_filter_value("amplitude_cutoff_maximum", **unit_filter_kwargs))
& (units["presence_ratio"] >= get_unit_filter_value("presence_ratio_minimum", **unit_filter_kwargs))
& (units["isi_violations"] <= get_unit_filter_value("isi_violations_maximum", **unit_filter_kwargs))
]
if "quality" in units.columns and filter_by_validity:
units = units[units["quality"] == "good"]
units.drop(columns="quality", inplace=True)
if "ecephys_structure_id" in units.columns and unit_filter_kwargs.get("filter_out_of_brain_units", True):
units = units[~(units["ecephys_structure_id"].isna())]
return units
def __init__(
self,
path,
probe_lfp_paths: Optional[Dict[int,
Callable[[],
pynwb.NWBFile]]] = None,
additional_unit_metrics=None,
external_channel_columns=None,
**kwargs):
self.filter_out_of_brain_units = kwargs.pop(
"filter_out_of_brain_units", True)
self.filter_by_validity = kwargs.pop("filter_by_validity", True)
self.amplitude_cutoff_maximum = get_unit_filter_value(
"amplitude_cutoff_maximum", **kwargs)
self.presence_ratio_minimum = get_unit_filter_value(
"presence_ratio_minimum", **kwargs)
self.isi_violations_maximum = get_unit_filter_value(
"isi_violations_maximum", **kwargs)
super(EcephysNwbSessionApi, self).__init__(path, **kwargs)
self.probe_lfp_paths = probe_lfp_paths
self.additional_unit_metrics = additional_unit_metrics
self.external_channel_columns = external_channel_columns
def get_channels(self,
channel_ids=None,
probe_ids=None,
session_ids=None,
**kwargs):
response = build_and_execute(
"""
{%- import 'postgres_macros' as pm -%}
select
ec.id as id,
ec.ecephys_probe_id,
ec.local_index,
ec.probe_vertical_position,
ec.probe_horizontal_position,
ec.manual_structure_id as ecephys_structure_id,
st.acronym as ecephys_structure_acronym,
pc.unit_count
from ecephys_channels ec
join ecephys_probes ep on ep.id = ec.ecephys_probe_id
join ecephys_sessions es on es.id = ep.ecephys_session_id
left join structures st on st.id = ec.manual_structure_id
join (
select ech.id as ecephys_channel_id,
count (distinct eun.id) as unit_count
from ecephys_channels ech
join ecephys_units eun on (
eun.ecephys_channel_id = ech.id
and eun.quality = 'good'
{{pm.optional_le('eun.amplitude_cutoff', amplitude_cutoff_maximum) -}}
{{pm.optional_ge('eun.presence_ratio', presence_ratio_minimum) -}}
{{pm.optional_le('eun.isi_violations', isi_violations_maximum) -}}
)
group by ech.id
) pc on ec.id = pc.ecephys_channel_id
where valid_data
and ep.workflow_state != 'failed'
and es.workflow_state != 'failed'
{{pm.optional_contains('ec.id', channel_ids) -}}
{{pm.optional_contains('ep.id', probe_ids) -}}
{{pm.optional_contains('es.id', session_ids) -}}
""",
base=postgres_macros(),
engine=self.postgres_engine.select,
channel_ids=channel_ids,
probe_ids=probe_ids,
session_ids=session_ids,
amplitude_cutoff_maximum=get_unit_filter_value(
"amplitude_cutoff_maximum", replace_none=False, **kwargs),
presence_ratio_minimum=get_unit_filter_value(
"presence_ratio_minimum", replace_none=False, **kwargs),
isi_violations_maximum=get_unit_filter_value(
"isi_violations_maximum", replace_none=False, **kwargs))
return response.set_index("id")
def get_units(self,
unit_ids=None,
channel_ids=None,
probe_ids=None,
session_ids=None,
quality="good",
**kwargs):
response = build_and_execute(
"""
{%- import 'postgres_macros' as pm -%}
{%- import 'macros' as m -%}
select eu.*
from ecephys_units eu
join ecephys_channels ec on ec.id = eu.ecephys_channel_id
join ecephys_probes ep on ep.id = ec.ecephys_probe_id
join ecephys_sessions es on es.id = ep.ecephys_session_id
where ec.valid_data
and ep.workflow_state != 'failed'
and es.workflow_state != 'failed'
{{pm.optional_equals('eu.quality', quality) -}}
{{pm.optional_contains('eu.id', unit_ids) -}}
{{pm.optional_contains('ec.id', channel_ids) -}}
{{pm.optional_contains('ep.id', probe_ids) -}}
{{pm.optional_contains('es.id', session_ids) -}}
{{pm.optional_le('eu.amplitude_cutoff', amplitude_cutoff_maximum) -}}
{{pm.optional_ge('eu.presence_ratio', presence_ratio_minimum) -}}
{{pm.optional_le('eu.isi_violations', isi_violations_maximum) -}}
""",
base=postgres_macros(),
engine=self.postgres_engine.select,
unit_ids=unit_ids,
channel_ids=channel_ids,
probe_ids=probe_ids,
session_ids=session_ids,
quality=f"'{quality}'" if quality is not None else quality,
amplitude_cutoff_maximum=get_unit_filter_value(
"amplitude_cutoff_maximum", replace_none=False, **kwargs),
presence_ratio_minimum=get_unit_filter_value(
"presence_ratio_minimum", replace_none=False, **kwargs),
isi_violations_maximum=get_unit_filter_value(
"isi_violations_maximum", replace_none=False, **kwargs))
response.set_index("id", inplace=True)
return response
def __init__(
self,
path,
probe_lfp_paths: Optional[Dict[int,
Callable[[],
pynwb.NWBFile]]] = None,
additional_unit_metrics=None,
external_channel_columns=None,
**kwargs):
self.filter_out_of_brain_units = kwargs.pop(
"filter_out_of_brain_units", True)
self.filter_by_validity = kwargs.pop("filter_by_validity", True)
self.amplitude_cutoff_maximum = get_unit_filter_value(
"amplitude_cutoff_maximum", **kwargs)
self.presence_ratio_minimum = get_unit_filter_value(
"presence_ratio_minimum", **kwargs)
self.isi_violations_maximum = get_unit_filter_value(
"isi_violations_maximum", **kwargs)
super(EcephysNwbSessionApi, self).__init__(path, **kwargs)
self.probe_lfp_paths = probe_lfp_paths
self.additional_unit_metrics = additional_unit_metrics
self.external_channel_columns = external_channel_columns
if hasattr(self, "path") and self.path:
check_nwbfile_version(
nwbfile_path=self.path,
desired_minimum_version="2.2.2",
warning_msg=(
f"It looks like the Visual Coding Neuropixels nwbfile "
f"you are trying to access ({self.path})"
f"was created by a previous (and incompatible) version of "
f"AllenSDK and pynwb. You will need to either 1) use "
f"AllenSDK version < 2.0.0 or 2) re-download an updated "
f"version of the nwbfile to access the desired data."))
def get_sessions(self,
session_ids=None,
workflow_states=("uploaded", ),
published=None,
habituation=False,
project_names=(
"BrainTV Neuropixels Visual Behavior",
"BrainTV Neuropixels Visual Coding",
),
**kwargs):
response = build_and_execute(
"""
{%- import 'postgres_macros' as pm -%}
{%- import 'macros' as m -%}
select
stimulus_name as session_type,
sp.id as specimen_id,
es.id as id,
dn.full_genotype as genotype,
gd.name as gender,
ages.days as age_in_days,
pr.code as project_code,
probe_count,
channel_count,
unit_count,
case
when nwb_id is not null then true
else false
end as has_nwb,
str.structure_acronyms as structure_acronyms
from ecephys_sessions es
join specimens sp on sp.id = es.specimen_id
join donors dn on dn.id = sp.donor_id
join genders gd on gd.id = dn.gender_id
join ages on ages.id = dn.age_id
join projects pr on pr.id = es.project_id
join (
select es.id as ecephys_session_id,
count (distinct epr.id) as probe_count,
count (distinct ech.id) as channel_count,
count (distinct eun.id) as unit_count
from ecephys_sessions es
join ecephys_probes epr on epr.ecephys_session_id = es.id
join ecephys_channels ech on (
ech.ecephys_probe_id = epr.id
and ech.valid_data
)
join ecephys_units eun on (
eun.ecephys_channel_id = ech.id
and eun.quality = 'good'
{{pm.optional_le('eun.amplitude_cutoff', amplitude_cutoff_maximum) -}}
{{pm.optional_ge('eun.presence_ratio', presence_ratio_minimum) -}}
{{pm.optional_le('eun.isi_violations', isi_violations_maximum) -}}
)
group by es.id
) pc on es.id = pc.ecephys_session_id
left join (
select ecephys_sessions.id as ecephys_session_id,
wkf.id as nwb_id
from ecephys_sessions
join ecephys_analysis_runs ear on (
ear.ecephys_session_id = ecephys_sessions.id
and ear.current
)
join well_known_files wkf on (
wkf.attachable_id = ear.id
and wkf.attachable_type = 'EcephysAnalysisRun'
)
join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id
where wkft.name = 'EcephysNwb'
) nwb on es.id = nwb.ecephys_session_id
left join (
select es.id as ecephys_session_id,
array_agg (st.id) as structure_ids,
array_agg (distinct st.acronym) as structure_acronyms
from ecephys_sessions es
join ecephys_probes epr on epr.ecephys_session_id = es.id
join ecephys_channels ech on (
ech.ecephys_probe_id = epr.id
and ech.valid_data
)
left join structures st on st.id = ech.manual_structure_id
group by es.id
) str on es.id = str.ecephys_session_id
where true
{{pm.optional_contains('es.id', session_ids) -}}
{{pm.optional_contains('es.workflow_state', workflow_states, True) -}}
{{pm.optional_equals('es.habituation', habituation) -}}
{{pm.optional_not_null('es.published_at', published) -}}
{{pm.optional_contains('pr.name', project_names, True) -}}
""",
base=postgres_macros(),
engine=self.postgres_engine.select,
session_ids=session_ids,
workflow_states=workflow_states,
published=published,
habituation=f"{habituation}".lower()
if habituation is not None else habituation,
project_names=project_names,
amplitude_cutoff_maximum=get_unit_filter_value(
"amplitude_cutoff_maximum", replace_none=False, **kwargs),
presence_ratio_minimum=get_unit_filter_value(
"presence_ratio_minimum", replace_none=False, **kwargs),
isi_violations_maximum=get_unit_filter_value(
"isi_violations_maximum", replace_none=False, **kwargs))
response.set_index("id", inplace=True)
response["genotype"].fillna("wt", inplace=True)
return response
def get_probes(self, probe_ids=None, session_ids=None, **kwargs):
response = build_and_execute(
"""
{%- import 'postgres_macros' as pm -%}
select
ep.id as id,
ep.ecephys_session_id,
ep.global_probe_sampling_rate,
ep.global_probe_lfp_sampling_rate,
total_time_shift,
channel_count,
unit_count,
case
when nwb_id is not null then true
else false
end as has_lfp_nwb,
str.structure_acronyms as structure_acronyms
from ecephys_probes ep
join ecephys_sessions es on es.id = ep.ecephys_session_id
join (
select epr.id as ecephys_probe_id,
count (distinct ech.id) as channel_count,
count (distinct eun.id) as unit_count
from ecephys_probes epr
join ecephys_channels ech on (
ech.ecephys_probe_id = epr.id
and ech.valid_data
)
join ecephys_units eun on (
eun.ecephys_channel_id = ech.id
and eun.quality = 'good'
{{pm.optional_le('eun.amplitude_cutoff', amplitude_cutoff_maximum) -}}
{{pm.optional_ge('eun.presence_ratio', presence_ratio_minimum) -}}
{{pm.optional_le('eun.isi_violations', isi_violations_maximum) -}}
)
group by epr.id
) chc on ep.id = chc.ecephys_probe_id
left join (
select
epr.id as ecephys_probe_id,
wkf.id as nwb_id
from ecephys_probes epr
join ecephys_analysis_runs ear on (
ear.ecephys_session_id = epr.ecephys_session_id
and ear.current
)
right join ecephys_analysis_run_probes earp on (
earp.ecephys_probe_id = epr.id
and earp.ecephys_analysis_run_id = ear.id
)
right join well_known_files wkf on (
wkf.attachable_id = earp.id
and wkf.attachable_type = 'EcephysAnalysisRunProbe'
)
join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id
where wkft.name = 'EcephysLfpNwb'
) nwb on ep.id = nwb.ecephys_probe_id
left join (
select epr.id as ecephys_probe_id,
array_agg (st.id) as structure_ids,
array_agg (distinct st.acronym) as structure_acronyms
from ecephys_probes epr
join ecephys_channels ech on (
ech.ecephys_probe_id = epr.id
and ech.valid_data
)
left join structures st on st.id = ech.manual_structure_id
group by epr.id
) str on ep.id = str.ecephys_probe_id
where true
and ep.workflow_state != 'failed'
and es.workflow_state != 'failed'
{{pm.optional_contains('ep.id', probe_ids) -}}
{{pm.optional_contains('es.id', session_ids) -}}
""",
base=postgres_macros(),
engine=self.postgres_engine.select,
probe_ids=probe_ids,
session_ids=session_ids,
amplitude_cutoff_maximum=get_unit_filter_value(
"amplitude_cutoff_maximum", replace_none=False, **kwargs),
presence_ratio_minimum=get_unit_filter_value(
"presence_ratio_minimum", replace_none=False, **kwargs),
isi_violations_maximum=get_unit_filter_value(
"isi_violations_maximum", replace_none=False, **kwargs))
response = response.set_index("id")
# Clarify name for external users
response.rename(columns={"use_lfp_data": "has_lfp_data"}, inplace=True)
return response
