def get_natural_scene_template(self, number):
return one_file_call_caching(
self.get_cache_path(None, self.NATURAL_SCENE_KEY, number),
partial(self.fetch_api.get_natural_scene_template, number=number),
self.stream_writer,
read_scene,
num_tries=self.fetch_tries)
def get_ophys_experiment_table(
self,
suppress: Optional[List[str]] = None,
as_df=True) -> Union[pd.DataFrame, SessionsTable]:
"""
Return summary table of all ophys_experiment_ids in the database.
:param suppress: optional list of columns to drop from the resulting
dataframe.
:type suppress: list of str
:param as_df: whether to return as df or as SessionsTable
:rtype: pd.DataFrame
"""
if isinstance(self.fetch_api, BehaviorProjectCloudApi):
return self.fetch_api.get_ophys_experiment_table()
if self.cache is not None:
path = self.cache.get_cache_path(None,
self.cache.OPHYS_EXPERIMENTS_KEY)
experiments = one_file_call_caching(
path, self.fetch_api.get_ophys_experiment_table, _write_json,
lambda path: _read_json(path, index_name='ophys_experiment_id'
))
else:
experiments = self.fetch_api.get_ophys_experiment_table()
# Merge behavior data in
behavior_sessions_table = self.get_behavior_session_table(
suppress=suppress, as_df=True, include_ophys_data=False)
experiments = behavior_sessions_table.merge(
experiments,
left_index=True,
right_on='behavior_session_id',
suffixes=('_behavior', '_ophys'))
experiments = ExperimentsTable(df=experiments, suppress=suppress)
return experiments.table if as_df else experiments
def _get_channels(self):
path = self.get_cache_path(None, self.CHANNELS_KEY)
return one_file_call_caching(path,
self.fetch_api.get_channels,
write_csv,
read_csv,
num_tries=self.fetch_tries)
def get_ophys_session_table(
self,
suppress: Optional[List[str]] = None,
index_column: str = "ophys_session_id",
as_df=True,
include_behavior_data=True,
passed_only=True) -> \
Union[pd.DataFrame, BehaviorOphysSessionsTable]:
"""
Return summary table of all ophys_session_ids in the database.
:param suppress: optional list of columns to drop from the resulting
dataframe.
:type suppress: list of str
:param index_column: (default="ophys_session_id"). Column to index
on, either
"ophys_session_id" or "ophys_experiment_id".
If index_column="ophys_experiment_id", then each row will only have
one experiment id, of type int (vs. an array of 1>more).
:type index_column: str
:param as_df: whether to return as df or as BehaviorOphysSessionsTable
:param include_behavior_data
Whether to include behavior data
:rtype: pd.DataFrame
"""
if isinstance(self.fetch_api, BehaviorProjectCloudApi):
return self.fetch_api.get_ophys_session_table()
if self.cache is not None:
path = self.cache.get_cache_path(None,
self.cache.OPHYS_SESSIONS_KEY)
ophys_sessions = one_file_call_caching(
path, self.fetch_api.get_ophys_session_table, _write_json,
lambda path: _read_json(path, index_name='ophys_session_id'))
else:
ophys_sessions = self.fetch_api.get_ophys_session_table()
if include_behavior_data:
# Merge behavior data in
behavior_sessions_table = self.get_behavior_session_table(
suppress=suppress, as_df=True, include_ophys_data=False)
ophys_sessions = behavior_sessions_table.merge(
ophys_sessions,
left_index=True,
right_on='behavior_session_id',
suffixes=('_behavior', '_ophys'))
sessions = BehaviorOphysSessionsTable(df=ophys_sessions,
suppress=suppress,
index_column=index_column)
if passed_only:
oet = self.get_ophys_experiment_table(passed_only=True)
for i in sessions.table.index:
sub_df = oet.query(f"ophys_session_id=={i}")
values = list(set(sub_df["ophys_container_id"].values))
values.sort()
sessions.table.at[i, "ophys_container_id"] = values
return sessions.table if as_df else sessions
def get_unit_analysis_metrics_by_session_type(
self,
session_type,
annotate: bool = True,
filter_by_validity: bool = True,
**unit_filter_kwargs):
""" Cache and return a table of analysis metrics calculated on each unit from a specified session type. See
get_all_session_types for a list of session types.
Parameters
----------
session_type : str
identifies the session type for which to fetch analysis metrics.
annotate : bool, optional
if True, information from the annotated units table will be merged onto the outputs
filter_by_validity : bool, optional
Filter units used by analysis so that only 'valid' units are returned, by default True
**unit_filter_kwargs :
Additional keyword arguments that can be used to filter units (for power users).
Returns
-------
metrics : pd.DataFrame
Each row corresponds to a single unit, describing a set of analysis metrics calculated on that unit.
"""
known_session_types = self.get_all_session_types()
if session_type not in known_session_types:
raise ValueError(
f"unrecognized session type: {session_type}. Available types: {known_session_types}"
)
path = self.get_cache_path(None, self.TYPEWISE_ANALYSIS_METRICS_KEY,
session_type)
fetch_metrics = partial(self.fetch_api.get_unit_analysis_metrics,
session_types=[session_type])
metrics = one_file_call_caching(path,
fetch_metrics,
write_metrics_csv,
read_metrics_csv,
num_tries=self.fetch_tries)
if annotate:
units = self.get_units(filter_by_validity=filter_by_validity,
**unit_filter_kwargs)
metrics = pd.merge(units,
metrics,
left_index=True,
right_index=True,
how="inner")
metrics.index.rename("ecephys_unit_id", inplace=True)
return metrics
def _get_probes(self):
path: str = self.get_cache_path(None, self.PROBES_KEY)
probes = one_file_call_caching(path,
self.fetch_api.get_probes,
write_csv,
read_csv,
num_tries=self.fetch_tries)
# Divide the lfp sampling by the subsampling factor for clearer presentation (if provided)
if all(c in list(probes) for c in
["lfp_sampling_rate", "lfp_temporal_subsampling_factor"]):
probes["lfp_sampling_rate"] = (
probes["lfp_sampling_rate"] /
probes["lfp_temporal_subsampling_factor"])
return probes
def get_unit_analysis_metrics_for_session(self,
session_id,
annotate: bool = True,
filter_by_validity: bool = True,
**unit_filter_kwargs):
""" Cache and return a table of analysis metrics calculated on each unit from a specified session. See
get_session_table for a list of sessions.
Parameters
----------
session_id : int
identifies the session from which to fetch analysis metrics.
annotate : bool, optional
if True, information from the annotated units table will be merged onto the outputs
filter_by_validity : bool, optional
Filter units used by analysis so that only 'valid' units are returned, by default True
**unit_filter_kwargs :
Additional keyword arguments that can be used to filter units (for power users).
Returns
-------
metrics : pd.DataFrame
Each row corresponds to a single unit, describing a set of analysis metrics calculated on that unit.
"""
path = self.get_cache_path(None, self.SESSION_ANALYSIS_METRICS_KEY,
session_id, session_id)
fetch_metrics = partial(self.fetch_api.get_unit_analysis_metrics,
ecephys_session_ids=[session_id])
metrics = one_file_call_caching(path,
fetch_metrics,
write_metrics_csv,
read_metrics_csv,
num_tries=self.fetch_tries)
if annotate:
units = self.get_units(filter_by_validity=filter_by_validity,
**unit_filter_kwargs)
units = units[units["ecephys_session_id"] == session_id]
metrics = pd.merge(units,
metrics,
left_index=True,
right_index=True,
how="inner")
metrics.index.rename("ecephys_unit_id", inplace=True)
return metrics
def get_ophys_cells_table(self) -> pd.DataFrame:
"""
Return summary table of all cells in this project cache
:rtype: pd.DataFrame
"""
if isinstance(self.fetch_api, BehaviorProjectCloudApi):
return self.fetch_api.get_ophys_cells_table()
if self.cache is not None:
path = self.cache.get_cache_path(None, self.cache.OPHyS_CELLS_KEY)
ophys_cells_table = one_file_call_caching(
path, self.fetch_api.get_ophys_cells_table, _write_json,
lambda path: _read_json(path, index_name='cell_roi_id'))
else:
ophys_cells_table = self.fetch_api.get_ophys_cells_table()
return ophys_cells_table
def _get_sessions(self):
path = self.get_cache_path(None, self.SESSIONS_KEY)
response = one_file_call_caching(path,
self.fetch_api.get_sessions,
write_csv,
read_csv,
num_tries=self.fetch_tries)
if "structure_acronyms" in response.columns: # unfortunately, structure_acronyms is a list of str
response["ecephys_structure_acronyms"] = [
ast.literal_eval(item)
for item in response["structure_acronyms"]
]
response.drop(columns=["structure_acronyms"], inplace=True)
return response
def get_session_data(self,
session_id: int,
filter_by_validity: bool = True,
**unit_filter_kwargs):
""" Obtain an EcephysSession object containing detailed data for a single session
"""
def read(_path):
session_api = self._build_nwb_api_for_session(
_path, session_id, filter_by_validity, **unit_filter_kwargs)
return EcephysSession(api=session_api, test=True)
return one_file_call_caching(self.get_cache_path(
None, self.SESSION_NWB_KEY, session_id, session_id),
partial(self.fetch_api.get_session_data,
session_id),
self.stream_writer,
read,
num_tries=self.fetch_tries)
def get_behavior_session_table(
self,
suppress: Optional[List[str]] = None,
as_df=True,
include_ophys_data=True,
passed_only=True) -> Union[pd.DataFrame, SessionsTable]:
"""
Return summary table of all behavior_session_ids in the database.
:param suppress: optional list of columns to drop from the resulting
dataframe.
:param as_df: whether to return as df or as SessionsTable
:param include_ophys_data
Whether to include ophys data
:type suppress: list of str
:rtype: pd.DataFrame
"""
if isinstance(self.fetch_api, BehaviorProjectCloudApi):
return self.fetch_api.get_behavior_session_table()
if self.cache is not None:
path = self.cache.get_cache_path(None,
self.cache.BEHAVIOR_SESSIONS_KEY)
sessions = one_file_call_caching(
path, self.fetch_api.get_behavior_session_table, _write_json,
lambda path: _read_json(path, index_name='behavior_session_id'
))
else:
sessions = self.fetch_api.get_behavior_session_table()
if include_ophys_data:
ophys_session_table = self.get_ophys_session_table(
suppress=suppress,
as_df=False,
include_behavior_data=False,
passed_only=passed_only)
else:
ophys_session_table = None
sessions = SessionsTable(df=sessions,
suppress=suppress,
fetch_api=self.fetch_api,
ophys_session_table=ophys_session_table)
return sessions.table if as_df else sessions
def test_one_file_call_caching(tmpdir_factory, existing):
tmpdir = str(tmpdir_factory.mktemp("foo"))
path = os.path.join(tmpdir, "baz.csv")
getter = get_data
data = getter()
if existing:
data.to_csv(path, index=False)
getter = lambda: "foo"
obtained = cu.one_file_call_caching(
path,
getter,
lambda path, df: df.to_csv(path, index=False),
lambda path: pd.read_csv(path),
num_tries=2
)
pd.testing.assert_frame_equal(get_data(), obtained, check_like=True, check_dtype=False)
def _get_units(self,
filter_by_validity: bool = True,
**unit_filter_kwargs) -> pd.DataFrame:
path = self.get_cache_path(None, self.UNITS_KEY)
units = one_file_call_caching(path,
self.fetch_api.get_units,
write_csv,
read_csv,
num_tries=self.fetch_tries)
units = units.rename(
columns={
'PT_ratio': 'waveform_PT_ratio',
'amplitude': 'waveform_amplitude',
'duration': 'waveform_duration',
'halfwidth': 'waveform_halfwidth',
'recovery_slope': 'waveform_recovery_slope',
'repolarization_slope': 'waveform_repolarization_slope',
'spread': 'waveform_spread',
'velocity_above': 'waveform_velocity_above',
'velocity_below': 'waveform_velocity_below',
'l_ratio': 'L_ratio',
})
units = units[(units["amplitude_cutoff"] <= get_unit_filter_value(
"amplitude_cutoff_maximum", **unit_filter_kwargs))
& (units["presence_ratio"] >= get_unit_filter_value(
"presence_ratio_minimum", **unit_filter_kwargs))
& (units["isi_violations"] <= get_unit_filter_value(
"isi_violations_maximum", **unit_filter_kwargs))]
if "quality" in units.columns and filter_by_validity:
units = units[units["quality"] == "good"]
units.drop(columns="quality", inplace=True)
if "ecephys_structure_id" in units.columns and unit_filter_kwargs.get(
"filter_out_of_brain_units", True):
units = units[~(units["ecephys_structure_id"].isna())]
return units
