def get_morphology_features(self, dataframe=False, file_name=None):
"""
Download morphology features for all cells with reconstructions in the database.
Parameters
----------
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
dataframe: boolean
Return the output as a Pandas DataFrame. If False, return
a list of dictionaries.
"""
file_name = self.get_cache_path(file_name,
self.MORPHOLOGY_FEATURES_KEY)
if self.cache:
if dataframe:
warnings.warn("dataframe argument is deprecated.")
args = Cache.cache_csv_dataframe()
else:
args = Cache.cache_csv_json()
else:
args = Cache.nocache_json()
args['strategy'] = 'lazy'
args['path'] = file_name
return self.api.get_morphology_features(**args)
def get_structure_tree(self, file_name=None, structure_graph_id=1):
"""
Read the list of adult mouse structures and return an StructureTree
instance.
Parameters
----------
file_name: string
File name to save/read the structures table. If file_name is None,
the file_name will be pulled out of the manifest. If caching
is disabled, no file will be saved. Default is None.
structure_graph_id: int
Build a tree using structure only from the identified structure graph.
"""
file_name = self.get_cache_path(file_name, self.STRUCTURE_TREE_KEY)
return OntologiesApi(self.api.api_url).get_structures_with_sets(
strategy='lazy',
path=file_name,
pre=StructureTree.clean_structures,
post=lambda x: StructureTree(StructureTree.clean_structures(x)),
structure_graph_ids=structure_graph_id,
**Cache.cache_json())
def test_file_kwarg(nrrd_read, safe_mkdir, mca, cache, file_exists):
with patch.object(mca, "retrieve_file_over_http") as mock_retrieve:
@cacheable(reader=nrrd_read,
pathfinder=Cache.pathfinder(file_name_position=3,
secondary_file_name_position=1,
path_keyword='save_file_path'))
def download_volumetric_data(data_path,
file_name,
voxel_resolution=None,
save_file_path=None,
release=None,
coordinate_framework=None):
url = mca.build_volumetric_data_download_url(
data_path, file_name, voxel_resolution, release,
coordinate_framework)
mca.retrieve_file_over_http(url, save_file_path)
with patch('os.path.exists',
Mock(name="os.path.exists",
return_value=file_exists)) as mkdir:
nrrd_read.reset_mock()
download_volumetric_data(MCA.AVERAGE_TEMPLATE,
'annotation_10.nrrd',
MCA.VOXEL_RESOLUTION_10_MICRONS,
'volumetric.nrrd',
MCA.CCF_2016,
strategy='file',
save_file_path='file.nrrd')
assert not mock_retrieve.called, 'server should not have been called'
assert not safe_mkdir.called, 'safe_mkdir should not have been called.'
nrrd_read.assert_called_once_with('file.nrrd')
def test_file_download_server(nrrd_read, safe_mkdir, mca, cache, file_exists):
with patch.object(mca, "retrieve_file_over_http") as mock_retrieve:
@cacheable(reader=nrrd_read,
pathfinder=Cache.pathfinder(file_name_position=3,
secondary_file_name_position=1))
def download_volumetric_data(data_path,
file_name,
voxel_resolution=None,
save_file_path=None,
release=None,
coordinate_framework=None):
url = mca.build_volumetric_data_download_url(
data_path, file_name, voxel_resolution, release,
coordinate_framework)
mca.retrieve_file_over_http(url, save_file_path)
with patch('os.path.exists',
Mock(name="os.path.exists",
return_value=file_exists)) as mkdir:
nrrd_read.reset_mock()
download_volumetric_data(MCA.AVERAGE_TEMPLATE,
'annotation_10.nrrd',
MCA.VOXEL_RESOLUTION_10_MICRONS,
'volumetric.nrrd',
MCA.CCF_2016,
strategy='create')
mock_retrieve.assert_called_once_with(
'http://download.alleninstitute.org/informatics-archive/annotation/ccf_2016/mouse_ccf/average_template/annotation_10.nrrd',
'volumetric.nrrd')
assert not safe_mkdir.called, 'safe_mkdir should not have been called.'
nrrd_read.assert_called_once_with('volumetric.nrrd')
def _get_stimulus_mappings(self, file_name=None):
""" Returns a mapping of which metrics are related to which stimuli. Internal use only. """
file_name = self.get_cache_path(file_name, self.STIMULUS_MAPPINGS_KEY)
mappings = self.api.get_stimulus_mappings(path=file_name,
strategy='lazy',
**Cache.cache_json())
return mappings
def get_affine_parameters(self,
section_data_set_id,
direction='trv',
file_name=None):
''' Extract the parameters of the 3D affine tranformation mapping this section data set's image-space stack to
CCF-space (or vice-versa).
Parameters
----------
section_data_set_id : int
download the parameters for this data set.
direction : str, optional
Valid options are:
trv : "transform from reference to volume". Maps CCF points to image space points. If you are
resampling data into CCF, this is the direction you want.
tvr : "transform from volume to reference". Maps image space points to CCF points.
file_name : str
If provided, store the downloaded file here.
Returns
-------
alignment : numpy.ndarray
4 X 3 matrix. In order to transform a point [X_1, X_2, X_3] run
np.dot([X_1, X_2, X_3, 1], alignment). In
to build a SimpleITK affine transform run:
transform = sitk.AffineTransform(3)
transform.SetParameters(alignment.flatten())
'''
if not direction in ('trv', 'tvr'):
raise ArgumentError(
'invalid direction: {}. direction must be one of tvr, trv'.
format(direction))
file_name = self.get_cache_path(file_name, self.ALIGNMENT3D_KEY)
raw_alignment = self.api.download_alignment3d(
strategy='lazy',
path=file_name,
section_data_set_id=section_data_set_id,
**Cache.cache_json())
alignment_re = re.compile('{}_(?P<index>\d+)'.format(direction))
alignment = np.zeros((4, 3), dtype=float)
for entry, value in raw_alignment.items():
match = alignment_re.match(entry)
if match is not None:
alignment.flat[int(match.group('index'))] = value
return alignment
def test_cacheable_no_cache_csv(read_csv, ju_read_url_get, ju_read, ju_write):
@cacheable()
def get_hemispheres():
return RmaApi().model_query(model='Hemisphere')
df = get_hemispheres(path='/xyz/abc/example.csv',
strategy='file',
**Cache.cache_csv())
assert df.loc[:, 'whatever'][0]
assert not ju_read_url_get.called
read_csv.assert_called_once_with('/xyz/abc/example.csv', parse_dates=True)
assert not ju_write.called, 'json write should not have been called'
assert not ju_read.called, 'json read should not have been called'
def get_cells(self,
file_name=None,
require_morphology=False,
require_reconstruction=False,
reporter_status=None,
species=None,
simple=True):
"""
Download metadata for all cells in the database and optionally return a
subset filtered by whether or not they have a morphology or reconstruction.
Parameters
----------
file_name: string
File name to save/read the cell metadata as JSON. If file_name is None,
the file_name will be pulled out of the manifest. If caching
is disabled, no file will be saved. Default is None.
require_morphology: boolean
Filter out cells that have no morphological images.
require_reconstruction: boolean
Filter out cells that have no morphological reconstructions.
reporter_status: list
Filter for cells that have one or more cell reporter statuses.
species: list
Filter for cells that belong to one or more species. If None, return all.
Must be one of [ CellTypesApi.MOUSE, CellTypesApi.HUMAN ].
"""
file_name = self.get_cache_path(file_name, self.CELLS_KEY)
cells = self.api.list_cells_api(path=file_name,
strategy='lazy',
**Cache.cache_json())
if isinstance(reporter_status, string_types):
reporter_status = [reporter_status]
# filter the cells on the way out
cells = self.api.filter_cells_api(cells, require_morphology,
require_reconstruction,
reporter_status, species, simple)
return cells
def test_cacheable_no_save_dataframe(to_csv, read_csv, ju_read_url_get,
ju_read, ju_write):
@cacheable()
def get_hemispheres():
return RmaApi().model_query(model='Hemisphere')
df = get_hemispheres(**Cache.nocache_dataframe())
assert df.loc[:, 'whatever'][0]
ju_read_url_get.assert_called_once_with(
'http://api.brain-map.org/api/v2/data/query.json?q=model::Hemisphere')
assert not to_csv.called, 'to_csv should not have been called'
assert not read_csv.called, 'read_csv should not have been called'
assert not ju_write.called, 'json write should not have been called'
assert not ju_read.called, 'json read should not have been called'
def test_cacheable_json(read_csv, mkdir, ju_read_url_get, ju_read, ju_write):
@cacheable()
def get_hemispheres():
return RmaApi().model_query(model='Hemisphere')
df = get_hemispheres(path='/xyz/abc/example.json',
strategy='create',
**Cache.cache_json())
assert 'whatever' in df[0]
ju_read_url_get.assert_called_once_with(
'http://api.brain-map.org/api/v2/data/query.json?q=model::Hemisphere')
assert not read_csv.called, 'read_csv should not have been called'
ju_write.assert_called_once_with('/xyz/abc/example.json', _msg)
ju_read.assert_called_once_with('/xyz/abc/example.json')
def test_cacheable_lazy_csv_file_exists(read_csv, ju_read_url_get, ju_read,
ju_write):
@cacheable()
def get_hemispheres():
return RmaApi().model_query(model='Hemisphere')
with patch('os.path.exists', MagicMock(return_value=True)) as ope:
df = get_hemispheres(path='/xyz/abc/example.csv',
strategy='lazy',
**Cache.cache_csv())
assert df.loc[:, 'whatever'][0]
assert not ju_read_url_get.called
read_csv.assert_called_once_with('/xyz/abc/example.csv', parse_dates=True)
assert not ju_write.called, 'json write should not have been called'
assert not ju_read.called, 'json read should not have been called'
def test_excpt(mkdir, ju_read_url_get, ju_read, ju_write):
@cacheable()
def get_hemispheres_excpt():
return RmaApi().model_query(model='Hemisphere', excpt=['symbol'])
df = get_hemispheres_excpt(path='/xyz/abc/example.json',
strategy='create',
**Cache.cache_json())
assert 'whatever' in df[0]
ju_read_url_get.assert_called_once_with(
'http://api.brain-map.org/api/v2/data/query.json?q=model::Hemisphere,rma::options%5Bexcept$eqsymbol%5D'
)
ju_write.assert_called_once_with('/xyz/abc/example.json', _msg)
ju_read.assert_called_once_with('/xyz/abc/example.json')
mkdir.assert_called_once_with('/xyz/abc')
def test_cacheable_json_dataframe(mkdir, ju_read_url_get, ju_read, ju_write,
read_csv, mock_read_json):
@cacheable()
def get_hemispheres():
return RmaApi().model_query(model='Hemisphere')
df = get_hemispheres(path='/xyz/abc/example.json',
strategy='create',
**Cache.cache_json_dataframe())
assert df.loc[:, 'whatever'][0]
ju_read_url_get.assert_called_once_with(
'http://api.brain-map.org/api/v2/data/query.json?q=model::Hemisphere')
assert not read_csv.called, 'read_csv should not have been called'
mock_read_json.assert_called_once_with('/xyz/abc/example.json',
orient='records')
ju_write.assert_called_once_with('/xyz/abc/example.json', _msg)
assert not ju_read.called, 'json read should not have been called'
mkdir.assert_called_once_with('/xyz/abc')
def get_ephys_sweeps(self, specimen_id, file_name=None):
"""
Download sweep metadata for a single cell specimen.
Parameters
----------
specimen_id: int
ID of a cell.
"""
file_name = self.get_cache_path(file_name, self.EPHYS_SWEEPS_KEY,
specimen_id)
sweeps = self.api.get_ephys_sweeps(specimen_id,
strategy='lazy',
path=file_name,
**Cache.cache_json())
return sweeps
def test_cacheable_lazy_csv_no_file(mkdir, dictwriter, ju_read_url_get,
ju_read, ju_write, read_csv):
@cacheable()
def get_hemispheres():
return RmaApi().model_query(model='Hemisphere')
with patch('os.path.exists', MagicMock(return_value=False)) as ope:
with patch(builtins.__name__ + '.open', mock_open(),
create=True) as open_mock:
open_mock.return_value.write = MagicMock()
df = get_hemispheres(path='/xyz/abc/example.csv',
strategy='lazy',
**Cache.cache_csv())
assert df.loc[:, 'whatever'][0]
ju_read_url_get.assert_called_once_with(
'http://api.brain-map.org/api/v2/data/query.json?q=model::Hemisphere')
open_mock.assert_called_once_with('/xyz/abc/example.csv', 'w')
dictwriter.return_value.writerow.assert_called()
read_csv.assert_called_once_with('/xyz/abc/example.csv', parse_dates=True)
assert not ju_write.called, 'json write should not have been called'
assert not ju_read.called, 'json read should not have been called'
def test_cacheable_csv_dataframe(read_csv, dictwriter, ju_read_url_get,
ju_read, ju_write):
@cacheable()
def get_hemispheres():
return RmaApi().model_query(model='Hemisphere')
with patch('allensdk.config.manifest.Manifest.safe_mkdir') as mkdir:
with patch(builtins.__name__ + '.open', mock_open(),
create=True) as open_mock:
open_mock.return_value.write = MagicMock()
df = get_hemispheres(path='/xyz/abc/example.txt',
strategy='create',
**Cache.cache_csv_dataframe())
assert df.loc[:, 'whatever'][0]
ju_read_url_get.assert_called_once_with(
'http://api.brain-map.org/api/v2/data/query.json?q=model::Hemisphere')
read_csv.assert_called_once_with('/xyz/abc/example.txt', parse_dates=True)
assert not ju_write.called, 'write should not have been called'
assert not ju_read.called, 'read should not have been called'
mkdir.assert_called_once_with('/xyz/abc')
open_mock.assert_called_once_with('/xyz/abc/example.txt', 'w')
def test_cacheable_csv_json(mkdir, dictwriter, ju_read_url_get, ju_read,
ju_write, read_csv, mock_read_json):
@cacheable()
def get_hemispheres():
return RmaApi().model_query(model='Hemisphere')
with patch(builtins.__name__ + '.open', mock_open(),
create=True) as open_mock:
open_mock.return_value.write = MagicMock()
df = get_hemispheres(path='/xyz/example.csv',
strategy='create',
**Cache.cache_csv_json())
assert 'whatever' in df[0]
ju_read_url_get.assert_called_once_with(
'http://api.brain-map.org/api/v2/data/query.json?q=model::Hemisphere')
read_csv.assert_called_once_with('/xyz/example.csv', parse_dates=True)
dictwriter.return_value.writerow.assert_called()
assert not mock_read_json.called, 'pj.read_json should not have been called'
assert not ju_write.called, 'ju.write should not have been called'
assert not ju_read.called, 'json read should not have been called'
mkdir.assert_called_once_with('/xyz')
open_mock.assert_called_once_with('/xyz/example.csv', 'w')
summary_structures[['id', 'parent_structure_id',
'acronym']].to_csv('summary_structures.csv',
index_label='structure_id')
reread = pd.read_csv('summary_structures.csv')
#===============================================================================
# example 10
#===============================================================================
for id, name, parent_structure_id in summary_structures[[
'name', 'parent_structure_id'
]].itertuples():
print("%d %s %d" % (id, name, parent_structure_id))
#===============================================================================
# example 11
#===============================================================================
from allensdk.api.warehouse_cache.cache import Cache
cache_writer = Cache()
do_cache = True
structures_from_api = \
cache_writer.wrap(rma.model_query,
path='summary.csv',
cache=do_cache,
model='Structure',
criteria='[graph_id$eq1]',
num_rows='all')
class ReferenceSpaceApi(RmaApi):
AVERAGE_TEMPLATE = 'average_template'
ARA_NISSL = 'ara_nissl'
MOUSE_2011 = 'annotation/mouse_2011'
DEVMOUSE_2012 = 'annotation/devmouse_2012'
CCF_2015 = 'annotation/ccf_2015'
CCF_2016 = 'annotation/ccf_2016'
CCF_2017 = 'annotation/ccf_2017'
CCF_VERSION_DEFAULT = CCF_2017
VOXEL_RESOLUTION_10_MICRONS = 10
VOXEL_RESOLUTION_25_MICRONS = 25
VOXEL_RESOLUTION_50_MICRONS = 50
VOXEL_RESOLUTION_100_MICRONS = 100
def __init__(self, base_uri=None):
super(ReferenceSpaceApi, self).__init__(base_uri=base_uri)
@cacheable(strategy='create',
reader=nrrd.read,
pathfinder=Cache.pathfinder(file_name_position=3,
path_keyword='file_name'))
def download_annotation_volume(self, ccf_version, resolution, file_name):
'''
Download the annotation volume at a particular resolution.
Parameters
----------
ccf_version: string
Which reference space version to download. Defaults to "annotation/ccf_2017"
resolution: int
Desired resolution to download in microns.
Must be 10, 25, 50, or 100.
file_name: string
Where to save the annotation volume.
Note: the parameters must be used as positional parameters, not keywords
'''
if ccf_version is None:
ccf_version = ReferenceSpaceApi.CCF_VERSION_DEFAULT
self.download_volumetric_data(ccf_version,
'annotation_%d.nrrd' % resolution,
save_file_path=file_name)
@cacheable(strategy='create',
reader=sitk_utilities.read_ndarray_with_sitk,
pathfinder=Cache.pathfinder(file_name_position=3,
path_keyword='file_name'))
def download_mouse_atlas_volume(self, age, volume_type, file_name):
'''Download a reference volume (annotation, grid annotation, atlas volume)
from the mouse brain atlas project
Parameters
----------
age : str
Specify a mouse age for which to download the reference volume
volume_type : str
Specify the type of volume to download
file_name : str
Specify the path to the downloaded volume
'''
remote_file_name = '{}_{}.zip'.format(age, volume_type)
url = '/'.join([
self.informatics_archive_endpoint, 'current-release',
'mouse_annotation', remote_file_name
])
self.retrieve_file_over_http(url, file_name, zipped=True)
@cacheable(strategy='create',
reader=nrrd.read,
pathfinder=Cache.pathfinder(file_name_position=2,
path_keyword='file_name'))
def download_template_volume(self, resolution, file_name):
'''
Download the registration template volume at a particular resolution.
Parameters
----------
resolution: int
Desired resolution to download in microns. Must be 10, 25, 50, or 100.
file_name: string
Where to save the registration template volume.
'''
self.download_volumetric_data(ReferenceSpaceApi.AVERAGE_TEMPLATE,
'average_template_%d.nrrd' % resolution,
save_file_path=file_name)
@cacheable(strategy='create',
reader=nrrd.read,
pathfinder=Cache.pathfinder(file_name_position=4,
path_keyword='file_name'))
def download_structure_mask(self, structure_id, ccf_version, resolution,
file_name):
'''Download an indicator mask for a specific structure.
Parameters
----------
structure_id : int
Unique identifier for the annotated structure
ccf_version : string
Which reference space version to download. Defaults to "annotation/ccf_2017"
resolution : int
Desired resolution to download in microns. Must be 10, 25, 50, or 100.
file_name : string
Where to save the downloaded mask.
'''
if ccf_version is None:
ccf_version = ReferenceSpaceApi.CCF_VERSION_DEFAULT
structure_mask_dir = 'structure_masks_{0}'.format(resolution)
data_path = '{0}/{1}/{2}'.format(ccf_version, 'structure_masks',
structure_mask_dir)
remote_file_name = 'structure_{0}.nrrd'.format(structure_id)
try:
self.download_volumetric_data(data_path,
remote_file_name,
save_file_path=file_name)
except Exception as e:
self._file_download_log.error(
'''We weren't able to download a structure mask for structure {0}.
You can instead build the mask locally using
ReferenceSpace.many_structure_masks'''
)
raise
@cacheable(strategy='create',
reader=read_obj,
pathfinder=Cache.pathfinder(file_name_position=3,
path_keyword='file_name'))
def download_structure_mesh(self, structure_id, ccf_version, file_name):
'''Download a Wavefront obj file containing a triangulated 3d mesh built
from an annotated structure.
Parameters
----------
structure_id : int
Unique identifier for the annotated structure
ccf_version : string
Which reference space version to download. Defaults to "annotation/ccf_2017"
file_name : string
Where to save the downloaded mask.
'''
if ccf_version is None:
ccf_version = ReferenceSpaceApi.CCF_VERSION_DEFAULT
data_path = '{0}/{1}'.format(ccf_version, 'structure_meshes')
remote_file_name = '{0}.obj'.format(structure_id)
try:
self.download_volumetric_data(data_path,
remote_file_name,
save_file_path=file_name)
except Exception as e:
self._file_download_log.error(
'unable to download a structure mesh for structure {0}.'.
format(structure_id))
raise
def build_volumetric_data_download_url(self,
data_path,
file_name,
voxel_resolution=None,
release=None,
coordinate_framework=None):
'''Construct url to download 3D reference model in NRRD format.
Parameters
----------
data_path : string
'average_template', 'ara_nissl', 'annotation/ccf_{year}',
'annotation/mouse_2011', or 'annotation/devmouse_2012'
voxel_resolution : int
10, 25, 50 or 100
coordinate_framework : string
'mouse_ccf' (default) or 'mouse_annotation'
Notes
-----
See: `3-D Reference Models <http://help.brain-map.org/display/mouseconnectivity/API#API-3DReferenceModels>`_
for additional documentation.
'''
if voxel_resolution is None:
voxel_resolution = ReferenceSpaceApi.VOXEL_RESOLUTION_10_MICRONS
if release is None:
release = 'current-release'
if coordinate_framework is None:
coordinate_framework = 'mouse_ccf'
url = ''.join([
self.informatics_archive_endpoint,
'/%s/%s/' % (release, coordinate_framework), data_path, '/',
file_name
])
return url
def download_volumetric_data(self,
data_path,
file_name,
voxel_resolution=None,
save_file_path=None,
release=None,
coordinate_framework=None):
'''Download 3D reference model in NRRD format.
Parameters
----------
data_path : string
'average_template', 'ara_nissl', 'annotation/ccf_{year}',
'annotation/mouse_2011', or 'annotation/devmouse_2012'
file_name : string
server-side file name. 'annotation_10.nrrd' for example.
voxel_resolution : int
10, 25, 50 or 100
coordinate_framework : string
'mouse_ccf' (default) or 'mouse_annotation'
Notes
-----
See: `3-D Reference Models <http://help.brain-map.org/display/mouseconnectivity/API#API-3DReferenceModels>`_
for additional documentation.
'''
url = self.build_volumetric_data_download_url(data_path, file_name,
voxel_resolution,
release,
coordinate_framework)
if save_file_path is None:
save_file_path = file_name
if save_file_path is None:
save_file_path = 'volumetric_data.nrrd'
self.retrieve_file_over_http(url, save_file_path)
class MouseAtlasApi(ReferenceSpaceApi, GridDataApi):
''' Downloads Mouse Brain Atlas grid data, reference volumes, and metadata.
'''
MOUSE_ATLAS_PRODUCTS = (1, )
DEVMOUSE_ATLAS_PRODUCTS = (3, )
MOUSE_ORGANISM = (2, )
HUMAN_ORGANISM = (1, )
@cacheable()
@pageable(num_rows=2000, total_rows='all')
def get_section_data_sets(self, gene_ids=None, product_ids=None, **kwargs):
''' Download a list of section data sets (experiments) from the Mouse Brain
Atlas project.
Parameters
----------
gene_ids : list of int, optional
Filter results based on the genes whose expression was characterized
in each experiment. Default is all.
product_ids : list of int, optional
Filter results to a subset of products. Default is the Mouse Brain Atlas.
Returns
-------
list of dict :
Each element is a section data set record, with one or more gene
records nested in a list.
'''
if product_ids is None:
product_ids = list(self.MOUSE_ATLAS_PRODUCTS)
criteria = 'products[id$in{}]'.format(','.join(map(str, product_ids)))
if gene_ids is not None:
criteria += ',genes[id$in{}]'.format(','.join(map(str, gene_ids)))
order = kwargs.pop('order', ['\'id\''])
return self.model_query(model='SectionDataSet',
criteria=criteria,
include='genes',
order=order,
**kwargs)
@cacheable()
@pageable(num_rows=2000, total_rows='all')
def get_genes(self, organism_ids=None, chromosome_ids=None, **kwargs):
''' Download a list of genes
Parameters
----------
organism_ids : list of int, optional
Filter genes to those appearing in these organisms. Defaults to mouse (2).
chromosome_ids : list of int, optional
Filter genes to those appearing on these chromosomes. Defaults to all.
Returns
-------
list of dict:
Each element is a gene record, with a nested chromosome record (also a dict).
'''
if organism_ids is None:
organism_ids = list(self.MOUSE_ORGANISM)
criteria = '[organism_id$in{}]'.format(','.join(map(str,
organism_ids)))
if chromosome_ids is not None:
criteria += ',[chromosome_id$in{}]'.format(','.join(
map(str, chromosome_ids)))
order = kwargs.pop('order', ['\'id\''])
return self.model_query(model='Gene',
criteria=criteria,
include='chromosome',
order=order,
**kwargs)
@cacheable(strategy='create',
reader=sitk_utilities.read_ndarray_with_sitk,
pathfinder=Cache.pathfinder(file_name_position=1,
path_keyword='path'))
def download_expression_density(self, path, experiment_id):
self.download_gene_expression_grid_data(experiment_id,
GridDataApi.DENSITY, path)
@cacheable(strategy='create',
reader=sitk_utilities.read_ndarray_with_sitk,
pathfinder=Cache.pathfinder(file_name_position=1,
path_keyword='path'))
def download_expression_energy(self, path, experiment_id):
self.download_gene_expression_grid_data(experiment_id,
GridDataApi.ENERGY, path)
@cacheable(strategy='create',
reader=sitk_utilities.read_ndarray_with_sitk,
pathfinder=Cache.pathfinder(file_name_position=1,
path_keyword='path'))
def download_expression_intensity(self, path, experiment_id):
self.download_gene_expression_grid_data(experiment_id,
GridDataApi.INTENSITY, path)
class BrainObservatoryApi(RmaTemplate):
_log = logging.getLogger('allensdk.api.queries.brain_observatory_api')
NWB_FILE_TYPE = 'NWBOphys'
OPHYS_ANALYSIS_FILE_TYPE = 'OphysExperimentCellRoiMetricsFile'
OPHYS_EVENTS_FILE_TYPE = 'ObservatoryEventsFile'
CELL_MAPPING_ID = 590985414
rma_templates = \
{"brain_observatory_queries": [
{'name': 'list_isi_experiments',
'description': 'see name',
'model': 'IsiExperiment',
'num_rows': 'all',
'count': False,
'criteria_params': []
},
{'name': 'isi_experiment_by_ids',
'description': 'see name',
'model': 'IsiExperiment',
'criteria': '[id$in{{ isi_experiment_ids }}]',
'include': 'experiment_container(ophys_experiments,targeted_structure)',
'num_rows': 'all',
'count': False,
'criteria_params': ['isi_experiment_ids']
},
{'name': 'ophys_experiment_by_ids',
'description': 'see name',
'model': 'OphysExperiment',
'criteria': '{% if ophys_experiment_ids is defined %}[id$in{{ ophys_experiment_ids }}]{%endif%}',
'include': 'experiment_container,well_known_files(well_known_file_type),targeted_structure,specimen(donor(age,transgenic_lines))',
'num_rows': 'all',
'count': False,
'criteria_params': ['ophys_experiment_ids']
},
{'name': 'ophys_experiment_data',
'description': 'see name',
'model': 'WellKnownFile',
'criteria': '[attachable_id$eq{{ ophys_experiment_id }}],well_known_file_type[name$eq%s]' % NWB_FILE_TYPE,
'num_rows': 'all',
'count': False,
'criteria_params': ['ophys_experiment_id']
},
{'name': 'ophys_analysis_file',
'description': 'see name',
'model': 'WellKnownFile',
'criteria': '[attachable_id$eq{{ ophys_experiment_id }}],well_known_file_type[name$eq%s]' % OPHYS_ANALYSIS_FILE_TYPE,
'num_rows': 'all',
'count': False,
'criteria_params': ['ophys_experiment_id']
},
{'name': 'ophys_events_file',
'description': 'see name',
'model': 'WellKnownFile',
'criteria': '[attachable_id$eq{{ ophys_experiment_id }}],well_known_file_type[name$eq%s]' % OPHYS_EVENTS_FILE_TYPE,
'num_rows': 'all',
'count': False,
'criteria_params': ['ophys_experiment_id']
},
{'name': 'column_definitions',
'description': 'see name',
'model': 'ApiColumnDefinition',
'criteria': '[api_class_name$eq{{ api_class_name }}]',
'num_rows': 'all',
'count': False,
'criteria_params': ['api_class_name']
},
{'name': 'column_definition_class_names',
'description': 'see name',
'model': 'ApiColumnDefinition',
'only': ['api_class_name'],
'num_rows': 'all',
'count': False,
},
{'name': 'stimulus_mapping',
'description': 'see name',
'model': 'ApiCamStimulusMapping',
'criteria': '{% if stimulus_mapping_ids is defined %}[id$in{{ stimulus_mapping_ids }}]{%endif%}',
'num_rows': 'all',
'count': False,
'criteria_params': ['stimulus_mapping_ids']
},
{'name': 'experiment_container',
'description': 'see name',
'model': 'ExperimentContainer',
'criteria': '{% if experiment_container_ids is defined %}[id$in{{ experiment_container_ids }}]{%endif%}',
'include': 'ophys_experiments,isi_experiment,specimen(donor(conditions,age,transgenic_lines)),targeted_structure',
'num_rows': 'all',
'count': False,
'criteria_params': ['experiment_container_ids']
},
{'name': 'experiment_container_metric',
'description': 'see name',
'model': 'ApiCamExperimentContainerMetric',
'criteria': '{% if experiment_container_metric_ids is defined %}[id$in{{ experiment_container_metric_ids }}]{%endif%}',
'num_rows': 'all',
'count': False,
'criteria_params': ['experiment_container_metric_ids']
},
{'name': 'cell_metric',
'description': 'see name',
'model': 'ApiCamCellMetric',
'criteria': '{% if cell_specimen_ids is defined %}[cell_specimen_id$in{{ cell_specimen_ids }}]{%endif%}',
'criteria_params': ['cell_specimen_ids']
},
{'name': 'cell_specimen_id_mapping_table',
'description': 'see name',
'model': 'WellKnownFile',
'criteria': '[id$eq{{ mapping_table_id }}],well_known_file_type[name$eqOphysCellSpecimenIdMapping]',
'num_rows': 'all',
'count': False,
'criteria_params': ['mapping_table_id']
},
{'name': 'eye_gaze_mapping_file',
'description': 'h5 file containing mouse eye gaze mapped onto screen coordinates (as well as pupil and eye sizes)',
'model': 'WellKnownFile',
'criteria': '[attachable_id$eq{{ ophys_session_id }}],well_known_file_type[name$eqEyeDlcScreenMapping]',
'num_rows': 'all',
'count': False,
'criteria_params': ['ophys_session_id']
},
# NOTE: 'all_eye_mapping_files' query is for facilitating an ugly
# hack to get around lack of relationship between experiment id
# and session id in current warehouse. This should be removed when
# the relationship is added.
{'name': 'all_eye_mapping_files',
'description': 'Get a list of dictionaries for all eye mapping wkfs',
'model': 'WellKnownFile',
'criteria': 'well_known_file_type[name$eqEyeDlcScreenMapping]',
'num_rows': 'all',
'count': False
}
]}
_QUERY_TEMPLATES = {
"=": '({0} == {1})',
"<": '({0} < {1})',
">": '({0} > {1})',
"<=": '({0} <= {1})',
">=": '({0} >= {1})',
"between": '({0} >= {1}) and ({0} <= {2})',
"in": '({0} == {1})',
"is": '({0} == {1})'
}
def __init__(self, base_uri=None, datacube_uri=None):
super(BrainObservatoryApi, self).__init__(base_uri,
query_manifest=BrainObservatoryApi.rma_templates)
self.datacube_uri = datacube_uri
@cacheable()
def get_ophys_experiments(self, ophys_experiment_ids=None):
''' Get OPhys Experiments by id
Parameters
----------
ophys_experiment_ids : integer or list of integers, optional
only select specific experiments.
Returns
-------
dict : ophys experiment metadata
'''
data = self.template_query('brain_observatory_queries',
'ophys_experiment_by_ids',
ophys_experiment_ids=ophys_experiment_ids)
return data
def get_isi_experiments(self, isi_experiment_ids=None):
''' Get ISI Experiments by id
Parameters
----------
isi_experiment_ids : integer or list of integers, optional
only select specific experiments.
Returns
-------
dict : isi experiment metadata
'''
data = self.template_query('brain_observatory_queries',
'isi_experiment_by_ids',
isi_experiment_ids=isi_experiment_ids)
return data
def list_isi_experiments(self, isi_ids=None):
'''List ISI experiments available through the Allen Institute API
Parameters
----------
neuronal_model_ids : integer or list of integers, optional
only select specific isi experiments.
Returns
-------
dict : neuronal model metadata
'''
data = self.template_query('brain_observatory_queries',
'list_isi_experiments')
return data
def list_column_definition_class_names(self):
''' Get column definitions
Parameters
----------
Returns
-------
list : api class name strings
'''
data = self.template_query('brain_observatory_queries',
'column_definition_class_names')
names = list(set([n['api_class_name'] for n in data]))
return names
def get_column_definitions(self, api_class_name=None):
''' Get column definitions
Parameters
----------
api_class_names : string or list of strings, optional
only select specific column definition records.
Returns
-------
dict : column definition metadata
'''
data = self.template_query('brain_observatory_queries',
'column_definitions',
api_class_name=api_class_name)
return data
@cacheable()
def get_stimulus_mappings(self, stimulus_mapping_ids=None):
''' Get stimulus mappings by id
Parameters
----------
stimulus_mapping_ids : integer or list of integers, optional
only select specific stimulus mapping records.
Returns
-------
dict : stimulus mapping metadata
'''
data = self.template_query('brain_observatory_queries',
'stimulus_mapping',
stimulus_mapping_ids=stimulus_mapping_ids)
return data
@cacheable()
@pageable(num_rows=2000, total_rows='all')
def get_cell_metrics(self, cell_specimen_ids=None, *args, **kwargs):
''' Get cell metrics by id
Parameters
----------
cell_metrics_ids : integer or list of integers, optional
only select specific cell metric records.
Returns
-------
dict : cell metric metadata
'''
order = kwargs.pop('order', ['\'cell_specimen_id\''])
data = self.template_query('brain_observatory_queries',
'cell_metric',
cell_specimen_ids=cell_specimen_ids,
order=order,
*args,
**kwargs)
return data
@cacheable()
def get_experiment_containers(self, experiment_container_ids=None):
''' Get experiment container by id
Parameters
----------
experiment_container_ids : integer or list of integers, optional
only select specific experiment containers.
Returns
-------
dict : experiment container metadata
'''
data = self.template_query('brain_observatory_queries',
'experiment_container',
experiment_container_ids=experiment_container_ids)
return data
def get_experiment_container_metrics(self, experiment_container_metric_ids=None):
''' Get experiment container metrics by id
Parameters
----------
isi_experiment_ids : integer or list of integers, optional
only select specific experiments.
Returns
-------
dict : isi experiment metadata
'''
data = self.template_query('brain_observatory_queries',
'experiment_container_metric',
experiment_container_metric_ids=experiment_container_metric_ids)
return data
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=2,
path_keyword='file_name'))
def save_ophys_experiment_data(self, ophys_experiment_id, file_name):
data = self.template_query('brain_observatory_queries',
'ophys_experiment_data',
ophys_experiment_id=ophys_experiment_id)
try:
file_url = data[0]['download_link']
except Exception as _:
raise Exception("ophys experiment %d has no data file" %
ophys_experiment_id)
self._log.warning(
"Downloading ophys_experiment %d NWB. This can take some time." % ophys_experiment_id)
self.retrieve_file_over_http(self.api_url + file_url, file_name)
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=2,
path_keyword='file_name'))
def save_ophys_experiment_analysis_data(self, ophys_experiment_id, file_name):
data = self.template_query('brain_observatory_queries',
'ophys_analysis_file',
ophys_experiment_id=ophys_experiment_id)
try:
file_url = data[0]['download_link']
except Exception as _:
raise Exception("ophys experiment %d has no %s analysis file" %
(ophys_experiment_id, ))
self._log.warning(
"Downloading ophys_experiment %d analysis file. This can take some time." % (ophys_experiment_id, ))
self.retrieve_file_over_http(self.api_url + file_url, file_name)
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=2,
path_keyword='file_name'))
def save_ophys_experiment_event_data(self, ophys_experiment_id, file_name):
data = self.template_query('brain_observatory_queries',
'ophys_events_file',
ophys_experiment_id=ophys_experiment_id)
try:
file_url = data[0]['download_link']
except Exception:
raise Exception("ophys experiment %d has no events file" %
ophys_experiment_id)
self._log.warning(
"Downloading ophys_experiment %d events file. This can take some time." % ophys_experiment_id)
self.retrieve_file_over_http(self.api_url + file_url, file_name)
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=3,
path_keyword='file_name'))
def save_ophys_experiment_eye_gaze_data(self,
ophys_experiment_id: int,
ophys_session_id: int,
file_name: str):
data = self.template_query('brain_observatory_queries',
'eye_gaze_mapping_file',
ophys_session_id=ophys_session_id)
experiment_session_string = f"ophys_experiment '{ophys_experiment_id}' (session '{ophys_session_id}')"
try:
file_url = data[0]['download_link']
except Exception:
raise Exception(f"{experiment_session_string} has no eye gaze mapping file")
self._log.warning(
f"Downloading {experiment_session_string} gaze mapping file. This can take some time."
)
self.retrieve_file_over_http(self.api_url + file_url, file_name)
def filter_experiments_and_containers(self, objs,
ids=None,
targeted_structures=None,
imaging_depths=None,
cre_lines=None,
reporter_lines=None,
transgenic_lines=None,
include_failed=False):
if not include_failed:
objs = [o for o in objs if not o.get('failed', False)]
if ids is not None:
objs = [o for o in objs if o['id'] in ids]
if targeted_structures is not None:
objs = [o for o in objs if o[
'targeted_structure']['acronym'] in targeted_structures]
if imaging_depths is not None:
objs = [o for o in objs if o[
'imaging_depth'] in imaging_depths]
if cre_lines is not None:
tls = [ tl.lower() for tl in cre_lines ]
obj_tls = [ find_specimen_cre_line(o['specimen']) for o in objs ]
obj_tls = [ o.lower() if o else None for o in obj_tls ]
objs = [o for i,o in enumerate(objs) if obj_tls[i] in tls]
if reporter_lines is not None:
tls = [ tl.lower() for tl in reporter_lines ]
obj_tls = [ find_specimen_reporter_line(o['specimen']) for o in objs ]
obj_tls = [ o.lower() if o else None for o in obj_tls ]
objs = [o for i,o in enumerate(objs) if obj_tls[i] in tls]
if transgenic_lines is not None:
tls = set([ tl.lower() for tl in transgenic_lines ])
objs = [ o for o in objs
if len(tls & set([ tl.lower()
for tl in find_specimen_transgenic_lines(o['specimen']) ]) ) ]
return objs
def filter_experiment_containers(self, containers,
ids=None,
targeted_structures=None,
imaging_depths=None,
cre_lines=None,
reporter_lines=None,
transgenic_lines=None,
include_failed=False,
simple=False):
containers = self.filter_experiments_and_containers(containers,
ids=ids,
targeted_structures=targeted_structures,
imaging_depths=imaging_depths,
cre_lines=cre_lines,
reporter_lines=reporter_lines,
transgenic_lines=transgenic_lines,
include_failed=include_failed)
if simple:
containers = self.simplify_experiment_containers(containers)
return containers
def filter_ophys_experiments(self, experiments,
ids=None,
experiment_container_ids=None,
targeted_structures=None,
imaging_depths=None,
cre_lines=None,
reporter_lines=None,
transgenic_lines=None,
stimuli=None,
session_types=None,
include_failed=False,
require_eye_tracking=False,
simple=False):
experiments = self.filter_experiments_and_containers(experiments,
ids=ids,
targeted_structures=targeted_structures,
imaging_depths=imaging_depths,
cre_lines=cre_lines,
reporter_lines=reporter_lines,
transgenic_lines=transgenic_lines)
if require_eye_tracking:
experiments = [e for e in experiments
if e.get('fail_eye_tracking', None) is False]
if not include_failed:
experiments = [e for e in experiments
if not e.get('experiment_container',{}).get('failed', False)]
if experiment_container_ids is not None:
experiments = [e for e in experiments if e[
'experiment_container_id'] in experiment_container_ids]
if session_types is not None:
experiments = [e for e in experiments if e[
'stimulus_name'] in session_types]
if stimuli is not None:
experiments = [e for e in experiments
if len(set(stimuli) & set(stimulus_info.stimuli_in_session(e['stimulus_name']))) > 0]
if simple:
experiments = self.simplify_ophys_experiments(experiments)
return experiments
def filter_cell_specimens(self, cell_specimens,
ids=None,
experiment_container_ids=None,
include_failed=False,
filters=None):
"""
Filter a list of cell specimen records returned from the get_cell_metrics method according
some of their properties.
Parameters
----------
cell_specimens: list of dicts
List of records returned by the get_cell_metrics method.
ids: list of integers
Return only records for cells with cell specimen ids in this list
experiment_container_ids: list of integers
Return only records for cells that belong to experiment container ids in this list
include_failed: bool
Whether to include cells from failed experiment containers
filters: list of dicts
Custom query used to reproduce filter sets created in the Allen Brain Observatory
web application. The general form is a list of dictionaries each of which
describes a filtering operation based on a metric. For more information, see
dataframe_query.
"""
if not include_failed:
cell_specimens = [c for c in cell_specimens if not c.get(
'failed_experiment_container', False)]
if ids is not None:
cell_specimens = [c for c in cell_specimens if c[
'cell_specimen_id'] in ids]
if experiment_container_ids is not None:
cell_specimens = [c for c in cell_specimens if c[
'experiment_container_id'] in experiment_container_ids]
if filters is not None:
cell_specimens = self.dataframe_query(cell_specimens,
filters,
'cell_specimen_id')
return cell_specimens
def dataframe_query_string(self,
filters):
"""
Convert a list of cell metric filter dictionaries into a
Pandas query string.
"""
def _quote_string(v):
if isinstance(v, string_types):
return "'%s'" % (v)
else:
return str(v)
def _filter_clause(op, field, value):
if op == 'in':
query_args = [field, str(value)]
elif type(value) is list:
query_args = [field] + list(map(_quote_string, value))
else:
query_args = [field, str(value)]
cluster_string = self._QUERY_TEMPLATES[op].\
format(*query_args)
return cluster_string
query_string = ' & '.join(_filter_clause(f['op'],
f['field'],
f['value']) for f in filters)
return query_string
def dataframe_query(self,
data,
filters,
primary_key):
"""
Given a list of dictionary records and a list of filter dictionaries,
filter the records using Pandas and return the filtered set of records.
Parameters
----------
data: list of dicts
List of dictionaries
filters: list of dicts
Each dictionary describes a filtering operation on a field in the dictionary.
The general form is { 'field': <field>, 'op': <operation>, 'value': <filter_value(s)> }.
For example, you can apply a threshold on the "osi_dg" column with something like this:
{ 'field': 'osi_dg', 'op': '>', 'value': 1.0 }. See _QUERY_TEMPLATES for a full list
of operators.
"""
if len(filters) == 0:
return data
queries = self.dataframe_query_string(filters)
result_dataframe = pd.DataFrame(data)
result_dataframe = result_dataframe.query(queries)
result_keys = set(result_dataframe[primary_key])
result = [d for d in data
if d[primary_key]
in result_keys]
return result
def get_cell_specimen_id_mapping(self, file_name, mapping_table_id=None):
'''Download mapping table from old to new cell specimen IDs.
The mapping table is a CSV file that maps cell specimen ids
that have changed between processing runs of the Brain
Observatory pipeline.
Parameters
----------
file_name : string
Filename to save locally.
mapping_table_id : integer
ID of the mapping table file. Defaults to the most recent
mapping table.
Returns
-------
pandas.DataFrame
Mapping table as a DataFrame.
'''
if mapping_table_id is None:
mapping_table_id = self.CELL_MAPPING_ID
data = self.template_query('brain_observatory_queries',
'cell_specimen_id_mapping_table',
mapping_table_id=mapping_table_id)
try:
file_url = data[0]['download_link']
except Exception as _:
raise Exception("No OphysCellSpecimenIdMapping file found.")
self.retrieve_file_over_http(self.api_url + file_url, file_name)
return pd.read_csv(file_name)
def simplify_experiment_containers(self, containers):
return [{
'id': c['id'],
'imaging_depth': c['imaging_depth'],
'targeted_structure': c['targeted_structure']['acronym'],
'cre_line': find_specimen_cre_line(c['specimen']),
'reporter_line': find_specimen_reporter_line(c['specimen']),
'donor_name': c['specimen']['donor']['external_donor_name'],
'specimen_name': c['specimen']['name'],
'tags': find_container_tags(c),
'failed': c['failed']
} for c in containers]
def simplify_ophys_experiments(self, exps):
return [{
'id': e['id'],
'imaging_depth': e['imaging_depth'],
'targeted_structure': e['targeted_structure']['acronym'],
'cre_line': find_specimen_cre_line(e['specimen']),
'reporter_line': find_specimen_reporter_line(e['specimen']),
'acquisition_age_days': find_experiment_acquisition_age(e),
'experiment_container_id': e['experiment_container_id'],
'session_type': e['stimulus_name'],
'donor_name': e['specimen']['donor']['external_donor_name'],
'specimen_name': e['specimen']['name'],
'fail_eye_tracking': e.get('fail_eye_tracking', None)
} for e in exps]
class VoxelModelApi(MouseConnectivityApi):
'''HTTP Client extending MouseConnectivityApi to download model data.
'''
HTTP_MODEL_DIRECTORY = "http://download.alleninstitute.org/publications/"\
"A_high_resolution_data-driven_model_of_the_mouse_connectome/"
NODES_FILE = "nodes.csv.gz"
WEIGHTS_FILE = "weights.csv.gz"
SOURCE_MASK_FILE = "source_mask_params.json"
TARGET_MASK_FILE = "target_mask_params.json"
CONNECTION_DENSITY_FILE = 'connection_density.csv.gz'
CONNECTION_STRENGTH_FILE = 'connection_strength.csv.gz'
NORMALIZED_CONNECTION_DENSITY_FILE = 'normalized_connection_density.csv.gz'
NORMALIZED_CONNECTION_STRENGTH_FILE = 'normalized_connection_strength.csv.gz'
def download_model_files(self, file_name, save_file_path=None):
"""Download data.
Parameters
----------
file_name : string, optional
save_file_path : string, optional
File name to save as.
"""
url = self.HTTP_MODEL_DIRECTORY + file_name
self.retrieve_file_over_http(url, save_file_path)
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=1, path_keyword='path'))
def download_nodes(self, file_name):
self.download_model_files(self.NODES_FILE, file_name)
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=1, path_keyword='path'))
def download_weights(self, file_name):
self.download_model_files(self.WEIGHTS_FILE, file_name)
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=1, path_keyword='path'))
def download_source_mask_params(self, file_name):
self.download_model_files(self.SOURCE_MASK_FILE, file_name)
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=1, path_keyword='path'))
def download_target_mask_params(self, file_name):
self.download_model_files(self.TARGET_MASK_FILE, file_name)
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=1, path_keyword='path'))
def download_connection_density(self, file_name):
self.download_model_files(self.CONNECTION_DENSITY_FILE, file_name)
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=1, path_keyword='path'))
def download_connection_strength(self, file_name):
self.download_model_files(self.CONNECTION_STRENGTH_FILE, file_name)
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=1, path_keyword='path'))
def download_normalized_connection_density(self, file_name):
self.download_model_files(self.NORMALIZED_CONNECTION_DENSITY_FILE, file_name)
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=1, path_keyword='path'))
def download_normalized_connection_strength(self, file_name):
self.download_model_files(self.NORMALIZED_CONNECTION_STRENGTH_FILE, file_name)
class MouseConnectivityApiPrerelease(MouseConnectivityApi):
'''Client for retrieving prereleased mouse connectivity data from lims.
Parameters
----------
base_uri : string, optional
Does not affect pulling from lims.
file_name : string, optional
File name to save/read storage_directories dict. Passed to
GridDataApiPrerelease constructor.
'''
def __init__(self,
storage_directories_file_name,
cache_storage_directories=True,
base_uri=None):
super(MouseConnectivityApiPrerelease, self).__init__(base_uri=base_uri)
self.grid_data_api = GridDataApiPrerelease.from_file_name(
storage_directories_file_name, cache=cache_storage_directories)
@cacheable()
def get_experiments(self):
query_result = lu.query(_EXPERIMENT_QUERY)
experiments = []
for row in query_result:
if str(row[b'id']) in self.grid_data_api.storage_directories:
exp_dict = _experiment_dict(row)
experiments.append(exp_dict)
return experiments
#@cacheable()
def get_structure_unionizes(self):
raise NotImplementedError()
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=1,
path_keyword='path'))
def download_injection_density(self, path, experiment_id, resolution):
file_name = "%s_%s.nrrd" % (GridDataApi.INJECTION_DENSITY, resolution)
self.grid_data_api.download_projection_grid_data(
path, experiment_id, file_name)
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=1,
path_keyword='path'))
def download_projection_density(self, path, experiment_id, resolution):
file_name = "%s_%s.nrrd" % (GridDataApi.PROJECTION_DENSITY, resolution)
self.grid_data_api.download_projection_grid_data(
path, experiment_id, file_name)
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=1,
path_keyword='path'))
def download_injection_fraction(self, path, experiment_id, resolution):
file_name = "%s_%s.nrrd" % (GridDataApi.INJECTION_FRACTION, resolution)
self.grid_data_api.download_projection_grid_data(
path, experiment_id, file_name)
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=1,
path_keyword='path'))
def download_data_mask(self, path, experiment_id, resolution):
file_name = "%s_%s.nrrd" % (GridDataApi.DATA_MASK, resolution)
self.grid_data_api.download_projection_grid_data(
path, experiment_id, file_name)
def get_cell_specimens(self,
file_name=None,
ids=None,
experiment_container_ids=None,
include_failed=False,
simple=True,
filters=None):
""" Return cell specimens that have certain properies.
Parameters
----------
file_name: string
File name to save/read the cell specimens. If file_name is None,
the file_name will be pulled out of the manifest. If caching
is disabled, no file will be saved. Default is None.
ids: list
List of cell specimen ids.
experiment_container_ids: list
List of experiment container ids.
include_failed: bool
Whether to include cells from failed experiment containers
simple: boolean
Whether or not to simplify the dictionary properties returned by this method
to a more concise subset.
filters: list of dicts
List of filter dictionaries. The Allen Brain Observatory web site can
generate filters in this format to reproduce a filtered set of cells
found there. To see what these look like, visit
http://observatory.brain-map.org/visualcoding, perform a cell search
and apply some filters (e.g. find cells in a particular area), then
click the "view these cells in the AllenSDK" link on the bottom-left
of the search results page. This will take you to a page that contains
a code sample you can use to apply those same filters via this argument.
For more detail on the filter syntax, see BrainObservatoryApi.dataframe_query.
Returns
-------
list of dictionaries
"""
file_name = self.get_cache_path(file_name, self.CELL_SPECIMENS_KEY)
cell_specimens = self.api.get_cell_metrics(
path=file_name,
strategy='lazy',
pre=lambda x: [y for y in x],
**Cache.cache_json())
cell_specimens = self.api.filter_cell_specimens(
cell_specimens,
ids=ids,
experiment_container_ids=experiment_container_ids,
include_failed=include_failed,
filters=filters)
# drop the thumbnail columns
if simple:
mappings = self._get_stimulus_mappings()
thumbnails = [
m['item'] for m in mappings
if m['item_type'] == 'T' and m['level'] == 'R'
]
for cs in cell_specimens:
for t in thumbnails:
del cs[t]
return cell_specimens
def get_ophys_experiments(self,
file_name=None,
ids=None,
experiment_container_ids=None,
targeted_structures=None,
imaging_depths=None,
cre_lines=None,
reporter_lines=None,
transgenic_lines=None,
stimuli=None,
session_types=None,
cell_specimen_ids=None,
include_failed=False,
require_eye_tracking=False,
simple=True):
""" Get a list of ophys experiments matching certain criteria.
Parameters
----------
file_name: string
File name to save/read the ophys experiments. If file_name is None,
the file_name will be pulled out of the manifest. If caching
is disabled, no file will be saved. Default is None.
ids: list
List of ophys experiment ids.
experiment_container_ids: list
List of experiment container ids.
targeted_structures: list
List of structure acronyms. Must be in the list returned by
BrainObservatoryCache.get_all_targeted_structures().
imaging_depths: list
List of imaging depths. Must be in the list returned by
BrainObservatoryCache.get_all_imaging_depths().
cre_lines: list
List of cre lines. Must be in the list returned by
BrainObservatoryCache.get_all_cre_lines().
reporter_lines: list
List of reporter lines. Must be in the list returned by
BrainObservatoryCache.get_all_reporter_lines().
transgenic_lines: list
List of transgenic lines. Must be in the list returned by
BrainObservatoryCache.get_all_cre_lines() or.
BrainObservatoryCache.get_all_reporter_lines().
stimuli: list
List of stimulus names. Must be in the list returned by
BrainObservatoryCache.get_all_stimuli().
session_types: list
List of stimulus session type names. Must be in the list returned by
BrainObservatoryCache.get_all_session_types().
cell_specimen_ids: list
Only include experiments that contain cells with these ids.
include_failed: boolean
Whether or not to include experiments from failed experiment containers.
simple: boolean
Whether or not to simplify the dictionary properties returned by this method
to a more concise subset.
require_eye_tracking: boolean
If True, only return experiments that have eye tracking results. Default: False.
Returns
-------
list of dictionaries
"""
_assert_not_string(targeted_structures, "targeted_structures")
_assert_not_string(cre_lines, "cre_lines")
_assert_not_string(reporter_lines, "reporter_lines")
_assert_not_string(transgenic_lines, "transgenic_lines")
_assert_not_string(stimuli, "stimuli")
_assert_not_string(session_types, "session_types")
file_name = self.get_cache_path(file_name, self.EXPERIMENTS_KEY)
exps = self.api.get_ophys_experiments(path=file_name,
strategy='lazy',
**Cache.cache_json())
# NOTE: Ugly hack to update the 'fail_eye_tracking' field
# which is using True/False values for the previous eye mapping
# implementation. This will also need to be fixed in warehouse.
# ----- Start of ugly hack -----
response = self.api.template_query('brain_observatory_queries',
'all_eye_mapping_files')
session_ids_with_eye_tracking: set = {
entry['attachable_id']
for entry in response if entry['attachable_type'] == "OphysSession"
}
for indx, exp in enumerate(exps):
try:
ophys_session_id = ophys_experiment_session_id_map[exp['id']]
if ophys_session_id in session_ids_with_eye_tracking:
exps[indx]['fail_eye_tracking'] = False
else:
exps[indx]['fail_eye_tracking'] = True
except KeyError:
exps[indx]['fail_eye_tracking'] = True
# ----- End of ugly hack -----
if cell_specimen_ids is not None:
cells = self.get_cell_specimens(ids=cell_specimen_ids)
cell_container_ids = set(
[cell['experiment_container_id'] for cell in cells])
if experiment_container_ids is not None:
experiment_container_ids = list(
set(experiment_container_ids) - cell_container_ids)
else:
experiment_container_ids = list(cell_container_ids)
exps = self.api.filter_ophys_experiments(
exps,
ids=ids,
experiment_container_ids=experiment_container_ids,
targeted_structures=targeted_structures,
imaging_depths=imaging_depths,
cre_lines=cre_lines,
reporter_lines=reporter_lines,
transgenic_lines=transgenic_lines,
stimuli=stimuli,
session_types=session_types,
include_failed=include_failed,
require_eye_tracking=require_eye_tracking,
simple=simple)
return exps
def get_experiment_containers(self,
file_name=None,
ids=None,
targeted_structures=None,
imaging_depths=None,
cre_lines=None,
reporter_lines=None,
transgenic_lines=None,
include_failed=False,
simple=True):
""" Get a list of experiment containers matching certain criteria.
Parameters
----------
file_name: string
File name to save/read the experiment containers. If file_name is None,
the file_name will be pulled out of the manifest. If caching
is disabled, no file will be saved. Default is None.
ids: list
List of experiment container ids.
targeted_structures: list
List of structure acronyms. Must be in the list returned by
BrainObservatoryCache.get_all_targeted_structures().
imaging_depths: list
List of imaging depths. Must be in the list returned by
BrainObservatoryCache.get_all_imaging_depths().
cre_lines: list
List of cre lines. Must be in the list returned by
BrainObservatoryCache.get_all_cre_lines().
reporter_lines: list
List of reporter lines. Must be in the list returned by
BrainObservatoryCache.get_all_reporter_lines().
transgenic_lines: list
List of transgenic lines. Must be in the list returned by
BrainObservatoryCache.get_all_cre_lines() or.
BrainObservatoryCache.get_all_reporter_lines().
include_failed: boolean
Whether or not to include failed experiment containers.
simple: boolean
Whether or not to simplify the dictionary properties returned by this method
to a more concise subset.
Returns
-------
list of dictionaries
"""
_assert_not_string(targeted_structures, "targeted_structures")
_assert_not_string(cre_lines, "cre_lines")
_assert_not_string(reporter_lines, "reporter_lines")
_assert_not_string(transgenic_lines, "transgenic_lines")
file_name = self.get_cache_path(file_name,
self.EXPERIMENT_CONTAINERS_KEY)
containers = self.api.get_experiment_containers(path=file_name,
strategy='lazy',
**Cache.cache_json())
containers = self.api.filter_experiment_containers(
containers,
ids=ids,
targeted_structures=targeted_structures,
imaging_depths=imaging_depths,
cre_lines=cre_lines,
reporter_lines=reporter_lines,
transgenic_lines=transgenic_lines,
include_failed=include_failed,
simple=simple)
return containers
def cache():
return Cache()
class MouseConnectivityApi(ReferenceSpaceApi, GridDataApi):
'''
HTTP Client for the Allen Mouse Brain Connectivity Atlas.
See: `Mouse Connectivity API <http://help.brain-map.org/display/mouseconnectivity/API>`_
'''
PRODUCT_IDS = [5, 31]
def __init__(self, base_uri=None):
super(MouseConnectivityApi, self).__init__(base_uri=base_uri)
@cacheable()
def get_experiments(self, structure_ids, **kwargs):
'''
Fetch experiment metadata from the Mouse Brain Connectivity Atlas.
Parameters
----------
structure_ids : integer or list, optional
injection structure
Returns
-------
url : string
The constructed URL
'''
criteria_list = [
'[failed$eqfalse]',
'products[id$in%s]' %
(','.join(str(i) for i in MouseConnectivityApi.PRODUCT_IDS))
]
if structure_ids is not None:
if type(structure_ids) is not list:
structure_ids = [structure_ids]
criteria_list.append('[id$in%s]' %
','.join(str(i) for i in structure_ids))
criteria_string = ','.join(criteria_list)
return self.model_query('SectionDataSet',
criteria=criteria_string,
**kwargs)
@cacheable()
def get_experiments_api(self):
'''
Fetch experiment metadata from the Mouse Brain Connectivity Atlas via the ApiConnectivity table.
Returns
-------
url : string
The constructed URL
'''
return self.model_query('ApiConnectivity', num_rows='all')
@cacheable()
def get_manual_injection_summary(self, experiment_id):
''' Retrieve manual injection summary. '''
criteria = '[id$in%d]' % (experiment_id)
include = [
'specimen(donor(transgenic_mouse(transgenic_lines)),',
'injections(structure,age)),', 'equalization,products'
]
only = [
'id', 'failed', 'storage_directory', 'red_lower', 'red_upper',
'green_lower', 'green_upper', 'blue_lower', 'blue_upper',
'products.id', 'specimen_id', 'structure_id', 'reference_space_id',
'primary_injection_structure_id', 'registration_point',
'coordinates_ap', 'coordinates_dv', 'coordinates_ml', 'angle',
'sex', 'strain', 'injection_materials', 'acronym',
'structures.name', 'days', 'transgenic_mice.name',
'transgenic_lines.name', 'transgenic_lines.description',
'transgenic_lines.id', 'donors.id'
]
return self.model_query('SectionDataSet',
criteria=criteria,
include=include,
only=only)
@cacheable()
def get_experiment_detail(self, experiment_id):
'''Retrieve the experiments data.'''
criteria = '[id$eq%d]' % (experiment_id)
include = [
'specimen(stereotaxic_injections(primary_injection_structure,structures,stereotaxic_injection_coordinates)),',
'equalization,', 'sub_images'
]
order = ["'sub_images.section_number$asc'"]
return self.model_query('SectionDataSet',
criteria=criteria,
include=include,
order=order)
@cacheable()
def get_projection_image_info(self, experiment_id, section_number):
'''Fetch meta-information of one projection image.
Parameters
----------
experiment_id : integer
section_number : integer
Notes
-----
See: image examples under
`Experimental Overview and Metadata <http://help.brain-map.org/display/mouseconnectivity/API##API-ExperimentalOverviewandMetadata>`_
for additional documentation.
Download the image using :py:meth:`allensdk.api.queries.image_download_api.ImageDownloadApi.download_section_image`
'''
criteria = '[id$eq%d]' % (experiment_id)
include = [
'equalization,sub_images[section_number$eq%d]' % (section_number)
]
return self.model_query('SectionDataSet',
criteria=criteria,
include=include)
def download_reference_aligned_image_channel_volumes(
self, data_set_id, save_file_path=None):
'''
Returns
-------
The well known file is downloaded
'''
well_known_file_url = self.get_reference_aligned_image_channel_volumes_url(
data_set_id)
if save_file_path is None:
save_file_path = str(data_set_id) + '.zip'
self.retrieve_file_over_http(well_known_file_url, save_file_path)
def build_reference_aligned_image_channel_volumes_url(self, data_set_id):
'''Construct url to download the red, green, and blue channels
aligned to the 25um adult mouse brain reference space volume.
Parameters
----------
data_set_id : integerallensdk.api.queries
aka attachable_id
Notes
-----
See: `Reference-aligned Image Channel Volumes <http://help.brain-map.org/display/mouseconnectivity/API#API-ReferencealignedImageChannelVolumes>`_
for additional documentation.
'''
criteria = [
'well_known_file_type', "[name$eq'ImagesResampledTo25MicronARA']",
"[attachable_id$eq%d]" % (data_set_id)
]
model_stage = self.model_stage('WellKnownFile', criteria=criteria)
url = self.build_query_url([model_stage])
return url
def get_reference_aligned_image_channel_volumes_url(self, data_set_id):
'''Retrieve the download link for a specific data set.\
Notes
-----
See `Reference-aligned Image Channel Volumes <http://help.brain-map.org/display/mouseconnectivity/API#API-ReferencealignedImageChannelVolumes>`_
for additional documentation.
'''
download_link = self.do_query(
self.build_reference_aligned_image_channel_volumes_url,
lambda parsed_json: str(parsed_json['msg'][0]['download_link']),
data_set_id)
url = self.api_url + download_link
return url
def experiment_source_search(self, **kwargs):
'''Search over the whole projection signal statistics dataset
to find experiments with specific projection profiles.
Parameters
----------
injection_structures : list of integers or strings
Integer Structure.id or String Structure.acronym.
target_domain : list of integers or strings, optional
Integer Structure.id or String Structure.acronym.
injection_hemisphere : string, optional
'right' or 'left', Defaults to both hemispheres.
target_hemisphere : string, optional
'right' or 'left', Defaults to both hemispheres.
transgenic_lines : list of integers or strings, optional
Integer TransgenicLine.id or String TransgenicLine.name. Specify ID 0 to exclude all TransgenicLines.
injection_domain : list of integers or strings, optional
Integer Structure.id or String Structure.acronym.
primary_structure_only : boolean, optional
product_ids : list of integers, optional
Integer Product.id
start_row : integer, optional
For paging purposes. Defaults to 0.
num_rows : integer, optional
For paging purposes. Defaults to 2000.
Notes
-----
See `Source Search <http://help.brain-map.org/display/mouseconnectivity/API#API-SourceSearch>`_,
`Target Search <http://help.brain-map.org/display/mouseconnectivity/API#API-TargetSearch>`_,
and
`service::mouse_connectivity_injection_structure <http://help.brain-map.org/display/api/Connected+Services+and+Pipes#ConnectedServicesandPipes-service%3A%3Amouseconnectivityinjectionstructure>`_.
'''
tuples = [(k, v) for k, v in six.iteritems(kwargs)]
return self.service_query('mouse_connectivity_injection_structure',
parameters=tuples)
def experiment_spatial_search(self, **kwargs):
'''Displays all SectionDataSets
with projection signal density >= 0.1 at the seed point.
This service also returns the path
along the most dense pixels from the seed point
to the center of each injection site..
Parameters
----------
seed_point : list of floats
The coordinates of a point in 3-D SectionDataSet space.
transgenic_lines : list of integers or strings, optional
Integer TransgenicLine.id or String TransgenicLine.name. Specify ID 0 to exclude all TransgenicLines.
section_data_sets : list of integers, optional
Ids to filter the results.
injection_structures : list of integers or strings, optional
Integer Structure.id or String Structure.acronym.
primary_structure_only : boolean, optional
product_ids : list of integers, optional
Integer Product.id
start_row : integer, optional
For paging purposes. Defaults to 0.
num_rows : integer, optional
For paging purposes. Defaults to 2000.
Notes
-----
See `Spatial Search <http://help.brain-map.org/display/mouseconnectivity/API#API-SpatialSearch>`_
and
`service::mouse_connectivity_target_spatial <http://help.brain-map.org/display/api/Connected+Services+and+Pipes#ConnectedServicesandPipes-service%3A%3Amouseconnectivitytargetspatial>`_.
'''
tuples = [(k, v) for k, v in six.iteritems(kwargs)]
return self.service_query('mouse_connectivity_target_spatial',
parameters=tuples)
def experiment_injection_coordinate_search(self, **kwargs):
'''User specifies a seed location within the 3D reference space.
The service returns a rank list of experiments
by distance of its injection site to the specified seed location.
Parameters
----------
seed_point : list of floats
The coordinates of a point in 3-D SectionDataSet space.
transgenic_lines : list of integers or strings, optional
Integer TransgenicLine.id or String TransgenicLine.name. Specify ID 0 to exclude all TransgenicLines.
injection_structures : list of integers or strings, optional
Integer Structure.id or String Structure.acronym.
primary_structure_only : boolean, optional
product_ids : list of integers, optional
Integer Product.id
start_row : integer, optional
For paging purposes. Defaults to 0.
num_rows : integer, optional
For paging purposes. Defaults to 2000.
Notes
-----
See `Injection Coordinate Search <http://help.brain-map.org/display/mouseconnectivity/API#API-InjectionCoordinateSearch>`_
and
`service::mouse_connectivity_injection_coordinate <http://help.brain-map.org/display/api/Connected+Services+and+Pipes#ConnectedServicesandPipes-service%3A%3Amouseconnectivityinjectioncoordinate>`_.
'''
tuples = [(k, v) for k, v in six.iteritems(kwargs)]
return self.service_query('mouse_connectivity_injection_coordinate',
parameters=tuples)
def experiment_correlation_search(self, **kwargs):
'''Select a seed experiment and a domain over
which the similarity comparison is to be made.
Parameters
----------
row : integer
SectionDataSet.id to correlate against.
structures : list of integers or strings, optional
Integer Structure.id or String Structure.acronym.
hemisphere : string, optional
Use 'right' or 'left'. Defaults to both hemispheres.
transgenic_lines : list of integers or strings, optional
Integer TransgenicLine.id or String TransgenicLine.name. Specify ID 0 to exclude all TransgenicLines.
injection_structures : list of integers or strings, optional
Integer Structure.id or String Structure.acronym.
primary_structure_only : boolean, optional
product_ids : list of integers, optional
Integer Product.id
start_row : integer, optional
For paging purposes. Defaults to 0.
num_rows : integer, optional
For paging purposes. Defaults to 2000.
Notes
-----
See `Correlation Search <http://help.brain-map.org/display/mouseconnectivity/API#API-CorrelationSearch>`_
and
`service::mouse_connectivity_correlation <http://help.brain-map.org/display/api/Connected+Services+and+Pipes#ConnectedServicesandPipes-service%3A%3Amouseconnectivitycorrelation>`_.
'''
tuples = sorted(six.iteritems(kwargs))
return self.service_query('mouse_connectivity_correlation',
parameters=tuples)
@cacheable()
def get_structure_unionizes(self,
experiment_ids,
is_injection=None,
structure_name=None,
structure_ids=None,
hemisphere_ids=None,
normalized_projection_volume_limit=None,
include=None,
debug=None,
order=None):
experiment_filter = '[section_data_set_id$in%s]' %\
','.join(str(i) for i in experiment_ids)
if is_injection is True:
is_injection_filter = '[is_injection$eqtrue]'
elif is_injection is False:
is_injection_filter = '[is_injection$eqfalse]'
else:
is_injection_filter = ''
if normalized_projection_volume_limit is not None:
volume_filter = '[normalized_projection_volume$gt%f]' %\
(normalized_projection_volume_limit)
else:
volume_filter = ''
if hemisphere_ids is not None:
hemisphere_filter = '[hemisphere_id$in%s]' %\
','.join(str(h) for h in hemisphere_ids)
else:
hemisphere_filter = ''
if structure_name is not None:
structure_filter = ",structure[name$eq'%s']" % (structure_name)
elif structure_ids is not None:
structure_filter = '[structure_id$in%s]' %\
','.join(str(i) for i in structure_ids)
else:
structure_filter = ''
return self.model_query('ProjectionStructureUnionize',
criteria=''.join([
experiment_filter, is_injection_filter,
volume_filter, hemisphere_filter,
structure_filter
]),
include=include,
order=order,
num_rows='all',
debug=debug,
count=False)
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=1,
path_keyword='path'))
def download_injection_density(self, path, experiment_id, resolution):
self.download_projection_grid_data(experiment_id,
[GridDataApi.INJECTION_DENSITY],
resolution, path)
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=1,
path_keyword='path'))
def download_projection_density(self, path, experiment_id, resolution):
self.download_projection_grid_data(experiment_id,
[GridDataApi.PROJECTION_DENSITY],
resolution, path)
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=1,
path_keyword='path'))
def download_injection_fraction(self, path, experiment_id, resolution):
self.download_projection_grid_data(experiment_id,
[GridDataApi.INJECTION_FRACTION],
resolution, path)
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=1,
path_keyword='path'))
def download_data_mask(self, path, experiment_id, resolution):
self.download_projection_grid_data(experiment_id,
[GridDataApi.DATA_MASK], resolution,
path)
def calculate_injection_centroid(self,
injection_density,
injection_fraction,
resolution=25):
'''
Compute the centroid of an injection site.
Parameters
----------
injection_density: np.ndarray
The injection density volume of an experiment
injection_fraction: np.ndarray
The injection fraction volume of an experiment
'''
# find all voxels with injection_fraction > 0
injection_voxels = np.nonzero(injection_fraction)
injection_density_computed = np.multiply(
injection_density[injection_voxels],
injection_fraction[injection_voxels])
sum_density = np.sum(injection_density_computed)
# compute centroid in CCF coordinates
if sum_density > 0:
centroid = np.dot(
injection_density_computed, list(
zip(*injection_voxels))) / sum_density * resolution
else:
centroid = None
return centroid
class CellTypesApi(RmaApi):
NWB_FILE_TYPE = 'NWBDownload'
SWC_FILE_TYPE = '3DNeuronReconstruction'
MARKER_FILE_TYPE = '3DNeuronMarker'
MOUSE = 'Mus musculus'
HUMAN = 'H**o Sapiens'
def __init__(self, base_uri=None):
super(CellTypesApi, self).__init__(base_uri)
@cacheable()
def list_cells_api(self,
id=None,
require_morphology=False,
require_reconstruction=False,
reporter_status=None,
species=None):
criteria = None
if id:
criteria = "[specimen__id$eq%d]" % id
cells = self.model_query('ApiCellTypesSpecimenDetail',
criteria=criteria,
num_rows='all')
return cells
@deprecated("please use list_cells_api instead")
def list_cells(self,
id=None,
require_morphology=False,
require_reconstruction=False,
reporter_status=None,
species=None):
"""
Query the API for a list of all cells in the Cell Types Database.
Parameters
----------
id: int
ID of a cell. If not provided returns all matching cells.
require_morphology: boolean
Only return cells that have morphology images.
require_reconstruction: boolean
Only return cells that have morphological reconstructions.
reporter_status: list
Return cells that have a particular cell reporter status.
species: list
Filter for cells that belong to one or more species. If None, return all.
Must be one of [ CellTypesApi.MOUSE, CellTypesApi.HUMAN ].
Returns
-------
list
Meta data for all cells.
"""
if id:
criteria = "[id$eq'%d']" % id
else:
criteria = "[is_cell_specimen$eq'true'],products[name$in'Mouse Cell Types','Human Cell Types'],ephys_result[failed$eqfalse]"
include = (
'structure,cortex_layer,donor(transgenic_lines,organism,conditions),specimen_tags,cell_soma_locations,'
+ 'ephys_features,data_sets,neuron_reconstructions,cell_reporter')
cells = self.model_query('Specimen',
criteria=criteria,
include=include,
num_rows='all')
for cell in cells:
# specimen tags
for tag in cell['specimen_tags']:
tag_name, tag_value = tag['name'].split(' - ')
tag_name = tag_name.replace(' ', '_')
cell[tag_name] = tag_value
# morphology and reconstuction
cell['has_reconstruction'] = len(
cell['neuron_reconstructions']) > 0
cell['has_morphology'] = len(cell['data_sets']) > 0
# transgenic line
cell['transgenic_line'] = None
for tl in cell['donor']['transgenic_lines']:
if tl['transgenic_line_type_name'] == 'driver':
cell['transgenic_line'] = tl['name']
# cell reporter status
cell['reporter_status'] = cell.get('cell_reporter',
{}).get('name', None)
# species
cell['species'] = cell.get('donor', {}).get('organism',
{}).get('name', None)
# conditions (whitelist)
condition_types = ['disease categories']
condition_keys = dict(
zip(condition_types,
[ct.replace(' ', '_') for ct in condition_types]))
for ct, ck in condition_keys.items():
cell[ck] = []
conditions = cell.get('donor', {}).get('conditions', [])
for condition in conditions:
c_type, c_val = condition['name'].split(' - ')
if c_type in condition_keys:
cell[condition_keys[c_type]].append(c_val)
result = self.filter_cells(cells, require_morphology,
require_reconstruction, reporter_status,
species)
return result
def get_cell(self, id):
'''
Query the API for a one cells in the Cell Types Database.
Returns
-------
list
Meta data for one cell.
'''
cells = self.list_cells_api(id=id)
cell = None if not cells else cells[0]
return cell
@cacheable()
def get_ephys_sweeps(self, specimen_id):
"""
Query the API for a list of sweeps for a particular cell in the Cell Types Database.
Parameters
----------
specimen_id: int
Specimen ID of a cell.
Returns
-------
list: List of sweep dictionaries belonging to a cell
"""
criteria = "[specimen_id$eq%d]" % specimen_id
sweeps = self.model_query('EphysSweep',
criteria=criteria,
num_rows='all')
return sorted(sweeps, key=lambda x: x['sweep_number'])
@deprecated("please use filter_cells_api")
def filter_cells(self, cells, require_morphology, require_reconstruction,
reporter_status, species):
"""
Filter a list of cell specimens to those that optionally have morphologies
or have morphological reconstructions.
Parameters
----------
cells: list
List of cell metadata dictionaries to be filtered
require_morphology: boolean
Filter out cells that have no morphological images.
require_reconstruction: boolean
Filter out cells that have no morphological reconstructions.
reporter_status: list
Filter for cells that have a particular cell reporter status
species: list
Filter for cells that belong to one or more species. If None, return all.
Must be one of [ CellTypesApi.MOUSE, CellTypesApi.HUMAN ].
"""
if require_morphology:
cells = [c for c in cells if c['has_morphology']]
if require_reconstruction:
cells = [c for c in cells if c['has_reconstruction']]
if reporter_status:
cells = [
c for c in cells if c['reporter_status'] in reporter_status
]
if species:
species_lower = [s.lower() for s in species]
cells = [
c for c in cells
if c['donor']['organism']['name'].lower() in species_lower
]
return cells
def filter_cells_api(self,
cells,
require_morphology=False,
require_reconstruction=False,
reporter_status=None,
species=None,
simple=True):
"""
"""
if require_morphology or require_reconstruction:
cells = [
c for c in cells
if c.get('nr__reconstruction_type') is not None
]
if reporter_status:
cells = [
c for c in cells
if c.get('cell_reporter_status') in reporter_status
]
if species:
species_lower = [s.lower() for s in species]
cells = [
c for c in cells
if c.get('donor__species', "").lower() in species_lower
]
if simple:
cells = self.simplify_cells_api(cells)
return cells
def simplify_cells_api(self, cells):
return [{
'reporter_status':
cell['cell_reporter_status'],
'cell_soma_location':
[cell['csl__x'], cell['csl__y'], cell['csl__z']],
'species':
cell['donor__species'],
'id':
cell['specimen__id'],
'name':
cell['specimen__name'],
'structure_layer_name':
cell['structure__layer'],
'structure_area_id':
cell['structure_parent__id'],
'structure_area_abbrev':
cell['structure_parent__acronym'],
'transgenic_line':
cell['line_name'],
'dendrite_type':
cell['tag__dendrite_type'],
'apical':
cell['tag__apical'],
'reconstruction_type':
cell['nr__reconstruction_type'],
'disease_state':
cell['donor__disease_state'],
'donor_id':
cell['donor__id'],
'structure_hemisphere':
cell['specimen__hemisphere'],
'normalized_depth':
cell['csl__normalized_depth']
} for cell in cells]
@cacheable()
def get_ephys_features(self):
"""
Query the API for the full table of EphysFeatures for all cells.
"""
return self.model_query(
'EphysFeature',
criteria='specimen(ephys_result[failed$eqfalse])',
num_rows='all')
@cacheable()
def get_morphology_features(self):
"""
Query the API for the full table of morphology features for all cells
Notes
-----
by default the tags column is removed because it isn't useful
"""
return self.model_query(
'NeuronReconstruction',
criteria="specimen(ephys_result[failed$eqfalse])",
excpt='tags',
num_rows='all')
@cacheable(strategy='create',
pathfinder=Cache.pathfinder(file_name_position=2,
path_keyword='file_name'))
def save_ephys_data(self, specimen_id, file_name):
"""
Save the electrophysology recordings for a cell as an NWB file.
Parameters
----------
specimen_id: int
ID of the specimen, from the Specimens database model in the Allen Institute API.
file_name: str
Path to save the NWB file.
"""
criteria = '[id$eq%d],ephys_result(well_known_files(well_known_file_type[name$eq%s]))' % (
specimen_id, self.NWB_FILE_TYPE)
includes = 'ephys_result(well_known_files(well_known_file_type))'
results = self.model_query('Specimen',
criteria=criteria,
include=includes,
num_rows='all')
try:
file_url = results[0]['ephys_result']['well_known_files'][0][
'download_link']
except Exception as _:
raise Exception("Specimen %d has no ephys data" % specimen_id)
self.retrieve_file_over_http(self.api_url + file_url, file_name)
def save_reconstruction(self, specimen_id, file_name):
"""
Save the morphological reconstruction of a cell as an SWC file.
Parameters
----------
specimen_id: int
ID of the specimen, from the Specimens database model in the Allen Institute API.
file_name: str
Path to save the SWC file.
"""
Manifest.safe_make_parent_dirs(file_name)
criteria = '[id$eq%d],neuron_reconstructions(well_known_files)' % specimen_id
includes = 'neuron_reconstructions(well_known_files(well_known_file_type[name$eq\'%s\']))' % self.SWC_FILE_TYPE
results = self.model_query('Specimen',
criteria=criteria,
include=includes,
num_rows='all')
try:
file_url = results[0]['neuron_reconstructions'][0][
'well_known_files'][0]['download_link']
except:
raise Exception("Specimen %d has no reconstruction" % specimen_id)
self.retrieve_file_over_http(self.api_url + file_url, file_name)
def save_reconstruction_markers(self, specimen_id, file_name):
"""
Save the marker file for the morphological reconstruction of a cell. These are
comma-delimited files indicating points of interest in a reconstruction (truncation
points, early tracing termination, etc).
Parameters
----------
specimen_id: int
ID of the specimen, from the Specimens database model in the Allen Institute API.
file_name: str
Path to save the marker file.
"""
Manifest.safe_make_parent_dirs(file_name)
criteria = '[id$eq%d],neuron_reconstructions(well_known_files)' % specimen_id
includes = 'neuron_reconstructions(well_known_files(well_known_file_type[name$eq\'%s\']))' % self.MARKER_FILE_TYPE
results = self.model_query('Specimen',
criteria=criteria,
include=includes,
num_rows='all')
try:
file_url = results[0]['neuron_reconstructions'][0][
'well_known_files'][0]['download_link']
except:
raise LookupError("Specimen %d has no marker file" % specimen_id)
self.retrieve_file_over_http(self.api_url + file_url, file_name)
def get_experiments(self,
dataframe=False,
file_name=None,
cre=None,
injection_structure_ids=None):
"""
Read a list of experiments that match certain criteria. If caching is enabled,
this will save the whole (unfiltered) list of experiments to a file.
Parameters
----------
dataframe: boolean
Return the list of experiments as a Pandas DataFrame. If False,
return a list of dictionaries. Default False.
file_name: string
File name to save/read the structures table. If file_name is None,
the file_name will be pulled out of the manifest. If caching
is disabled, no file will be saved. Default is None.
cre: boolean or list
If True, return only cre-positive experiments. If False, return only
cre-negative experiments. If None, return all experients. If list, return
all experiments with cre line names in the supplied list. Default None.
injection_structure_ids: list
Only return experiments that were injected in the structures provided here.
If None, return all experiments. Default None.
"""
file_name = self.get_cache_path(file_name, self.EXPERIMENTS_KEY)
experiments = self.api.get_experiments_api(path=file_name,
strategy='lazy',
**Cache.cache_json())
for e in experiments:
# renaming id
e['id'] = e['data_set_id']
del e['data_set_id']
# simplify trangsenic line
tl = e.get('transgenic_line', None)
if tl:
e['transgenic_line'] = tl['name']
# parse the injection structures
injs = [int(i) for i in e['injection_structures'].split('/')]
e['injection_structures'] = injs
e['primary_injection_structure'] = injs[0]
# remove storage dir
del e['storage_directory']
# filter the read/downloaded list of experiments
experiments = self.filter_experiments(experiments, cre,
injection_structure_ids)
if dataframe:
experiments = pd.DataFrame(experiments)
experiments.set_index(['id'], inplace=True, drop=False)
return experiments