def __init__(self,
cache=True,
manifest_file='cell_types_manifest.json',
base_uri=None):
super(CellTypesCache, self).__init__(manifest=manifest_file,
cache=cache)
self.api = CellTypesApi(base_uri=base_uri)
def get_observation(dataset_id, kind, cached=True, quiet=False):
"""
Gets an observation of kind 'kind' from the dataset with id 'dataset_id',
optionally using the cached value retrieved previously.
"""
db = shelve.open('aibs-cache') if cached else {}
identifier = '%d_%s' % (dataset_id,kind)
if identifier in db:
print("Getting %s cached data value for from AIBS dataset %s" \
% (kind.title(),dataset_id))
value = db[identifier]
else:
print("Getting %s data value for from AIBS dataset %s" \
% (kind.title(),dataset_id))
ct = CellTypesApi()
cmd = ct.get_cell(dataset_id) # Cell metadata
if kind == 'rheobase':
sweep_id = cmd['ephys_features'][0]['rheobase_sweep_id']
sp = get_sweep_params(dataset_id, sweep_id)
if kind == 'rheobase':
value = sp['stimulus_absolute_amplitude']
value = np.round(value,2) # Round to nearest hundredth of a pA.
value *= pq.pA # Apply units.
db[identifier] = value
if cached:
db.close()
return {'value': value}
def __init__(self, cache=True, manifest_file=None, base_uri=None):
if manifest_file is None:
manifest_file = get_default_manifest_file('cell_types')
super(CellTypesCache, self).__init__(manifest=manifest_file,
cache=cache,
version=self.MANIFEST_VERSION)
self.api = CellTypesApi(base_uri=base_uri)
def get_sweep_params(dataset_id,sweep_id):
ct = CellTypesApi()
experiment_params = ct.get_ephys_sweeps(dataset_id)
for sp in experiment_params:
if sp['id']==sweep_id:
sweep_num = sp['sweep_number']
break
if sweep_num is None:
raise Exception('Sweep with ID %d not found in dataset with ID %d.' % (sweep_id, dataset_id))
return sp
def test_list_cells_mocked(mock_cells):
ctapi = CellTypesApi()
ctapi.model_query = MagicMock(return_value=mock_cells)
cells = ctapi.list_cells()
assert len(cells) == 3
flu_cells = [
cell for cell in cells if cell['disease_categories'] == [('influenza')]
]
assert len(flu_cells) == 1
cells = ctapi.list_cells(require_reconstruction=True)
assert len(cells) == 1
cells = ctapi.list_cells(require_morphology=True)
assert len(cells) == 1
cells = ctapi.list_cells(reporter_status=['bob'])
assert len(cells) == 1
cells = ctapi.list_cells(species=['H**O SAPIENS'])
assert len(cells) == 1
cells = ctapi.list_cells(species=['mus musculus'])
assert len(cells) == 1
def fetch_pipeline_file(request):
specimen_id = request.param
nwb_file_name = '{}.nwb'.format(specimen_id)
nwb_file_full_path = os.path.join(TEST_DATA_PATH, nwb_file_name)
if not os.path.exists(nwb_file_full_path):
ct = CellTypesApi()
ct.save_ephys_data(specimen_id, nwb_file_full_path)
return nwb_file_full_path
def __init__(self, archive_dir=None):
self.bp = BiophysicalApi('http://api.brain-map.org')
self.bp.cache_stimulus = True # change to False to not download the large stimulus NWB file
self.cta = CellTypesApi()
self.rma = RmaApi()
self.neuronal_model_download_endpoint = 'http://celltypes.brain-map.org/neuronal_model/download/'
self.template_names = {}
self.nwb_list = []
if archive_dir == None:
archive_dir = '.'
self.archive_dir = archive_dir
def get_observation(dataset_id,kind,cached=True):
ct = CellTypesApi()
cmd = ct.get_cell(dataset_id) # Cell metadata
sweep_num = None
if kind == 'rheobase':
sweep_id = cmd['ephys_features'][0]['rheobase_sweep_id']
sp = get_sweep_params(dataset_id, sweep_id)
if kind == 'rheobase':
value = sp['stimulus_absolute_amplitude']
value = np.round(value,2) # Round to nearest hundredth of a pA.
value *= pq.pA # Apply units.
return {'value': value}
def extract_data(ephys_cell, sweep_number):
"""Data extraction from AllenDB
Parameters
----------
ephys_cell : int
Cell identity from AllenDB
sweep_number : int
Stimulus identity for cell ephys_cell from AllenDB
"""
t_offset = 815.
duration = 1450.
real_data_path = 'ephys_cell_{}_sweep_number_{}.pkl'.format(
ephys_cell, sweep_number)
if not os.path.isfile(real_data_path):
from allensdk.core.cell_types_cache import CellTypesCache
from allensdk.api.queries.cell_types_api import CellTypesApi
manifest_file = 'cell_types/manifest.json'
cta = CellTypesApi()
ctc = CellTypesCache(manifest_file=manifest_file)
data_set = ctc.get_ephys_data(ephys_cell)
sweep_data = data_set.get_sweep(
sweep_number) # works with python2 and fails with python3
sweeps = cta.get_ephys_sweeps(ephys_cell)
sweep = sweeps[sweep_number]
index_range = sweep_data["index_range"]
i = sweep_data["stimulus"][0:index_range[1] + 1] # in A
v = sweep_data["response"][0:index_range[1] + 1] # in V
sampling_rate = sweep_data["sampling_rate"] # in Hz
dt = 1e3 / sampling_rate # in ms
i *= 1e6 # to muA
v *= 1e3 # to mV
v = v[int(t_offset / dt):int((t_offset + duration) / dt)]
i = i[int(t_offset / dt):int((t_offset + duration) / dt)]
real_data_obs = np.array(v).reshape(1, -1, 1)
I_real_data = np.array(i).reshape(-1)
t_on = int(
sweep['stimulus_start_time'] * sampling_rate) * dt - t_offset
t_off = int(
(sweep['stimulus_start_time'] + sweep['stimulus_duration']) *
sampling_rate) * dt - t_offset
f = open(real_data_path, 'wb')
pickle.dump((real_data_obs, I_real_data, dt, t_on, t_off), f)
f.close()
def get_sweep_params(dataset_id, sweep_id):
"""
Gets sweep parameters corresponding to the sweep with id 'sweep_id' from
the dataset with id 'dataset_id'.
"""
ct = CellTypesApi()
experiment_params = ct.get_ephys_sweeps(dataset_id)
sp = None
for sp in experiment_params:
if sp['id']==sweep_id:
sweep_num = sp['sweep_number']
if sweep_num is None:
raise Exception('Sweep with ID %d not found in dataset with ID %d.' % (sweep_id, dataset_id))
break
return sp
def cell_types_api():
endpoint = None
if 'TEST_API_ENDPOINT' in os.environ:
endpoint = os.environ['TEST_API_ENDPOINT']
return CellTypesApi(endpoint)
else:
return None
def __init__(self, cache=True, manifest_file=None, base_uri=None):
if manifest_file is None:
manifest_file = get_default_manifest_file('cell_types')
super(CellTypesCache, self).__init__(
manifest=manifest_file, cache=cache, version=self.MANIFEST_VERSION)
self.api = CellTypesApi(base_uri=base_uri)
class BiophysicalArchiver(object):
def __init__(self, archive_dir=None):
self.bp = BiophysicalApi('http://api.brain-map.org')
self.bp.cache_stimulus = True # change to False to not download the large stimulus NWB file
self.cta = CellTypesApi()
self.rma = RmaApi()
self.neuronal_model_download_endpoint = 'http://celltypes.brain-map.org/neuronal_model/download/'
self.template_names = {}
self.nwb_list = []
if archive_dir == None:
archive_dir = '.'
self.archive_dir = archive_dir
def get_template_names(self):
template_response = self.rma.model_query('NeuronalModelTemplate')
self.template_names = { t['id']: str(t['name']).replace(' ', '_') for t in template_response}
def get_cells(self):
return self.cta.list_cells(True, True)
def get_neuronal_models(self, specimen_ids):
return self.rma.model_query('NeuronalModel',
criteria='specimen[id$in%s]' % ','.join(str(i) for i in specimen_ids),
include='specimen',
num_rows='all')
def get_stimulus_file(self, neuronal_model_id):
result = self.rma.model_query('NeuronalModel',
criteria='[id$eq%d]' % (neuronal_model_id),
include="specimen(ephys_result(well_known_files(well_known_file_type[name$il'NWB*'])))",
tabular=['path'])
stimulus_filename = result[0]['path']
return stimulus_filename
stimulus_filename = os.path.basename(result[0]['path'])
return stimulus_filename
def archive_cell(self, ephys_result_id, specimen_id, template, neuronal_model_id):
url = self.neuronal_model_download_endpoint + "/%d" % (neuronal_model_id)
file_name = os.path.join(self.archive_dir, 'ephys_result_%d_specimen_%d_%s_neuronal_model_%d.zip' % (ephys_result_id,
specimen_id,
template,
neuronal_model_id))
self.rma.retrieve_file_over_http(url, file_name)
nwb_file = self.get_stimulus_file(neuronal_model_id)
shutil.copy(nwb_file, self.archive_dir)
self.nwb_list.append("%s\t%s" % (os.path.basename(nwb_file),
file_name))
def get_observation(dataset_id, kind, cached=True, quiet=False):
"""Get an observation.
Get an observation of kind 'kind' from the dataset with id 'dataset_id'.
optionally using the cached value retrieved previously.
"""
db = shelve.open('aibs-cache') if cached else {}
identifier = '%d_%s' % (dataset_id, kind)
if identifier in db:
print("Getting %s cached data value for from AIBS dataset %s" %
(kind.title(), dataset_id))
value = db[identifier]
else:
print("Getting %s data value for from AIBS dataset %s" %
(kind.title(), dataset_id))
ct = CellTypesApi()
cmd = ct.get_cell(dataset_id) # Cell metadata
if kind == 'rheobase':
if 'ephys_features' in cmd:
value = cmd['ephys_features'][0][
'threshold_i_long_square'] # newer API
else:
value = cmd['ef__threshold_i_long_square'] # older API
value = np.round(value, 2) # Round to nearest hundredth of a pA.
value *= pq.pA # Apply units.
else:
value = cmd[kind]
db[identifier] = value
if cached:
db.close()
return {'value': value}
def test_list_cells_api_mocked(mock_cells_api):
with patch.object(CellTypesApi, "model_query", return_value=mock_cells_api):
ctapi = CellTypesApi()
cells = ctapi.list_cells_api()
assert len(cells) == 2
fcells = ctapi.filter_cells_api(cells, require_reconstruction=True)
assert len(fcells) == 1
fcells = ctapi.filter_cells_api(cells, require_morphology=True)
assert len(fcells) == 1
fcells = ctapi.filter_cells_api(cells, species=['taco'])
assert len(fcells) == 2
fcells = ctapi.filter_cells_api(cells, reporter_status=['fish'])
assert len(fcells) == 1
# Based on https://github.com/stripathy/AIBS_cell_types/blob/master/Allen_ephys_playground.ipynb
'''
This script can be used to download data from the CellTypes Database
'''
from allensdk.api.queries.cell_types_api import CellTypesApi
import os
import sys
ct = CellTypesApi()
from data_helper import CURRENT_DATASETS, DATASET_TARGET_SWEEPS
test = '-test' in sys.argv
dataset_ids = CURRENT_DATASETS
if test:
dataset_ids = [479704527]
sweep_numbers_for_data = DATASET_TARGET_SWEEPS
for dataset_id in dataset_ids:
raw_ephys_file_name = '%d_raw_data.nwb' % dataset_id
if not os.path.isfile(raw_ephys_file_name):
print('Downloading data: %s' % raw_ephys_file_name)
ct.save_ephys_data(dataset_id, raw_ephys_file_name)
print('Saved: %s' % raw_ephys_file_name)
else:
all_experiments = mcc.get_experiments(dataframe=True)
from allensdk.api.queries.biophysical_perisomatic_api import \
BiophysicalPerisomaticApi
from allensdk.api.queries.cell_types_api import CellTypesApi
import allensdk.core.swc as swc
import os
from allensdk.core.nwb_data_set import NwbDataSet
from IPython import __main__
bp = BiophysicalPerisomaticApi('http://api.brain-map.org')
ct = CellTypesApi()
cells = ct.list_cells()
#Below is one way, I can create URL, queries and get the result in a local python dictionary.
import allensdk
import mpi4py as MPI
import numpy as np
import csv
import pickle
import sys
#import glob
def __main__():
print 'main'
def allen_obs_data(ephys_cell=464212183,
sweep_number=33,
A_soma=np.pi * (70. * 1e-4)**2):
"""Data for x_o. Cell from AllenDB
Parameters
----------
ephys_cell : int
Cell identity from AllenDB
sweep_number : int
Stimulus identity for cell ephys_cell from AllenDB
"""
t_offset = 815.
duration = 1450.
dir_cache = './support_files'
real_data_path = dir_cache + '/ephys_cell_{}_sweep_number_{}.pkl'.format(
ephys_cell, sweep_number)
if not os.path.isfile(real_data_path):
from allensdk.core.cell_types_cache import CellTypesCache
from allensdk.api.queries.cell_types_api import CellTypesApi
manifest_file = 'cell_types/manifest.json'
cta = CellTypesApi()
ctc = CellTypesCache(manifest_file=manifest_file)
data_set = ctc.get_ephys_data(ephys_cell)
sweep_data = data_set.get_sweep(
sweep_number) # works with python2 and fails with python3
sweeps = cta.get_ephys_sweeps(ephys_cell)
sweep = sweeps[sweep_number]
index_range = sweep_data["index_range"]
i = sweep_data["stimulus"][0:index_range[1] + 1] # in A
v = sweep_data["response"][0:index_range[1] + 1] # in V
sampling_rate = sweep_data["sampling_rate"] # in Hz
dt = 1e3 / sampling_rate # in ms
i *= 1e6 # to muA
v *= 1e3 # to mV
v = v[int(t_offset / dt):int((t_offset + duration) / dt)]
i = i[int(t_offset / dt):int((t_offset + duration) / dt)]
real_data_obs = np.array(v).reshape(1, -1, 1)
I_real_data = np.array(i).reshape(-1)
t_on = int(
sweep['stimulus_start_time'] * sampling_rate) * dt - t_offset
t_off = int(
(sweep['stimulus_start_time'] + sweep['stimulus_duration']) *
sampling_rate) * dt - t_offset
io.save((real_data_obs, I_real_data, dt, t_on, t_off), real_data_path)
else:
def pickle_load(file):
"""Loads data from file."""
f = open(file, 'rb')
data = pickle.load(f, encoding='latin1')
f.close()
return data
real_data_obs, I_real_data, dt, t_on, t_off = pickle_load(
real_data_path)
t = np.arange(0, duration, dt)
# external current
I = I_real_data / A_soma # muA/cm2
# return real_data_obs, I_obs
return {
'data': real_data_obs.reshape(-1),
'time': t,
'dt': dt,
'I': I.reshape(-1),
't_on': t_on,
't_off': t_off
}
import numpy as np
from allensdk.api.queries.cell_types_api import CellTypesApi
from allensdk import __version__ as allensdk_ver
import time
import sys
import data_helper as DH
from pyelectro import utils
from pyelectro import __version__ as pyel_ver
import pprint
pp = pprint.PrettyPrinter(indent=4)
ct = CellTypesApi()
plot = False # not '-nogui' in sys.argv
test = '-test' in sys.argv
def extract_info_from_nwb_file(dataset_id, raw_ephys_file_name):
info = {}
import h5py
import numpy as np
h5f = h5py.File(raw_ephys_file_name, "r")
metas = [
'aibs_cre_line', 'aibs_dendrite_type',
'intracellular_ephys/Electrode 1/location'
class CellTypesCache(Cache):
"""
Cache class for storing and accessing data from the Cell Types Database.
By default, this class will cache any downloaded metadata or files in
well known locations defined in a manifest file. This behavior can be
disabled.
Attributes
----------
api: CellTypesApi instance
The object used for making API queries related to the Cell Types Database
Parameters
----------
cache: boolean
Whether the class should save results of API queries to locations specified
in the manifest file. Queries for files (as opposed to metadata) must have a
file location. If caching is disabled, those locations must be specified
in the function call (e.g. get_ephys_data(file_name='file.nwb')).
manifest_file: string
File name of the manifest to be read. Default is "cell_types_manifest.json".
"""
# manifest keys
CELLS_KEY = 'CELLS'
EPHYS_FEATURES_KEY = 'EPHYS_FEATURES'
MORPHOLOGY_FEATURES_KEY = 'MORPHOLOGY_FEATURES'
EPHYS_DATA_KEY = 'EPHYS_DATA'
EPHYS_SWEEPS_KEY = 'EPHYS_SWEEPS'
RECONSTRUCTION_KEY = 'RECONSTRUCTION'
MARKER_KEY = 'MARKER'
MANIFEST_VERSION = "1.1"
def __init__(self, cache=True, manifest_file=None, base_uri=None):
if manifest_file is None:
manifest_file = get_default_manifest_file('cell_types')
super(CellTypesCache, self).__init__(
manifest=manifest_file, cache=cache, version=self.MANIFEST_VERSION)
self.api = CellTypesApi(base_uri=base_uri)
def get_cells(self, file_name=None,
require_morphology=False,
require_reconstruction=False,
reporter_status=None,
species=None,
simple=True):
"""
Download metadata for all cells in the database and optionally return a
subset filtered by whether or not they have a morphology or reconstruction.
Parameters
----------
file_name: string
File name to save/read the cell metadata as JSON. If file_name is None,
the file_name will be pulled out of the manifest. If caching
is disabled, no file will be saved. Default is None.
require_morphology: boolean
Filter out cells that have no morphological images.
require_reconstruction: boolean
Filter out cells that have no morphological reconstructions.
reporter_status: list
Filter for cells that have one or more cell reporter statuses.
species: list
Filter for cells that belong to one or more species. If None, return all.
Must be one of [ CellTypesApi.MOUSE, CellTypesApi.HUMAN ].
"""
file_name = self.get_cache_path(file_name, self.CELLS_KEY)
cells = self.api.list_cells_api(path=file_name,
strategy='lazy',
**Cache.cache_json())
if isinstance(reporter_status, string_types):
reporter_status = [reporter_status]
# filter the cells on the way out
cells = self.api.filter_cells_api(cells,
require_morphology,
require_reconstruction,
reporter_status,
species,
simple)
return cells
def get_ephys_sweeps(self, specimen_id, file_name=None):
"""
Download sweep metadata for a single cell specimen.
Parameters
----------
specimen_id: int
ID of a cell.
"""
file_name = self.get_cache_path(
file_name, self.EPHYS_SWEEPS_KEY, specimen_id)
sweeps = self.api.get_ephys_sweeps(specimen_id,
strategy='lazy',
path=file_name,
**Cache.cache_json())
return sweeps
def get_ephys_features(self, dataframe=False, file_name=None):
"""
Download electrophysiology features for all cells in the database.
Parameters
----------
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
dataframe: boolean
Return the output as a Pandas DataFrame. If False, return
a list of dictionaries.
"""
file_name = self.get_cache_path(file_name, self.EPHYS_FEATURES_KEY)
if self.cache:
if dataframe:
warnings.warn("dataframe argument is deprecated.")
args = Cache.cache_csv_dataframe()
else:
args = Cache.cache_csv_json()
args['strategy'] = 'lazy'
else:
args = Cache.nocache_json()
features_df = self.api.get_ephys_features(path=file_name,
**args)
return features_df
def get_morphology_features(self, dataframe=False, file_name=None):
"""
Download morphology features for all cells with reconstructions in the database.
Parameters
----------
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
dataframe: boolean
Return the output as a Pandas DataFrame. If False, return
a list of dictionaries.
"""
file_name = self.get_cache_path(
file_name, self.MORPHOLOGY_FEATURES_KEY)
if self.cache:
if dataframe:
warnings.warn("dataframe argument is deprecated.")
args = Cache.cache_csv_dataframe()
else:
args = Cache.cache_csv_json()
else:
args = Cache.nocache_json()
args['strategy'] = 'lazy'
args['path'] = file_name
return self.api.get_morphology_features(**args)
def get_all_features(self, dataframe=False, require_reconstruction=True):
"""
Download morphology and electrophysiology features for all cells and merge them
into a single table.
Parameters
----------
dataframe: boolean
Return the output as a Pandas DataFrame. If False, return
a list of dictionaries.
require_reconstruction: boolean
Only return ephys and morphology features for cells that have
reconstructions. Default True.
"""
ephys_features = pd.DataFrame(self.get_ephys_features())
morphology_features = pd.DataFrame(self.get_morphology_features())
how = 'inner' if require_reconstruction else 'outer'
all_features = ephys_features.merge(morphology_features,
how=how,
on='specimen_id')
if dataframe:
warnings.warn("dataframe argument is deprecated.")
return all_features
else:
return all_features.to_dict('records')
def get_ephys_data(self, specimen_id, file_name=None):
"""
Download electrophysiology traces for a single cell in the database.
Parameters
----------
specimen_id: int
The ID of a cell specimen to download.
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
Returns
-------
NwbDataSet
A class instance with helper methods for retrieving stimulus
and response traces out of an NWB file.
"""
file_name = self.get_cache_path(
file_name, self.EPHYS_DATA_KEY, specimen_id)
self.api.save_ephys_data(specimen_id, file_name, strategy='lazy')
return NwbDataSet(file_name)
def get_reconstruction(self, specimen_id, file_name=None):
"""
Download and open a reconstruction for a single cell in the database.
Parameters
----------
specimen_id: int
The ID of a cell specimen to download.
file_name: string
File name to save/read the reconstruction SWC.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
Returns
-------
Morphology
A class instance with methods for accessing morphology compartments.
"""
file_name = self.get_cache_path(
file_name, self.RECONSTRUCTION_KEY, specimen_id)
if file_name is None:
raise Exception(
"Please enable caching (CellTypes.cache = True) or specify a save_file_name.")
if not os.path.exists(file_name):
self.api.save_reconstruction(specimen_id, file_name)
return swc.read_swc(file_name)
def get_reconstruction_markers(self, specimen_id, file_name=None):
"""
Download and open a reconstruction marker file for a single cell in the database.
Parameters
----------
specimen_id: int
The ID of a cell specimen to download.
file_name: string
File name to save/read the reconstruction marker.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
Returns
-------
Morphology
A class instance with methods for accessing morphology compartments.
"""
file_name = self.get_cache_path(
file_name, self.MARKER_KEY, specimen_id)
if file_name is None:
raise Exception(
"Please enable caching (CellTypes.cache = True) or specify a save_file_name.")
if not os.path.exists(file_name):
try:
self.api.save_reconstruction_markers(specimen_id, file_name)
except LookupError as e:
logging.warning(e.args)
return []
return swc.read_marker_file(file_name)
def build_manifest(self, file_name):
"""
Construct a manifest for this Cache class and save it in a file.
Parameters
----------
file_name: string
File location to save the manifest.
"""
mb = ManifestBuilder()
mb.set_version(self.MANIFEST_VERSION)
mb.add_path('BASEDIR', '.')
mb.add_path(self.CELLS_KEY, 'cells.json',
typename='file', parent_key='BASEDIR')
mb.add_path(self.EPHYS_DATA_KEY, 'specimen_%d/ephys.nwb',
typename='file', parent_key='BASEDIR')
mb.add_path(self.EPHYS_FEATURES_KEY, 'ephys_features.csv',
typename='file', parent_key='BASEDIR')
mb.add_path(self.MORPHOLOGY_FEATURES_KEY, 'morphology_features.csv',
typename='file', parent_key='BASEDIR')
mb.add_path(self.RECONSTRUCTION_KEY, 'specimen_%d/reconstruction.swc',
typename='file', parent_key='BASEDIR')
mb.add_path(self.MARKER_KEY, 'specimen_%d/reconstruction.marker',
typename='file', parent_key='BASEDIR')
mb.add_path(self.EPHYS_SWEEPS_KEY, 'specimen_%d/ephys_sweeps.json',
typename='file', parent_key='BASEDIR')
mb.write_json_file(file_name)
def __init__(self, cache=True, manifest_file='cell_types_manifest.json', base_uri=None):
super(CellTypesCache, self).__init__(
manifest=manifest_file, cache=cache)
self.api = CellTypesApi(base_uri=base_uri)
import os
from allensdk.api.queries.cell_types_api import CellTypesApi
import allensdk.core.swc as swc
import matplotlib.pyplot as plt
import numpy as np
import math
import pprint
import csv
#output_file = 'sholl_all.csv'
#output_file = 'sholl_apical_dendrite_only.csv'
output_file = 'sholl_basal_dendrite_only.csv'
ct = CellTypesApi()
cells = ct.list_cells(require_reconstruction=True)
print "Cells with reconstructions: ", len(cells)
download_file = False
allow_types = [3]
radii = np.array(range(0, 1000, 10))
radii_square = np.square(radii)
#
# Computation based on line-sphere intersection formula
# https://en.wikipedia.org/wiki/Line%E2%80%93sphere_intersection
#
with open(output_file, 'wb') as csvfile:
cwriter = csv.writer(csvfile, delimiter=',', quotechar='"')
from allensdk.api.queries.cell_types_api import CellTypesApi ct = CellTypesApi() # a list of dictionaries containing metadata for cells with reconstructions cells = ct.list_cells(require_reconstruction=True) # download the electrophysiology data for one cell ct.save_ephys_data(cells[0]['id'], 'example.nwb') # download the reconstruction for the same cell ct.save_reconstruction(cells[0]['id'], 'example.swc')
class CellTypesCache(Cache):
'''
Cache class for storing and accessing data from the Cell Types Database.
By default, this class will cache any downloaded metadata or files in
well known locations defined in a manifest file. This behavior can be
disabled.
Attributes
----------
api: CellTypesApi instance
The object used for making API queries related to the Cell Types Database
Parameters
----------
cache: boolean
Whether the class should save results of API queries to locations specified
in the manifest file. Queries for files (as opposed to metadata) must have a
file location. If caching is disabled, those locations must be specified
in the function call (e.g. get_ephys_data(file_name='file.nwb')).
manifest_file: string
File name of the manifest to be read. Default is "manifest.json".
'''
CELLS_KEY = 'CELLS'
EPHYS_FEATURES_KEY = 'EPHYS_FEATURES'
MORPHOLOGY_FEATURES_KEY = 'MORPHOLOGY_FEATURES'
EPHYS_DATA_KEY = 'EPHYS_DATA'
EPHYS_SWEEPS_KEY = 'EPHYS_SWEEPS'
RECONSTRUCTION_KEY = 'RECONSTRUCTION'
def __init__(self, cache=True, manifest_file='manifest.json'):
super(CellTypesCache, self).__init__(manifest=manifest_file, cache=cache)
self.api = CellTypesApi()
def get_cells(self, file_name=None, require_morphology=False, require_reconstruction=False):
'''
Download metadata for all cells in the database and optionally return a
subset filtered by whether or not they have a morphology or reconstruction.
Parameters
----------
file_name: string
File name to save/read the cell metadata as JSON. If file_name is None,
the file_name will be pulled out of the manifest. If caching
is disabled, no file will be saved. Default is None.
require_morphology: boolean
Filter out cells that have no morphological images.
require_reconstruction: boolean
Filter out cells that have no morphological reconstructions.
'''
file_name = self.get_cache_path(file_name, self.CELLS_KEY)
if os.path.exists(file_name):
cells = json_utilities.read(file_name)
else:
cells = self.api.list_cells(False, False)
if self.cache:
json_utilities.write(file_name, cells)
# filter the cells on the way out
return self.api.filter_cells(cells, require_morphology, require_reconstruction)
def get_ephys_sweeps(self, specimen_id, file_name=None):
'''
Download sweep metadata for a single cell specimen.
Parameters
----------
specimen_id: int
ID of a cell.
'''
file_name = self.get_cache_path(file_name, self.EPHYS_SWEEPS_KEY, specimen_id)
if os.path.exists(file_name):
sweeps = json_utilities.read(file_name)
else:
sweeps = self.api.get_ephys_sweeps(specimen_id)
if self.cache:
json_utilities.write(file_name, sweeps)
return sweeps
def get_ephys_features(self, dataframe=False, file_name=None):
'''
Download electrophysiology features for all cells in the database.
Parameters
----------
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
dataframe: boolean
Return the output as a Pandas DataFrame. If False, return
a list of dictionaries.
'''
file_name = self.get_cache_path(file_name, self.EPHYS_FEATURES_KEY)
if os.path.exists(file_name):
features_df = pd.DataFrame.from_csv(file_name)
else:
features_df = self.api.get_ephys_features(dataframe=True)
if self.cache:
features_df.to_csv(file_name)
if dataframe:
return features_df
else:
return features_df.to_dict('records')
def get_morphology_features(self, dataframe=False, file_name=None):
'''
Download morphology features for all cells with reconstructions in the database.
Parameters
----------
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
dataframe: boolean
Return the output as a Pandas DataFrame. If False, return
a list of dictionaries.
'''
file_name = self.get_cache_path(file_name, self.MORPHOLOGY_FEATURES_KEY)
if os.path.exists(file_name):
features_df = pd.DataFrame.from_csv(file_name)
else:
features_df = self.api.get_morphology_features(dataframe=True)
if self.cache:
features_df.to_csv(file_name)
if dataframe:
return features_df
else:
return features_df.to_dict('records')
def get_all_features(self, dataframe=False, require_reconstruction=True):
'''
Download morphology and electrophysiology features for all cells and merge them
into a single table.
Parameters
----------
dataframe: boolean
Return the output as a Pandas DataFrame. If False, return
a list of dictionaries.
require_reconstruction: boolean
Only return ephys and morphology features for cells that have
reconstructions. Default True.
'''
ephys_features = self.get_ephys_features(dataframe=True)
morphology_features = self.get_morphology_features(dataframe=True)
how = 'inner' if require_reconstruction else 'outer'
all_features = ephys_features.merge(morphology_features,
how=how,
on='specimen_id')
if dataframe:
return all_features
else:
return all_features.to_dict('records')
def get_ephys_data(self, specimen_id, file_name=None):
'''
Download electrophysiology traces for a single cell in the database.
Parameters
----------
specimen_id: int
The ID of a cell specimen to download.
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
Returns
-------
NwbDataSet
A class instance with helper methods for retrieving stimulus
and response traces out of an NWB file.
'''
file_name = self.get_cache_path(file_name, self.EPHYS_DATA_KEY, specimen_id)
if not os.path.exists(file_name):
self.api.save_ephys_data(specimen_id, file_name)
return NwbDataSet(file_name)
def get_reconstruction(self, specimen_id, file_name=None):
'''
Download and open a reconstruction for a single cell in the database.
Parameters
----------
specimen_id: int
The ID of a cell specimen to download.
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
Returns
-------
Morphology
A class instance with methods for accessing morphology compartments.
'''
file_name = self.get_cache_path(file_name, self.RECONSTRUCTION_KEY, specimen_id)
if file_name is None:
raise Exception("Please enable caching (CellTypes.cache = True) or specify a save_file_name.")
if not os.path.exists(file_name):
self.api.save_reconstruction(specimen_id, file_name)
return swc.read_swc(file_name)
def build_manifest(self, file_name):
'''
Construct a manifest for this Cache class and save it in a file.
Parameters
----------
file_name: string
File location to save the manifest.
'''
mb = ManifestBuilder()
mb.add_path('BASEDIR', '.')
mb.add_path(self.CELLS_KEY, 'cells.json', typename='file', parent_key='BASEDIR')
mb.add_path(self.EPHYS_DATA_KEY, 'specimen_%d/ephys.nwb', typename='file', parent_key='BASEDIR')
mb.add_path(self.EPHYS_FEATURES_KEY, 'ephys_features.csv', typename='file', parent_key='BASEDIR')
mb.add_path(self.MORPHOLOGY_FEATURES_KEY, 'morphology_features.csv', typename='file', parent_key='BASEDIR')
mb.add_path(self.RECONSTRUCTION_KEY, 'specimen_%d/reconstruction.swc', typename='file', parent_key='BASEDIR')
mb.add_path(self.EPHYS_SWEEPS_KEY, 'specimen_%d/ephys_sweeps.json', typename='file', parent_key='BASEDIR')
mb.write_json_file(file_name)
import os
from allensdk.api.queries.cell_types_api import CellTypesApi
import allensdk.core.swc as swc
import matplotlib.pyplot as plt
import numpy as np
import math
import pprint
import csv
#output_file = 'sholl_all.csv'
#output_file = 'sholl_apical_dendrite_only.csv'
output_file = 'sholl_basal_dendrite_only.csv'
ct = CellTypesApi()
cells = ct.list_cells(require_reconstruction=True)
print "Cells with reconstructions: ", len(cells)
download_file = False
allow_types = [3]
radii = np.array(range(0,1000,10))
radii_square = np.square( radii )
#
# Computation based on line-sphere intersection formula
# https://en.wikipedia.org/wiki/Line%E2%80%93sphere_intersection
#
with open( output_file, 'wb' ) as csvfile :
cwriter = csv.writer( csvfile, delimiter=',', quotechar='"' )
def __init__(self, cache=True, manifest_file='manifest.json'):
super(CellTypesCache, self).__init__(manifest=manifest_file, cache=cache)
self.api = CellTypesApi()
class CellTypesCache(Cache):
"""
Cache class for storing and accessing data from the Cell Types Database.
By default, this class will cache any downloaded metadata or files in
well known locations defined in a manifest file. This behavior can be
disabled.
Attributes
----------
api: CellTypesApi instance
The object used for making API queries related to the Cell Types Database
Parameters
----------
cache: boolean
Whether the class should save results of API queries to locations specified
in the manifest file. Queries for files (as opposed to metadata) must have a
file location. If caching is disabled, those locations must be specified
in the function call (e.g. get_ephys_data(file_name='file.nwb')).
manifest_file: string
File name of the manifest to be read. Default is "cell_types_manifest.json".
"""
# manifest keys
CELLS_KEY = 'CELLS'
EPHYS_FEATURES_KEY = 'EPHYS_FEATURES'
MORPHOLOGY_FEATURES_KEY = 'MORPHOLOGY_FEATURES'
EPHYS_DATA_KEY = 'EPHYS_DATA'
EPHYS_SWEEPS_KEY = 'EPHYS_SWEEPS'
RECONSTRUCTION_KEY = 'RECONSTRUCTION'
MARKER_KEY = 'MARKER'
MANIFEST_VERSION = "1.1"
def __init__(self, cache=True, manifest_file=None, base_uri=None):
if manifest_file is None:
manifest_file = get_default_manifest_file('cell_types')
super(CellTypesCache, self).__init__(manifest=manifest_file,
cache=cache,
version=self.MANIFEST_VERSION)
self.api = CellTypesApi(base_uri=base_uri)
def get_cells(self,
file_name=None,
require_morphology=False,
require_reconstruction=False,
reporter_status=None,
species=None,
simple=True):
"""
Download metadata for all cells in the database and optionally return a
subset filtered by whether or not they have a morphology or reconstruction.
Parameters
----------
file_name: string
File name to save/read the cell metadata as JSON. If file_name is None,
the file_name will be pulled out of the manifest. If caching
is disabled, no file will be saved. Default is None.
require_morphology: boolean
Filter out cells that have no morphological images.
require_reconstruction: boolean
Filter out cells that have no morphological reconstructions.
reporter_status: list
Filter for cells that have one or more cell reporter statuses.
species: list
Filter for cells that belong to one or more species. If None, return all.
Must be one of [ CellTypesApi.MOUSE, CellTypesApi.HUMAN ].
"""
file_name = self.get_cache_path(file_name, self.CELLS_KEY)
cells = self.api.list_cells_api(path=file_name,
strategy='lazy',
**Cache.cache_json())
if isinstance(reporter_status, string_types):
reporter_status = [reporter_status]
# filter the cells on the way out
cells = self.api.filter_cells_api(cells, require_morphology,
require_reconstruction,
reporter_status, species, simple)
return cells
def get_ephys_sweeps(self, specimen_id, file_name=None):
"""
Download sweep metadata for a single cell specimen.
Parameters
----------
specimen_id: int
ID of a cell.
"""
file_name = self.get_cache_path(file_name, self.EPHYS_SWEEPS_KEY,
specimen_id)
sweeps = self.api.get_ephys_sweeps(specimen_id,
strategy='lazy',
path=file_name,
**Cache.cache_json())
return sweeps
def get_ephys_features(self, dataframe=False, file_name=None):
"""
Download electrophysiology features for all cells in the database.
Parameters
----------
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
dataframe: boolean
Return the output as a Pandas DataFrame. If False, return
a list of dictionaries.
"""
file_name = self.get_cache_path(file_name, self.EPHYS_FEATURES_KEY)
if self.cache:
if dataframe:
warnings.warn("dataframe argument is deprecated.")
args = Cache.cache_csv_dataframe()
else:
args = Cache.cache_csv_json()
args['strategy'] = 'lazy'
else:
args = Cache.nocache_json()
features_df = self.api.get_ephys_features(path=file_name, **args)
return features_df
def get_morphology_features(self, dataframe=False, file_name=None):
"""
Download morphology features for all cells with reconstructions in the database.
Parameters
----------
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
dataframe: boolean
Return the output as a Pandas DataFrame. If False, return
a list of dictionaries.
"""
file_name = self.get_cache_path(file_name,
self.MORPHOLOGY_FEATURES_KEY)
if self.cache:
if dataframe:
warnings.warn("dataframe argument is deprecated.")
args = Cache.cache_csv_dataframe()
else:
args = Cache.cache_csv_json()
else:
args = Cache.nocache_json()
args['strategy'] = 'lazy'
args['path'] = file_name
return self.api.get_morphology_features(**args)
def get_all_features(self, dataframe=False, require_reconstruction=True):
"""
Download morphology and electrophysiology features for all cells and merge them
into a single table.
Parameters
----------
dataframe: boolean
Return the output as a Pandas DataFrame. If False, return
a list of dictionaries.
require_reconstruction: boolean
Only return ephys and morphology features for cells that have
reconstructions. Default True.
"""
ephys_features = pd.DataFrame(self.get_ephys_features())
morphology_features = pd.DataFrame(self.get_morphology_features())
how = 'inner' if require_reconstruction else 'outer'
all_features = ephys_features.merge(morphology_features,
how=how,
on='specimen_id')
if dataframe:
warnings.warn("dataframe argument is deprecated.")
return all_features
else:
return all_features.to_dict('records')
def get_ephys_data(self, specimen_id, file_name=None):
"""
Download electrophysiology traces for a single cell in the database.
Parameters
----------
specimen_id: int
The ID of a cell specimen to download.
file_name: string
File name to save/read the ephys features metadata as CSV.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
Returns
-------
NwbDataSet
A class instance with helper methods for retrieving stimulus
and response traces out of an NWB file.
"""
file_name = self.get_cache_path(file_name, self.EPHYS_DATA_KEY,
specimen_id)
self.api.save_ephys_data(specimen_id, file_name, strategy='lazy')
return NwbDataSet(file_name)
def get_reconstruction(self, specimen_id, file_name=None):
"""
Download and open a reconstruction for a single cell in the database.
Parameters
----------
specimen_id: int
The ID of a cell specimen to download.
file_name: string
File name to save/read the reconstruction SWC.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
Returns
-------
Morphology
A class instance with methods for accessing morphology compartments.
"""
file_name = self.get_cache_path(file_name, self.RECONSTRUCTION_KEY,
specimen_id)
if file_name is None:
raise Exception(
"Please enable caching (CellTypes.cache = True) or specify a save_file_name."
)
if not os.path.exists(file_name):
self.api.save_reconstruction(specimen_id, file_name)
return swc.read_swc(file_name)
def get_reconstruction_markers(self, specimen_id, file_name=None):
"""
Download and open a reconstruction marker file for a single cell in the database.
Parameters
----------
specimen_id: int
The ID of a cell specimen to download.
file_name: string
File name to save/read the reconstruction marker.
If file_name is None, the file_name will be pulled out of the
manifest. If caching is disabled, no file will be saved.
Default is None.
Returns
-------
Morphology
A class instance with methods for accessing morphology compartments.
"""
file_name = self.get_cache_path(file_name, self.MARKER_KEY,
specimen_id)
if file_name is None:
raise Exception(
"Please enable caching (CellTypes.cache = True) or specify a save_file_name."
)
if not os.path.exists(file_name):
try:
self.api.save_reconstruction_markers(specimen_id, file_name)
except LookupError as e:
logging.warning(e.args)
return []
return swc.read_marker_file(file_name)
def build_manifest(self, file_name):
"""
Construct a manifest for this Cache class and save it in a file.
Parameters
----------
file_name: string
File location to save the manifest.
"""
mb = ManifestBuilder()
mb.set_version(self.MANIFEST_VERSION)
mb.add_path('BASEDIR', '.')
mb.add_path(self.CELLS_KEY,
'cells.json',
typename='file',
parent_key='BASEDIR')
mb.add_path(self.EPHYS_DATA_KEY,
'specimen_%d/ephys.nwb',
typename='file',
parent_key='BASEDIR')
mb.add_path(self.EPHYS_FEATURES_KEY,
'ephys_features.csv',
typename='file',
parent_key='BASEDIR')
mb.add_path(self.MORPHOLOGY_FEATURES_KEY,
'morphology_features.csv',
typename='file',
parent_key='BASEDIR')
mb.add_path(self.RECONSTRUCTION_KEY,
'specimen_%d/reconstruction.swc',
typename='file',
parent_key='BASEDIR')
mb.add_path(self.MARKER_KEY,
'specimen_%d/reconstruction.marker',
typename='file',
parent_key='BASEDIR')
mb.add_path(self.EPHYS_SWEEPS_KEY,
'specimen_%d/ephys_sweeps.json',
typename='file',
parent_key='BASEDIR')
mb.write_json_file(file_name)
import pprint
import subprocess
from neuroml import NeuroMLDocument
from neuroml import Network
from neuroml import Population
from neuroml import Location
from neuroml import Instance
from neuroml import IncludeType
import neuroml.writers as writers
pp = pprint.PrettyPrinter(indent=4)
ct = CellTypesApi()
# use the SDK API to get a list of single cells and corresponding metadata
cell_metadata_list = ct.list_cells(require_reconstruction=True)
# convert that list of cells' metadata to a python dictionary for easy access
cell_meta_dict = {}
dataset_ids = []
for i in range(len(cell_metadata_list)):
id = cell_metadata_list[i]['id']
dataset_ids.append(id)
cell_meta_dict[id] = cell_metadata_list[i]
cells = ['467703703', '323452245']
cells = ['467703703', '471141261', '320207387', '471141261', '324493977', '464326095']
from allensdk.api.queries.cell_types_api import CellTypesApi
from allensdk.ephys.extract_cell_features import extract_cell_features
from collections import defaultdict
from allensdk.core.nwb_data_set import NwbDataSet
# pick a cell to analyze
specimen_id = 324257146
nwb_file = 'ephys.nwb'
# download the ephys data and sweep metadata
cta = CellTypesApi()
sweeps = cta.get_ephys_sweeps(specimen_id)
cta.save_ephys_data(specimen_id, nwb_file)
# group the sweeps by stimulus
sweep_numbers = defaultdict(list)
for sweep in sweeps:
sweep_numbers[sweep['stimulus_name']].append(sweep['sweep_number'])
# calculate features
cell_features = extract_cell_features(NwbDataSet(nwb_file),
sweep_numbers['Ramp'],
sweep_numbers['Short Square'],
sweep_numbers['Long Square'])
# Based on https://github.com/stripathy/AIBS_cell_types/blob/master/Allen_ephys_playground.ipynb
from allensdk.api.queries.cell_types_api import CellTypesApi
import os
ct = CellTypesApi()
from data_helper import CURRENT_DATASETS, DATASET_TARGET_SWEEPS
dataset_ids = CURRENT_DATASETS
sweep_numbers_for_data = DATASET_TARGET_SWEEPS
for dataset_id in dataset_ids:
raw_ephys_file_name = '%d_raw_data.nwb' % dataset_id
if not os.path.isfile(raw_ephys_file_name):
print('Downloading data: %s'%raw_ephys_file_name)
ct.save_ephys_data(dataset_id, raw_ephys_file_name)
print('Saved: %s'%raw_ephys_file_name)
else:
print('File: %s already present...'%raw_ephys_file_name)
print('Loading data from: %s'%raw_ephys_file_name)
from allensdk.core.nwb_data_set import NwbDataSet
data_set = NwbDataSet(raw_ephys_file_name)
"""
Single sweep detection
=================================
Detect spikes for a single sweep
"""
import os
import matplotlib.pyplot as plt
from allensdk.api.queries.cell_types_api import CellTypesApi
from ipfx.aibs_data_set import AibsDataSet
from ipfx.ephys_extractor import SpikeExtractor
# Download and access the experimental data
ct = CellTypesApi()
specimen_id = 595570553
nwb_file = "%d.nwb" % specimen_id
sweep_info = ct.get_ephys_sweeps(specimen_id)
if not os.path.exists(nwb_file):
ct.save_ephys_data(specimen_id, nwb_file)
# Get the data for the sweep into a format we can use
dataset = AibsDataSet(sweep_info=sweep_info, nwb_file=nwb_file)
sweep_number = 39
sweep = dataset.sweep(sweep_number)
# Extract information about the spikes
ext = SpikeExtractor()
results = ext.process(t=sweep.t, v=sweep.v, i=sweep.i)
from allensdk.api.queries.cell_types_api import CellTypesApi
from allensdk.ephys.extract_cell_features import extract_cell_features
from collections import defaultdict
# pick a cell to analyze
specimen_id = 324257146
nwb_file = 'ephys.nwb'
# download the ephys data and sweep metadata
cta = CellTypesApi()
sweeps = cta.get_ephys_sweeps(specimen_id)
cta.save_ephys_data(specimen_id, nwb_file)
# group the sweeps by stimulus
sweep_numbers = defaultdict(list)
for sweep in sweeps:
sweep_numbers[sweep['stimulus_name']].append(sweep['sweep_number'])
# calculate features
cell_features = extract_cell_features(nwb_file,
sweep_numbers['Long Square'],
sweep_numbers['Short Square'],
sweep_numbers['Ramp'])
import pprint
import subprocess
from neuroml import NeuroMLDocument
from neuroml import Network
from neuroml import Population
from neuroml import Location
from neuroml import Instance
from neuroml import IncludeType
import neuroml.writers as writers
pp = pprint.PrettyPrinter(indent=4)
ct = CellTypesApi()
# use the SDK API to get a list of single cells and corresponding metadata
cell_metadata_list = ct.list_cells(require_reconstruction=True)
# convert that list of cells' metadata to a python dictionary for easy access
cell_meta_dict = {}
dataset_ids = []
for i in range(len(cell_metadata_list)):
id = cell_metadata_list[i]['id']
dataset_ids.append(id)
cell_meta_dict[id] = cell_metadata_list[i]
cells = ['467703703', '323452245']
cells = [
def download_glif_models(cell_ids,
base_dir,
incl_ephys=True,
force_overwrite=False):
"""Goes through the list of cell_ids and downloads cell config and ephys data in base_dir/cell_<ID>. Then looks up
all possible models and downloads model files int base_dir/cell_<ID>/<MODEL_TYPE>_<MODEL_ID>/
"""
# Determine the best url for connecting to cell-types db
try:
# see if we can connect to interal cell-types db
request = requests.get('http://icelltypes/')
if request.status_code == 200:
base_uri = 'http://icelltypes/'
else:
base_uri = None
except Exception:
base_uri = None # use the default url
base_dir = base_dir if base_dir.endswith('/') else base_dir + '/'
valid_cells = []
ct_api = CellTypesApi(base_uri)
for cell in ct_api.list_cells():
if cell['id'] in cell_ids:
# create directory for cell
cell_home = '{}cell_{}/'.format(base_dir, cell['id'])
if not os.path.exists(cell_home):
os.makedirs(cell_home)
# save metadata
cell_metadata_file = cell_home + 'cell_metadata.json'
if force_overwrite or not os.path.exists(cell_metadata_file):
print('Saving metadata for cell {} in {}'.format(
cell['id'], cell_metadata_file))
json_utilities.write(cell_metadata_file, cell)
else:
print('File {} already exists. Skipping'.format(
cell_metadata_file))
# save ephys data
if incl_ephys:
cell_ephys_file = cell_home + 'ephys_data.nwb'
if force_overwrite or not os.path.exists(cell_ephys_file):
print('Saving ephys data for cell {} in {}'.format(
cell['id'], cell_ephys_file))
ct_api.save_ephys_data(cell['id'], cell_ephys_file)
else:
print('File {} already exists. Skipping'.format(
cell_ephys_file))
# save sweeps file
sweeps_file = cell_home + 'ephys_sweeps.json'
if force_overwrite or not os.path.exists(sweeps_file):
print('- Saving sweeps file to {}'.format(sweeps_file))
ephys_sweeps = ct_api.get_ephys_sweeps(cell['id'])
json_utilities.write(sweeps_file, ephys_sweeps)
else:
print('- File {} already exits. Skipping.'.format(sweeps_file))
# keep track of valid ids
valid_cells.append(cell['id'])
cell_ids.remove(cell['id'])
for cid in cell_ids:
print('Warning: cell #{} was not found in cell-types database'.format(
cid))
# Iterate through each all available models and find ones correspoding to cell list
glif_models = {} # map model-id to their directory
glif_api = GlifApi(base_uri=base_uri)
for model in glif_api.list_neuronal_models():
if model['specimen_id'] in valid_cells:
# save model files <BASE_DIR>/cell_<CELL_ID>/<MODEL_TYPE>_<MODEL_ID>/
cell_id = model['specimen_id']
model_id = model['id']
model_type = model[
'neuronal_model_template_id'] #['id'] # type of model, GLIF-LIF, GLIF-ASC, etc
type_name = model_id2name.get(model_type, None)
if type_name is None:
print(
'Warning: Unknown model type {} ({}) for cell/model {}/{}'.
format(model_type,
model['neuronal_model_template']['name'], cell_id,
model_id))
type_name = model_type
model_home_dir = '{}cell_{}/{}_{}/'.format(base_dir, cell_id,
type_name, model_id)
glif_models[model_id] = model_home_dir
# go through all the found models, download necessary files
n_models = len(glif_models)
for i, (gid, home_dir) in enumerate(glif_models.iteritems()):
print('Processing model {} ({} of {})'.format(gid, (i + 1), n_models))
model_metadata = glif_api.get_neuronal_model(gid)
if not os.path.exists(home_dir):
os.makedirs(home_dir)
# save model metadata
metadata_file = home_dir + 'metadata.json'
if force_overwrite or not os.path.exists(metadata_file):
print('- Saving metadata file to {}'.format(metadata_file))
#print type(metadata_file)
with open(metadata_file, 'wb') as fp:
json.dump(model_metadata, fp, indent=2)
else:
print('- File {} already exits. Skipping.'.format(metadata_file))
# get neuron configuration file
config_file = home_dir + 'config.json'
if force_overwrite or not os.path.exists(config_file):
print('- Saving configuration file to {}'.format(config_file))
neuron_config = glif_api.get_neuron_config()
json_utilities.write(config_file, neuron_config)
else:
print('- File {} already exits. Skipping.'.format(config_file))