def get_sweep_params(dataset_id,sweep_id): ct = CellTypesApi() experiment_params = ct.get_ephys_sweeps(dataset_id) for sp in experiment_params: if sp['id']==sweep_id: sweep_num = sp['sweep_number'] break if sweep_num is None: raise Exception('Sweep with ID %d not found in dataset with ID %d.' % (sweep_id, dataset_id)) return sp
def extract_data(ephys_cell, sweep_number): """Data extraction from AllenDB Parameters ---------- ephys_cell : int Cell identity from AllenDB sweep_number : int Stimulus identity for cell ephys_cell from AllenDB """ t_offset = 815. duration = 1450. real_data_path = 'ephys_cell_{}_sweep_number_{}.pkl'.format( ephys_cell, sweep_number) if not os.path.isfile(real_data_path): from allensdk.core.cell_types_cache import CellTypesCache from allensdk.api.queries.cell_types_api import CellTypesApi manifest_file = 'cell_types/manifest.json' cta = CellTypesApi() ctc = CellTypesCache(manifest_file=manifest_file) data_set = ctc.get_ephys_data(ephys_cell) sweep_data = data_set.get_sweep( sweep_number) # works with python2 and fails with python3 sweeps = cta.get_ephys_sweeps(ephys_cell) sweep = sweeps[sweep_number] index_range = sweep_data["index_range"] i = sweep_data["stimulus"][0:index_range[1] + 1] # in A v = sweep_data["response"][0:index_range[1] + 1] # in V sampling_rate = sweep_data["sampling_rate"] # in Hz dt = 1e3 / sampling_rate # in ms i *= 1e6 # to muA v *= 1e3 # to mV v = v[int(t_offset / dt):int((t_offset + duration) / dt)] i = i[int(t_offset / dt):int((t_offset + duration) / dt)] real_data_obs = np.array(v).reshape(1, -1, 1) I_real_data = np.array(i).reshape(-1) t_on = int( sweep['stimulus_start_time'] * sampling_rate) * dt - t_offset t_off = int( (sweep['stimulus_start_time'] + sweep['stimulus_duration']) * sampling_rate) * dt - t_offset f = open(real_data_path, 'wb') pickle.dump((real_data_obs, I_real_data, dt, t_on, t_off), f) f.close()
def get_sweep_params(dataset_id, sweep_id): """ Gets sweep parameters corresponding to the sweep with id 'sweep_id' from the dataset with id 'dataset_id'. """ ct = CellTypesApi() experiment_params = ct.get_ephys_sweeps(dataset_id) sp = None for sp in experiment_params: if sp['id']==sweep_id: sweep_num = sp['sweep_number'] if sweep_num is None: raise Exception('Sweep with ID %d not found in dataset with ID %d.' % (sweep_id, dataset_id)) break return sp
class CellTypesCache(Cache): """ Cache class for storing and accessing data from the Cell Types Database. By default, this class will cache any downloaded metadata or files in well known locations defined in a manifest file. This behavior can be disabled. Attributes ---------- api: CellTypesApi instance The object used for making API queries related to the Cell Types Database Parameters ---------- cache: boolean Whether the class should save results of API queries to locations specified in the manifest file. Queries for files (as opposed to metadata) must have a file location. If caching is disabled, those locations must be specified in the function call (e.g. get_ephys_data(file_name='file.nwb')). manifest_file: string File name of the manifest to be read. Default is "cell_types_manifest.json". """ # manifest keys CELLS_KEY = 'CELLS' EPHYS_FEATURES_KEY = 'EPHYS_FEATURES' MORPHOLOGY_FEATURES_KEY = 'MORPHOLOGY_FEATURES' EPHYS_DATA_KEY = 'EPHYS_DATA' EPHYS_SWEEPS_KEY = 'EPHYS_SWEEPS' RECONSTRUCTION_KEY = 'RECONSTRUCTION' MARKER_KEY = 'MARKER' MANIFEST_VERSION = "1.1" def __init__(self, cache=True, manifest_file=None, base_uri=None): if manifest_file is None: manifest_file = get_default_manifest_file('cell_types') super(CellTypesCache, self).__init__( manifest=manifest_file, cache=cache, version=self.MANIFEST_VERSION) self.api = CellTypesApi(base_uri=base_uri) def get_cells(self, file_name=None, require_morphology=False, require_reconstruction=False, reporter_status=None, species=None, simple=True): """ Download metadata for all cells in the database and optionally return a subset filtered by whether or not they have a morphology or reconstruction. Parameters ---------- file_name: string File name to save/read the cell metadata as JSON. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. require_morphology: boolean Filter out cells that have no morphological images. require_reconstruction: boolean Filter out cells that have no morphological reconstructions. reporter_status: list Filter for cells that have one or more cell reporter statuses. species: list Filter for cells that belong to one or more species. If None, return all. Must be one of [ CellTypesApi.MOUSE, CellTypesApi.HUMAN ]. """ file_name = self.get_cache_path(file_name, self.CELLS_KEY) cells = self.api.list_cells_api(path=file_name, strategy='lazy', **Cache.cache_json()) if isinstance(reporter_status, string_types): reporter_status = [reporter_status] # filter the cells on the way out cells = self.api.filter_cells_api(cells, require_morphology, require_reconstruction, reporter_status, species, simple) return cells def get_ephys_sweeps(self, specimen_id, file_name=None): """ Download sweep metadata for a single cell specimen. Parameters ---------- specimen_id: int ID of a cell. """ file_name = self.get_cache_path( file_name, self.EPHYS_SWEEPS_KEY, specimen_id) sweeps = self.api.get_ephys_sweeps(specimen_id, strategy='lazy', path=file_name, **Cache.cache_json()) return sweeps def get_ephys_features(self, dataframe=False, file_name=None): """ Download electrophysiology features for all cells in the database. Parameters ---------- file_name: string File name to save/read the ephys features metadata as CSV. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. dataframe: boolean Return the output as a Pandas DataFrame. If False, return a list of dictionaries. """ file_name = self.get_cache_path(file_name, self.EPHYS_FEATURES_KEY) if self.cache: if dataframe: warnings.warn("dataframe argument is deprecated.") args = Cache.cache_csv_dataframe() else: args = Cache.cache_csv_json() args['strategy'] = 'lazy' else: args = Cache.nocache_json() features_df = self.api.get_ephys_features(path=file_name, **args) return features_df def get_morphology_features(self, dataframe=False, file_name=None): """ Download morphology features for all cells with reconstructions in the database. Parameters ---------- file_name: string File name to save/read the ephys features metadata as CSV. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. dataframe: boolean Return the output as a Pandas DataFrame. If False, return a list of dictionaries. """ file_name = self.get_cache_path( file_name, self.MORPHOLOGY_FEATURES_KEY) if self.cache: if dataframe: warnings.warn("dataframe argument is deprecated.") args = Cache.cache_csv_dataframe() else: args = Cache.cache_csv_json() else: args = Cache.nocache_json() args['strategy'] = 'lazy' args['path'] = file_name return self.api.get_morphology_features(**args) def get_all_features(self, dataframe=False, require_reconstruction=True): """ Download morphology and electrophysiology features for all cells and merge them into a single table. Parameters ---------- dataframe: boolean Return the output as a Pandas DataFrame. If False, return a list of dictionaries. require_reconstruction: boolean Only return ephys and morphology features for cells that have reconstructions. Default True. """ ephys_features = pd.DataFrame(self.get_ephys_features()) morphology_features = pd.DataFrame(self.get_morphology_features()) how = 'inner' if require_reconstruction else 'outer' all_features = ephys_features.merge(morphology_features, how=how, on='specimen_id') if dataframe: warnings.warn("dataframe argument is deprecated.") return all_features else: return all_features.to_dict('records') def get_ephys_data(self, specimen_id, file_name=None): """ Download electrophysiology traces for a single cell in the database. Parameters ---------- specimen_id: int The ID of a cell specimen to download. file_name: string File name to save/read the ephys features metadata as CSV. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. Returns ------- NwbDataSet A class instance with helper methods for retrieving stimulus and response traces out of an NWB file. """ file_name = self.get_cache_path( file_name, self.EPHYS_DATA_KEY, specimen_id) self.api.save_ephys_data(specimen_id, file_name, strategy='lazy') return NwbDataSet(file_name) def get_reconstruction(self, specimen_id, file_name=None): """ Download and open a reconstruction for a single cell in the database. Parameters ---------- specimen_id: int The ID of a cell specimen to download. file_name: string File name to save/read the reconstruction SWC. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. Returns ------- Morphology A class instance with methods for accessing morphology compartments. """ file_name = self.get_cache_path( file_name, self.RECONSTRUCTION_KEY, specimen_id) if file_name is None: raise Exception( "Please enable caching (CellTypes.cache = True) or specify a save_file_name.") if not os.path.exists(file_name): self.api.save_reconstruction(specimen_id, file_name) return swc.read_swc(file_name) def get_reconstruction_markers(self, specimen_id, file_name=None): """ Download and open a reconstruction marker file for a single cell in the database. Parameters ---------- specimen_id: int The ID of a cell specimen to download. file_name: string File name to save/read the reconstruction marker. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. Returns ------- Morphology A class instance with methods for accessing morphology compartments. """ file_name = self.get_cache_path( file_name, self.MARKER_KEY, specimen_id) if file_name is None: raise Exception( "Please enable caching (CellTypes.cache = True) or specify a save_file_name.") if not os.path.exists(file_name): try: self.api.save_reconstruction_markers(specimen_id, file_name) except LookupError as e: logging.warning(e.args) return [] return swc.read_marker_file(file_name) def build_manifest(self, file_name): """ Construct a manifest for this Cache class and save it in a file. Parameters ---------- file_name: string File location to save the manifest. """ mb = ManifestBuilder() mb.set_version(self.MANIFEST_VERSION) mb.add_path('BASEDIR', '.') mb.add_path(self.CELLS_KEY, 'cells.json', typename='file', parent_key='BASEDIR') mb.add_path(self.EPHYS_DATA_KEY, 'specimen_%d/ephys.nwb', typename='file', parent_key='BASEDIR') mb.add_path(self.EPHYS_FEATURES_KEY, 'ephys_features.csv', typename='file', parent_key='BASEDIR') mb.add_path(self.MORPHOLOGY_FEATURES_KEY, 'morphology_features.csv', typename='file', parent_key='BASEDIR') mb.add_path(self.RECONSTRUCTION_KEY, 'specimen_%d/reconstruction.swc', typename='file', parent_key='BASEDIR') mb.add_path(self.MARKER_KEY, 'specimen_%d/reconstruction.marker', typename='file', parent_key='BASEDIR') mb.add_path(self.EPHYS_SWEEPS_KEY, 'specimen_%d/ephys_sweeps.json', typename='file', parent_key='BASEDIR') mb.write_json_file(file_name)
Detect spikes for a single sweep """ import os import matplotlib.pyplot as plt from allensdk.api.queries.cell_types_api import CellTypesApi from ipfx.aibs_data_set import AibsDataSet from ipfx.ephys_extractor import SpikeExtractor # Download and access the experimental data ct = CellTypesApi() specimen_id = 595570553 nwb_file = "%d.nwb" % specimen_id sweep_info = ct.get_ephys_sweeps(specimen_id) if not os.path.exists(nwb_file): ct.save_ephys_data(specimen_id, nwb_file) # Get the data for the sweep into a format we can use dataset = AibsDataSet(sweep_info=sweep_info, nwb_file=nwb_file) sweep_number = 39 sweep = dataset.sweep(sweep_number) # Extract information about the spikes ext = SpikeExtractor() results = ext.process(t=sweep.t, v=sweep.v, i=sweep.i) # Plot the results, showing two features of the detected spikes plt.plot(sweep.t, sweep.v)
class CellTypesCache(Cache): """ Cache class for storing and accessing data from the Cell Types Database. By default, this class will cache any downloaded metadata or files in well known locations defined in a manifest file. This behavior can be disabled. Attributes ---------- api: CellTypesApi instance The object used for making API queries related to the Cell Types Database Parameters ---------- cache: boolean Whether the class should save results of API queries to locations specified in the manifest file. Queries for files (as opposed to metadata) must have a file location. If caching is disabled, those locations must be specified in the function call (e.g. get_ephys_data(file_name='file.nwb')). manifest_file: string File name of the manifest to be read. Default is "cell_types_manifest.json". """ # manifest keys CELLS_KEY = 'CELLS' EPHYS_FEATURES_KEY = 'EPHYS_FEATURES' MORPHOLOGY_FEATURES_KEY = 'MORPHOLOGY_FEATURES' EPHYS_DATA_KEY = 'EPHYS_DATA' EPHYS_SWEEPS_KEY = 'EPHYS_SWEEPS' RECONSTRUCTION_KEY = 'RECONSTRUCTION' MARKER_KEY = 'MARKER' MANIFEST_VERSION = "1.1" def __init__(self, cache=True, manifest_file=None, base_uri=None): if manifest_file is None: manifest_file = get_default_manifest_file('cell_types') super(CellTypesCache, self).__init__(manifest=manifest_file, cache=cache, version=self.MANIFEST_VERSION) self.api = CellTypesApi(base_uri=base_uri) def get_cells(self, file_name=None, require_morphology=False, require_reconstruction=False, reporter_status=None, species=None, simple=True): """ Download metadata for all cells in the database and optionally return a subset filtered by whether or not they have a morphology or reconstruction. Parameters ---------- file_name: string File name to save/read the cell metadata as JSON. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. require_morphology: boolean Filter out cells that have no morphological images. require_reconstruction: boolean Filter out cells that have no morphological reconstructions. reporter_status: list Filter for cells that have one or more cell reporter statuses. species: list Filter for cells that belong to one or more species. If None, return all. Must be one of [ CellTypesApi.MOUSE, CellTypesApi.HUMAN ]. """ file_name = self.get_cache_path(file_name, self.CELLS_KEY) cells = self.api.list_cells_api(path=file_name, strategy='lazy', **Cache.cache_json()) if isinstance(reporter_status, string_types): reporter_status = [reporter_status] # filter the cells on the way out cells = self.api.filter_cells_api(cells, require_morphology, require_reconstruction, reporter_status, species, simple) return cells def get_ephys_sweeps(self, specimen_id, file_name=None): """ Download sweep metadata for a single cell specimen. Parameters ---------- specimen_id: int ID of a cell. """ file_name = self.get_cache_path(file_name, self.EPHYS_SWEEPS_KEY, specimen_id) sweeps = self.api.get_ephys_sweeps(specimen_id, strategy='lazy', path=file_name, **Cache.cache_json()) return sweeps def get_ephys_features(self, dataframe=False, file_name=None): """ Download electrophysiology features for all cells in the database. Parameters ---------- file_name: string File name to save/read the ephys features metadata as CSV. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. dataframe: boolean Return the output as a Pandas DataFrame. If False, return a list of dictionaries. """ file_name = self.get_cache_path(file_name, self.EPHYS_FEATURES_KEY) if self.cache: if dataframe: warnings.warn("dataframe argument is deprecated.") args = Cache.cache_csv_dataframe() else: args = Cache.cache_csv_json() args['strategy'] = 'lazy' else: args = Cache.nocache_json() features_df = self.api.get_ephys_features(path=file_name, **args) return features_df def get_morphology_features(self, dataframe=False, file_name=None): """ Download morphology features for all cells with reconstructions in the database. Parameters ---------- file_name: string File name to save/read the ephys features metadata as CSV. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. dataframe: boolean Return the output as a Pandas DataFrame. If False, return a list of dictionaries. """ file_name = self.get_cache_path(file_name, self.MORPHOLOGY_FEATURES_KEY) if self.cache: if dataframe: warnings.warn("dataframe argument is deprecated.") args = Cache.cache_csv_dataframe() else: args = Cache.cache_csv_json() else: args = Cache.nocache_json() args['strategy'] = 'lazy' args['path'] = file_name return self.api.get_morphology_features(**args) def get_all_features(self, dataframe=False, require_reconstruction=True): """ Download morphology and electrophysiology features for all cells and merge them into a single table. Parameters ---------- dataframe: boolean Return the output as a Pandas DataFrame. If False, return a list of dictionaries. require_reconstruction: boolean Only return ephys and morphology features for cells that have reconstructions. Default True. """ ephys_features = pd.DataFrame(self.get_ephys_features()) morphology_features = pd.DataFrame(self.get_morphology_features()) how = 'inner' if require_reconstruction else 'outer' all_features = ephys_features.merge(morphology_features, how=how, on='specimen_id') if dataframe: warnings.warn("dataframe argument is deprecated.") return all_features else: return all_features.to_dict('records') def get_ephys_data(self, specimen_id, file_name=None): """ Download electrophysiology traces for a single cell in the database. Parameters ---------- specimen_id: int The ID of a cell specimen to download. file_name: string File name to save/read the ephys features metadata as CSV. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. Returns ------- NwbDataSet A class instance with helper methods for retrieving stimulus and response traces out of an NWB file. """ file_name = self.get_cache_path(file_name, self.EPHYS_DATA_KEY, specimen_id) self.api.save_ephys_data(specimen_id, file_name, strategy='lazy') return NwbDataSet(file_name) def get_reconstruction(self, specimen_id, file_name=None): """ Download and open a reconstruction for a single cell in the database. Parameters ---------- specimen_id: int The ID of a cell specimen to download. file_name: string File name to save/read the reconstruction SWC. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. Returns ------- Morphology A class instance with methods for accessing morphology compartments. """ file_name = self.get_cache_path(file_name, self.RECONSTRUCTION_KEY, specimen_id) if file_name is None: raise Exception( "Please enable caching (CellTypes.cache = True) or specify a save_file_name." ) if not os.path.exists(file_name): self.api.save_reconstruction(specimen_id, file_name) return swc.read_swc(file_name) def get_reconstruction_markers(self, specimen_id, file_name=None): """ Download and open a reconstruction marker file for a single cell in the database. Parameters ---------- specimen_id: int The ID of a cell specimen to download. file_name: string File name to save/read the reconstruction marker. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. Returns ------- Morphology A class instance with methods for accessing morphology compartments. """ file_name = self.get_cache_path(file_name, self.MARKER_KEY, specimen_id) if file_name is None: raise Exception( "Please enable caching (CellTypes.cache = True) or specify a save_file_name." ) if not os.path.exists(file_name): try: self.api.save_reconstruction_markers(specimen_id, file_name) except LookupError as e: logging.warning(e.args) return [] return swc.read_marker_file(file_name) def build_manifest(self, file_name): """ Construct a manifest for this Cache class and save it in a file. Parameters ---------- file_name: string File location to save the manifest. """ mb = ManifestBuilder() mb.set_version(self.MANIFEST_VERSION) mb.add_path('BASEDIR', '.') mb.add_path(self.CELLS_KEY, 'cells.json', typename='file', parent_key='BASEDIR') mb.add_path(self.EPHYS_DATA_KEY, 'specimen_%d/ephys.nwb', typename='file', parent_key='BASEDIR') mb.add_path(self.EPHYS_FEATURES_KEY, 'ephys_features.csv', typename='file', parent_key='BASEDIR') mb.add_path(self.MORPHOLOGY_FEATURES_KEY, 'morphology_features.csv', typename='file', parent_key='BASEDIR') mb.add_path(self.RECONSTRUCTION_KEY, 'specimen_%d/reconstruction.swc', typename='file', parent_key='BASEDIR') mb.add_path(self.MARKER_KEY, 'specimen_%d/reconstruction.marker', typename='file', parent_key='BASEDIR') mb.add_path(self.EPHYS_SWEEPS_KEY, 'specimen_%d/ephys_sweeps.json', typename='file', parent_key='BASEDIR') mb.write_json_file(file_name)
def allen_obs_data(ephys_cell=464212183, sweep_number=33, A_soma=np.pi * (70. * 1e-4)**2): """Data for x_o. Cell from AllenDB Parameters ---------- ephys_cell : int Cell identity from AllenDB sweep_number : int Stimulus identity for cell ephys_cell from AllenDB """ t_offset = 815. duration = 1450. dir_cache = './support_files' real_data_path = dir_cache + '/ephys_cell_{}_sweep_number_{}.pkl'.format( ephys_cell, sweep_number) if not os.path.isfile(real_data_path): from allensdk.core.cell_types_cache import CellTypesCache from allensdk.api.queries.cell_types_api import CellTypesApi manifest_file = 'cell_types/manifest.json' cta = CellTypesApi() ctc = CellTypesCache(manifest_file=manifest_file) data_set = ctc.get_ephys_data(ephys_cell) sweep_data = data_set.get_sweep( sweep_number) # works with python2 and fails with python3 sweeps = cta.get_ephys_sweeps(ephys_cell) sweep = sweeps[sweep_number] index_range = sweep_data["index_range"] i = sweep_data["stimulus"][0:index_range[1] + 1] # in A v = sweep_data["response"][0:index_range[1] + 1] # in V sampling_rate = sweep_data["sampling_rate"] # in Hz dt = 1e3 / sampling_rate # in ms i *= 1e6 # to muA v *= 1e3 # to mV v = v[int(t_offset / dt):int((t_offset + duration) / dt)] i = i[int(t_offset / dt):int((t_offset + duration) / dt)] real_data_obs = np.array(v).reshape(1, -1, 1) I_real_data = np.array(i).reshape(-1) t_on = int( sweep['stimulus_start_time'] * sampling_rate) * dt - t_offset t_off = int( (sweep['stimulus_start_time'] + sweep['stimulus_duration']) * sampling_rate) * dt - t_offset io.save((real_data_obs, I_real_data, dt, t_on, t_off), real_data_path) else: def pickle_load(file): """Loads data from file.""" f = open(file, 'rb') data = pickle.load(f, encoding='latin1') f.close() return data real_data_obs, I_real_data, dt, t_on, t_off = pickle_load( real_data_path) t = np.arange(0, duration, dt) # external current I = I_real_data / A_soma # muA/cm2 # return real_data_obs, I_obs return { 'data': real_data_obs.reshape(-1), 'time': t, 'dt': dt, 'I': I.reshape(-1), 't_on': t_on, 't_off': t_off }
from allensdk.api.queries.cell_types_api import CellTypesApi from allensdk.ephys.extract_cell_features import extract_cell_features from collections import defaultdict # pick a cell to analyze specimen_id = 324257146 nwb_file = 'ephys.nwb' # download the ephys data and sweep metadata cta = CellTypesApi() sweeps = cta.get_ephys_sweeps(specimen_id) cta.save_ephys_data(specimen_id, nwb_file) # group the sweeps by stimulus sweep_numbers = defaultdict(list) for sweep in sweeps: sweep_numbers[sweep['stimulus_name']].append(sweep['sweep_number']) # calculate features cell_features = extract_cell_features(nwb_file, sweep_numbers['Long Square'], sweep_numbers['Short Square'], sweep_numbers['Ramp'])
def download_glif_models(cell_ids, base_dir, incl_ephys=True, force_overwrite=False): """Goes through the list of cell_ids and downloads cell config and ephys data in base_dir/cell_<ID>. Then looks up all possible models and downloads model files int base_dir/cell_<ID>/<MODEL_TYPE>_<MODEL_ID>/ """ # Determine the best url for connecting to cell-types db try: # see if we can connect to interal cell-types db request = requests.get('http://icelltypes/') if request.status_code == 200: base_uri = 'http://icelltypes/' else: base_uri = None except Exception: base_uri = None # use the default url base_dir = base_dir if base_dir.endswith('/') else base_dir + '/' valid_cells = [] ct_api = CellTypesApi(base_uri) for cell in ct_api.list_cells(): if cell['id'] in cell_ids: # create directory for cell cell_home = '{}cell_{}/'.format(base_dir, cell['id']) if not os.path.exists(cell_home): os.makedirs(cell_home) # save metadata cell_metadata_file = cell_home + 'cell_metadata.json' if force_overwrite or not os.path.exists(cell_metadata_file): print('Saving metadata for cell {} in {}'.format( cell['id'], cell_metadata_file)) json_utilities.write(cell_metadata_file, cell) else: print('File {} already exists. Skipping'.format( cell_metadata_file)) # save ephys data if incl_ephys: cell_ephys_file = cell_home + 'ephys_data.nwb' if force_overwrite or not os.path.exists(cell_ephys_file): print('Saving ephys data for cell {} in {}'.format( cell['id'], cell_ephys_file)) ct_api.save_ephys_data(cell['id'], cell_ephys_file) else: print('File {} already exists. Skipping'.format( cell_ephys_file)) # save sweeps file sweeps_file = cell_home + 'ephys_sweeps.json' if force_overwrite or not os.path.exists(sweeps_file): print('- Saving sweeps file to {}'.format(sweeps_file)) ephys_sweeps = ct_api.get_ephys_sweeps(cell['id']) json_utilities.write(sweeps_file, ephys_sweeps) else: print('- File {} already exits. Skipping.'.format(sweeps_file)) # keep track of valid ids valid_cells.append(cell['id']) cell_ids.remove(cell['id']) for cid in cell_ids: print('Warning: cell #{} was not found in cell-types database'.format( cid)) # Iterate through each all available models and find ones correspoding to cell list glif_models = {} # map model-id to their directory glif_api = GlifApi(base_uri=base_uri) for model in glif_api.list_neuronal_models(): if model['specimen_id'] in valid_cells: # save model files <BASE_DIR>/cell_<CELL_ID>/<MODEL_TYPE>_<MODEL_ID>/ cell_id = model['specimen_id'] model_id = model['id'] model_type = model[ 'neuronal_model_template_id'] #['id'] # type of model, GLIF-LIF, GLIF-ASC, etc type_name = model_id2name.get(model_type, None) if type_name is None: print( 'Warning: Unknown model type {} ({}) for cell/model {}/{}'. format(model_type, model['neuronal_model_template']['name'], cell_id, model_id)) type_name = model_type model_home_dir = '{}cell_{}/{}_{}/'.format(base_dir, cell_id, type_name, model_id) glif_models[model_id] = model_home_dir # go through all the found models, download necessary files n_models = len(glif_models) for i, (gid, home_dir) in enumerate(glif_models.iteritems()): print('Processing model {} ({} of {})'.format(gid, (i + 1), n_models)) model_metadata = glif_api.get_neuronal_model(gid) if not os.path.exists(home_dir): os.makedirs(home_dir) # save model metadata metadata_file = home_dir + 'metadata.json' if force_overwrite or not os.path.exists(metadata_file): print('- Saving metadata file to {}'.format(metadata_file)) #print type(metadata_file) with open(metadata_file, 'wb') as fp: json.dump(model_metadata, fp, indent=2) else: print('- File {} already exits. Skipping.'.format(metadata_file)) # get neuron configuration file config_file = home_dir + 'config.json' if force_overwrite or not os.path.exists(config_file): print('- Saving configuration file to {}'.format(config_file)) neuron_config = glif_api.get_neuron_config() json_utilities.write(config_file, neuron_config) else: print('- File {} already exits. Skipping.'.format(config_file))
class CellTypesCache(Cache): ''' Cache class for storing and accessing data from the Cell Types Database. By default, this class will cache any downloaded metadata or files in well known locations defined in a manifest file. This behavior can be disabled. Attributes ---------- api: CellTypesApi instance The object used for making API queries related to the Cell Types Database Parameters ---------- cache: boolean Whether the class should save results of API queries to locations specified in the manifest file. Queries for files (as opposed to metadata) must have a file location. If caching is disabled, those locations must be specified in the function call (e.g. get_ephys_data(file_name='file.nwb')). manifest_file: string File name of the manifest to be read. Default is "manifest.json". ''' CELLS_KEY = 'CELLS' EPHYS_FEATURES_KEY = 'EPHYS_FEATURES' MORPHOLOGY_FEATURES_KEY = 'MORPHOLOGY_FEATURES' EPHYS_DATA_KEY = 'EPHYS_DATA' EPHYS_SWEEPS_KEY = 'EPHYS_SWEEPS' RECONSTRUCTION_KEY = 'RECONSTRUCTION' def __init__(self, cache=True, manifest_file='manifest.json'): super(CellTypesCache, self).__init__(manifest=manifest_file, cache=cache) self.api = CellTypesApi() def get_cells(self, file_name=None, require_morphology=False, require_reconstruction=False): ''' Download metadata for all cells in the database and optionally return a subset filtered by whether or not they have a morphology or reconstruction. Parameters ---------- file_name: string File name to save/read the cell metadata as JSON. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. require_morphology: boolean Filter out cells that have no morphological images. require_reconstruction: boolean Filter out cells that have no morphological reconstructions. ''' file_name = self.get_cache_path(file_name, self.CELLS_KEY) if os.path.exists(file_name): cells = json_utilities.read(file_name) else: cells = self.api.list_cells(False, False) if self.cache: json_utilities.write(file_name, cells) # filter the cells on the way out return self.api.filter_cells(cells, require_morphology, require_reconstruction) def get_ephys_sweeps(self, specimen_id, file_name=None): ''' Download sweep metadata for a single cell specimen. Parameters ---------- specimen_id: int ID of a cell. ''' file_name = self.get_cache_path(file_name, self.EPHYS_SWEEPS_KEY, specimen_id) if os.path.exists(file_name): sweeps = json_utilities.read(file_name) else: sweeps = self.api.get_ephys_sweeps(specimen_id) if self.cache: json_utilities.write(file_name, sweeps) return sweeps def get_ephys_features(self, dataframe=False, file_name=None): ''' Download electrophysiology features for all cells in the database. Parameters ---------- file_name: string File name to save/read the ephys features metadata as CSV. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. dataframe: boolean Return the output as a Pandas DataFrame. If False, return a list of dictionaries. ''' file_name = self.get_cache_path(file_name, self.EPHYS_FEATURES_KEY) if os.path.exists(file_name): features_df = pd.DataFrame.from_csv(file_name) else: features_df = self.api.get_ephys_features(dataframe=True) if self.cache: features_df.to_csv(file_name) if dataframe: return features_df else: return features_df.to_dict('records') def get_morphology_features(self, dataframe=False, file_name=None): ''' Download morphology features for all cells with reconstructions in the database. Parameters ---------- file_name: string File name to save/read the ephys features metadata as CSV. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. dataframe: boolean Return the output as a Pandas DataFrame. If False, return a list of dictionaries. ''' file_name = self.get_cache_path(file_name, self.MORPHOLOGY_FEATURES_KEY) if os.path.exists(file_name): features_df = pd.DataFrame.from_csv(file_name) else: features_df = self.api.get_morphology_features(dataframe=True) if self.cache: features_df.to_csv(file_name) if dataframe: return features_df else: return features_df.to_dict('records') def get_all_features(self, dataframe=False, require_reconstruction=True): ''' Download morphology and electrophysiology features for all cells and merge them into a single table. Parameters ---------- dataframe: boolean Return the output as a Pandas DataFrame. If False, return a list of dictionaries. require_reconstruction: boolean Only return ephys and morphology features for cells that have reconstructions. Default True. ''' ephys_features = self.get_ephys_features(dataframe=True) morphology_features = self.get_morphology_features(dataframe=True) how = 'inner' if require_reconstruction else 'outer' all_features = ephys_features.merge(morphology_features, how=how, on='specimen_id') if dataframe: return all_features else: return all_features.to_dict('records') def get_ephys_data(self, specimen_id, file_name=None): ''' Download electrophysiology traces for a single cell in the database. Parameters ---------- specimen_id: int The ID of a cell specimen to download. file_name: string File name to save/read the ephys features metadata as CSV. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. Returns ------- NwbDataSet A class instance with helper methods for retrieving stimulus and response traces out of an NWB file. ''' file_name = self.get_cache_path(file_name, self.EPHYS_DATA_KEY, specimen_id) if not os.path.exists(file_name): self.api.save_ephys_data(specimen_id, file_name) return NwbDataSet(file_name) def get_reconstruction(self, specimen_id, file_name=None): ''' Download and open a reconstruction for a single cell in the database. Parameters ---------- specimen_id: int The ID of a cell specimen to download. file_name: string File name to save/read the ephys features metadata as CSV. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. Returns ------- Morphology A class instance with methods for accessing morphology compartments. ''' file_name = self.get_cache_path(file_name, self.RECONSTRUCTION_KEY, specimen_id) if file_name is None: raise Exception("Please enable caching (CellTypes.cache = True) or specify a save_file_name.") if not os.path.exists(file_name): self.api.save_reconstruction(specimen_id, file_name) return swc.read_swc(file_name) def build_manifest(self, file_name): ''' Construct a manifest for this Cache class and save it in a file. Parameters ---------- file_name: string File location to save the manifest. ''' mb = ManifestBuilder() mb.add_path('BASEDIR', '.') mb.add_path(self.CELLS_KEY, 'cells.json', typename='file', parent_key='BASEDIR') mb.add_path(self.EPHYS_DATA_KEY, 'specimen_%d/ephys.nwb', typename='file', parent_key='BASEDIR') mb.add_path(self.EPHYS_FEATURES_KEY, 'ephys_features.csv', typename='file', parent_key='BASEDIR') mb.add_path(self.MORPHOLOGY_FEATURES_KEY, 'morphology_features.csv', typename='file', parent_key='BASEDIR') mb.add_path(self.RECONSTRUCTION_KEY, 'specimen_%d/reconstruction.swc', typename='file', parent_key='BASEDIR') mb.add_path(self.EPHYS_SWEEPS_KEY, 'specimen_%d/ephys_sweeps.json', typename='file', parent_key='BASEDIR') mb.write_json_file(file_name)
from allensdk.api.queries.cell_types_api import CellTypesApi from allensdk.ephys.extract_cell_features import extract_cell_features from collections import defaultdict from allensdk.core.nwb_data_set import NwbDataSet # pick a cell to analyze specimen_id = 324257146 nwb_file = 'ephys.nwb' # download the ephys data and sweep metadata cta = CellTypesApi() sweeps = cta.get_ephys_sweeps(specimen_id) cta.save_ephys_data(specimen_id, nwb_file) # group the sweeps by stimulus sweep_numbers = defaultdict(list) for sweep in sweeps: sweep_numbers[sweep['stimulus_name']].append(sweep['sweep_number']) # calculate features cell_features = extract_cell_features(NwbDataSet(nwb_file), sweep_numbers['Ramp'], sweep_numbers['Short Square'], sweep_numbers['Long Square'])