class BiophysicalArchiver(object): def __init__(self, archive_dir=None): self.bp = BiophysicalApi('http://api.brain-map.org') self.bp.cache_stimulus = True # change to False to not download the large stimulus NWB file self.cta = CellTypesApi() self.rma = RmaApi() self.neuronal_model_download_endpoint = 'http://celltypes.brain-map.org/neuronal_model/download/' self.template_names = {} self.nwb_list = [] if archive_dir == None: archive_dir = '.' self.archive_dir = archive_dir def get_template_names(self): template_response = self.rma.model_query('NeuronalModelTemplate') self.template_names = { t['id']: str(t['name']).replace(' ', '_') for t in template_response} def get_cells(self): return self.cta.list_cells(True, True) def get_neuronal_models(self, specimen_ids): return self.rma.model_query('NeuronalModel', criteria='specimen[id$in%s]' % ','.join(str(i) for i in specimen_ids), include='specimen', num_rows='all') def get_stimulus_file(self, neuronal_model_id): result = self.rma.model_query('NeuronalModel', criteria='[id$eq%d]' % (neuronal_model_id), include="specimen(ephys_result(well_known_files(well_known_file_type[name$il'NWB*'])))", tabular=['path']) stimulus_filename = result[0]['path'] return stimulus_filename stimulus_filename = os.path.basename(result[0]['path']) return stimulus_filename def archive_cell(self, ephys_result_id, specimen_id, template, neuronal_model_id): url = self.neuronal_model_download_endpoint + "/%d" % (neuronal_model_id) file_name = os.path.join(self.archive_dir, 'ephys_result_%d_specimen_%d_%s_neuronal_model_%d.zip' % (ephys_result_id, specimen_id, template, neuronal_model_id)) self.rma.retrieve_file_over_http(url, file_name) nwb_file = self.get_stimulus_file(neuronal_model_id) shutil.copy(nwb_file, self.archive_dir) self.nwb_list.append("%s\t%s" % (os.path.basename(nwb_file), file_name))
def test_list_cells_mocked(mock_cells): with patch.object(CellTypesApi, "model_query", return_value=mock_cells): ctapi = CellTypesApi() cells = ctapi.list_cells() assert len(cells) == 3 flu_cells = [ cell for cell in cells if cell['disease_categories'] == [('influenza')] ] assert len(flu_cells) == 1 cells = ctapi.list_cells(require_reconstruction=True) assert len(cells) == 1 cells = ctapi.list_cells(require_morphology=True) assert len(cells) == 1 cells = ctapi.list_cells(reporter_status=['bob']) assert len(cells) == 1 cells = ctapi.list_cells(species=['H**O SAPIENS']) assert len(cells) == 1 cells = ctapi.list_cells(species=['mus musculus']) assert len(cells) == 1
class CellTypesCache(Cache): """ Cache class for storing and accessing data from the Cell Types Database. By default, this class will cache any downloaded metadata or files in well known locations defined in a manifest file. This behavior can be disabled. Attributes ---------- api: CellTypesApi instance The object used for making API queries related to the Cell Types Database Parameters ---------- cache: boolean Whether the class should save results of API queries to locations specified in the manifest file. Queries for files (as opposed to metadata) must have a file location. If caching is disabled, those locations must be specified in the function call (e.g. get_ephys_data(file_name='file.nwb')). manifest_file: string File name of the manifest to be read. Default is "cell_types_manifest.json". """ # manifest keys CELLS_KEY = 'CELLS' EPHYS_FEATURES_KEY = 'EPHYS_FEATURES' MORPHOLOGY_FEATURES_KEY = 'MORPHOLOGY_FEATURES' EPHYS_DATA_KEY = 'EPHYS_DATA' EPHYS_SWEEPS_KEY = 'EPHYS_SWEEPS' RECONSTRUCTION_KEY = 'RECONSTRUCTION' MARKER_KEY = 'MARKER' def __init__(self, cache=True, manifest_file='cell_types_manifest.json', base_uri=None): super(CellTypesCache, self).__init__( manifest=manifest_file, cache=cache) self.api = CellTypesApi(base_uri=base_uri) def get_cells(self, file_name=None, require_morphology=False, require_reconstruction=False, reporter_status=None): """ Download metadata for all cells in the database and optionally return a subset filtered by whether or not they have a morphology or reconstruction. Parameters ---------- file_name: string File name to save/read the cell metadata as JSON. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. require_morphology: boolean Filter out cells that have no morphological images. require_reconstruction: boolean Filter out cells that have no morphological reconstructions. reporter_status: list Filter for cells that have one or more cell reporter statuses. """ file_name = self.get_cache_path(file_name, self.CELLS_KEY) if os.path.exists(file_name): cells = json_utilities.read(file_name) else: cells = self.api.list_cells(False, False) if self.cache: json_utilities.write(file_name, cells) if isinstance(reporter_status, string_types): reporter_status = [reporter_status] # filter the cells on the way out return self.api.filter_cells(cells, require_morphology, require_reconstruction, reporter_status) def get_ephys_sweeps(self, specimen_id, file_name=None): """ Download sweep metadata for a single cell specimen. Parameters ---------- specimen_id: int ID of a cell. """ file_name = self.get_cache_path( file_name, self.EPHYS_SWEEPS_KEY, specimen_id) if os.path.exists(file_name): sweeps = json_utilities.read(file_name) else: sweeps = self.api.get_ephys_sweeps(specimen_id) if self.cache: json_utilities.write(file_name, sweeps) return sweeps def get_ephys_features(self, dataframe=False, file_name=None): """ Download electrophysiology features for all cells in the database. Parameters ---------- file_name: string File name to save/read the ephys features metadata as CSV. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. dataframe: boolean Return the output as a Pandas DataFrame. If False, return a list of dictionaries. """ file_name = self.get_cache_path(file_name, self.EPHYS_FEATURES_KEY) if os.path.exists(file_name): features_df = pd.DataFrame.from_csv(file_name) else: features_df = self.api.get_ephys_features(dataframe=True) if self.cache: features_df.to_csv(file_name) if dataframe: return features_df else: return features_df.to_dict('records') def get_morphology_features(self, dataframe=False, file_name=None): """ Download morphology features for all cells with reconstructions in the database. Parameters ---------- file_name: string File name to save/read the ephys features metadata as CSV. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. dataframe: boolean Return the output as a Pandas DataFrame. If False, return a list of dictionaries. """ file_name = self.get_cache_path( file_name, self.MORPHOLOGY_FEATURES_KEY) if os.path.exists(file_name): features_df = pd.DataFrame.from_csv(file_name) else: features_df = self.api.get_morphology_features(dataframe=True) if self.cache: features_df.to_csv(file_name) if dataframe: return features_df else: return features_df.to_dict('records') def get_all_features(self, dataframe=False, require_reconstruction=True): """ Download morphology and electrophysiology features for all cells and merge them into a single table. Parameters ---------- dataframe: boolean Return the output as a Pandas DataFrame. If False, return a list of dictionaries. require_reconstruction: boolean Only return ephys and morphology features for cells that have reconstructions. Default True. """ ephys_features = self.get_ephys_features(dataframe=True) morphology_features = self.get_morphology_features(dataframe=True) how = 'inner' if require_reconstruction else 'outer' all_features = ephys_features.merge(morphology_features, how=how, on='specimen_id') if dataframe: return all_features else: return all_features.to_dict('records') def get_ephys_data(self, specimen_id, file_name=None): """ Download electrophysiology traces for a single cell in the database. Parameters ---------- specimen_id: int The ID of a cell specimen to download. file_name: string File name to save/read the ephys features metadata as CSV. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. Returns ------- NwbDataSet A class instance with helper methods for retrieving stimulus and response traces out of an NWB file. """ file_name = self.get_cache_path( file_name, self.EPHYS_DATA_KEY, specimen_id) if not os.path.exists(file_name): self.api.save_ephys_data(specimen_id, file_name) return NwbDataSet(file_name) def get_reconstruction(self, specimen_id, file_name=None): """ Download and open a reconstruction for a single cell in the database. Parameters ---------- specimen_id: int The ID of a cell specimen to download. file_name: string File name to save/read the reconstruction SWC. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. Returns ------- Morphology A class instance with methods for accessing morphology compartments. """ file_name = self.get_cache_path( file_name, self.RECONSTRUCTION_KEY, specimen_id) if file_name is None: raise Exception( "Please enable caching (CellTypes.cache = True) or specify a save_file_name.") if not os.path.exists(file_name): self.api.save_reconstruction(specimen_id, file_name) return swc.read_swc(file_name) def get_reconstruction_markers(self, specimen_id, file_name=None): """ Download and open a reconstruction marker file for a single cell in the database. Parameters ---------- specimen_id: int The ID of a cell specimen to download. file_name: string File name to save/read the reconstruction marker. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. Returns ------- Morphology A class instance with methods for accessing morphology compartments. """ file_name = self.get_cache_path( file_name, self.MARKER_KEY, specimen_id) if file_name is None: raise Exception( "Please enable caching (CellTypes.cache = True) or specify a save_file_name.") if not os.path.exists(file_name): try: self.api.save_reconstruction_markers(specimen_id, file_name) except LookupError as e: logging.warning(e.args) return [] return swc.read_marker_file(file_name) def build_manifest(self, file_name): """ Construct a manifest for this Cache class and save it in a file. Parameters ---------- file_name: string File location to save the manifest. """ mb = ManifestBuilder() mb.add_path('BASEDIR', '.') mb.add_path(self.CELLS_KEY, 'cells.json', typename='file', parent_key='BASEDIR') mb.add_path(self.EPHYS_DATA_KEY, 'specimen_%d/ephys.nwb', typename='file', parent_key='BASEDIR') mb.add_path(self.EPHYS_FEATURES_KEY, 'ephys_features.csv', typename='file', parent_key='BASEDIR') mb.add_path(self.MORPHOLOGY_FEATURES_KEY, 'morphology_features.csv', typename='file', parent_key='BASEDIR') mb.add_path(self.RECONSTRUCTION_KEY, 'specimen_%d/reconstruction.swc', typename='file', parent_key='BASEDIR') mb.add_path(self.MARKER_KEY, 'specimen_%d/reconstruction.marker', typename='file', parent_key='BASEDIR') mb.add_path(self.EPHYS_SWEEPS_KEY, 'specimen_%d/ephys_sweeps.json', typename='file', parent_key='BASEDIR') mb.write_json_file(file_name)
from allensdk.api.queries.biophysical_perisomatic_api import \ BiophysicalPerisomaticApi from allensdk.api.queries.cell_types_api import CellTypesApi import allensdk.core.swc as swc import os from allensdk.core.nwb_data_set import NwbDataSet from IPython import __main__ bp = BiophysicalPerisomaticApi('http://api.brain-map.org') ct = CellTypesApi() cells = ct.list_cells() #Below is one way, I can create URL, queries and get the result in a local python dictionary. import allensdk import mpi4py as MPI import numpy as np import csv import pickle import sys #import glob def __main__(): print 'main'
import os from allensdk.api.queries.cell_types_api import CellTypesApi import allensdk.core.swc as swc import matplotlib.pyplot as plt import numpy as np import math import pprint import csv #output_file = 'sholl_all.csv' #output_file = 'sholl_apical_dendrite_only.csv' output_file = 'sholl_basal_dendrite_only.csv' ct = CellTypesApi() cells = ct.list_cells(require_reconstruction=True) print "Cells with reconstructions: ", len(cells) download_file = False allow_types = [3] radii = np.array(range(0,1000,10)) radii_square = np.square( radii ) # # Computation based on line-sphere intersection formula # https://en.wikipedia.org/wiki/Line%E2%80%93sphere_intersection # with open( output_file, 'wb' ) as csvfile : cwriter = csv.writer( csvfile, delimiter=',', quotechar='"' )
def download_glif_models(cell_ids, base_dir, incl_ephys=True, force_overwrite=False): """Goes through the list of cell_ids and downloads cell config and ephys data in base_dir/cell_<ID>. Then looks up all possible models and downloads model files int base_dir/cell_<ID>/<MODEL_TYPE>_<MODEL_ID>/ """ # Determine the best url for connecting to cell-types db try: # see if we can connect to interal cell-types db request = requests.get('http://icelltypes/') if request.status_code == 200: base_uri = 'http://icelltypes/' else: base_uri = None except Exception: base_uri = None # use the default url base_dir = base_dir if base_dir.endswith('/') else base_dir + '/' valid_cells = [] ct_api = CellTypesApi(base_uri) for cell in ct_api.list_cells(): if cell['id'] in cell_ids: # create directory for cell cell_home = '{}cell_{}/'.format(base_dir, cell['id']) if not os.path.exists(cell_home): os.makedirs(cell_home) # save metadata cell_metadata_file = cell_home + 'cell_metadata.json' if force_overwrite or not os.path.exists(cell_metadata_file): print('Saving metadata for cell {} in {}'.format( cell['id'], cell_metadata_file)) json_utilities.write(cell_metadata_file, cell) else: print('File {} already exists. Skipping'.format( cell_metadata_file)) # save ephys data if incl_ephys: cell_ephys_file = cell_home + 'ephys_data.nwb' if force_overwrite or not os.path.exists(cell_ephys_file): print('Saving ephys data for cell {} in {}'.format( cell['id'], cell_ephys_file)) ct_api.save_ephys_data(cell['id'], cell_ephys_file) else: print('File {} already exists. Skipping'.format( cell_ephys_file)) # save sweeps file sweeps_file = cell_home + 'ephys_sweeps.json' if force_overwrite or not os.path.exists(sweeps_file): print('- Saving sweeps file to {}'.format(sweeps_file)) ephys_sweeps = ct_api.get_ephys_sweeps(cell['id']) json_utilities.write(sweeps_file, ephys_sweeps) else: print('- File {} already exits. Skipping.'.format(sweeps_file)) # keep track of valid ids valid_cells.append(cell['id']) cell_ids.remove(cell['id']) for cid in cell_ids: print('Warning: cell #{} was not found in cell-types database'.format( cid)) # Iterate through each all available models and find ones correspoding to cell list glif_models = {} # map model-id to their directory glif_api = GlifApi(base_uri=base_uri) for model in glif_api.list_neuronal_models(): if model['specimen_id'] in valid_cells: # save model files <BASE_DIR>/cell_<CELL_ID>/<MODEL_TYPE>_<MODEL_ID>/ cell_id = model['specimen_id'] model_id = model['id'] model_type = model[ 'neuronal_model_template_id'] #['id'] # type of model, GLIF-LIF, GLIF-ASC, etc type_name = model_id2name.get(model_type, None) if type_name is None: print( 'Warning: Unknown model type {} ({}) for cell/model {}/{}'. format(model_type, model['neuronal_model_template']['name'], cell_id, model_id)) type_name = model_type model_home_dir = '{}cell_{}/{}_{}/'.format(base_dir, cell_id, type_name, model_id) glif_models[model_id] = model_home_dir # go through all the found models, download necessary files n_models = len(glif_models) for i, (gid, home_dir) in enumerate(glif_models.iteritems()): print('Processing model {} ({} of {})'.format(gid, (i + 1), n_models)) model_metadata = glif_api.get_neuronal_model(gid) if not os.path.exists(home_dir): os.makedirs(home_dir) # save model metadata metadata_file = home_dir + 'metadata.json' if force_overwrite or not os.path.exists(metadata_file): print('- Saving metadata file to {}'.format(metadata_file)) #print type(metadata_file) with open(metadata_file, 'wb') as fp: json.dump(model_metadata, fp, indent=2) else: print('- File {} already exits. Skipping.'.format(metadata_file)) # get neuron configuration file config_file = home_dir + 'config.json' if force_overwrite or not os.path.exists(config_file): print('- Saving configuration file to {}'.format(config_file)) neuron_config = glif_api.get_neuron_config() json_utilities.write(config_file, neuron_config) else: print('- File {} already exits. Skipping.'.format(config_file))
from allensdk.api.queries.cell_types_api import CellTypesApi ct = CellTypesApi() # a list of dictionaries containing metadata for cells with reconstructions cells = ct.list_cells(require_reconstruction=True) # download the electrophysiology data for one cell ct.save_ephys_data(cells[0]['id'], 'example.nwb') # download the reconstruction for the same cell ct.save_reconstruction(cells[0]['id'], 'example.swc')
from neuroml import NeuroMLDocument from neuroml import Network from neuroml import Population from neuroml import Location from neuroml import Instance from neuroml import IncludeType import neuroml.writers as writers pp = pprint.PrettyPrinter(indent=4) ct = CellTypesApi() # use the SDK API to get a list of single cells and corresponding metadata cell_metadata_list = ct.list_cells(require_reconstruction=True) # convert that list of cells' metadata to a python dictionary for easy access cell_meta_dict = {} dataset_ids = [] for i in range(len(cell_metadata_list)): id = cell_metadata_list[i]['id'] dataset_ids.append(id) cell_meta_dict[id] = cell_metadata_list[i] cells = ['467703703', '323452245'] cells = ['467703703', '471141261', '320207387', '471141261', '324493977', '464326095'] cells = None # i.e. all... # Why? Good question...
from neuroml import NeuroMLDocument from neuroml import Network from neuroml import Population from neuroml import Location from neuroml import Instance from neuroml import IncludeType import neuroml.writers as writers pp = pprint.PrettyPrinter(indent=4) ct = CellTypesApi() # use the SDK API to get a list of single cells and corresponding metadata cell_metadata_list = ct.list_cells(require_reconstruction=True) # convert that list of cells' metadata to a python dictionary for easy access cell_meta_dict = {} dataset_ids = [] for i in range(len(cell_metadata_list)): id = cell_metadata_list[i]['id'] dataset_ids.append(id) cell_meta_dict[id] = cell_metadata_list[i] cells = ['467703703', '323452245'] cells = [ '467703703', '471141261', '320207387', '471141261', '324493977', '464326095' ]