def __init__(self, cache=True, manifest_file='cell_types_manifest.json', base_uri=None): super(CellTypesCache, self).__init__(manifest=manifest_file, cache=cache) self.api = CellTypesApi(base_uri=base_uri)
def get_observation(dataset_id, kind, cached=True, quiet=False): """ Gets an observation of kind 'kind' from the dataset with id 'dataset_id', optionally using the cached value retrieved previously. """ db = shelve.open('aibs-cache') if cached else {} identifier = '%d_%s' % (dataset_id,kind) if identifier in db: print("Getting %s cached data value for from AIBS dataset %s" \ % (kind.title(),dataset_id)) value = db[identifier] else: print("Getting %s data value for from AIBS dataset %s" \ % (kind.title(),dataset_id)) ct = CellTypesApi() cmd = ct.get_cell(dataset_id) # Cell metadata if kind == 'rheobase': sweep_id = cmd['ephys_features'][0]['rheobase_sweep_id'] sp = get_sweep_params(dataset_id, sweep_id) if kind == 'rheobase': value = sp['stimulus_absolute_amplitude'] value = np.round(value,2) # Round to nearest hundredth of a pA. value *= pq.pA # Apply units. db[identifier] = value if cached: db.close() return {'value': value}
def test_list_cells_mocked(mock_cells): with patch.object(CellTypesApi, "model_query", return_value=mock_cells): ctapi = CellTypesApi() cells = ctapi.list_cells() assert len(cells) == 3 flu_cells = [ cell for cell in cells if cell['disease_categories'] == [('influenza')] ] assert len(flu_cells) == 1 cells = ctapi.list_cells(require_reconstruction=True) assert len(cells) == 1 cells = ctapi.list_cells(require_morphology=True) assert len(cells) == 1 cells = ctapi.list_cells(reporter_status=['bob']) assert len(cells) == 1 cells = ctapi.list_cells(species=['H**O SAPIENS']) assert len(cells) == 1 cells = ctapi.list_cells(species=['mus musculus']) assert len(cells) == 1
def cell_types_api(): endpoint = None if 'TEST_API_ENDPOINT' in os.environ: endpoint = os.environ['TEST_API_ENDPOINT'] return CellTypesApi(endpoint) else: return None
def __init__(self, cache=True, manifest_file=None, base_uri=None): if manifest_file is None: manifest_file = get_default_manifest_file('cell_types') super(CellTypesCache, self).__init__(manifest=manifest_file, cache=cache, version=self.MANIFEST_VERSION) self.api = CellTypesApi(base_uri=base_uri)
def fetch_pipeline_file(request): specimen_id = request.param nwb_file_name = '{}.nwb'.format(specimen_id) nwb_file_full_path = os.path.join(TEST_DATA_PATH, nwb_file_name) if not os.path.exists(nwb_file_full_path): ct = CellTypesApi() ct.save_ephys_data(specimen_id, nwb_file_full_path) return nwb_file_full_path
def __init__(self, archive_dir=None): self.bp = BiophysicalApi('http://api.brain-map.org') self.bp.cache_stimulus = True # change to False to not download the large stimulus NWB file self.cta = CellTypesApi() self.rma = RmaApi() self.neuronal_model_download_endpoint = 'http://celltypes.brain-map.org/neuronal_model/download/' self.template_names = {} self.nwb_list = [] if archive_dir == None: archive_dir = '.' self.archive_dir = archive_dir
def extract_data(ephys_cell, sweep_number): """Data extraction from AllenDB Parameters ---------- ephys_cell : int Cell identity from AllenDB sweep_number : int Stimulus identity for cell ephys_cell from AllenDB """ t_offset = 815. duration = 1450. real_data_path = 'ephys_cell_{}_sweep_number_{}.pkl'.format( ephys_cell, sweep_number) if not os.path.isfile(real_data_path): from allensdk.core.cell_types_cache import CellTypesCache from allensdk.api.queries.cell_types_api import CellTypesApi manifest_file = 'cell_types/manifest.json' cta = CellTypesApi() ctc = CellTypesCache(manifest_file=manifest_file) data_set = ctc.get_ephys_data(ephys_cell) sweep_data = data_set.get_sweep( sweep_number) # works with python2 and fails with python3 sweeps = cta.get_ephys_sweeps(ephys_cell) sweep = sweeps[sweep_number] index_range = sweep_data["index_range"] i = sweep_data["stimulus"][0:index_range[1] + 1] # in A v = sweep_data["response"][0:index_range[1] + 1] # in V sampling_rate = sweep_data["sampling_rate"] # in Hz dt = 1e3 / sampling_rate # in ms i *= 1e6 # to muA v *= 1e3 # to mV v = v[int(t_offset / dt):int((t_offset + duration) / dt)] i = i[int(t_offset / dt):int((t_offset + duration) / dt)] real_data_obs = np.array(v).reshape(1, -1, 1) I_real_data = np.array(i).reshape(-1) t_on = int( sweep['stimulus_start_time'] * sampling_rate) * dt - t_offset t_off = int( (sweep['stimulus_start_time'] + sweep['stimulus_duration']) * sampling_rate) * dt - t_offset f = open(real_data_path, 'wb') pickle.dump((real_data_obs, I_real_data, dt, t_on, t_off), f) f.close()
def get_sweep_params(dataset_id, sweep_id): """ Gets sweep parameters corresponding to the sweep with id 'sweep_id' from the dataset with id 'dataset_id'. """ ct = CellTypesApi() experiment_params = ct.get_ephys_sweeps(dataset_id) sp = None for sp in experiment_params: if sp['id']==sweep_id: sweep_num = sp['sweep_number'] if sweep_num is None: raise Exception('Sweep with ID %d not found in dataset with ID %d.' % (sweep_id, dataset_id)) break return sp
def test_list_cells_api_mocked(mock_cells_api): with patch.object(CellTypesApi, "model_query", return_value=mock_cells_api): ctapi = CellTypesApi() cells = ctapi.list_cells_api() assert len(cells) == 2 fcells = ctapi.filter_cells_api(cells, require_reconstruction=True) assert len(fcells) == 1 fcells = ctapi.filter_cells_api(cells, require_morphology=True) assert len(fcells) == 1 fcells = ctapi.filter_cells_api(cells, species=['taco']) assert len(fcells) == 2 fcells = ctapi.filter_cells_api(cells, reporter_status=['fish']) assert len(fcells) == 1
def get_observation(dataset_id, kind, cached=True, quiet=False): """Get an observation. Get an observation of kind 'kind' from the dataset with id 'dataset_id'. optionally using the cached value retrieved previously. """ db = shelve.open('aibs-cache') if cached else {} identifier = '%d_%s' % (dataset_id, kind) if identifier in db: print("Getting %s cached data value for from AIBS dataset %s" % (kind.title(), dataset_id)) value = db[identifier] else: print("Getting %s data value for from AIBS dataset %s" % (kind.title(), dataset_id)) ct = CellTypesApi() cmd = ct.get_cell(dataset_id) # Cell metadata if kind == 'rheobase': if 'ephys_features' in cmd: value = cmd['ephys_features'][0][ 'threshold_i_long_square'] # newer API else: value = cmd['ef__threshold_i_long_square'] # older API value = np.round(value, 2) # Round to nearest hundredth of a pA. value *= pq.pA # Apply units. else: value = cmd[kind] db[identifier] = value if cached: db.close() return {'value': value}
""" Single sweep detection ================================= Detect spikes for a single sweep """ import os import matplotlib.pyplot as plt from allensdk.api.queries.cell_types_api import CellTypesApi from ipfx.aibs_data_set import AibsDataSet from ipfx.ephys_extractor import SpikeExtractor # Download and access the experimental data ct = CellTypesApi() specimen_id = 595570553 nwb_file = "%d.nwb" % specimen_id sweep_info = ct.get_ephys_sweeps(specimen_id) if not os.path.exists(nwb_file): ct.save_ephys_data(specimen_id, nwb_file) # Get the data for the sweep into a format we can use dataset = AibsDataSet(sweep_info=sweep_info, nwb_file=nwb_file) sweep_number = 39 sweep = dataset.sweep(sweep_number) # Extract information about the spikes ext = SpikeExtractor() results = ext.process(t=sweep.t, v=sweep.v, i=sweep.i)
def allen_obs_data(ephys_cell=464212183, sweep_number=33, A_soma=np.pi * (70. * 1e-4)**2): """Data for x_o. Cell from AllenDB Parameters ---------- ephys_cell : int Cell identity from AllenDB sweep_number : int Stimulus identity for cell ephys_cell from AllenDB """ t_offset = 815. duration = 1450. dir_cache = './support_files' real_data_path = dir_cache + '/ephys_cell_{}_sweep_number_{}.pkl'.format( ephys_cell, sweep_number) if not os.path.isfile(real_data_path): from allensdk.core.cell_types_cache import CellTypesCache from allensdk.api.queries.cell_types_api import CellTypesApi manifest_file = 'cell_types/manifest.json' cta = CellTypesApi() ctc = CellTypesCache(manifest_file=manifest_file) data_set = ctc.get_ephys_data(ephys_cell) sweep_data = data_set.get_sweep( sweep_number) # works with python2 and fails with python3 sweeps = cta.get_ephys_sweeps(ephys_cell) sweep = sweeps[sweep_number] index_range = sweep_data["index_range"] i = sweep_data["stimulus"][0:index_range[1] + 1] # in A v = sweep_data["response"][0:index_range[1] + 1] # in V sampling_rate = sweep_data["sampling_rate"] # in Hz dt = 1e3 / sampling_rate # in ms i *= 1e6 # to muA v *= 1e3 # to mV v = v[int(t_offset / dt):int((t_offset + duration) / dt)] i = i[int(t_offset / dt):int((t_offset + duration) / dt)] real_data_obs = np.array(v).reshape(1, -1, 1) I_real_data = np.array(i).reshape(-1) t_on = int( sweep['stimulus_start_time'] * sampling_rate) * dt - t_offset t_off = int( (sweep['stimulus_start_time'] + sweep['stimulus_duration']) * sampling_rate) * dt - t_offset io.save((real_data_obs, I_real_data, dt, t_on, t_off), real_data_path) else: def pickle_load(file): """Loads data from file.""" f = open(file, 'rb') data = pickle.load(f, encoding='latin1') f.close() return data real_data_obs, I_real_data, dt, t_on, t_off = pickle_load( real_data_path) t = np.arange(0, duration, dt) # external current I = I_real_data / A_soma # muA/cm2 # return real_data_obs, I_obs return { 'data': real_data_obs.reshape(-1), 'time': t, 'dt': dt, 'I': I.reshape(-1), 't_on': t_on, 't_off': t_off }
def download_glif_models(cell_ids, base_dir, incl_ephys=True, force_overwrite=False): """Goes through the list of cell_ids and downloads cell config and ephys data in base_dir/cell_<ID>. Then looks up all possible models and downloads model files int base_dir/cell_<ID>/<MODEL_TYPE>_<MODEL_ID>/ """ # Determine the best url for connecting to cell-types db try: # see if we can connect to interal cell-types db request = requests.get('http://icelltypes/') if request.status_code == 200: base_uri = 'http://icelltypes/' else: base_uri = None except Exception: base_uri = None # use the default url base_dir = base_dir if base_dir.endswith('/') else base_dir + '/' valid_cells = [] ct_api = CellTypesApi(base_uri) for cell in ct_api.list_cells(): if cell['id'] in cell_ids: # create directory for cell cell_home = '{}cell_{}/'.format(base_dir, cell['id']) if not os.path.exists(cell_home): os.makedirs(cell_home) # save metadata cell_metadata_file = cell_home + 'cell_metadata.json' if force_overwrite or not os.path.exists(cell_metadata_file): print('Saving metadata for cell {} in {}'.format( cell['id'], cell_metadata_file)) json_utilities.write(cell_metadata_file, cell) else: print('File {} already exists. Skipping'.format( cell_metadata_file)) # save ephys data if incl_ephys: cell_ephys_file = cell_home + 'ephys_data.nwb' if force_overwrite or not os.path.exists(cell_ephys_file): print('Saving ephys data for cell {} in {}'.format( cell['id'], cell_ephys_file)) ct_api.save_ephys_data(cell['id'], cell_ephys_file) else: print('File {} already exists. Skipping'.format( cell_ephys_file)) # save sweeps file sweeps_file = cell_home + 'ephys_sweeps.json' if force_overwrite or not os.path.exists(sweeps_file): print('- Saving sweeps file to {}'.format(sweeps_file)) ephys_sweeps = ct_api.get_ephys_sweeps(cell['id']) json_utilities.write(sweeps_file, ephys_sweeps) else: print('- File {} already exits. Skipping.'.format(sweeps_file)) # keep track of valid ids valid_cells.append(cell['id']) cell_ids.remove(cell['id']) for cid in cell_ids: print('Warning: cell #{} was not found in cell-types database'.format( cid)) # Iterate through each all available models and find ones correspoding to cell list glif_models = {} # map model-id to their directory glif_api = GlifApi(base_uri=base_uri) for model in glif_api.list_neuronal_models(): if model['specimen_id'] in valid_cells: # save model files <BASE_DIR>/cell_<CELL_ID>/<MODEL_TYPE>_<MODEL_ID>/ cell_id = model['specimen_id'] model_id = model['id'] model_type = model[ 'neuronal_model_template_id'] #['id'] # type of model, GLIF-LIF, GLIF-ASC, etc type_name = model_id2name.get(model_type, None) if type_name is None: print( 'Warning: Unknown model type {} ({}) for cell/model {}/{}'. format(model_type, model['neuronal_model_template']['name'], cell_id, model_id)) type_name = model_type model_home_dir = '{}cell_{}/{}_{}/'.format(base_dir, cell_id, type_name, model_id) glif_models[model_id] = model_home_dir # go through all the found models, download necessary files n_models = len(glif_models) for i, (gid, home_dir) in enumerate(glif_models.iteritems()): print('Processing model {} ({} of {})'.format(gid, (i + 1), n_models)) model_metadata = glif_api.get_neuronal_model(gid) if not os.path.exists(home_dir): os.makedirs(home_dir) # save model metadata metadata_file = home_dir + 'metadata.json' if force_overwrite or not os.path.exists(metadata_file): print('- Saving metadata file to {}'.format(metadata_file)) #print type(metadata_file) with open(metadata_file, 'wb') as fp: json.dump(model_metadata, fp, indent=2) else: print('- File {} already exits. Skipping.'.format(metadata_file)) # get neuron configuration file config_file = home_dir + 'config.json' if force_overwrite or not os.path.exists(config_file): print('- Saving configuration file to {}'.format(config_file)) neuron_config = glif_api.get_neuron_config() json_utilities.write(config_file, neuron_config) else: print('- File {} already exits. Skipping.'.format(config_file))
from allensdk.api.queries.cell_types_api import CellTypesApi from allensdk.ephys.extract_cell_features import extract_cell_features from collections import defaultdict from allensdk.core.nwb_data_set import NwbDataSet # pick a cell to analyze specimen_id = 324257146 nwb_file = 'ephys.nwb' # download the ephys data and sweep metadata cta = CellTypesApi() sweeps = cta.get_ephys_sweeps(specimen_id) cta.save_ephys_data(specimen_id, nwb_file) # group the sweeps by stimulus sweep_numbers = defaultdict(list) for sweep in sweeps: sweep_numbers[sweep['stimulus_name']].append(sweep['sweep_number']) # calculate features cell_features = extract_cell_features(NwbDataSet(nwb_file), sweep_numbers['Ramp'], sweep_numbers['Short Square'], sweep_numbers['Long Square'])