def get_events_all_bob_experiments(): save_dir = '/local1/Documents/projects/cam_analysis/events' if not os.path.exists(save_dir): os.mkdir(save_dir) boc = BrainObservatoryCache(manifest_file='boc/manifest.json') ecs = boc.get_experiment_containers(simple=False) ec_ids = [ec['id'] for ec in ecs] exps = boc.get_ophys_experiments(experiment_container_ids=ec_ids) expt_list = [exp['id'] for exp in exps] bad_expt_list = [563226901] expt_list = [exp for exp in expt_list if exp not in bad_expt_list] for exp_num, expt in enumerate(expt_list): # [:1]: print 'experiment #'+str(exp_num)+'/'+str(len(expt_list)) data_set = boc.get_ophys_experiment_data(ophys_experiment_id=expt) dffTime, dff = data_set.get_dff_traces() N, T = dff.shape events = [[] for i in range(N)] for i in range(N): if np.mod(i, 50) == 0: print 'roi #'+str(i)+'/'+str(N) times, heights = get_events_derivative(dff[i], smooth_window=5, plot=False, smooth_weight=0.2) events[i] = concatenate_adjacent_events(times, heights) savefile = os.path.join(save_dir, str(expt) + '.pkl') pickle.dump(events, open(savefile, 'wb'))
def test_json_files(manifest_file): boc = BrainObservatoryCache(manifest_file=manifest_file, base_uri="http://swarehouse/") exps = boc.get_ophys_experiments() assert len( exps) == 1296, "Wrong number of ophys experiments. Found " + str( len(exps)) print "Passed: ", len(exps), " experiments found (expecting 1296)." conts = boc.get_experiment_containers() assert len( conts) == 432, "Wrong number of experiment containers. Found " + str( len(conts)) print "Passed: ", len( conts), " experiment containers found (expecting 432)." cells = boc.get_cell_specimens() assert len( cells) == 61371, "Wrong number of cell specimens. Found " + str( len(cells)) print "Passed: ", len(cells), " cell specimens found (expecting 61371)." import pandas as pd cell_df = pd.DataFrame(cells) assert len(cell_df) == len(cell_df.cell_specimen_id.unique( )), "len(cell_specimen_df)!=number of unique csids." print "Passed: len(cell_specimen_df) is of consistent length."
def get_L0_events(session_id): '''gets the L0 event time series for a given session Parameters ---------- session_id (int) Returns ------- l0 event traces (numpy array) ''' # event_path = get_event_path() # event_file = os.path.join(event_path, str(session_id)+'.npz') # print "Loading L0 events from: ", event_file # events = np.load(event_file)['ev'] # return events print("Loading L0 events for: " + str(session_id)) manifest_path = get_manifest_path() from allensdk.core.brain_observatory_cache import BrainObservatoryCache boc = BrainObservatoryCache(manifest_file=manifest_path) data_set = boc.get_ophys_experiment_data(session_id) from l0_analysis import L0_analysis l0 = L0_analysis(data_set) events = l0.get_events() return events
def example(): boc = BrainObservatoryCache(manifest_file='boc/manifest.json') ecs = boc.get_experiment_containers(simple=False) # ecs = ecs[:1] ec_ids = [ec['id'] for ec in ecs] exps = boc.get_ophys_experiments(experiment_container_ids=ec_ids, simple=False) expt_list = [exp['id'] for exp in exps] expt = expt_list[0] data_set = boc.get_ophys_experiment_data(ophys_experiment_id=expt) dffTime, dff = data_set.get_dff_traces() N, T = dff.shape ### do for one cell times, heights = get_events_derivative(dff[0], smooth_window=5, plot=False, smooth_weight=0.2) train = np.zeros((T)) train[times] = heights times_new, heights_new = concatenate_adjacent_events(times, heights) train_new = np.zeros((T)) train_new[times_new] = heights_new plt.figure() plt.plot(smooth(dff[0], 5), 'k') # plt.plot(dff[0], 'k') # plt.plot(dff[0], 'k', linewidth=1, alpha=0.8) # plt.plot(denoise_tv_chambolle(dff[0], weight=0.2), 'k') plt.plot(train, 'b') plt.plot(train_new, 'r')
def get_evoked_responses(): save_path = core.get_save_path() manifest_path = core.get_manifest_path() boc = BrainObservatoryCache(manifest_file=manifest_path) exps = pd.DataFrame(boc.get_ophys_experiments(include_failed=False)) for i,a in enumerate(exps.experiment_container_id.unique()): if np.mod(i,50)==0: print i subset = exps[exps.experiment_container_id==a] session_A = subset[subset.session_type=='three_session_A'].id.values[0] dataset = boc.get_ophys_experiment_data(session_A) cell_ids_A = dataset.get_cell_specimen_ids() data_file_dg = os.path.join(save_path, 'DriftingGratings', str(session_A)+"_dg_events_analysis.h5") f = h5py.File(data_file_dg) response_dg = f['response_events'].value f.close() numbercells = response_dg.shape[2] evoked_response = pd.DataFrame(columns=('cell_specimen_id','max_evoked_response_dg'), index=range(numbercells)) evoked_response['cell_specimen_id'] = cell_ids_A response2 = response_dg.reshape(48,numbercells,3) evoked_response.max_evoked_response_dg = np.nanmax(response2[:,:,0], axis=0) if i==0: evoked_response_all = evoked_response.copy() else: evoked_response_all = evoked_response_all.append(evoked_response) return evoked_response_all
def get_boc_json_files(manifest_file): # We need 'base_uri' until all of the data is publicly released boc = BrainObservatoryCache(manifest_file=manifest_file, base_uri="http://swarehouse/") exps = boc.get_ophys_experiments() cells = boc.get_cell_specimens() ecs = boc.get_experiment_containers()
def __init__(self, dataset, event_min_size=2., noise_scale=.1, median_filter_1=5401, median_filter_2=101, halflife_ms=None, sample_rate_hz=30, genotype='Unknown', L0_constrain=False, use_cache=True, use_bisection=True): manifest_file = core.get_manifest_path() if type(dataset) is int: if manifest_file is None: boc = BrainObservatoryCache() else: boc = BrainObservatoryCache(manifest_file=manifest_file) dataset = boc.get_ophys_experiment_data( ophys_experiment_id=dataset) try: self.metadata = dataset.get_metadata() # dff_traces = dataset.get_dff_traces()[1] self.corrected_fluorescence_traces = dataset.get_corrected_fluorescence_traces( )[1] except: self.metadata = {'genotype': genotype, 'ophys_experiment_id': 999} self.corrected_fluorescence_traces = dataset self.num_cells = self.corrected_fluorescence_traces.shape[0] self.sample_rate_hz = sample_rate_hz self.event_min_size = event_min_size self.noise_scale = noise_scale if halflife_ms is None: self.halflife = self.get_halflife() else: self.halflife = halflife_ms self.use_cache = use_cache self.use_bisection = use_bisection self.median_filter_1 = median_filter_1 self.median_filter_2 = median_filter_2 self.L0_constrain = L0_constrain self.cache_directory = core.get_cache_path() self._noise_stds = None self._num_small_baseline_frames = None self._dff_traces = None self._fit_params = None self._gamma = None self.lambdas = [] self.l0_func = None
def generate_events_for_expt(eid): boc = BrainObservatoryCache(manifest_file=MANIFEST_FILE) print "Processing events for ", eid data = boc.get_ophys_experiment_data(eid) l0 = L0_analysis(data) events = l0.get_events()
def merge_all_metrics(): save_path = core.get_save_path() manifest_path = core.get_manifest_path() boc = BrainObservatoryCache(manifest_file=manifest_path) exps = pd.DataFrame(boc.get_ophys_experiments(include_failed=False)) for i, a in enumerate(exps.experiment_container_id.unique()): subset = exps[exps.experiment_container_id == a] session_A = subset[subset.session_type == 'three_session_A'].id.values[0] session_B = subset[subset.session_type == 'three_session_B'].id.values[0] data_file_dg = os.path.join(save_path, 'Drifting Gratings', str(session_A) + "_dg_events_analysis.h5") peak_dg = pd.read_hdf(data_file_dg, 'peak') data_file_nm = os.path.join(save_path, 'Natural Movies', str(session_A) + "_nm_events_analysis.h5") peak_nm = pd.read_hdf(data_file_nm, 'peak') data_file_sg = os.path.join(save_path, 'Static Gratings', str(session_B) + "_sg_events_analysis.h5") peak_sg = pd.read_hdf(data_file_sg, 'peak') data_file_ns = os.path.join(save_path, 'Natural Scenes', str(session_B) + "_ns_events_analysis.h5") peak_ns = pd.read_hdf(data_file_ns, 'peak') peak_all = pd.merge(peak_dg, peak_nm, on='cell_specimen_id', how='outer') peak_all = pd.merge(peak_all, peak_sg, on='cell_specimen_id', how='outer') peak_all = pd.merge(peak_all, peak_ns, on='cell_specimen_id', how='outer') # peak_all = pd.merge(peak_all, peak_lsn, on='cell_specimen_id', how='outer') peak_all['experiment_container_id'] = a peak_all['tld1_name'] = subset.cre_line.iloc[0] peak_all['area'] = subset.targeted_structure.iloc[0] peak_all['imaging_depth'] = subset.imaging_depth.iloc[0] peak_all.to_csv( os.path.join(save_path, 'Metrics', str(a) + '_all_metrics.csv')) if i == 0: metrics = peak_all.copy() else: metrics = metrics.append(peak_all) metrics.to_csv(os.path.join(save_path, 'Metrics', 'metrics.csv')) return metrics
def get_all_dg(): save_path = core.get_save_path() manifest_path = core.get_manifest_path() boc = BrainObservatoryCache(manifest_file=manifest_path) exps = pd.DataFrame(boc.get_ophys_experiments(include_failed=False)) missing = [] for i, a in enumerate(exps.experiment_container_id.unique()): subset = exps[exps.experiment_container_id == a] try: session_A = subset[subset.session_type == 'three_session_A'].id.values[0] data_file_dg = os.path.join( save_path, 'DriftingGratings', str(session_A) + "_dg_events_analysis.h5") peak_dg = pd.read_hdf(data_file_dg, 'peak') peak_dg['experiment_container_id'] = a peak_dg['tld1_name'] = subset.cre_line.iloc[0] peak_dg['area'] = subset.targeted_structure.iloc[0] peak_dg['imaging_depth'] = subset.imaging_depth.iloc[0] if subset.cre_line.iloc[0] in ['Scnn1a-Tg3-Cre', 'Nr5a1-Cre']: depth_range = 200 elif subset.cre_line.iloc[0] in ['Fezf2-CreER']: depth_range = 300 else: depth_range = 100 * ( (np.floor(subset.imaging_depth.iloc[0] / 100)).astype(int)) peak_dg['depth_range'] = depth_range peak_dg['cre_depth'] = peak_dg[['tld1_name', 'depth_range']].apply(tuple, axis=1) if i == 0: peak_all = peak_dg.copy() else: peak_all = peak_all.append(peak_dg) except: missing.append(a) peak_all.reset_index(inplace=True) peak_all['type'] = 'E' peak_all.ix[peak_all.tld1_name == 'Sst-IRES-Cre', 'type'] = 'I' peak_all.ix[peak_all.tld1_name == 'Vip-IRES-Cre', 'type'] = 'I' # peak_all['depth_range'] =np.floor(peak_all.imaging_depth/100) # peak_all[['depth_range']] = peak_all[['depth_range']].astype(int) # peak_all.depth_range*=100 # peak_all.loc[peak_all.tld1_name=='Scnn1a-Tg3-Cre','depth_range'] = 200 # peak_all.loc[peak_all.tld1_name=='Nr5a1-Cre','depth_range'] = 200 # peak_all.loc[peak_all.tld1_name=='Fezf2-CreER', 'depth_range'] = 300 peak_all.to_hdf(os.path.join(save_path, 'Metrics', 'metrics_dg.h5'), 'peak_all') return peak_all, missing
def get_table(stimulus_type): session_type = SESSION_DICT[stimulus_type] boc = BrainObservatoryCache(manifest_file=MANIFEST_FILE) expt_C_id = boc.get_ophys_experiments( session_types=[session_type])[0]['id'] expt_C = boc.get_ophys_experiment_data(expt_C_id) stim_template = expt_C.get_stimulus_table(stimulus_type) return stim_template
def get_stimulus_template_brain_observatory(stimulus, data_path=default_data_path, manifest_file=default_manifest): boc = BrainObservatoryCache(manifest_file=manifest_file) dataset_path = get_stimulus_path(stimulus, data_path=data_path) if os.path.exists(dataset_path): data = np.load(dataset_path) else: oeid = stimulus_oeid_dict[stimulus] data_set = boc.get_ophys_experiment_data(oeid) data = data_set.get_stimulus_template(stimulus) assert hash_dict[stimulus] == get_hash(data) np.save(dataset_path, data) assert hash_dict[stimulus] == get_hash(data) return data
def observatory_variables(data_path=drive_path): #creates boc object and returns it #input your own filepath for the data if you are not using AWS manifest_file = os.path.join(drive_path, 'manifest.json') boc = BrainObservatoryCache(manifest_file=manifest_file) print('yes') return (boc)
def boc(tmpdir_factory): manifest_file = tmpdir_factory.mktemp('data').join( os.path.join('boc', 'manifest.json')) endpoint = os.environ[ 'TEST_API_ENDPOINT'] if 'TEST_API_ENDPOINT' in os.environ else 'http://api.brain-map.org' return BrainObservatoryCache(manifest_file=str(manifest_file), base_uri=endpoint)
def get_running_speed(session_id): '''uses allenSDK to get the running speed for a specified session Parameters ---------- session_id Returns ------- running speed ''' manifest_path = get_manifest_path() from allensdk.core.brain_observatory_cache import BrainObservatoryCache boc = BrainObservatoryCache(manifest_file=manifest_path) data_set = boc.get_ophys_experiment_data(session_id) running_speed,_ = data_set.get_running_speed() return running_speed
def prePareAllenStim_for_othermodels(exp_id): boc = BrainObservatoryCache(manifest_file='boc/manifest.json') data_set = boc.get_ophys_experiment_data(exp_id) scenes = data_set.get_stimulus_template('natural_scenes') movie = data_set.get_stimulus_template('natural_movie_one') numImages = scenes.shape[0] data = np.ndarray((numImages,1,224,224))#scenes.shape[1],scenes.shape[1] for n in range(0,numImages): thisImage = np.array(scenes[n,:,0:918]) thisImage = cv2.resize(thisImage,(224,224)) thisImage = (thisImage - np.mean(thisImage))/np.std(thisImage) data[n,0,:,:] = thisImage data_colored = np.concatenate((data,data,data),axis=1) return data_colored
def get_stimulus_template(session_id, stimulus): '''uses allenSDK to get the stimulus template for the specific stimulus. Parameters ---------- session_id (int) stimulus name (string) Returns ------- stimulus template (numpy array) ''' manifest_path = get_manifest_path() from allensdk.core.brain_observatory_cache import BrainObservatoryCache boc = BrainObservatoryCache(manifest_file=manifest_path) data_set = boc.get_ophys_experiment_data(session_id) stim_template = data_set.get_stimulus_template(stimulus) return stim_template
def get_boc(drive_path='/Volumes/Brain2016/'): """ Grab BrainObservatoryCache :param drive_path: :return: boc """ manifest_path = os.path.join(drive_path, 'BrainObservatory/manifest.json') boc = BrainObservatoryCache(manifest_file=manifest_path) return boc
def get_L0_events(session_id): '''gets the L0 event time series for a given session Parameters ---------- session_id (int) Returns ------- l0 event traces (numpy array) ''' manifest_path = get_manifest_path() from allensdk.core.brain_observatory_cache import BrainObservatoryCache boc = BrainObservatoryCache(manifest_file=manifest_path) print("Loading L0 events for: " + str(session_id)) # we're now using the builtin events files from the SDK return boc.get_ophys_experiment_events(session_id)
def main(initialize_database, start_exp=None, end_exp=None): """Main function to process Allen data.""" config = Config() boc = BrainObservatoryCache(manifest_file=config.manifest_file) timestamp = py_utils.timestamp() log_file = os.path.join(config.log_dir, timestamp) log = logger.get(log_file) if initialize_database: data_db.initialize_database() log.info('Initialized DB.') populate_db(config, boc, log, timestamp, start_exp, end_exp)
def generate_events(n): boc = BrainObservatoryCache(manifest_file=MANIFEST_FILE) expt_list = boc.get_ophys_experiments() total = len(expt_list) batch_size = total/40 + (1 if total%40!=0 else 0) lower = batch_size*n upper = lower + batch_size for expt in expt_list[lower:upper]: eid = expt['id'] print "Processing events for ", eid data = boc.get_ophys_experiment_data(eid) l0 = L0_analysis(data) events = l0.get_events() print "Done"
def get_stim_table(session_id, stimulus): '''uses allenSDK to get stimulus table, specimen IDs, and the number of cells in specific session. Parameters ---------- session_id (int) stimulus name (string) Returns ------- stim_table (pandas DataFrame) numbercells specimen IDs ''' manifest_path = get_manifest_path() from allensdk.core.brain_observatory_cache import BrainObservatoryCache boc = BrainObservatoryCache(manifest_file=manifest_path) data_set = boc.get_ophys_experiment_data(session_id) specimen_ids = data_set.get_cell_specimen_ids() stim_table = data_set.get_stimulus_table(stimulus) numbercells = len(specimen_ids) return stim_table, numbercells, specimen_ids
def main(): manifest_file = '/data/dynamic-brain-workshop/brain_observatory_cache/brain_observatory_manifest.json' boc = BrainObservatoryCache(manifest_file=manifest_file) stim = si.NATURAL_MOVIE_THREE session = si.THREE_SESSION_A metric = 'reliability_nm3' threshold = .5 out_file = 'sparkles_nm3_2d.npz' template_file = 'nm3.npz' download_data(boc, stim, session, metric, threshold, out_file, template_file)
def prePareAllenStim_for_CPC(exp_id,blocksize): boc = BrainObservatoryCache(manifest_file='boc/manifest.json') data_set = boc.get_ophys_experiment_data(exp_id) scenes = data_set.get_stimulus_template('natural_scenes') movie = data_set.get_stimulus_template('natural_movie_one') numImages = scenes.shape[0] data = np.ndarray((1,numImages,1,blocksize,184,184))#scenes.shape[1],scenes.shape[1] for n in range(0,numImages): for b in range(0,blocksize): thisImage = np.array(scenes[n,:,0:918]) thisImage = cv2.resize(thisImage,(184,184)) thisImage = (thisImage - np.mean(thisImage))/np.std(thisImage) data[0,n,0,b,:,:] = thisImage data_colored = np.concatenate((data,data,data),axis=2) numFrames = movie.shape[0] numBlocks = int(numFrames/(3*blocksize)) data = np.ndarray((1,numBlocks,1,blocksize,184,184))#scenes.shape[1],scenes.shape[1] f = 0 nidx = np.arange(numBlocks) for n in range(0,numBlocks): for b in range(0,blocksize): thisImage = np.array(movie[f,:,0:303]) thisImage = cv2.resize(thisImage,(184,184)) thisImage = (thisImage - np.mean(thisImage))/np.std(thisImage) data[0,nidx[n],0,b,:,:] = thisImage f = f + 3 data2_colored = np.concatenate((data,data,data),axis=2) return data_colored, data2_colored
def brain_observatory_cache(): boc = None with patch('os.path.exists') as m: m.return_value = True with patch("__builtin__.open", mock_open(read_data=CACHE_MANIFEST)): # Download a list of all targeted areas boc = BrainObservatoryCache(manifest_file='boc/manifest.json', base_uri='http://testwarehouse:9000') boc.api.json_msg_query = MagicMock(name='json_msg_query') return boc
def test_pre_release_get_experiments(tmpdir): # Values from original boc/api: temp_dir_base = os.path.join(str(tmpdir), 'base-api') outfile_base = os.path.join(temp_dir_base, 'manifest.json') boc_base = BrainObservatoryCache(manifest_file=outfile_base) experiments_base = boc_base.get_ophys_experiments() # # For development: print key/val pairs needed to be populated by adapter: # for key, val in sorted(experiments_base[0].items(), key=lambda x: x[0]): # print key, val # raise try: temp_dir_extended = os.path.join(str(tmpdir), 'extended-api') outfile_extended = os.path.join(temp_dir_extended, 'manifest.json') boc_extended = BrainObservatoryCache( manifest_file=outfile_extended, api=BrainObservatoryApiPreRelease()) except TypeError: import allensdk raise RuntimeError( 'Allensdk out-of-date; upgrade with "pip install --force --upgrade allensdk"' ) experiments_extended = boc_extended.get_ophys_experiments() # # For development: print key/val pairs actually populated by adapter: # for key, val in sorted(containers_extended[0].items(), key=lambda x: x[0]): # print key, val # raise check_key_list = [ 'acquisition_age_days', 'cre_line', 'donor_name', 'experiment_container_id', 'fail_eye_tracking', 'id', 'imaging_depth', 'reporter_line', 'session_type', 'specimen_name', 'targeted_structure' ] for key in check_key_list: assert key in experiments_extended[0] assert len(experiments_extended) > 0 assert set(experiments_base[0].keys()) == set( experiments_extended[0].keys()) id_experiment_dict = {} for c_e in experiments_extended: curr_id = c_e['id'] id_experiment_dict[curr_id] = c_e for c_b in experiments_base: c_e = id_experiment_dict[c_b['id']] for key in c_e: # assert c_e[key] == c_b[key] if not c_e[key] == c_b[key]: print(key, c_e[key], c_b[key]) raise Exception()
def brain_observatory_cache(fn_temp_dir): boc = None try: manifest_data = bytes(CACHE_MANIFEST, 'UTF-8') # Python 3 except: manifest_data = bytes(CACHE_MANIFEST) # Python 2.7 with patch('os.path.exists', return_value=True): with patch(builtins.__name__ + ".open", mock_open(read_data=manifest_data)): manifest_file = os.path.join(fn_temp_dir, "boc", "manifest.json") boc = BrainObservatoryCache(manifest_file=manifest_file, base_uri='http://api.brain-map.org') return boc
def brain_observatory_cache(): boc = None try: manifest_data = bytes(CACHE_MANIFEST, 'UTF-8') # Python 3 except: manifest_data = bytes(CACHE_MANIFEST) # Python 2.7 with patch('os.path.exists', return_value=True): with patch(builtins.__name__ + ".open", mock_open(read_data=manifest_data)): # Download a list of all targeted areas boc = BrainObservatoryCache( manifest_file="some_path/manifest.json", base_uri='http://api.brain-map.org') return boc
def test_build_manifest(): with patch('os.path.exists') as m: m.return_value = False with patch('allensdk.config.manifest.Manifest.safe_mkdir') as mkdir: with patch( 'allensdk.config.manifest_builder.' 'ManifestBuilder.write_json_file', MagicMock(name='write_json_file')) as mock_write_json: with patch("__builtin__.open", mock_open(read_data=CACHE_MANIFEST)): brain_observatory_cache = BrainObservatoryCache( manifest_file='boc/manifest.json', base_uri='http://testwarehouse:9000') mkdir.assert_called_once_with('boc') mock_write_json.assert_called_once_with( 'boc/manifest.json')
def brain_observatory_cache(): boc = None try: manifest_data = bytes(CACHE_MANIFEST, 'UTF-8') # Python 3 except: manifest_data = bytes(CACHE_MANIFEST) # Python 2.7 with patch('os.path.exists', return_value=True): with patch(builtins.__name__ + ".open", mock_open(read_data=manifest_data)): # Download a list of all targeted areas boc = BrainObservatoryCache(manifest_file='boc/manifest.json', base_uri='http://testwarehouse:9000') boc.api.json_msg_query = MagicMock(name='json_msg_query') return boc
def download_data(region, cre_line, stimulus = None): ''' region = [reg1, reg2, ...] cre_line = [line1, line2, ...] ''' boc = BrainObservatoryCache(manifest_file='boc/manifest.json') ecs = boc.get_experiment_containers(targeted_structures=region, cre_lines=cre_line) ec_ids = [ ec['id'] for ec in ecs ] exp = boc.get_ophys_experiments(experiment_container_ids=ec_ids) if stimulus == None: exp = boc.get_ophys_experiments(experiment_container_ids=ec_ids) else: exp = boc.get_ophys_experiments(experiment_container_ids=ec_ids, stimuli = stimulus) exp_id_list = [ec['id'] for ec in exp] data_set = {exp_id:boc.get_ophys_experiment_data(exp_id) for exp_id in exp_id_list} return data_set
# -*- coding: utf-8 -*- """ Created on Mon Aug 29 21:28:20 2016 Use Saskia's analysis files to get info without importing the whole freakin dataset. @author: ttruszko 511510855 """ from __future__ import print_function #this is the same old thing, getting the cell specimen ids from the ophys_experiment_data... from allensdk.core.brain_observatory_cache import BrainObservatoryCache exp_id= 510705057 #this is session ID boc = BrainObservatoryCache() data_set = boc.get_ophys_experiment_data(exp_id) csids = data_set.get_cell_specimen_ids() # In[] this is Saskia's example code. Works as is but it's better to use the open_h5_file function. import h5py import pandas as pd #this loads the response array and the peak dataframe from the analysis file ## edit this for Mac!! And for whatever session you want. path = 'D:/BrainObservatory/ophys_analysis/510705057_three_session_B_analysis.h5' f = h5py.File(path, 'r') response = f['analysis']['response_ns'].value f.close() sweep_response = pd.read_hdf(path, 'analysis/sweep_response_ns') mean_sweep_response=pd.read_hdf(path, 'analysis/mean_sweep_response_ns')
# We need to import these modules to get started drive_path = 'e/' import numpy as np import pandas as pd import osp import sys from decoder import * from allensdk.core.brain_observatory_cache import BrainObservatoryCache from allensdk.brain_observatory.static_gratings import StaticGratings manifest_path = os.path.join(drive_path,'BrainObservatory/manifest.json') boc = BrainObservatoryCache(manifest_file=manifest_path) #label extractor def labeled_data_extractor(stim_info, type_of_labels): '''Assignes lables for eah stimuli, Type_of_lable can be ['orientation'] or ['temporal_frequency'] or both. stim_info is a panda data_frame produces by BrainObservatoryCashe ''' si = stim_info.copy(); if len(type_of_labels) == 1: temp_si = si[type_of_labels[0]].replace(si[type_of_labels[0]].unique(), range(len(si[type_of_labels[0]].unique()))); return(temp_si.values.astype(int)) elif ['temporal_frequency', 'orientation'] in type_of_labels: 0; # Pick an experiment container (:)-- experiment_container_v1_cux2 = boc.get_experiment_containers(targeted_structures=['VISp'],cre_lines=['Cux2-CreERT2'],imaging_depths=[275]); #experiment_container_v1_cux2 = boc.get_experiment_containers(targeted_structures=['VISp'],cre_lines=['Cux2-CreERT2']); # choose an animal animal_number=0; experiment_container_id = experiment_container_v1_cux2[animal_number]['id']; experiment_information = boc.get_ophys_experiments(experiment_container_ids=[experiment_container_id]);
def BOC_init(stimuli_to_use={ 'drifting_gratings', 'locally_sparse_noise', 'spontaneous', 'static_gratings' }, selectivity_csv_filename='image_selectivity_dataframe.csv', areas={ 'VISp' }, discard_nan={ 'selectivity_ns', 'osi_sg', 'osi_dg', 'time_to_peak_ns', 'time_to_peak_sg', } ): """"Returns a BrainObservatoryCache initialized using the data on the external harddrives provided by the Allen Institute Example usage: boc, specimens_with_selectivity_S, VISp_cells_with_numbers = BOC_init() Input: stimuli_stimuli_to_use : Which stimuli we are interested in. By default, this is ['drifting_gratings', 'locally_sparse_noise', 'spontaneous', 'static_gratings'] selectivity_csv_filename : The path (including filename) to the image_selectivity_data.csv file. By default, this is just 'image_selectivity_dataframe.csv', which assumes that the file is in the same directory as the script you are running. areas : We will restrict our attention to the areas in this list. By default this is the list ['VISp'], because we are visual cortex chauvanists here at the Allen Institute. discard_nana : One of the outputs of this function is a dataframe that's been filtered to discard rows that have NaN values in the columns specified here. Output: boc : the BrainObservatoryCache object from AllenSDK, initialized using the data from the external harddrive. specimens_with_selectivity_S : A dataframe the includes the output of boc.get_cell_specimens as well as a column for the selectivity index S See Decoding Visual Inputs From Multiple Neurons in the Human Temporal Lobe, J. Neurophys 2007, by Quiroga et al. VISp_cells_with_numbers : a dataframe that filters specimens_with_selectivity_S by discarding rows with NaN values in the columns specified above. """ # The dynamic brain database is on an external hard drive. drive_path = '/Volumes/Brain2016' if sys.platform.startswith('w'): drive_path = 'd:' manifest_path = os.path.join(drive_path, 'BrainObservatory/manifest.json') boc = BrainObservatoryCache(manifest_file=manifest_path) expt_containers = boc.get_experiment_containers() all_expts_df = pd.DataFrame(boc.get_ophys_experiments(stimuli=list(stimuli_to_use))) # this has headers: # age_days cre_line experiment_container_id id imaging_depth session_type targeted_structure specimens_df = pd.DataFrame( boc.get_cell_specimens(experiment_container_ids=all_expts_df.experiment_container_id.values)) # this has headers: # area cell_specimen_id dsi_dg experiment_container_id imaging_depth osi_dg osi_sg p_dg p_ns p_sg # pref_dir_dg pref_image_ns pref_ori_sg pref_phase_sg pref_sf_sg pref_tf_dg time_to_peak_ns time_to_peak_sg # tld1_id tld1_name tld2_id tld2_name tlr1_id tlr1_name # There's also a handy bit of data from Saskia, in the form of a measurement called S. See # Decoding Visual Inputs From Multiple Neurons in the Human Temporal Lobe, J. Neurophys 2007, by Quiroga et al selectivity_S_df = pd.read_csv(selectivity_csv_filename, index_col=0) selectivity_S_df = selectivity_S_df[['cell_specimen_id', 'selectivity_ns']] specimens_with_selectivity_S = specimens_df.merge(selectivity_S_df, how='outer', on='cell_specimen_id') # This is all cells in VISp that have a value for the specified parameters (i.e not NaN) VISp_cells_with_numbers = specimens_with_selectivity_S for area_name in areas: VISp_cells_with_numbers = VISp_cells_with_numbers[VISp_cells_with_numbers.area == area_name] for col_name in discard_nan: VISp_cells_with_numbers = VISp_cells_with_numbers[np.isnan(VISp_cells_with_numbers[col_name]) == False] return boc, specimens_with_selectivity_S, VISp_cells_with_numbers