def get_events_all_bob_experiments():
save_dir = '/local1/Documents/projects/cam_analysis/events'
if not os.path.exists(save_dir):
os.mkdir(save_dir)
boc = BrainObservatoryCache(manifest_file='boc/manifest.json')
ecs = boc.get_experiment_containers(simple=False)
ec_ids = [ec['id'] for ec in ecs]
exps = boc.get_ophys_experiments(experiment_container_ids=ec_ids)
expt_list = [exp['id'] for exp in exps]
bad_expt_list = [563226901]
expt_list = [exp for exp in expt_list if exp not in bad_expt_list]
for exp_num, expt in enumerate(expt_list): # [:1]:
print 'experiment #'+str(exp_num)+'/'+str(len(expt_list))
data_set = boc.get_ophys_experiment_data(ophys_experiment_id=expt)
dffTime, dff = data_set.get_dff_traces()
N, T = dff.shape
events = [[] for i in range(N)]
for i in range(N):
if np.mod(i, 50) == 0:
print 'roi #'+str(i)+'/'+str(N)
times, heights = get_events_derivative(dff[i], smooth_window=5, plot=False, smooth_weight=0.2)
events[i] = concatenate_adjacent_events(times, heights)
savefile = os.path.join(save_dir, str(expt) + '.pkl')
pickle.dump(events, open(savefile, 'wb'))
def test_json_files(manifest_file):
boc = BrainObservatoryCache(manifest_file=manifest_file,
base_uri="http://swarehouse/")
exps = boc.get_ophys_experiments()
assert len(
exps) == 1296, "Wrong number of ophys experiments. Found " + str(
len(exps))
print "Passed: ", len(exps), " experiments found (expecting 1296)."
conts = boc.get_experiment_containers()
assert len(
conts) == 432, "Wrong number of experiment containers. Found " + str(
len(conts))
print "Passed: ", len(
conts), " experiment containers found (expecting 432)."
cells = boc.get_cell_specimens()
assert len(
cells) == 61371, "Wrong number of cell specimens. Found " + str(
len(cells))
print "Passed: ", len(cells), " cell specimens found (expecting 61371)."
import pandas as pd
cell_df = pd.DataFrame(cells)
assert len(cell_df) == len(cell_df.cell_specimen_id.unique(
)), "len(cell_specimen_df)!=number of unique csids."
print "Passed: len(cell_specimen_df) is of consistent length."
def get_L0_events(session_id):
'''gets the L0 event time series for a given session
Parameters
----------
session_id (int)
Returns
-------
l0 event traces (numpy array)
'''
# event_path = get_event_path()
# event_file = os.path.join(event_path, str(session_id)+'.npz')
# print "Loading L0 events from: ", event_file
# events = np.load(event_file)['ev']
# return events
print("Loading L0 events for: " + str(session_id))
manifest_path = get_manifest_path()
from allensdk.core.brain_observatory_cache import BrainObservatoryCache
boc = BrainObservatoryCache(manifest_file=manifest_path)
data_set = boc.get_ophys_experiment_data(session_id)
from l0_analysis import L0_analysis
l0 = L0_analysis(data_set)
events = l0.get_events()
return events
def example():
boc = BrainObservatoryCache(manifest_file='boc/manifest.json')
ecs = boc.get_experiment_containers(simple=False)
# ecs = ecs[:1]
ec_ids = [ec['id'] for ec in ecs]
exps = boc.get_ophys_experiments(experiment_container_ids=ec_ids, simple=False)
expt_list = [exp['id'] for exp in exps]
expt = expt_list[0]
data_set = boc.get_ophys_experiment_data(ophys_experiment_id=expt)
dffTime, dff = data_set.get_dff_traces()
N, T = dff.shape
### do for one cell
times, heights = get_events_derivative(dff[0], smooth_window=5, plot=False, smooth_weight=0.2)
train = np.zeros((T))
train[times] = heights
times_new, heights_new = concatenate_adjacent_events(times, heights)
train_new = np.zeros((T))
train_new[times_new] = heights_new
plt.figure()
plt.plot(smooth(dff[0], 5), 'k')
# plt.plot(dff[0], 'k')
# plt.plot(dff[0], 'k', linewidth=1, alpha=0.8)
# plt.plot(denoise_tv_chambolle(dff[0], weight=0.2), 'k')
plt.plot(train, 'b')
plt.plot(train_new, 'r')
def get_evoked_responses():
save_path = core.get_save_path()
manifest_path = core.get_manifest_path()
boc = BrainObservatoryCache(manifest_file=manifest_path)
exps = pd.DataFrame(boc.get_ophys_experiments(include_failed=False))
for i,a in enumerate(exps.experiment_container_id.unique()):
if np.mod(i,50)==0:
print i
subset = exps[exps.experiment_container_id==a]
session_A = subset[subset.session_type=='three_session_A'].id.values[0]
dataset = boc.get_ophys_experiment_data(session_A)
cell_ids_A = dataset.get_cell_specimen_ids()
data_file_dg = os.path.join(save_path, 'DriftingGratings', str(session_A)+"_dg_events_analysis.h5")
f = h5py.File(data_file_dg)
response_dg = f['response_events'].value
f.close()
numbercells = response_dg.shape[2]
evoked_response = pd.DataFrame(columns=('cell_specimen_id','max_evoked_response_dg'), index=range(numbercells))
evoked_response['cell_specimen_id'] = cell_ids_A
response2 = response_dg.reshape(48,numbercells,3)
evoked_response.max_evoked_response_dg = np.nanmax(response2[:,:,0], axis=0)
if i==0:
evoked_response_all = evoked_response.copy()
else:
evoked_response_all = evoked_response_all.append(evoked_response)
return evoked_response_all
def get_boc_json_files(manifest_file):
# We need 'base_uri' until all of the data is publicly released
boc = BrainObservatoryCache(manifest_file=manifest_file,
base_uri="http://swarehouse/")
exps = boc.get_ophys_experiments()
cells = boc.get_cell_specimens()
ecs = boc.get_experiment_containers()
def __init__(self,
dataset,
event_min_size=2.,
noise_scale=.1,
median_filter_1=5401,
median_filter_2=101,
halflife_ms=None,
sample_rate_hz=30,
genotype='Unknown',
L0_constrain=False,
use_cache=True,
use_bisection=True):
manifest_file = core.get_manifest_path()
if type(dataset) is int:
if manifest_file is None:
boc = BrainObservatoryCache()
else:
boc = BrainObservatoryCache(manifest_file=manifest_file)
dataset = boc.get_ophys_experiment_data(
ophys_experiment_id=dataset)
try:
self.metadata = dataset.get_metadata()
# dff_traces = dataset.get_dff_traces()[1]
self.corrected_fluorescence_traces = dataset.get_corrected_fluorescence_traces(
)[1]
except:
self.metadata = {'genotype': genotype, 'ophys_experiment_id': 999}
self.corrected_fluorescence_traces = dataset
self.num_cells = self.corrected_fluorescence_traces.shape[0]
self.sample_rate_hz = sample_rate_hz
self.event_min_size = event_min_size
self.noise_scale = noise_scale
if halflife_ms is None:
self.halflife = self.get_halflife()
else:
self.halflife = halflife_ms
self.use_cache = use_cache
self.use_bisection = use_bisection
self.median_filter_1 = median_filter_1
self.median_filter_2 = median_filter_2
self.L0_constrain = L0_constrain
self.cache_directory = core.get_cache_path()
self._noise_stds = None
self._num_small_baseline_frames = None
self._dff_traces = None
self._fit_params = None
self._gamma = None
self.lambdas = []
self.l0_func = None
def generate_events_for_expt(eid):
boc = BrainObservatoryCache(manifest_file=MANIFEST_FILE)
print "Processing events for ", eid
data = boc.get_ophys_experiment_data(eid)
l0 = L0_analysis(data)
events = l0.get_events()
def merge_all_metrics():
save_path = core.get_save_path()
manifest_path = core.get_manifest_path()
boc = BrainObservatoryCache(manifest_file=manifest_path)
exps = pd.DataFrame(boc.get_ophys_experiments(include_failed=False))
for i, a in enumerate(exps.experiment_container_id.unique()):
subset = exps[exps.experiment_container_id == a]
session_A = subset[subset.session_type ==
'three_session_A'].id.values[0]
session_B = subset[subset.session_type ==
'three_session_B'].id.values[0]
data_file_dg = os.path.join(save_path, 'Drifting Gratings',
str(session_A) + "_dg_events_analysis.h5")
peak_dg = pd.read_hdf(data_file_dg, 'peak')
data_file_nm = os.path.join(save_path, 'Natural Movies',
str(session_A) + "_nm_events_analysis.h5")
peak_nm = pd.read_hdf(data_file_nm, 'peak')
data_file_sg = os.path.join(save_path, 'Static Gratings',
str(session_B) + "_sg_events_analysis.h5")
peak_sg = pd.read_hdf(data_file_sg, 'peak')
data_file_ns = os.path.join(save_path, 'Natural Scenes',
str(session_B) + "_ns_events_analysis.h5")
peak_ns = pd.read_hdf(data_file_ns, 'peak')
peak_all = pd.merge(peak_dg,
peak_nm,
on='cell_specimen_id',
how='outer')
peak_all = pd.merge(peak_all,
peak_sg,
on='cell_specimen_id',
how='outer')
peak_all = pd.merge(peak_all,
peak_ns,
on='cell_specimen_id',
how='outer')
# peak_all = pd.merge(peak_all, peak_lsn, on='cell_specimen_id', how='outer')
peak_all['experiment_container_id'] = a
peak_all['tld1_name'] = subset.cre_line.iloc[0]
peak_all['area'] = subset.targeted_structure.iloc[0]
peak_all['imaging_depth'] = subset.imaging_depth.iloc[0]
peak_all.to_csv(
os.path.join(save_path, 'Metrics',
str(a) + '_all_metrics.csv'))
if i == 0:
metrics = peak_all.copy()
else:
metrics = metrics.append(peak_all)
metrics.to_csv(os.path.join(save_path, 'Metrics', 'metrics.csv'))
return metrics
def get_all_dg():
save_path = core.get_save_path()
manifest_path = core.get_manifest_path()
boc = BrainObservatoryCache(manifest_file=manifest_path)
exps = pd.DataFrame(boc.get_ophys_experiments(include_failed=False))
missing = []
for i, a in enumerate(exps.experiment_container_id.unique()):
subset = exps[exps.experiment_container_id == a]
try:
session_A = subset[subset.session_type ==
'three_session_A'].id.values[0]
data_file_dg = os.path.join(
save_path, 'DriftingGratings',
str(session_A) + "_dg_events_analysis.h5")
peak_dg = pd.read_hdf(data_file_dg, 'peak')
peak_dg['experiment_container_id'] = a
peak_dg['tld1_name'] = subset.cre_line.iloc[0]
peak_dg['area'] = subset.targeted_structure.iloc[0]
peak_dg['imaging_depth'] = subset.imaging_depth.iloc[0]
if subset.cre_line.iloc[0] in ['Scnn1a-Tg3-Cre', 'Nr5a1-Cre']:
depth_range = 200
elif subset.cre_line.iloc[0] in ['Fezf2-CreER']:
depth_range = 300
else:
depth_range = 100 * (
(np.floor(subset.imaging_depth.iloc[0] / 100)).astype(int))
peak_dg['depth_range'] = depth_range
peak_dg['cre_depth'] = peak_dg[['tld1_name',
'depth_range']].apply(tuple,
axis=1)
if i == 0:
peak_all = peak_dg.copy()
else:
peak_all = peak_all.append(peak_dg)
except:
missing.append(a)
peak_all.reset_index(inplace=True)
peak_all['type'] = 'E'
peak_all.ix[peak_all.tld1_name == 'Sst-IRES-Cre', 'type'] = 'I'
peak_all.ix[peak_all.tld1_name == 'Vip-IRES-Cre', 'type'] = 'I'
# peak_all['depth_range'] =np.floor(peak_all.imaging_depth/100)
# peak_all[['depth_range']] = peak_all[['depth_range']].astype(int)
# peak_all.depth_range*=100
# peak_all.loc[peak_all.tld1_name=='Scnn1a-Tg3-Cre','depth_range'] = 200
# peak_all.loc[peak_all.tld1_name=='Nr5a1-Cre','depth_range'] = 200
# peak_all.loc[peak_all.tld1_name=='Fezf2-CreER', 'depth_range'] = 300
peak_all.to_hdf(os.path.join(save_path, 'Metrics', 'metrics_dg.h5'),
'peak_all')
return peak_all, missing
def get_table(stimulus_type):
session_type = SESSION_DICT[stimulus_type]
boc = BrainObservatoryCache(manifest_file=MANIFEST_FILE)
expt_C_id = boc.get_ophys_experiments(
session_types=[session_type])[0]['id']
expt_C = boc.get_ophys_experiment_data(expt_C_id)
stim_template = expt_C.get_stimulus_table(stimulus_type)
return stim_template
def get_stimulus_template_brain_observatory(stimulus, data_path=default_data_path, manifest_file=default_manifest):
boc = BrainObservatoryCache(manifest_file=manifest_file)
dataset_path = get_stimulus_path(stimulus, data_path=data_path)
if os.path.exists(dataset_path):
data = np.load(dataset_path)
else:
oeid = stimulus_oeid_dict[stimulus]
data_set = boc.get_ophys_experiment_data(oeid)
data = data_set.get_stimulus_template(stimulus)
assert hash_dict[stimulus] == get_hash(data)
np.save(dataset_path, data)
assert hash_dict[stimulus] == get_hash(data)
return data
def observatory_variables(data_path=drive_path):
#creates boc object and returns it
#input your own filepath for the data if you are not using AWS
manifest_file = os.path.join(drive_path, 'manifest.json')
boc = BrainObservatoryCache(manifest_file=manifest_file)
print('yes')
return (boc)
def boc(tmpdir_factory):
manifest_file = tmpdir_factory.mktemp('data').join(
os.path.join('boc', 'manifest.json'))
endpoint = os.environ[
'TEST_API_ENDPOINT'] if 'TEST_API_ENDPOINT' in os.environ else 'http://api.brain-map.org'
return BrainObservatoryCache(manifest_file=str(manifest_file),
base_uri=endpoint)
def get_running_speed(session_id):
'''uses allenSDK to get the running speed for a specified session
Parameters
----------
session_id
Returns
-------
running speed
'''
manifest_path = get_manifest_path()
from allensdk.core.brain_observatory_cache import BrainObservatoryCache
boc = BrainObservatoryCache(manifest_file=manifest_path)
data_set = boc.get_ophys_experiment_data(session_id)
running_speed,_ = data_set.get_running_speed()
return running_speed
def prePareAllenStim_for_othermodels(exp_id):
boc = BrainObservatoryCache(manifest_file='boc/manifest.json')
data_set = boc.get_ophys_experiment_data(exp_id)
scenes = data_set.get_stimulus_template('natural_scenes')
movie = data_set.get_stimulus_template('natural_movie_one')
numImages = scenes.shape[0]
data = np.ndarray((numImages,1,224,224))#scenes.shape[1],scenes.shape[1]
for n in range(0,numImages):
thisImage = np.array(scenes[n,:,0:918])
thisImage = cv2.resize(thisImage,(224,224))
thisImage = (thisImage - np.mean(thisImage))/np.std(thisImage)
data[n,0,:,:] = thisImage
data_colored = np.concatenate((data,data,data),axis=1)
return data_colored
def get_stimulus_template(session_id, stimulus):
'''uses allenSDK to get the stimulus template for the specific stimulus.
Parameters
----------
session_id (int)
stimulus name (string)
Returns
-------
stimulus template (numpy array)
'''
manifest_path = get_manifest_path()
from allensdk.core.brain_observatory_cache import BrainObservatoryCache
boc = BrainObservatoryCache(manifest_file=manifest_path)
data_set = boc.get_ophys_experiment_data(session_id)
stim_template = data_set.get_stimulus_template(stimulus)
return stim_template
def get_boc(drive_path='/Volumes/Brain2016/'):
"""
Grab BrainObservatoryCache
:param drive_path:
:return: boc
"""
manifest_path = os.path.join(drive_path, 'BrainObservatory/manifest.json')
boc = BrainObservatoryCache(manifest_file=manifest_path)
return boc
def get_L0_events(session_id):
'''gets the L0 event time series for a given session
Parameters
----------
session_id (int)
Returns
-------
l0 event traces (numpy array)
'''
manifest_path = get_manifest_path()
from allensdk.core.brain_observatory_cache import BrainObservatoryCache
boc = BrainObservatoryCache(manifest_file=manifest_path)
print("Loading L0 events for: " + str(session_id))
# we're now using the builtin events files from the SDK
return boc.get_ophys_experiment_events(session_id)
def main(initialize_database, start_exp=None, end_exp=None):
"""Main function to process Allen data."""
config = Config()
boc = BrainObservatoryCache(manifest_file=config.manifest_file)
timestamp = py_utils.timestamp()
log_file = os.path.join(config.log_dir, timestamp)
log = logger.get(log_file)
if initialize_database:
data_db.initialize_database()
log.info('Initialized DB.')
populate_db(config, boc, log, timestamp, start_exp, end_exp)
def generate_events(n):
boc = BrainObservatoryCache(manifest_file=MANIFEST_FILE)
expt_list = boc.get_ophys_experiments()
total = len(expt_list)
batch_size = total/40 + (1 if total%40!=0 else 0)
lower = batch_size*n
upper = lower + batch_size
for expt in expt_list[lower:upper]:
eid = expt['id']
print "Processing events for ", eid
data = boc.get_ophys_experiment_data(eid)
l0 = L0_analysis(data)
events = l0.get_events()
print "Done"
def get_stim_table(session_id, stimulus):
'''uses allenSDK to get stimulus table, specimen IDs, and the number of cells in specific session.
Parameters
----------
session_id (int)
stimulus name (string)
Returns
-------
stim_table (pandas DataFrame)
numbercells
specimen IDs
'''
manifest_path = get_manifest_path()
from allensdk.core.brain_observatory_cache import BrainObservatoryCache
boc = BrainObservatoryCache(manifest_file=manifest_path)
data_set = boc.get_ophys_experiment_data(session_id)
specimen_ids = data_set.get_cell_specimen_ids()
stim_table = data_set.get_stimulus_table(stimulus)
numbercells = len(specimen_ids)
return stim_table, numbercells, specimen_ids
def main():
manifest_file = '/data/dynamic-brain-workshop/brain_observatory_cache/brain_observatory_manifest.json'
boc = BrainObservatoryCache(manifest_file=manifest_file)
stim = si.NATURAL_MOVIE_THREE
session = si.THREE_SESSION_A
metric = 'reliability_nm3'
threshold = .5
out_file = 'sparkles_nm3_2d.npz'
template_file = 'nm3.npz'
download_data(boc, stim, session, metric, threshold, out_file,
template_file)
def prePareAllenStim_for_CPC(exp_id,blocksize):
boc = BrainObservatoryCache(manifest_file='boc/manifest.json')
data_set = boc.get_ophys_experiment_data(exp_id)
scenes = data_set.get_stimulus_template('natural_scenes')
movie = data_set.get_stimulus_template('natural_movie_one')
numImages = scenes.shape[0]
data = np.ndarray((1,numImages,1,blocksize,184,184))#scenes.shape[1],scenes.shape[1]
for n in range(0,numImages):
for b in range(0,blocksize):
thisImage = np.array(scenes[n,:,0:918])
thisImage = cv2.resize(thisImage,(184,184))
thisImage = (thisImage - np.mean(thisImage))/np.std(thisImage)
data[0,n,0,b,:,:] = thisImage
data_colored = np.concatenate((data,data,data),axis=2)
numFrames = movie.shape[0]
numBlocks = int(numFrames/(3*blocksize))
data = np.ndarray((1,numBlocks,1,blocksize,184,184))#scenes.shape[1],scenes.shape[1]
f = 0
nidx = np.arange(numBlocks)
for n in range(0,numBlocks):
for b in range(0,blocksize):
thisImage = np.array(movie[f,:,0:303])
thisImage = cv2.resize(thisImage,(184,184))
thisImage = (thisImage - np.mean(thisImage))/np.std(thisImage)
data[0,nidx[n],0,b,:,:] = thisImage
f = f + 3
data2_colored = np.concatenate((data,data,data),axis=2)
return data_colored, data2_colored
def brain_observatory_cache():
boc = None
with patch('os.path.exists') as m:
m.return_value = True
with patch("__builtin__.open", mock_open(read_data=CACHE_MANIFEST)):
# Download a list of all targeted areas
boc = BrainObservatoryCache(manifest_file='boc/manifest.json',
base_uri='http://testwarehouse:9000')
boc.api.json_msg_query = MagicMock(name='json_msg_query')
return boc
def test_pre_release_get_experiments(tmpdir):
# Values from original boc/api:
temp_dir_base = os.path.join(str(tmpdir), 'base-api')
outfile_base = os.path.join(temp_dir_base, 'manifest.json')
boc_base = BrainObservatoryCache(manifest_file=outfile_base)
experiments_base = boc_base.get_ophys_experiments()
# # For development: print key/val pairs needed to be populated by adapter:
# for key, val in sorted(experiments_base[0].items(), key=lambda x: x[0]):
# print key, val
# raise
try:
temp_dir_extended = os.path.join(str(tmpdir), 'extended-api')
outfile_extended = os.path.join(temp_dir_extended, 'manifest.json')
boc_extended = BrainObservatoryCache(
manifest_file=outfile_extended,
api=BrainObservatoryApiPreRelease())
except TypeError:
import allensdk
raise RuntimeError(
'Allensdk out-of-date; upgrade with "pip install --force --upgrade allensdk"'
)
experiments_extended = boc_extended.get_ophys_experiments()
# # For development: print key/val pairs actually populated by adapter:
# for key, val in sorted(containers_extended[0].items(), key=lambda x: x[0]):
# print key, val
# raise
check_key_list = [
'acquisition_age_days', 'cre_line', 'donor_name',
'experiment_container_id', 'fail_eye_tracking', 'id', 'imaging_depth',
'reporter_line', 'session_type', 'specimen_name', 'targeted_structure'
]
for key in check_key_list:
assert key in experiments_extended[0]
assert len(experiments_extended) > 0
assert set(experiments_base[0].keys()) == set(
experiments_extended[0].keys())
id_experiment_dict = {}
for c_e in experiments_extended:
curr_id = c_e['id']
id_experiment_dict[curr_id] = c_e
for c_b in experiments_base:
c_e = id_experiment_dict[c_b['id']]
for key in c_e:
# assert c_e[key] == c_b[key]
if not c_e[key] == c_b[key]:
print(key, c_e[key], c_b[key])
raise Exception()
def brain_observatory_cache(fn_temp_dir):
boc = None
try:
manifest_data = bytes(CACHE_MANIFEST, 'UTF-8') # Python 3
except:
manifest_data = bytes(CACHE_MANIFEST) # Python 2.7
with patch('os.path.exists', return_value=True):
with patch(builtins.__name__ + ".open",
mock_open(read_data=manifest_data)):
manifest_file = os.path.join(fn_temp_dir, "boc", "manifest.json")
boc = BrainObservatoryCache(manifest_file=manifest_file,
base_uri='http://api.brain-map.org')
return boc
def brain_observatory_cache():
boc = None
try:
manifest_data = bytes(CACHE_MANIFEST, 'UTF-8') # Python 3
except:
manifest_data = bytes(CACHE_MANIFEST) # Python 2.7
with patch('os.path.exists', return_value=True):
with patch(builtins.__name__ + ".open",
mock_open(read_data=manifest_data)):
# Download a list of all targeted areas
boc = BrainObservatoryCache(
manifest_file="some_path/manifest.json",
base_uri='http://api.brain-map.org')
return boc
def test_build_manifest():
with patch('os.path.exists') as m:
m.return_value = False
with patch('allensdk.config.manifest.Manifest.safe_mkdir') as mkdir:
with patch(
'allensdk.config.manifest_builder.'
'ManifestBuilder.write_json_file',
MagicMock(name='write_json_file')) as mock_write_json:
with patch("__builtin__.open",
mock_open(read_data=CACHE_MANIFEST)):
brain_observatory_cache = BrainObservatoryCache(
manifest_file='boc/manifest.json',
base_uri='http://testwarehouse:9000')
mkdir.assert_called_once_with('boc')
mock_write_json.assert_called_once_with(
'boc/manifest.json')
def brain_observatory_cache():
boc = None
try:
manifest_data = bytes(CACHE_MANIFEST, 'UTF-8') # Python 3
except:
manifest_data = bytes(CACHE_MANIFEST) # Python 2.7
with patch('os.path.exists', return_value=True):
with patch(builtins.__name__ + ".open",
mock_open(read_data=manifest_data)):
# Download a list of all targeted areas
boc = BrainObservatoryCache(manifest_file='boc/manifest.json',
base_uri='http://testwarehouse:9000')
boc.api.json_msg_query = MagicMock(name='json_msg_query')
return boc
def download_data(region, cre_line, stimulus = None):
'''
region = [reg1, reg2, ...]
cre_line = [line1, line2, ...]
'''
boc = BrainObservatoryCache(manifest_file='boc/manifest.json')
ecs = boc.get_experiment_containers(targeted_structures=region, cre_lines=cre_line)
ec_ids = [ ec['id'] for ec in ecs ]
exp = boc.get_ophys_experiments(experiment_container_ids=ec_ids)
if stimulus == None:
exp = boc.get_ophys_experiments(experiment_container_ids=ec_ids)
else:
exp = boc.get_ophys_experiments(experiment_container_ids=ec_ids, stimuli = stimulus)
exp_id_list = [ec['id'] for ec in exp]
data_set = {exp_id:boc.get_ophys_experiment_data(exp_id) for exp_id in exp_id_list}
return data_set
# -*- coding: utf-8 -*- """ Created on Mon Aug 29 21:28:20 2016 Use Saskia's analysis files to get info without importing the whole freakin dataset. @author: ttruszko 511510855 """ from __future__ import print_function #this is the same old thing, getting the cell specimen ids from the ophys_experiment_data... from allensdk.core.brain_observatory_cache import BrainObservatoryCache exp_id= 510705057 #this is session ID boc = BrainObservatoryCache() data_set = boc.get_ophys_experiment_data(exp_id) csids = data_set.get_cell_specimen_ids() # In[] this is Saskia's example code. Works as is but it's better to use the open_h5_file function. import h5py import pandas as pd #this loads the response array and the peak dataframe from the analysis file ## edit this for Mac!! And for whatever session you want. path = 'D:/BrainObservatory/ophys_analysis/510705057_three_session_B_analysis.h5' f = h5py.File(path, 'r') response = f['analysis']['response_ns'].value f.close() sweep_response = pd.read_hdf(path, 'analysis/sweep_response_ns') mean_sweep_response=pd.read_hdf(path, 'analysis/mean_sweep_response_ns')
# We need to import these modules to get started
drive_path = 'e/'
import numpy as np
import pandas as pd
import osp
import sys
from decoder import *
from allensdk.core.brain_observatory_cache import BrainObservatoryCache
from allensdk.brain_observatory.static_gratings import StaticGratings
manifest_path = os.path.join(drive_path,'BrainObservatory/manifest.json')
boc = BrainObservatoryCache(manifest_file=manifest_path)
#label extractor
def labeled_data_extractor(stim_info, type_of_labels):
'''Assignes lables for eah stimuli, Type_of_lable can be ['orientation'] or ['temporal_frequency'] or both. stim_info is a panda data_frame produces by BrainObservatoryCashe '''
si = stim_info.copy();
if len(type_of_labels) == 1:
temp_si = si[type_of_labels[0]].replace(si[type_of_labels[0]].unique(), range(len(si[type_of_labels[0]].unique())));
return(temp_si.values.astype(int))
elif ['temporal_frequency', 'orientation'] in type_of_labels:
0;
# Pick an experiment container (:)--
experiment_container_v1_cux2 = boc.get_experiment_containers(targeted_structures=['VISp'],cre_lines=['Cux2-CreERT2'],imaging_depths=[275]);
#experiment_container_v1_cux2 = boc.get_experiment_containers(targeted_structures=['VISp'],cre_lines=['Cux2-CreERT2']);
# choose an animal
animal_number=0;
experiment_container_id = experiment_container_v1_cux2[animal_number]['id'];
experiment_information = boc.get_ophys_experiments(experiment_container_ids=[experiment_container_id]);
def BOC_init(stimuli_to_use={
'drifting_gratings',
'locally_sparse_noise',
'spontaneous',
'static_gratings'
},
selectivity_csv_filename='image_selectivity_dataframe.csv',
areas={
'VISp'
},
discard_nan={
'selectivity_ns',
'osi_sg',
'osi_dg',
'time_to_peak_ns',
'time_to_peak_sg',
}
):
""""Returns a BrainObservatoryCache initialized using the data on the external harddrives provided by the Allen Institute
Example usage:
boc, specimens_with_selectivity_S, VISp_cells_with_numbers = BOC_init()
Input:
stimuli_stimuli_to_use : Which stimuli we are interested in. By default, this is
['drifting_gratings', 'locally_sparse_noise', 'spontaneous', 'static_gratings']
selectivity_csv_filename : The path (including filename) to the image_selectivity_data.csv file. By default, this is just 'image_selectivity_dataframe.csv', which assumes that the file is in the same directory as the script you are running.
areas : We will restrict our attention to the areas in this list. By default this is the list ['VISp'], because we are visual cortex chauvanists here at the Allen Institute.
discard_nana : One of the outputs of this function is a dataframe that's been filtered to discard rows that have NaN values in the columns specified here.
Output:
boc : the BrainObservatoryCache object from AllenSDK, initialized using the data from the external harddrive.
specimens_with_selectivity_S : A dataframe the includes the output of boc.get_cell_specimens as well as a column for the selectivity index S
See Decoding Visual Inputs From Multiple Neurons in the Human Temporal Lobe, J. Neurophys 2007, by Quiroga et al.
VISp_cells_with_numbers : a dataframe that filters specimens_with_selectivity_S by discarding rows with NaN values in the columns specified above.
"""
# The dynamic brain database is on an external hard drive.
drive_path = '/Volumes/Brain2016'
if sys.platform.startswith('w'):
drive_path = 'd:'
manifest_path = os.path.join(drive_path, 'BrainObservatory/manifest.json')
boc = BrainObservatoryCache(manifest_file=manifest_path)
expt_containers = boc.get_experiment_containers()
all_expts_df = pd.DataFrame(boc.get_ophys_experiments(stimuli=list(stimuli_to_use)))
# this has headers:
# age_days cre_line experiment_container_id id imaging_depth session_type targeted_structure
specimens_df = pd.DataFrame(
boc.get_cell_specimens(experiment_container_ids=all_expts_df.experiment_container_id.values))
# this has headers:
# area cell_specimen_id dsi_dg experiment_container_id imaging_depth osi_dg osi_sg p_dg p_ns p_sg
# pref_dir_dg pref_image_ns pref_ori_sg pref_phase_sg pref_sf_sg pref_tf_dg time_to_peak_ns time_to_peak_sg
# tld1_id tld1_name tld2_id tld2_name tlr1_id tlr1_name
# There's also a handy bit of data from Saskia, in the form of a measurement called S. See
# Decoding Visual Inputs From Multiple Neurons in the Human Temporal Lobe, J. Neurophys 2007, by Quiroga et al
selectivity_S_df = pd.read_csv(selectivity_csv_filename, index_col=0)
selectivity_S_df = selectivity_S_df[['cell_specimen_id', 'selectivity_ns']]
specimens_with_selectivity_S = specimens_df.merge(selectivity_S_df, how='outer', on='cell_specimen_id')
# This is all cells in VISp that have a value for the specified parameters (i.e not NaN)
VISp_cells_with_numbers = specimens_with_selectivity_S
for area_name in areas:
VISp_cells_with_numbers = VISp_cells_with_numbers[VISp_cells_with_numbers.area == area_name]
for col_name in discard_nan:
VISp_cells_with_numbers = VISp_cells_with_numbers[np.isnan(VISp_cells_with_numbers[col_name]) == False]
return boc, specimens_with_selectivity_S, VISp_cells_with_numbers
