def Main():
filter_obj = {
'dendrite_type': 'spiny',
'structure_layer_name': '5',
'structure_area_abbrev': 'VISp'
}
ctc = CellTypesCache()
cells = ctc.get_cells(species=['Mus musculus'])
cells_df = pd.DataFrame(cells)
for filt_key, filt_val in filter_obj.items():
cells_df = cells_df.loc[cells_df[filt_key] == filt_val, :]
cell_ids = list(cells_df['id'].values)
rc = Client(profile=os.getenv('IPYTHON_PROFILE'))
logger.debug('Using ipyparallel with %d engines', len(rc))
lview = rc.load_balanced_view()
func = partial(get_fi_data, ctc)
filter_fi_data = lview.map_sync(func, cell_ids)
filter_fi_data = [data for data in filter_fi_data if data is not None]
file_name = 'fi_data.pkl'
with open(file_name, 'wb') as fh:
pickle.dump(filter_fi_data, fh)
plot_fi_data(filter_fi_data)
rc.shutdown(hub=True)
class AllenMorphology(Paths):
def __init__(self, *args, **kwargs):
if not connected_to_internet():
raise ConnectionError("You will need to be connected to the internet to use the AllenMorphology class")
Paths.__init__(self, *args, **kwargs)
# Create a Cache for the Cell Types Cache API
self.ctc = CellTypesCache(manifest_file=os.path.join(self.morphology_allen, 'manifest.json'))
# Get a list of cell metadata for neurons with reconstructions, download if necessary
self.neurons = pd.DataFrame(self.ctc.get_cells(species=[CellTypesApi.MOUSE], require_reconstruction = True))
self.n_neurons = len(self.neurons)
if not self.n_neurons: raise ValueError("Something went wrong and couldn't get neurons metadata from Allen")
self.downloaded_neurons = self.get_downloaded_neurons()
def get_downloaded_neurons(self):
return [os.path.join(self.morphology_allen, f) for f in os.listdir(self.morphology_allen) if ".swc" in f]
def download_neurons(self, ids):
if isinstance(ids, np.ndarray):
ids = list(ids)
if not isinstance(ids, (list)): ids = [ids]
for neuron_id in ids:
neuron_file = os.path.join(self.morphology_allen, "{}.swc".format(neuron_id))
neuron = self.ctc.get_reconstruction(neuron_id, file_name=neuron_file)
def main_all_human():
parser = argparse.ArgumentParser()
parser.add_argument('output_dir', default='.')
args = parser.parse_args()
from allensdk.core.cell_types_cache import CellTypesCache
from allensdk.api.queries.cell_types_api import CellTypesApi
ctc = CellTypesCache(
manifest_file=os.path.join(args.output_dir, "ctc", "manifest.json"))
cells = ctc.get_cells(require_reconstruction=True,
species=[CellTypesApi.HUMAN])
for cell in cells:
morphology = fetch_aligned_morphology(specimen_id=cell['id'])
cell_dir = os.path.join(args.output_dir, str(cell['id']))
if not os.path.exists(cell_dir):
os.makedirs(cell_dir)
# swc_file = os.path.join(cell_dir, "recon.swc")
ply_file = os.path.join(cell_dir, "recon.ply")
vtk_file = os.path.join(cell_dir, "recon.vtk")
tube_pd = vtkmorph.generate_mesh(morphology.compartment_index,
morphology.root,
color_by_type,
6,
radius=None)
vtkmorph.write_ply(tube_pd, ply_file)
vtkmorph.write_vtk(tube_pd, vtk_file)
print(ply_file)
def __init__(self, *args, scene_kwargs={}, **kwargs):
"""
Initialise API interaction and fetch metadata of neurons in the Allen Database.
"""
if not connected_to_internet():
raise ConnectionError(
"You will need to be connected to the internet to use the AllenMorphology class"
)
Paths.__init__(self, *args, **kwargs)
self.scene = Scene(add_root=False, display_inset=False, **scene_kwargs)
# Create a Cache for the Cell Types Cache API
self.ctc = CellTypesCache(
manifest_file=os.path.join(self.morphology_allen, 'manifest.json'))
# Get a list of cell metadata for neurons with reconstructions, download if necessary
self.neurons = pd.DataFrame(
self.ctc.get_cells(species=[CellTypesApi.MOUSE],
require_reconstruction=True))
self.n_neurons = len(self.neurons)
if not self.n_neurons:
raise ValueError(
"Something went wrong and couldn't get neurons metadata from Allen"
)
self.downloaded_neurons = self.get_downloaded_neurons()
def get_cell_morphXYZ(cell_id):
from allensdk.core.cell_types_cache import CellTypesCache
from allensdk.core.swc import Marker
import pprint
ctc = CellTypesCache(manifest_file='cell_types/manifest.json')
morph = ctc.get_reconstruction(cell_id)
markers = ctc.get_reconstruction_markers(cell_id)
x = []
y = []
z = []
for n in morph.compartment_list:
#print(n['type']) #type=1, soma; type=2, axon; type=3, dendrite; type=4,apical dendrite
if n['type'] == 4 or n['type'] == 3 or n['type'] == 1:
x.append(n['x'] - morph.soma["x"])
y.append(n['y'] - morph.soma["y"])
z.append(n['z'] - morph.soma["z"])
morph_data = np.array(np.column_stack((x, y, z)))
morph_soma = [morph.soma["x"], morph.soma["y"], morph.soma["z"]]
return (morph_data, morph_soma)
def load_realdata_ephys(cell_specimen_id=464212183):
from allensdk.core.cell_types_cache import CellTypesCache
ctc = CellTypesCache(manifest_file='cell_types/manifest.json')
# this saves the NWB file to 'cell_types/specimen_464212183/ephys.nwb'
cell_specimen_id =cell_specimen_id#64212183#325941643#464212183 (good)
data_set = ctc.get_ephys_data(cell_specimen_id)
sweep_number = 35
sweep_data = data_set.get_sweep(sweep_number)
index_range = sweep_data["index_range"]
i = sweep_data["stimulus"][0:index_range[1]+1] # in A
v = sweep_data["response"][0:index_range[1]+1] # in V
i *= 1e6 # to mu A #1e12 # to pA
v *= 1e3 # to mV
sampling_rate = sweep_data["sampling_rate"] # in Hz
t = np.arange(0, len(v)) * (1.0 / sampling_rate)
D_latent= 2
signal_idx = np.logical_and(t >= 1.03, t <= 2.022)
V = v[signal_idx]
I_inj_values = i[signal_idx]
t = t[signal_idx]
t *= 1e3 #ms
downsample = 30 # downsample
V = V[::downsample]
I_inj_values = I_inj_values[::downsample]
t = t[::downsample]
return V, t, I_inj_values
def download():
ctc = CellTypesCache(manifest_file='ctc/manifest.json')
cells = ctc.get_cells()
for i, (cats, patches) in enumerate(sample_data_sets(cells, ctc, 100, 1000, 100, 4096)):
print(cats.shape)
np.save('patches/cats_%04d.npy' % i, cats)
np.save('patches/patches_%04d.npy' % i, patches)
def allen_id_to_sweeps(specimen_id):
ctc = CellTypesCache(manifest_file='cell_types/manifest.json')
specimen_id = int(specimen_id)
data_set = ctc.get_ephys_data(specimen_id)
sweeps = ctc.get_ephys_sweeps(specimen_id)
sweep_numbers = defaultdict(list)
for sweep in sweeps:
sweep_numbers[sweep['stimulus_name']].append(sweep['sweep_number'])
return sweep_numbers,data_set,sweeps
def load_realdata_morphology(cell_id = 464212183, desample_tree_factor=30):
from allensdk.core.cell_types_cache import CellTypesCache
# load real data
ctc = CellTypesCache(manifest_file='cell_types/manifest.json')
# load 3D morphology data
morphology = ctc.get_reconstruction(cell_id)
# extract
compartments = morphology.compartment_list
compartments_byid = { c['id']: c for c in compartments}
num_children = np.array([len(c['children']) for c in compartments])
branch_indexes = np.where(num_children>1)[0] # branch compartment index
# find all paths (terminate at branches)
paths = find_branch_paths(compartments_byid, branch_indexes)
# de-sample tree and paths
select_compartments = [] # index selected
path_compartments = [] # every connections between selected compartments (not limited to branches)
path_filtered = [] #per branch, only keep selected one
for k, path in enumerate(paths):
n = len(path)
if n < desample_tree_factor:
select_compartments.extend([path[0], path[-1]])
path_compartments.append([path[0], path[-1]])
path_filtered.append([path[0], path[-1]])
else:
use = [path[int(i)] for i in np.linspace(0, n - 1, n//desample_tree_factor+1)]
path_filtered.append(use)
select_compartments.extend(use)
for j in range(len(use) - 1):
path_compartments.append([use[j], use[j+1]])
select_compartments = np.unique(select_compartments)
assert len(np.setdiff1d(branch_indexes, select_compartments)) == 0
# map old id to new id
new_id = {select_compartments[i]:i for i in range(len(select_compartments))}
# calculate distance between selected compartments
from numpy import linalg as LA
C = len(select_compartments)
dist = np.full((C, C), np.inf)
for path in path_compartments:
ID1, xyz1, _ = get_info(compartments_byid[path[0]])
ID2, xyz2, _ = get_info(compartments_byid[path[1]])
dist[new_id[path[0]], new_id[path[1]]] = LA.norm(xyz1 - xyz2, 2)
dist[new_id[path[1]], new_id[path[0]]] = dist[new_id[path[0]], new_id[path[1]]]
network = 1/dist
return network, new_id, compartments_byid, paths, select_compartments, path_compartments, path_filtered, compartments, morphology
def cache_fixture():
# Instantiate the CellTypesCache instance. The manifest_file argument
# tells it where to store the manifest, which is a JSON file that tracks
# file paths. If you supply a relative path (like this), it will go
# into your current working directory
ctc = CellTypesCache(manifest_file='cell_types/cell_types_manifest.json')
specimen_id = 464212183
# this saves the NWB file to 'cell_types/specimen_464212183/ephys.nwb'
data_set = ctc.get_ephys_data(specimen_id)
return ctc, data_set
def cache_fixture():
# Instantiate the CellTypesCache instance. The manifest_file argument
# tells it where to store the manifest, which is a JSON file that tracks
# file paths. If you supply a relative path (like this), it will go
# into your current working directory
ctc = CellTypesCache(manifest_file='cell_types/cell_types_manifest.json')
specimen_id = 464212183
# this saves the NWB file to 'cell_types/specimen_464212183/ephys.nwb'
data_set = ctc.get_ephys_data(specimen_id)
return ctc, data_set
def cell_morphology_rot(cell_id, x_soma, y_soma, z_soma, theta):
theta_z = theta[2]
theta_y = theta[1]
theta_x = theta[0]
# download and open an SWC file
ctc = CellTypesCache(manifest_file='cell_types/manifest.json')
morph = ctc.get_reconstruction(cell_id)
#First applying a rotation angle around z axis
tr_rot_z = [
math.cos(theta_z), -math.sin(theta_z), 0,
math.sin(theta_z),
math.cos(theta_z), 0, 0, 0, 1, 0, 0, 0
]
#Second applying a rotation angle around y axis
tr_rot_y = [
math.cos(theta_y), 0,
math.sin(theta_y), 0, 1, 0, -math.sin(theta_y), 0,
math.cos(theta_y), 0, 0, 0
]
#Third applying a rotation angle around x axis
tr_rot_x = [
1, 0, 0, 0,
math.cos(theta_x), -math.sin(theta_x), 0,
math.sin(theta_x),
math.cos(theta_x), 0, 0, 0
]
morph.apply_affine(tr_rot_z)
morph.apply_affine(tr_rot_y)
morph.apply_affine(tr_rot_x)
# translate the soma location
tr_soma = [
1, 0, 0, 0, 1, 0, 0, 0, 1, -morph.soma["x"] + x_soma,
-morph.soma["y"] + y_soma, -morph.soma["z"] + z_soma
]
morph.apply_affine(tr_soma)
# plot xy and xz views
# fig, axes = plt.subplots(1, 2, sharey=True, sharex=True)
# vm.plot_morph_swc(morph,axes,color='r')
# fig, axes = plt.subplots(1, 2, sharey=True, sharex=True)
# Make a line drawing of x-y and y-z views
return morph
def extract_data(ephys_cell, sweep_number):
"""Data extraction from AllenDB
Parameters
----------
ephys_cell : int
Cell identity from AllenDB
sweep_number : int
Stimulus identity for cell ephys_cell from AllenDB
"""
t_offset = 815.
duration = 1450.
real_data_path = 'ephys_cell_{}_sweep_number_{}.pkl'.format(
ephys_cell, sweep_number)
if not os.path.isfile(real_data_path):
from allensdk.core.cell_types_cache import CellTypesCache
from allensdk.api.queries.cell_types_api import CellTypesApi
manifest_file = 'cell_types/manifest.json'
cta = CellTypesApi()
ctc = CellTypesCache(manifest_file=manifest_file)
data_set = ctc.get_ephys_data(ephys_cell)
sweep_data = data_set.get_sweep(
sweep_number) # works with python2 and fails with python3
sweeps = cta.get_ephys_sweeps(ephys_cell)
sweep = sweeps[sweep_number]
index_range = sweep_data["index_range"]
i = sweep_data["stimulus"][0:index_range[1] + 1] # in A
v = sweep_data["response"][0:index_range[1] + 1] # in V
sampling_rate = sweep_data["sampling_rate"] # in Hz
dt = 1e3 / sampling_rate # in ms
i *= 1e6 # to muA
v *= 1e3 # to mV
v = v[int(t_offset / dt):int((t_offset + duration) / dt)]
i = i[int(t_offset / dt):int((t_offset + duration) / dt)]
real_data_obs = np.array(v).reshape(1, -1, 1)
I_real_data = np.array(i).reshape(-1)
t_on = int(
sweep['stimulus_start_time'] * sampling_rate) * dt - t_offset
t_off = int(
(sweep['stimulus_start_time'] + sweep['stimulus_duration']) *
sampling_rate) * dt - t_offset
f = open(real_data_path, 'wb')
pickle.dump((real_data_obs, I_real_data, dt, t_on, t_off), f)
f.close()
def run_nwb(cell_id, model_type, neuron_config, stim_type='Ramp'):
"""Generates a injection current from nwb sweep file
Parameters
----------
cell_id : ID of cell speciment
model_type : LIF, LIF-R, LIF-R-ASC, LIF-ASC, LIF-R-ASC-A
stumilus_data : stumilus data from NWB file
"""
# get sweep/stimulus data
ctc = CellTypesCache()
ephys_sweeps = ctc.get_ephys_sweeps(cell_id)
ds = ctc.get_ephys_data(cell_id)
#ephys_sweep = next( s for s in ephys_sweeps if s['stimulus_name'] == stim_type )
ephys_sweep_stim = [
s for s in ephys_sweeps if s['stimulus_name'] == stim_type
]
ephys_sweep = ephys_sweep_stim[0]
stumilus_data = ds.get_sweep(ephys_sweep['sweep_number'])
ret = {}
n_steps = len(stumilus_data['stimulus'])
dt = 1.0 / stumilus_data['sampling_rate'] * 1.0e03
amp_times = [t * dt for t in xrange(n_steps)]
amp_values = stumilus_data['stimulus'].tolist()
total_time = n_steps * dt
output = runNestModel(model_type, neuron_config, amp_times, amp_values, dt,
total_time)
ret['nest'] = {
'times': output[0],
'voltages': output[1],
'spike_times': output[2]
}
output = runGlifNeuron(neuron_config, amp_values, dt)
ret['allen'] = {
'times': output[0],
'voltages': output[1],
'spike_times': output[2]
}
ret['I'] = amp_values
ret['dt'] = dt
return ret
def testAllDataSets(self):
manifest_file = '/local1/projects/FHL2015/cell_types/manifest.json'
if not os.path.exists(manifest_file):
print "Cannot run this test: manifest does not exist (%s)" % manifest_file
return True
self.cache = CellTypesCache(manifest_file=manifest_file)
cells = self.cache.get_cells()
for cell in cells:
data_set = self.cache.get_ephys_data(cell['id'])
sweeps = self.cache.get_ephys_sweeps(cell['id'])
for sweep in sweeps:
metadata = data_set.get_sweep_metadata(sweep['sweep_number'])
def main_specimen():
from allensdk.core.cell_types_cache import CellTypesCache
specimen_id = 485880739
ctc = CellTypesCache()
morphology = ctc.get_reconstruction(specimen_id)
for c in morphology.compartment_list:
c['z'] *= 3
tube_pd = vtkmorph.generate_tube(morphology.compartment_index,
morphology.root,
color_by_type,
6,
radius=1.5)
vtkmorph.write_ply(tube_pd, '%d.ply' % specimen_id)
vtkmorph.write_vtk(tube_pd, '%d.vtk' % specimen_id)
def get_data_sets(upper_bound=2, lower_bound=None):
try:
with open('../all_allen_cells.p', 'rb') as f:
cells = pickle.load(f)
except:
ctc = CellTypesCache(manifest_file='cell_types/manifest.json')
cells = ctc.get_cells()
with open('all_allen_cells.p', 'wb') as f:
pickle.dump(cells, f)
data = []
data_sets = []
path_name = 'data_nwbs'
try:
os.mkdir(path_name)
except:
print('directory already made.')
ids = [c['id'] for c in cells]
if upper_bound == None and lower_bound is None:
limited_range = ids[0:-1]
elif upper_bound is not None and lower_bound is not None:
limited_range = ids[lower_bound:upper_bound]
for specimen_id in limited_range:
temp_path = str(path_name) + str('/') + str(specimen_id) + '.p'
if os.path.exists(temp_path):
with open(temp_path, 'rb') as f:
(data_set_nwb, sweeps, specimen_id) = pickle.load(f)
data_sets.append((data_set_nwb, sweeps, specimen_id))
else:
data_set = ctc.get_ephys_data(specimen_id)
sweeps = ctc.get_ephys_sweeps(specimen_id)
file_name = 'cell_types/specimen_' + str(
specimen_id) + '/ephys.nwb'
data_set_nwb = NwbDataSet(file_name)
data_sets.append((data_set_nwb, sweeps, specimen_id))
with open(temp_path, 'wb') as f:
pickle.dump((data_set_nwb, sweeps, specimen_id), f)
#broken
return data_sets
class NwbDataSetTest(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(NwbDataSetTest, self).__init__(*args, **kwargs)
def testAllDataSets(self):
manifest_file = '/local1/projects/FHL2015/cell_types/manifest.json'
if not os.path.exists(manifest_file):
print "Cannot run this test: manifest does not exist (%s)" % manifest_file
return True
self.cache = CellTypesCache(manifest_file=manifest_file)
cells = self.cache.get_cells()
for cell in cells:
data_set = self.cache.get_ephys_data(cell['id'])
sweeps = self.cache.get_ephys_sweeps(cell['id'])
for sweep in sweeps:
metadata = data_set.get_sweep_metadata(sweep['sweep_number'])
def load_data(stim_names, reps=10, dur=3000, delay=200):
ctc = CellTypesCache(manifest_file="ctc/manifest.json")
cells = ctc.get_cells()
cell_id = cells[0]['id']
sweeps = ctc.get_ephys_sweeps(cell_id)
sweeps = [(sweep['sweep_number'], sweep['stimulus_name'])
for sweep in sweeps if sweep['stimulus_name'] in stim_names]
ds = ctc.get_ephys_data(cell_id)
vv, ii = [], []
for sn, st in sweeps:
v, i, t = load_sweep(ds, sn)
stim_start = np.argwhere(i != 0)[0][0]
for rep in range(reps):
idx0 = stim_start - delay - np.random.randint(0, dur // 2)
vr = v[idx0:]
ir = i[idx0:]
if st.startswith('Noise'):
offs = [0, 200000, 400000]
for off in offs:
vv.append(vr[off:off + dur])
ii.append(ir[off:off + dur])
else:
vv.append(vr[:dur])
ii.append(ir[:dur])
stims = np.vstack(ii)
resps = np.vstack(vv) + 74.0
print(stims.shape)
return stims, resps
def getExperiment(cell_id, stimtypes):
ctc = CellTypesCache()
ephys_sweeps = ctc.get_ephys_sweeps(specimen_id=cell_id)
sweeps_by_type = defaultdict(list)
sweeps = []
for sweep in ephys_sweeps:
if sweep['stimulus_name'] in stimtypes:
sweeps.append(sweep['sweep_number'])
sweeps_by_type[sweep['stimulus_name']].append(
sweep['sweep_number'])
ephys_filename = f'{cell_id}/{cell_id}_ephys.nwb'
ctc.get_ephys_data(cell_id, ephys_filename)
swc_filename = f'{cell_id}/{cell_id}.swc'
ctc.get_reconstruction(cell_id, swc_filename)
return ephys_filename, swc_filename, sweeps, sweeps_by_type
def save_cell_metadata(**cell_metadata):
cell_id = cell_metadata["cell_id"]
metadata_filename = 'cell_metadata_%s.json' % cell_id
# TODO: maybe move this to launch_optimjob?
machine_name = socket.gethostname()
machine_name = 'aws' if machine_name == 'master' else machine_name
cell_metadata['machine'] = machine_name
data_source = cell_metadata["data_source"]
if data_source == "web":
ctc = CellTypesCache(
manifest_file=os.path.join(data_dir, 'manifest.json'))
cell_metadata.update(get_data_web(cell_id, ctc))
cell_metadata.update(cell_props(cell_id, ctc))
cell_metadata.update(model_props(cell_id))
elif data_source == "lims":
cell_metadata.update(get_data_lims(cell_id))
utility.save_json(metadata_filename, cell_metadata)
return cell_metadata, metadata_filename
# First, we'll `import` the CellTypesCache module. This module provides tools to allow us to get information from the cell types database. We're giving it a **manifest** filename as well. CellTypesCache will create this manifest file, which contains metadata about the cache. You can look under cell_types in your directory, and take a look at the file. # # (If you're curious you can see the full documentation for the core package <a href="https://allensdk.readthedocs.io/en/latest/allensdk.core.html">here</a>.) # # <b>Note</b>: In order to run the line below, you need to have the AllenSDK installed. You can find information on how to do that <a href="http://alleninstitute.github.io/AllenSDK/install.html">here</a>. If you're running this on the UCSD Datahub, the Allen SDK has already been installed for you. # In[1]: #Import the "Cell Types Cache" from the AllenSDK core package from allensdk.core.cell_types_cache import CellTypesCache #Import CellTypesApi, which will allow us to query the database. from allensdk.api.queries.cell_types_api import CellTypesApi # We'll then initialize the cache as 'ctc' (cell types cache) ctc = CellTypesCache(manifest_file='cell_types/manifest.json') # ### Step One: Get Cells & Manipulate Dataframe # # The `get_cells` method downloads metadata for all cells in the database. The database contains human cells and mouse cells. Alternatively, one can filter out the database to only include cells collected from a certain species. # Look through <a href="https://allensdk.readthedocs.io/en/latest/allensdk.core.cell_types_cache.html">the documentation for the CellTypesCache</a> for more information on the `get_cells` method. # In[2]: all_cells = ctc.get_cells() print(all_cells) # As you can see, the output for the metadata of our cells is messy and difficutlt to interpret. To make our data easier to read and work with, we can convert `all_cells` into a pandas datadrame. # # Note: If you're having trouble with Pandas, it can help to look at <a href="https://pandas.pydata.org/pandas-docs/stable/user_guide/">the user guide</a>.
def sdk_nwb_information(specimen_id):
ctc = CellTypesCache()
nwb_data_set = ctc.get_ephys_data(specimen_id)
sweep_info = ctc.get_ephys_sweeps(specimen_id)
return nwb_data_set.file_name, sweep_info
'1':['574734127','475585413','536951541'],
# '2/3':['485184849','475515168','485468180','476087653','571306690'],
'2/3':['485184849','475515168','485468180'],
# '4':['483101699','602822298','490205998','569723367','324257146'],
'4':['483101699','602822298','569723367'],
'5':['479225052','607124114','515249852'],
'6a':['490259231','473564515','561985849'],
# '6b':['589128331','574993444','510136749','509881736','590558808']
'6b':['589128331']
}
highlight_cells = ['485184849', '479225052', '473564515']
# Get normalized depth metadata for individual cells
ctc = CellTypesCache()
cells_allensdk = ctc.get_cells(species = ['Mus musculus'],simple = False)
sdk_data = pd.DataFrame(cells_allensdk)
sdk_data['specimen__id'] = sdk_data['specimen__id'].astype(str)
ylim_min,ylim_max =-200, 1200
soma_loc_x = 0
sigma_layer = 50
soma_loc_displacement_x = 350
unique_layers = sorted(sdk_data.structure__layer.unique().tolist())
layer_dist = {layer_ : i*soma_loc_displacement_x for i,layer_ in enumerate(unique_layers)}
sns.set(style='whitegrid')
fig,ax = plt.subplots()
figname = os.path.join('figures','morph_layerwise.png')
'''Written by Corinne Teeter creates Supplementary Material Figure 4 and prints
out numbers reported in the article.
'''
import os
import numpy as np
import allensdk.core.json_utilities as json_utilities
import matplotlib.pyplot as plt
import sys
relative_path = os.path.dirname(os.getcwd())
sys.path.append(os.path.join(relative_path, 'libraries'))
from data_library import get_pp_path, get_ev_percent_from_calculated_file
from allensdk.core.cell_types_cache import CellTypesCache
ctc = CellTypesCache(
manifest_file=os.path.join(relative_path, 'cell_types_manifest.json'))
from data_library import get_ev_from_folder, check_and_organize_data, get_sweep_num_by_name
from pub_plot_library import distribution_plot
import pandas as pd
import statsmodels.api as sm
from scipy.stats import mannwhitneyu as manu
df = pd.read_csv('spikecut_standard_err.csv') #, delimiter='\t')
data_path = os.path.join(relative_path, 'mouse_struc_data_dir')
nwb_directory = os.path.join(relative_path, 'mouse_nwb')
folders = [os.path.join(data_path, f) for f in os.listdir(data_path)]
all_neurons = []
std_error_list = []
spike_length_list = []
reciprocal_num_sp_list = []
ev_LIFASC_list = []
from allensdk.core.cell_types_cache import CellTypesCache ctc = CellTypesCache() # a list of cell metadata for cells with reconstructions, download if necessary cells = ctc.get_cells(require_reconstruction=True) # open the electrophysiology data of one cell, download if necessary data_set = ctc.get_ephys_data(cells[0]['id']) # read the reconstruction, download if necessary reconstruction = ctc.get_reconstruction(cells[0]['id'])
from pandas import DataFrame, Series
import matplotlib.pyplot as plt
import seaborn as sns
import os.path
from multiprocessing import Pool
import lims_utils
from allensdk.core.nwb_data_set import NwbDataSet
from allensdk.ephys.ephys_extractor import EphysSweepFeatureExtractor
from allensdk.ephys.feature_extractor import EphysFeatureExtractor
from allensdk.core.cell_types_cache import CellTypesCache #Following jupyter notebook here
import pprint
pp = pprint.PrettyPrinter(indent=2)
ctc = CellTypesCache(manifest_file='cell_types/cell_types_manifest.json')
from allensdk.api.queries.cell_types_api import CellTypesApi
#%%
ephys_features = ctc.get_ephys_features()
data_set = ctc.get_ephys_data(
464212183
) # For one particular specimen (later find one that has all models, follows trend in plots)
#ct = CellTypesApi()
#cells = ct.list_cells(require_reconstruction=False)
#ct.save_ephys_data(cells[0]['476218657'], 'example.nwb')
exp_sweeps = []
''' Grab GLIF configuration (one for each GLIF model, *_neuron_config.json) and preprocessor files (one for each neuron *_preprocessor_values.json) from the Allen Institute Cell Types database and arranges them for use in the analysis code. A data folder (mouse_struc_data_dir or human_struc_data_dir) should appear within this directory. Note that that cleaned up versions of these directories are provided in this repository and are available in the root directory.''' import os import sys relative_path=os.path.dirname(os.getcwd()) from allensdk.api.queries.glif_api import GlifApi from allensdk.core.cell_types_cache import CellTypesCache ctc = CellTypesCache(manifest_file=os.path.join(relative_path,'cell_types_manifest.json')) import pandas as pd from shutil import copyfile import numpy as np from allensdk.config import enable_console_log enable_console_log() #--------------------------------------------------------------------------- #----------------------SPECIFY SPECIES-------------------------------------- #--------------------------------------------------------------------------- species="mouse" #species="human" #--------------------------------------------------------------------------- #--------------------------------------------------------------------------- #--------------------------------------------------------------------------- model_template_ids=[395310469, 395310479, 395310475, 471355161, 395310498] #model template ids associated with data in database model_names=['GLIF1', 'GLIF2', 'GLIF3', 'GLIF4', 'GLIF5']
def allen_obs_data(ephys_cell=464212183,
sweep_number=33,
A_soma=np.pi * (70. * 1e-4)**2):
"""Data for x_o. Cell from AllenDB
Parameters
----------
ephys_cell : int
Cell identity from AllenDB
sweep_number : int
Stimulus identity for cell ephys_cell from AllenDB
"""
t_offset = 815.
duration = 1450.
dir_cache = './support_files'
real_data_path = dir_cache + '/ephys_cell_{}_sweep_number_{}.pkl'.format(
ephys_cell, sweep_number)
if not os.path.isfile(real_data_path):
from allensdk.core.cell_types_cache import CellTypesCache
from allensdk.api.queries.cell_types_api import CellTypesApi
manifest_file = 'cell_types/manifest.json'
cta = CellTypesApi()
ctc = CellTypesCache(manifest_file=manifest_file)
data_set = ctc.get_ephys_data(ephys_cell)
sweep_data = data_set.get_sweep(
sweep_number) # works with python2 and fails with python3
sweeps = cta.get_ephys_sweeps(ephys_cell)
sweep = sweeps[sweep_number]
index_range = sweep_data["index_range"]
i = sweep_data["stimulus"][0:index_range[1] + 1] # in A
v = sweep_data["response"][0:index_range[1] + 1] # in V
sampling_rate = sweep_data["sampling_rate"] # in Hz
dt = 1e3 / sampling_rate # in ms
i *= 1e6 # to muA
v *= 1e3 # to mV
v = v[int(t_offset / dt):int((t_offset + duration) / dt)]
i = i[int(t_offset / dt):int((t_offset + duration) / dt)]
real_data_obs = np.array(v).reshape(1, -1, 1)
I_real_data = np.array(i).reshape(-1)
t_on = int(
sweep['stimulus_start_time'] * sampling_rate) * dt - t_offset
t_off = int(
(sweep['stimulus_start_time'] + sweep['stimulus_duration']) *
sampling_rate) * dt - t_offset
io.save((real_data_obs, I_real_data, dt, t_on, t_off), real_data_path)
else:
def pickle_load(file):
"""Loads data from file."""
f = open(file, 'rb')
data = pickle.load(f, encoding='latin1')
f.close()
return data
real_data_obs, I_real_data, dt, t_on, t_off = pickle_load(
real_data_path)
t = np.arange(0, duration, dt)
# external current
I = I_real_data / A_soma # muA/cm2
# return real_data_obs, I_obs
return {
'data': real_data_obs.reshape(-1),
'time': t,
'dt': dt,
'I': I.reshape(-1),
't_on': t_on,
't_off': t_off
}
from allensdk.core.cell_types_cache import CellTypesCache
import allensdk.core.json_utilities as json_utilities
neuronal_model_id = 566302806
# download model metadata
glif_api = GlifApi()
nm = glif_api.get_neuronal_models_by_id([neuronal_model_id])[0]
# download the model configuration file
nc = glif_api.get_neuron_configs([neuronal_model_id])[neuronal_model_id]
neuron_config = glif_api.get_neuron_configs([neuronal_model_id])
json_utilities.write('neuron_config.json', neuron_config)
# download information about the cell
ctc = CellTypesCache()
ctc.get_ephys_data(nm['specimen_id'], file_name='stimulus.nwb')
ctc.get_ephys_sweeps(nm['specimen_id'], file_name='ephys_sweeps.json')
#===============================================================================
# example 2
#===============================================================================
import allensdk.core.json_utilities as json_utilities
from allensdk.model.glif.glif_neuron import GlifNeuron
# initialize the neuron
neuron_config = json_utilities.read('neuron_config.json')['566302806']
neuron = GlifNeuron.from_dict(neuron_config)
# make a short square pulse. stimulus units should be in Amps.
#!/usr/bin/python
#coding:utf-8
#@ZHOU_YING
#2018-10-27
from allensdk.core.cell_types_cache import CellTypesCache
from allensdk.core.nwb_data_set import NwbDataSet
from allensdk.ephys.extract_cell_features import extract_cell_features
from collections import defaultdict
import pandas as pd
import os
ctc = CellTypesCache(manifest_file='/mnt/f/allen_cell_type/manifest.json')
path = '/mnt/f/temp/allen/'
os.chdir(path)
basename = "_ephys.nwb"
file_list = pd.read_csv('id.csv', header=None, usecols=[0], skiprows=1)
features_list = [
'tau', 'input_resistance', 'vm_for_sag', 'fi_fit_slope', 'sag',
'rheobase_i', 'v_baseline'
]
features = defaultdict(list)
for i in range(0, len(file_list)):
filename = str(file_list.loc[i, 0]) + basename
data_set = NwbDataSet(filename)
sweeps = ctc.get_ephys_sweeps(file_list.loc[i, 0])
# group the sweeps by stimulus
sweep_numbers = defaultdict(list)
for sweep in sweeps:
sweep_numbers[sweep['stimulus_name']].append(sweep['sweep_number'])
# calculate features
cell_features = extract_cell_features(data_set, sweep_numbers['Ramp'],
def get_files_from_LIMS_public(output_path, glif_sp_ids=None, type='mouse'):
'''This will grab cre positive data config files from LIMS and sort them and put them in
the specified output folder.
input:
output_path: string
specifies path for files to be placed in
glif_sp_ids: list of strings or integers
specimen ids of cells specifically want to grab. If none it will get all available on the
Allen Institue Cell Types Database.
type: string
can be 'mouse' or 'human'. Note that if mouse is specified is will only grab cre positive mouse cells
(code can be altered to get cre negative cells).
output:
Does not return values but creates the specified 'output_path' folder.
Inside the folder a series of folders are created with the name format:
specimenid_cre. Inside those inner folders are the neuron configs of
the available GLIF models along with the preprocessor files.
'''
glif_api = GlifApi()
ctc = CellTypesCache(manifest_file=os.path.join(relative_path,'cell_types_manifest.json'))
# select the specimen ids to grab from the data base (cre positive or human which have at least 1 GLIF model)
if glif_sp_ids==None: #if no specimen id's are specified grab all data in the cell types manifest
specimen_id_list = []
if type=='mouse':
for c in ctc.get_cells():
if c['reporter_status']=='cre reporter positive':
specimen_id_list.append(c['id'])
elif type=='human':
print 'getting human'
for c in ctc.get_cells(species=['H**o Sapiens']):
#print c
specimen_id_list.append(c['id'])
print specimen_id_list
# reduce list to cells that have a GLIF model
glif_sp_ids=[]
for sp in specimen_id_list:
models=glif_api.get_neuronal_models(sp)[0]
for m in models['neuronal_models']:
if 'LIF' in m['name']:
glif_sp_ids.append(m['specimen_id'])
glif_sp_ids=list(set(glif_sp_ids))
print len(glif_sp_ids), 'cre positive specimens with at least 1 LIF model'
# create the overall output directory if it doesn't exist
try:
os.makedirs(output_path)
except:
pass
# go get the files corresponding to the specimen ids from the Allen Cell Types Database
# and put them into a specified output directory
for id in glif_sp_ids:
model_query=glif_api.get_neuronal_models(id)[0]['neuronal_models']
df=pd.DataFrame(model_query)
for mt_id, short_name in zip(model_template_ids, model_names):
dff=df[df['neuronal_model_template_id']==mt_id]
if len(dff)>=2:
print dff
raise Exception("This is public data, there should not be more than 1 model")
elif len(dff)==1:
use_me=dff
#go get the file
path=use_me['well_known_files'].iloc[0][0]['path']
if type=='mouse':
cre=(str(use_me['name'].values).split(')_'))[1].split(';')[0]
elif type=='human':
cre='human'
else:
raise Exception('specified species not known')
# convert old non complete cre names
if 'Ntsr1-Cre' in cre:
cre='Ntsr1-Cre_GN220'
if 'Chat-IRES-Cre' in cre:
cre='Chat-IRES-Cre-neo'
dir_name=os.path.join(output_path, str(id)+'_'+cre)
try:
os.makedirs(dir_name)
except:
pass
if path.endswith('_neuron_config.json'):
pass
else:
print path
raise Exception('the file doesnt end with _neuron_config.json')
try:
copyfile(path, os.path.join(dir_name, str(id)+'_'+cre+'_'+short_name+'_neuron_config.json'))
except:
print 'couldnt make ', os.path.join(dir_name, str(id)+'_'+cre+'_'+short_name+'_neuron_config.json')
if mt_id==model_template_ids[0]:
model_path=os.path.dirname(path)
pp_path=os.path.join(model_path,
os.listdir(model_path)[np.where([fname.endswith('_preprocessor_values.json') for fname in os.listdir(model_path)])[0][0]])
try:
copyfile(pp_path, os.path.join(dir_name, str(id)+'_'+cre+'_preprocessor_values.json'))
except:
print 'couldnt make ', os.path.join(dir_name, str(id)+'_'+cre+'_preprocessor_values.json')
raise Exception('there should be a preprocessed file')
elif len(dff)<1:
use_me=pd.DataFrame()
path=None
def set_observation(self,observation):
self.observation = {}
self.observation['mean'] = observation
self.observation['std'] = observation
def set_prediction(self,prediction):
self.prediction = {}
self.prediction['mean'] = prediction
self.prediction['std'] = prediction
#df = pickle.load(open("onefive_df.pkl","rb"))
#allen_indexs = [ i for i in df.index if str("NML") not in str(i)]
ctc = CellTypesCache(manifest_file='manifest.json')
features = ctc.get_ephys_features()
'''
for i in allen_indexs:
for f in features[0].keys():
for c in df.columns:
at = AllenTest()
if str(f) in str(c):# and str(f) == str(c):
print(i,f,c)
try:
temp = np.mean(df.loc[i,c])
except:
temp = df.loc[i,f]
print(temp)
#=============================================================================== # example 1 #=============================================================================== from allensdk.core.cell_types_cache import CellTypesCache ctc = CellTypesCache(manifest_file='cell_types/manifest.json') # a list of cell metadata for cells with reconstructions, download if necessary cells = ctc.get_cells(require_reconstruction=True) # open the electrophysiology data of one cell, download if necessary data_set = ctc.get_ephys_data(cells[0]['id']) # read the reconstruction, download if necessary reconstruction = ctc.get_reconstruction(cells[0]['id']) #=============================================================================== # example 2 #=============================================================================== from allensdk.core.cell_types_cache import CellTypesCache from allensdk.ephys.extract_cell_features import extract_cell_features from collections import defaultdict # initialize the cache ctc = CellTypesCache(manifest_file='cell_types/manifest.json') # pick a cell to analyze specimen_id = 324257146
class AllenMorphology(Paths):
""" Handles the download and visualisation of neuronal morphology data from the Allen database. """
def __init__(self, *args, scene_kwargs={}, **kwargs):
"""
Initialise API interaction and fetch metadata of neurons in the Allen Database.
"""
if not connected_to_internet():
raise ConnectionError(
"You will need to be connected to the internet to use the AllenMorphology class"
)
Paths.__init__(self, *args, **kwargs)
self.scene = Scene(add_root=False, display_inset=False, **scene_kwargs)
# Create a Cache for the Cell Types Cache API
self.ctc = CellTypesCache(
manifest_file=os.path.join(self.morphology_allen, 'manifest.json'))
# Get a list of cell metadata for neurons with reconstructions, download if necessary
self.neurons = pd.DataFrame(
self.ctc.get_cells(species=[CellTypesApi.MOUSE],
require_reconstruction=True))
self.n_neurons = len(self.neurons)
if not self.n_neurons:
raise ValueError(
"Something went wrong and couldn't get neurons metadata from Allen"
)
self.downloaded_neurons = self.get_downloaded_neurons()
def get_downloaded_neurons(self):
"""
Get's the path to files of downloaded neurons
"""
return [
os.path.join(self.morphology_allen, f)
for f in os.listdir(self.morphology_allen) if ".swc" in f
]
def download_neurons(self, ids):
"""
Download neurons
:param ids: list of integers with neurons IDs
"""
if isinstance(ids, np.ndarray):
ids = list(ids)
if not isinstance(ids, (list)): ids = [ids]
neurons = []
for neuron_id in ids:
neuron_file = os.path.join(self.morphology_allen,
"{}.swc".format(neuron_id))
neurons.append(
self.ctc.get_reconstruction(neuron_id, file_name=neuron_file))
return neurons
def parse_neurons_swc_allen(self, morphology, color='blackboard', alpha=1):
"""
SWC parser for Allen neuron's morphology data, they're a bit different from the Mouse Light SWC
:param morphology: data with morphology
:param neuron_number: int, number of the neuron being rendered.
"""
# Create soma actor
radius = 1
neuron_actors = [
shapes.Sphere(pos=get_coords(morphology.soma)[::-1],
c=color,
r=radius * 3)
]
# loop over trees
for tree in morphology._tree_list:
tree = pd.DataFrame(tree)
branching_points = [
t.id for i, t in tree.iterrows()
if len(t.children) > 2 and t.id < len(tree)
]
branch_starts = []
for bp in branching_points:
branch_starts.extend(tree.iloc[bp].children)
for bp in branch_starts:
parent = tree.iloc[tree.iloc[bp].parent]
branch = [(parent.x, parent.y, parent.z)]
point = tree.iloc[bp]
while True:
branch.append((point.x, point.y, point.z))
if not point.children:
break
else:
try:
point = tree.iloc[point.children[0]]
except:
break
# Create actor
neuron_actors.append(
shapes.Tube(branch, r=radius, c='red', alpha=1, res=24))
actor = merge(*neuron_actors)
actor.color(color)
actor.alpha(alpha)
return actor
# # Todo load/save neurons??
def load_save_neuron(self, neuron_file, neuron=None):
neuron_name = os.path.split(neuron_file)[-1].split('.swc')[0]
savepath = os.path.join(self.morphology_cache, neuron_name + '.vtk')
if neuron is None and os.path.isfile(savepath):
return load(savepath)
elif neuron is None:
return None
elif neuron is not None:
neuron.write(savepath)
def parse_neuron_swc(self,
filepath,
color='blackboard',
alpha=1,
radius_multiplier=.1,
overwrite=False):
"""
Given an swc file, render the neuron
:param filepath: str with path to swc file
:param neuron_number: numnber of neuron being rendered
"""
# See if we rendered this neuron already
if not overwrite:
loaded = self.load_save_neuron(filepath)
if loaded is not None:
return loaded.color(color)
print(f"Parsing swc file: {filepath}")
# details on swc files: http://www.neuronland.org/NLMorphologyConverter/MorphologyFormats/SWC/Spec.html
_sample = namedtuple("sample", "sampleN structureID x y z r parent"
) # sampleN structureID x y z r parent
if not os.path.isfile(filepath) or not ".swc" in filepath.lower():
raise ValueError("unrecognized file path: {}".format(filepath))
try:
return self.parse_neurons_swc_allen(filepath)
except:
pass # the .swc file fas not generate with by allen
f = open(filepath)
content = f.readlines()
f.close()
content = [
sample.replace("\n", "") for sample in content if sample[0] != '#'
]
content = [sample for sample in content if len(sample) > 3]
# crate empty dicts for soma axon and dendrites
data = dict(id=[],
parentNumber=[],
radius=[],
sampleNumber=[],
x=[],
y=[],
z=[])
# start looping around samples
for sample in content:
s = _sample(
*[float(samp) for samp in sample.lstrip().rstrip().split(" ")])
# append data to dictionary
data['id'] = s.structureID
data['parentNumber'].append(int(s.parent))
data['radius'].append(s.r)
data['x'].append(s.x)
data['y'].append(s.y)
data['z'].append(s.z)
data['sampleNumber'].append(int(s.sampleN))
# Get branches and soma
print(" reconstructing neurites trees")
data = pd.DataFrame(data)
radius = data['radius'].values[0] * radius_multiplier
soma = data.iloc[0]
soma = shapes.Sphere(pos=[soma.x, soma.y, soma.z],
c=color,
r=radius * 4)
neuron_actors = [soma]
branches_end, branches_start = [], [] # Get branches start and end
for parent in data.parentNumber.values:
sons = data.loc[data.parentNumber == parent]
if len(sons) > 1:
branches_end.append(parent)
for i, son in sons.iterrows():
branches_start.append(son.sampleNumber)
print(" creating actors")
for start in branches_start:
node = data.loc[data.sampleNumber == start]
parent = data.loc[data.sampleNumber == node.parentNumber.values[0]]
branch = [(parent.x.values[0], parent.y.values[0],
parent.z.values[0])]
while True:
branch.append(
(node.x.values[0], node.y.values[0], node.z.values[0]))
node = data.loc[data.parentNumber ==
node.sampleNumber.values[0]]
if not len(node): break
if node.sampleNumber.values[0] in branches_end:
branch.append(
(node.x.values[0], node.y.values[0], node.z.values[0]))
break
neuron_actors.append(
shapes.Tube(branch, r=radius, c='red', alpha=1, res=24))
# Merge actors and save
actor = merge(*neuron_actors)
actor.color(color)
actor.alpha(alpha)
self.load_save_neuron(filepath, neuron=actor)
return actor
def add_neuron(self,
neuron,
shadow_axis=None,
shadow_offset=-20,
**kwargs):
if isinstance(neuron, list):
neurons = neuron
else:
if isinstance(neuron, str):
if os.path.isdir(neuron):
neurons = listdir(neuron)
else:
neurons = [neuron]
actors = []
for neuron in neurons:
if isinstance(neuron, str):
neuron = self.parse_neuron_swc(neuron, **kwargs)
elif isinstance(neuron, Morphology):
neuron = self.parse_neurons_swc_allen(neuron, **kwargs)
actor = self.scene.add_vtkactor(neuron)
# scals = actor.points()[:, 1]
# alphas = np.linspace(0.82, .83, 250)
# actor.pointColors(scals, alpha=alphas, cmap="Greens_r")
# actor.points()[:, 0] += np.random.normal(0, 2000)
# actor.points()[:, 2] += np.random.normal(0, 2000)
if shadow_axis == 'x':
actor.addShadow(x=shadow_offset)
elif shadow_axis == 'y':
actor.addShadow(y=shadow_offset)
elif shadow_axis == 'z':
actor.addShadow(z=shadow_offset)
actors.append(neuron)
return actors
def render(self, **kwargs):
self.scene.render(**kwargs)
