def main(output_dir, resolution, retain_transform_data, zoom,
parallel_downloads, parallel_processors):
mcc = MouseConnectivityCache(
manifest_file=
f'{output_dir}/connectivity/mouse_connectivity_manifest.json',
resolution=resolution)
mcc.get_annotation_volume()
mcc.get_reference_space()
experiments = [e['id'] for e in mcc.get_experiments(dataframe=False)]
experiments = sorted(list(set(experiments)))
try:
os.makedirs(f'{output_dir}/data/input')
for e in experiments:
os.makedirs(f'{output_dir}/data/input/{e}')
except FileExistsError:
for e in experiments:
if os.path.isdir(f'{output_dir}/data/input/{e}'):
shutil.rmtree(f'{output_dir}/data/input/{e}')
os.makedirs(f'{output_dir}/data/input/{e}')
downloads = [
subprocess.Popen([
"python", "./displacement_data_downloader.py", f"-o{output_dir}",
f"-r{resolution}", f"-n{i}"
]) for i in range(parallel_downloads)
]
processes = [
subprocess.Popen([
"python", "./segmentation_data_builder.py", f"-o{output_dir}",
f"-r{resolution}", f"-c{len(experiments)}", f"-z{zoom}"
] + (["-t"] if retain_transform_data else []) + [f"-n{i}"])
for i in range(parallel_processors)
]
exit_codes = [p.wait() for p in processes + downloads]
class SegmentationDataDownloader(DirWatcher):
def __init__(self, output_dir, number, resolution):
super().__init__(
*[
os.path.join(output_dir, d)
for d in ['data/input', 'data/dl', 'data/ready']
], f'downloader-{number}')
self.mcc = MouseConnectivityCache(
manifest_file=
f'{output_dir}/connectivity/mouse_connectivity_manifest.json',
resolution=resolution)
self.mcc.get_annotation_volume()
self.mcc.get_reference_space()
def process_item(self, item, directory):
self.logger.info(f"Download displacement data for {item}")
self.mcc.get_deformation_field(item,
header_path=f'{directory}/dfmfld.mhd',
voxel_path=f'{directory}/dfmfld.raw')
self.mcc.get_affine_parameters(item,
direction='trv',
file_name=f'{directory}/aff_param.txt')
def main():
mcc = MouseConnectivityCache(resolution=10)
rsp = mcc.get_reference_space()
# structures come from an rma query like this:
# http://api.brain-map.org/api/v2/data/query.json?q=model::Structure,rma::criteria,[graph_id$in1]
# which does the same thing as using structure_graph_download, but without nesting
# the graph id is 1
structures = rsp.structure_tree.nodes()
ontology_ids = [s['id'] for s in structures]
# the annotation comes from: http://download.alleninstitute.org/informatics-archive/current-release/mouse_ccf/annotation/ccf_2017/annotation_10.nrrd
annotation_ids = np.unique(rsp.annotation)
class SegmentationDataBuilder(DirWatcher):
def __init__(self, output_dir, resolution, retain_transform_data, zoom,
number, count):
super().__init__(
*[
os.path.join(output_dir, d)
for d in ['data/ready', 'data/proc', 'data/result']
], f'segmentation-data-builder-{number}')
self.count = count
self.zoom = zoom
self.retain_transform_data = retain_transform_data
self.resolution = resolution
self.output_dir = output_dir
self.mcc = MouseConnectivityCache(
manifest_file=
f'{output_dir}/connectivity/mouse_connectivity_manifest.json',
resolution=resolution)
self.anno, self.meta = self.mcc.get_annotation_volume()
self.rsp = self.mcc.get_reference_space()
self.rsp.remove_unassigned(
) # This removes ids that are not in this particular reference space
def process_item(self, item, directory):
item = int(item)
experiment = ExperimentSectionData(
self.mcc,
item,
directory,
self.anno,
self.meta,
self.rsp,
self.logger,
zoom=self.zoom,
remove_transform_data=not self.retain_transform_data)
experiment.create_section_data()
experiment.cleanup()
from collections import Counter
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from jaratoolbox import settings
from jaratoolbox import extraplots
reload(extraplots)
from jaratoolbox import colorpalette
from scipy import stats
import copy
import pandas as pd
import figparams
reload(figparams)
from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache
mcc = MouseConnectivityCache(resolution=25)
rsp = mcc.get_reference_space()
rspAnnotationVolumeRotated = np.rot90(rsp.annotation, 1, axes=(2, 0))
np.random.seed(0)
def jitter(arr, frac):
jitter = (np.random.random(len(arr))-0.5)*2*frac
jitteredArr = arr + jitter
return jitteredArr
def medline(ax, yval, midline, width, color='k', linewidth=3):
start = midline-(width/2)
end = midline+(width/2)
ax.plot([start, end], [yval, yval], color=color, lw=linewidth)
FIGNAME = 'figure_tagged_untagged'
def cell_locations(db):
"""
This function takes as argument a pandas DataFrame and adds new columns.
The filename should be the full path to where the database will be saved. If a filename is not specified, the database will not be saved.
This function computes the depths and cortical locations of all cells with suppression indices computed.
This function should be run in a virtual environment because the allensdk has weird dependencies that we don't want tainting our computers.
"""
# lapPath = os.path.join(settings.ATLAS_PATH, 'AllenCCF_25/coronal_laplacian_25.nrrd')
# lapPath = '/mnt/jarahubdata/tmp/coronal_laplacian_25.nrrd'
# TODO Edit the lapPath to be something more descriptive of what it actually is. Also should not use laplacian for non-cortical areas
# lapPath = settings.LAP_PATH
# lapData = nrrd.read(lapPath)
# lap = lapData[0]
mcc = MouseConnectivityCache(resolution=25)
rsp = mcc.get_reference_space()
rspAnnotationVolumeRotated = np.rot90(rsp.annotation, 1, axes=(2, 0))
tetrodetoshank = {
1: 1,
2: 1,
3: 2,
4: 2,
5: 3,
6: 3,
7: 4,
8: 4
} # hardcoded dictionary of tetrode to shank mapping for probe geometry used in this study
try:
bestCells = db.query(
'rsquaredFit>{}'.format(studyparams.R2_CUTOFF)
) # calculate depths for all the cells that we quantify as tuned
except pd.core.computation.ops.UndefinedVariableError:
bestCells = db
db['recordingSiteName'] = '' # prefill will empty strings so whole column is strings (no NaNs)
for dbIndex, dbRow in bestCells.iterrows():
subject = dbRow['subject']
try:
fileNameInfohist = os.path.join(settings.INFOHIST_PATH,
'{}_tracks.py'.format(subject))
tracks = imp.load_source('tracks_module', fileNameInfohist).tracks
except IOError:
print("No such tracks file: {}".format(fileNameInfohist))
else:
# TODO Replace this with a more generic way of finding the brain areas for histology saving.
brainArea = dbRow['brainArea']
if brainArea == 'left_AudStr':
brainArea = 'LeftAstr'
elif brainArea == 'right_AudStr':
brainArea = 'RightAstr'
tetrode = dbRow['tetrode']
shank = tetrodetoshank[tetrode]
recordingTrack = dbRow['info'][
0] # This line relies on someone putting track info first in the inforec
track = next(
(track
for track in tracks if (track['brainArea'] == brainArea) and (
track['shank'] == shank) and (
track['recordingTrack'] == recordingTrack)), None)
if track is not None:
histImage = track['histImage']
filenameSVG = ha.get_filename_registered_svg(
subject, brainArea, histImage, recordingTrack, shank)
if tetrode % 2 == 0:
depth = dbRow['depth']
else:
depth = dbRow['depth'] - 150.0 # odd tetrodes are higher
brainSurfCoords, tipCoords, siteCoords = ha.get_coords_from_svg(
filenameSVG, [depth], dbRow['maxDepth'])
siteCoords = siteCoords[0]
atlasZ = track['atlasZ']
# cortexDepthData = np.rot90(lap[:, :, atlasZ], -1)
#
# # We consider the points with depth > 0.95 to be the bottom surface of cortex
# bottomData = np.where(cortexDepthData > 0.95)
#
# # Top of cortex is less than 0.02 but greater than 0
# topData = np.where((cortexDepthData < 0.02) & (cortexDepthData > 0))
#
# # Distance between the cell and each point on the surface of the brain
# dXTop = topData[1] - siteCoords[0]
# dYTop = topData[0] - siteCoords[1]
# distanceTop = np.sqrt(dXTop**2 + dYTop**2)
#
# # The index and distance to the closest point on the top surface
# indMinTop = np.argmin(distanceTop)
# minDistanceTop = distanceTop.min()
#
# # Same for the distance from the cell to the bottom surface of cortex
# dXBottom = bottomData[1] - siteCoords[0]
# dYBottom = bottomData[0] - siteCoords[1]
# distanceBottom = np.sqrt(dXBottom**2 + dYBottom**2)
# minDistanceBottom = distanceBottom.min()
#
# # The metric we want is the relative distance from the top surface
# cellRatio = minDistanceTop / (minDistanceBottom + minDistanceTop)
# db.at[dbIndex, 'cortexRatioDepth'] = cellRatio
# use allen annotated atlas to figure out where recording site is
thisCoordID = rspAnnotationVolumeRotated[int(siteCoords[0]),
int(siteCoords[1]),
atlasZ]
structDict = rsp.structure_tree.get_structures_by_id(
[thisCoordID])
print("This is {}".format(str(structDict[0]['name'])))
db.at[dbIndex, 'recordingSiteName'] = structDict[0]['name']
# Saving the coordinates in the dataframe
db.at[dbIndex, 'x-coord'] = siteCoords[0]
db.at[dbIndex, 'y-coord'] = siteCoords[1]
db.at[dbIndex, 'z-coord'] = atlasZ
else:
print(subject, brainArea, shank, recordingTrack)
return db
def photoDB_cell_locations(db, filename=''):
'''
This function takes as argument a pandas DataFrame and adds new columns.
The filename should be the full path to where the database will be saved. If a filename is not specified, the database will not be saved.
This function computes the depths and cortical locations of all cells with suppression indices computed.
This function should be run in a virtual environment because the allensdk has weird dependencies that we don't want tainting our computers.
'''
from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache
# lapPath = os.path.join(settings.ATLAS_PATH, 'AllenCCF_25/coronal_laplacian_25.nrrd')
lapPath = '/mnt/jarahubdata/tmp/coronal_laplacian_25.nrrd'
lapData = nrrd.read(lapPath)
lap = lapData[0]
mcc = MouseConnectivityCache(resolution=25)
rsp = mcc.get_reference_space()
rspAnnotationVolumeRotated = np.rot90(rsp.annotation, 1, axes=(2, 0))
tetrodetoshank = {
1: 1,
2: 1,
3: 2,
4: 2,
5: 3,
6: 3,
7: 4,
8: 4
} #hardcoded dictionary of tetrode to shank mapping for probe geometry used in this study
bestCells = db[db['sustainedSuppressionIndex'].notnull(
)] #calculate depths for all the cells that we calculated SIs for
for dbIndex, dbRow in bestCells.iterrows():
subject = dbRow['subject']
try:
fileNameInfohist = os.path.join(settings.INFOHIST_PATH,
'{}_tracks.py'.format(subject))
tracks = imp.load_source('tracks_module', fileNameInfohist).tracks
except IOError:
print("No such tracks file: {}".format(fileNameInfohist))
else:
brainArea = dbRow['brainArea']
if brainArea == 'left_AC':
brainArea = 'LeftAC'
elif brainArea == 'right_AC':
brainArea = 'RightAC'
tetrode = dbRow['tetrode']
shank = tetrodetoshank[tetrode]
recordingTrack = dbRow['info'][0]
track = next(
(track
for track in tracks if (track['brainArea'] == brainArea) and (
track['shank'] == shank) and (
track['recordingTrack'] == recordingTrack)), None)
if track is not None:
histImage = track['histImage']
filenameSVG = ha.get_filename_registered_svg(
subject, brainArea, histImage, recordingTrack, shank)
if tetrode % 2 == 0:
depth = dbRow['depth']
else:
depth = dbRow['depth'] - 150.0 #odd tetrodes are higher
brainSurfCoords, tipCoords, siteCoords = ha.get_coords_from_svg(
filenameSVG, [depth], dbRow['maxDepth'])
siteCoords = siteCoords[0]
atlasZ = track['atlasZ']
cortexDepthData = np.rot90(lap[:, :, atlasZ], -1)
# We consider the points with depth > 0.95 to be the bottom surface of cortex
bottomData = np.where(cortexDepthData > 0.95)
# Top of cortex is less than 0.02 but greater than 0
topData = np.where((cortexDepthData < 0.02)
& (cortexDepthData > 0))
# Distance between the cell and each point on the surface of the brain
dXTop = topData[1] - siteCoords[0]
dYTop = topData[0] - siteCoords[1]
distanceTop = np.sqrt(dXTop**2 + dYTop**2)
# The index and distance to the closest point on the top surface
indMinTop = np.argmin(distanceTop)
minDistanceTop = distanceTop.min()
# Same for the distance from the cell to the bottom surface of cortex
dXBottom = bottomData[1] - siteCoords[0]
dYBottom = bottomData[0] - siteCoords[1]
distanceBottom = np.sqrt(dXBottom**2 + dYBottom**2)
minDistanceBottom = distanceBottom.min()
# The metric we want is the relative distance from the top surface
cellRatio = minDistanceTop / (minDistanceBottom +
minDistanceTop)
db.at[dbIndex, 'cortexRatioDepth'] = cellRatio
# use allen annotated atlas to figure out where recording site is
thisCoordID = rspAnnotationVolumeRotated[int(siteCoords[0]),
int(siteCoords[1]),
atlasZ]
structDict = rsp.structure_tree.get_structures_by_id(
[thisCoordID])
db.at[dbIndex, 'recordingSiteName'] = structDict[0]['name']
else:
print subject, brainArea, shank, recordingTrack
if len(filename) != 0:
celldatabase.save_hdf(db, filename)
print filename + " saved"
return db
