def main(output_dir, resolution, retain_transform_data, zoom, parallel_downloads, parallel_processors): mcc = MouseConnectivityCache( manifest_file= f'{output_dir}/connectivity/mouse_connectivity_manifest.json', resolution=resolution) mcc.get_annotation_volume() mcc.get_reference_space() experiments = [e['id'] for e in mcc.get_experiments(dataframe=False)] experiments = sorted(list(set(experiments))) try: os.makedirs(f'{output_dir}/data/input') for e in experiments: os.makedirs(f'{output_dir}/data/input/{e}') except FileExistsError: for e in experiments: if os.path.isdir(f'{output_dir}/data/input/{e}'): shutil.rmtree(f'{output_dir}/data/input/{e}') os.makedirs(f'{output_dir}/data/input/{e}') downloads = [ subprocess.Popen([ "python", "./displacement_data_downloader.py", f"-o{output_dir}", f"-r{resolution}", f"-n{i}" ]) for i in range(parallel_downloads) ] processes = [ subprocess.Popen([ "python", "./segmentation_data_builder.py", f"-o{output_dir}", f"-r{resolution}", f"-c{len(experiments)}", f"-z{zoom}" ] + (["-t"] if retain_transform_data else []) + [f"-n{i}"]) for i in range(parallel_processors) ] exit_codes = [p.wait() for p in processes + downloads]
class SegmentationDataDownloader(DirWatcher): def __init__(self, output_dir, number, resolution): super().__init__( *[ os.path.join(output_dir, d) for d in ['data/input', 'data/dl', 'data/ready'] ], f'downloader-{number}') self.mcc = MouseConnectivityCache( manifest_file= f'{output_dir}/connectivity/mouse_connectivity_manifest.json', resolution=resolution) self.mcc.get_annotation_volume() self.mcc.get_reference_space() def process_item(self, item, directory): self.logger.info(f"Download displacement data for {item}") self.mcc.get_deformation_field(item, header_path=f'{directory}/dfmfld.mhd', voxel_path=f'{directory}/dfmfld.raw') self.mcc.get_affine_parameters(item, direction='trv', file_name=f'{directory}/aff_param.txt')
def main(): mcc = MouseConnectivityCache(resolution=10) rsp = mcc.get_reference_space() # structures come from an rma query like this: # http://api.brain-map.org/api/v2/data/query.json?q=model::Structure,rma::criteria,[graph_id$in1] # which does the same thing as using structure_graph_download, but without nesting # the graph id is 1 structures = rsp.structure_tree.nodes() ontology_ids = [s['id'] for s in structures] # the annotation comes from: http://download.alleninstitute.org/informatics-archive/current-release/mouse_ccf/annotation/ccf_2017/annotation_10.nrrd annotation_ids = np.unique(rsp.annotation)
class SegmentationDataBuilder(DirWatcher): def __init__(self, output_dir, resolution, retain_transform_data, zoom, number, count): super().__init__( *[ os.path.join(output_dir, d) for d in ['data/ready', 'data/proc', 'data/result'] ], f'segmentation-data-builder-{number}') self.count = count self.zoom = zoom self.retain_transform_data = retain_transform_data self.resolution = resolution self.output_dir = output_dir self.mcc = MouseConnectivityCache( manifest_file= f'{output_dir}/connectivity/mouse_connectivity_manifest.json', resolution=resolution) self.anno, self.meta = self.mcc.get_annotation_volume() self.rsp = self.mcc.get_reference_space() self.rsp.remove_unassigned( ) # This removes ids that are not in this particular reference space def process_item(self, item, directory): item = int(item) experiment = ExperimentSectionData( self.mcc, item, directory, self.anno, self.meta, self.rsp, self.logger, zoom=self.zoom, remove_transform_data=not self.retain_transform_data) experiment.create_section_data() experiment.cleanup()
from collections import Counter import matplotlib.pyplot as plt import matplotlib.gridspec as gridspec from jaratoolbox import settings from jaratoolbox import extraplots reload(extraplots) from jaratoolbox import colorpalette from scipy import stats import copy import pandas as pd import figparams reload(figparams) from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache mcc = MouseConnectivityCache(resolution=25) rsp = mcc.get_reference_space() rspAnnotationVolumeRotated = np.rot90(rsp.annotation, 1, axes=(2, 0)) np.random.seed(0) def jitter(arr, frac): jitter = (np.random.random(len(arr))-0.5)*2*frac jitteredArr = arr + jitter return jitteredArr def medline(ax, yval, midline, width, color='k', linewidth=3): start = midline-(width/2) end = midline+(width/2) ax.plot([start, end], [yval, yval], color=color, lw=linewidth) FIGNAME = 'figure_tagged_untagged'
def cell_locations(db): """ This function takes as argument a pandas DataFrame and adds new columns. The filename should be the full path to where the database will be saved. If a filename is not specified, the database will not be saved. This function computes the depths and cortical locations of all cells with suppression indices computed. This function should be run in a virtual environment because the allensdk has weird dependencies that we don't want tainting our computers. """ # lapPath = os.path.join(settings.ATLAS_PATH, 'AllenCCF_25/coronal_laplacian_25.nrrd') # lapPath = '/mnt/jarahubdata/tmp/coronal_laplacian_25.nrrd' # TODO Edit the lapPath to be something more descriptive of what it actually is. Also should not use laplacian for non-cortical areas # lapPath = settings.LAP_PATH # lapData = nrrd.read(lapPath) # lap = lapData[0] mcc = MouseConnectivityCache(resolution=25) rsp = mcc.get_reference_space() rspAnnotationVolumeRotated = np.rot90(rsp.annotation, 1, axes=(2, 0)) tetrodetoshank = { 1: 1, 2: 1, 3: 2, 4: 2, 5: 3, 6: 3, 7: 4, 8: 4 } # hardcoded dictionary of tetrode to shank mapping for probe geometry used in this study try: bestCells = db.query( 'rsquaredFit>{}'.format(studyparams.R2_CUTOFF) ) # calculate depths for all the cells that we quantify as tuned except pd.core.computation.ops.UndefinedVariableError: bestCells = db db['recordingSiteName'] = '' # prefill will empty strings so whole column is strings (no NaNs) for dbIndex, dbRow in bestCells.iterrows(): subject = dbRow['subject'] try: fileNameInfohist = os.path.join(settings.INFOHIST_PATH, '{}_tracks.py'.format(subject)) tracks = imp.load_source('tracks_module', fileNameInfohist).tracks except IOError: print("No such tracks file: {}".format(fileNameInfohist)) else: # TODO Replace this with a more generic way of finding the brain areas for histology saving. brainArea = dbRow['brainArea'] if brainArea == 'left_AudStr': brainArea = 'LeftAstr' elif brainArea == 'right_AudStr': brainArea = 'RightAstr' tetrode = dbRow['tetrode'] shank = tetrodetoshank[tetrode] recordingTrack = dbRow['info'][ 0] # This line relies on someone putting track info first in the inforec track = next( (track for track in tracks if (track['brainArea'] == brainArea) and ( track['shank'] == shank) and ( track['recordingTrack'] == recordingTrack)), None) if track is not None: histImage = track['histImage'] filenameSVG = ha.get_filename_registered_svg( subject, brainArea, histImage, recordingTrack, shank) if tetrode % 2 == 0: depth = dbRow['depth'] else: depth = dbRow['depth'] - 150.0 # odd tetrodes are higher brainSurfCoords, tipCoords, siteCoords = ha.get_coords_from_svg( filenameSVG, [depth], dbRow['maxDepth']) siteCoords = siteCoords[0] atlasZ = track['atlasZ'] # cortexDepthData = np.rot90(lap[:, :, atlasZ], -1) # # # We consider the points with depth > 0.95 to be the bottom surface of cortex # bottomData = np.where(cortexDepthData > 0.95) # # # Top of cortex is less than 0.02 but greater than 0 # topData = np.where((cortexDepthData < 0.02) & (cortexDepthData > 0)) # # # Distance between the cell and each point on the surface of the brain # dXTop = topData[1] - siteCoords[0] # dYTop = topData[0] - siteCoords[1] # distanceTop = np.sqrt(dXTop**2 + dYTop**2) # # # The index and distance to the closest point on the top surface # indMinTop = np.argmin(distanceTop) # minDistanceTop = distanceTop.min() # # # Same for the distance from the cell to the bottom surface of cortex # dXBottom = bottomData[1] - siteCoords[0] # dYBottom = bottomData[0] - siteCoords[1] # distanceBottom = np.sqrt(dXBottom**2 + dYBottom**2) # minDistanceBottom = distanceBottom.min() # # # The metric we want is the relative distance from the top surface # cellRatio = minDistanceTop / (minDistanceBottom + minDistanceTop) # db.at[dbIndex, 'cortexRatioDepth'] = cellRatio # use allen annotated atlas to figure out where recording site is thisCoordID = rspAnnotationVolumeRotated[int(siteCoords[0]), int(siteCoords[1]), atlasZ] structDict = rsp.structure_tree.get_structures_by_id( [thisCoordID]) print("This is {}".format(str(structDict[0]['name']))) db.at[dbIndex, 'recordingSiteName'] = structDict[0]['name'] # Saving the coordinates in the dataframe db.at[dbIndex, 'x-coord'] = siteCoords[0] db.at[dbIndex, 'y-coord'] = siteCoords[1] db.at[dbIndex, 'z-coord'] = atlasZ else: print(subject, brainArea, shank, recordingTrack) return db
def photoDB_cell_locations(db, filename=''): ''' This function takes as argument a pandas DataFrame and adds new columns. The filename should be the full path to where the database will be saved. If a filename is not specified, the database will not be saved. This function computes the depths and cortical locations of all cells with suppression indices computed. This function should be run in a virtual environment because the allensdk has weird dependencies that we don't want tainting our computers. ''' from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache # lapPath = os.path.join(settings.ATLAS_PATH, 'AllenCCF_25/coronal_laplacian_25.nrrd') lapPath = '/mnt/jarahubdata/tmp/coronal_laplacian_25.nrrd' lapData = nrrd.read(lapPath) lap = lapData[0] mcc = MouseConnectivityCache(resolution=25) rsp = mcc.get_reference_space() rspAnnotationVolumeRotated = np.rot90(rsp.annotation, 1, axes=(2, 0)) tetrodetoshank = { 1: 1, 2: 1, 3: 2, 4: 2, 5: 3, 6: 3, 7: 4, 8: 4 } #hardcoded dictionary of tetrode to shank mapping for probe geometry used in this study bestCells = db[db['sustainedSuppressionIndex'].notnull( )] #calculate depths for all the cells that we calculated SIs for for dbIndex, dbRow in bestCells.iterrows(): subject = dbRow['subject'] try: fileNameInfohist = os.path.join(settings.INFOHIST_PATH, '{}_tracks.py'.format(subject)) tracks = imp.load_source('tracks_module', fileNameInfohist).tracks except IOError: print("No such tracks file: {}".format(fileNameInfohist)) else: brainArea = dbRow['brainArea'] if brainArea == 'left_AC': brainArea = 'LeftAC' elif brainArea == 'right_AC': brainArea = 'RightAC' tetrode = dbRow['tetrode'] shank = tetrodetoshank[tetrode] recordingTrack = dbRow['info'][0] track = next( (track for track in tracks if (track['brainArea'] == brainArea) and ( track['shank'] == shank) and ( track['recordingTrack'] == recordingTrack)), None) if track is not None: histImage = track['histImage'] filenameSVG = ha.get_filename_registered_svg( subject, brainArea, histImage, recordingTrack, shank) if tetrode % 2 == 0: depth = dbRow['depth'] else: depth = dbRow['depth'] - 150.0 #odd tetrodes are higher brainSurfCoords, tipCoords, siteCoords = ha.get_coords_from_svg( filenameSVG, [depth], dbRow['maxDepth']) siteCoords = siteCoords[0] atlasZ = track['atlasZ'] cortexDepthData = np.rot90(lap[:, :, atlasZ], -1) # We consider the points with depth > 0.95 to be the bottom surface of cortex bottomData = np.where(cortexDepthData > 0.95) # Top of cortex is less than 0.02 but greater than 0 topData = np.where((cortexDepthData < 0.02) & (cortexDepthData > 0)) # Distance between the cell and each point on the surface of the brain dXTop = topData[1] - siteCoords[0] dYTop = topData[0] - siteCoords[1] distanceTop = np.sqrt(dXTop**2 + dYTop**2) # The index and distance to the closest point on the top surface indMinTop = np.argmin(distanceTop) minDistanceTop = distanceTop.min() # Same for the distance from the cell to the bottom surface of cortex dXBottom = bottomData[1] - siteCoords[0] dYBottom = bottomData[0] - siteCoords[1] distanceBottom = np.sqrt(dXBottom**2 + dYBottom**2) minDistanceBottom = distanceBottom.min() # The metric we want is the relative distance from the top surface cellRatio = minDistanceTop / (minDistanceBottom + minDistanceTop) db.at[dbIndex, 'cortexRatioDepth'] = cellRatio # use allen annotated atlas to figure out where recording site is thisCoordID = rspAnnotationVolumeRotated[int(siteCoords[0]), int(siteCoords[1]), atlasZ] structDict = rsp.structure_tree.get_structures_by_id( [thisCoordID]) db.at[dbIndex, 'recordingSiteName'] = structDict[0]['name'] else: print subject, brainArea, shank, recordingTrack if len(filename) != 0: celldatabase.save_hdf(db, filename) print filename + " saved" return db