def get_structures(self, file_name=None): """ Read the list of adult mouse structures and return a Pandas DataFrame. Parameters ---------- file_name: string File name to save/read the structures table. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. """ file_name = self.get_cache_path(file_name, self.STRUCTURES_KEY) if os.path.exists(file_name): structures = pd.DataFrame.from_csv(file_name) else: structures = OntologiesApi(base_uri=self.api.api_url).get_structures(1) structures = pd.DataFrame(structures) if self.cache: Manifest.safe_make_parent_dirs(file_name) structures.to_csv(file_name) structures.set_index(["id"], inplace=True, drop=False) return structures
def get_template_volume(self, file_name=None): """ Read the template volume. Download it first if it doesn't exist. Parameters ---------- file_name: string File name to store the template volume. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. """ file_name = self.get_cache_path(file_name, self.TEMPLATE_KEY, self.resolution) if file_name is None: raise Exception("No save file provided for annotation volume.") if os.path.exists(file_name): annotation, info = nrrd.read(file_name) else: Manifest.safe_make_parent_dirs(file_name) annotation, info = self.api.download_template_volume( self.resolution, file_name) return annotation, info
def write_volume(volume, name, prefix=None, specify_resolution=None, extension='.nrrd', paths=None): if prefix is None: path = name else: path = os.path.join(prefix, name) if specify_resolution is not None: if isinstance(specify_resolution, (float, np.floating)) and \ specify_resolution % 1.0 == 0: specify_resolution = int(specify_resolution) path = path + '_{0}'.format(specify_resolution) path = path + extension logging.info('writing {0} volume to {1}'.format(name, path)) Manifest.safe_make_parent_dirs(path) volume.SetOrigin([0, 0, 0]) sitk.WriteImage(volume, str(path), True) if paths is not None: paths.append(path)
def debug(container_id, local=False, plots=None): SCRIPT = "/data/informatics/CAM/analysis/allensdk/allensdk/internal/pipeline_modules/run_observatory_container_thumbnails.py" SDK_PATH = "/data/informatics/CAM/analysis/allensdk/" OUTPUT_DIR = "/data/informatics/CAM/analysis/containers" container_dir = os.path.join(OUTPUT_DIR, str(container_id)) input_data = [] for exp in get_container_info(container_id): exp_data = robsth.get_input_data(exp['id']) exp_input_json = os.path.join(exp_data["output_directory"], "input.json") input_data.append( dict(input_json=exp_input_json, output_json=os.path.join(exp_data["output_directory"], "output.json"))) Manifest.safe_make_parent_dirs(exp_input_json) ju.write(exp_input_json, exp_data) run_module(SCRIPT, input_data, container_dir, sdk_path=SDK_PATH, pbs=dict(vmem=32, job_name="cthumbs_%d" % container_id, walltime="10:00:00"), local=local, optional_args=['--types=' + ','.join(plots)] if plots else None)
def save_reconstruction_markers(self, specimen_id, file_name): """ Save the marker file for the morphological reconstruction of a cell. These are comma-delimited files indicating points of interest in a reconstruction (truncation points, early tracing termination, etc). Parameters ---------- specimen_id: int ID of the specimen, from the Specimens database model in the Allen Institute API. file_name: str Path to save the marker file. """ Manifest.safe_make_parent_dirs(file_name) criteria = '[id$eq%d],neuron_reconstructions(well_known_files)' % specimen_id includes = 'neuron_reconstructions(well_known_files(well_known_file_type[name$eq\'%s\']))' % self.MARKER_FILE_TYPE results = self.model_query('Specimen', criteria=criteria, include=includes, num_rows='all') try: file_url = results[0]['neuron_reconstructions'][ 0]['well_known_files'][0]['download_link'] except: raise LookupError("Specimen %d has no marker file" % specimen_id) self.retrieve_file_over_http(self.api_url + file_url, file_name)
def save_reconstruction(self, specimen_id, file_name): """ Save the morphological reconstruction of a cell as an SWC file. Parameters ---------- specimen_id: int ID of the specimen, from the Specimens database model in the Allen Institute API. file_name: str Path to save the SWC file. """ Manifest.safe_make_parent_dirs(file_name) criteria = '[id$eq%d],neuron_reconstructions(well_known_files)' % specimen_id includes = 'neuron_reconstructions(well_known_files(well_known_file_type[name$eq\'%s\']))' % self.SWC_FILE_TYPE results = self.model_query('Specimen', criteria=criteria, include=includes, num_rows='all') try: file_url = results[0]['neuron_reconstructions'][ 0]['well_known_files'][0]['download_link'] except: raise Exception("Specimen %d has no reconstruction" % specimen_id) self.retrieve_file_over_http(self.api_url + file_url, file_name)
def save_ephys_data(self, specimen_id, file_name): """ Save the electrophysology recordings for a cell as an NWB file. Parameters ---------- specimen_id: int ID of the specimen, from the Specimens database model in the Allen Institute API. file_name: str Path to save the NWB file. """ Manifest.safe_make_parent_dirs(file_name) criteria = '[id$eq%d],ephys_result(well_known_files(well_known_file_type[name$eq%s]))' % ( specimen_id, self.NWB_FILE_TYPE) includes = 'ephys_result(well_known_files(well_known_file_type))' results = self.model_query('Specimen', criteria=criteria, include=includes, num_rows='all') try: file_url = results[0]['ephys_result'][ 'well_known_files'][0]['download_link'] except Exception as _: raise Exception("Specimen %d has no ephys data" % specimen_id) self.retrieve_file_over_http(self.api_url + file_url, file_name)
def get_structures(self, file_name=None): """ Read the list of adult mouse structures and return a Pandas DataFrame. Parameters ---------- file_name: string File name to save/read the structures table. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. """ file_name = self.get_cache_path(file_name, self.STRUCTURES_KEY) if os.path.exists(file_name): structures = pd.DataFrame.from_csv(file_name) else: structures = OntologiesApi( base_uri=self.api.api_url).get_structures(1) structures = pd.DataFrame(structures) if self.cache: Manifest.safe_make_parent_dirs(file_name) structures.to_csv(file_name) structures.set_index(['id'], inplace=True, drop=False) return structures
def get_injection_fraction(self, experiment_id, file_name=None): """ Read an injection fraction volume for a single experiment. Download it first if it doesn't exist. Injection fraction is the proportion of pixels in the injection site in a grid voxel in [0,1]. Parameters ---------- experiment_id: int ID of the experiment to download/read. This corresponds to section_data_set_id in the API. file_name: string File name to store the template volume. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. """ file_name = self.get_cache_path(file_name, self.INJECTION_FRACTION_KEY, experiment_id, self.resolution) if file_name is None: raise Exception("No file name to save volume.") if not os.path.exists(file_name): Manifest.safe_make_parent_dirs(file_name) self.api.download_injection_fraction(file_name, experiment_id, self.resolution) return nrrd.read(file_name)
def get_data_mask(self, experiment_id, file_name=None): """ Read a data mask volume for a single experiment. Download it first if it doesn't exist. Data mask is a binary mask of voxels that have valid data. Only use valid data in analysis! Parameters ---------- experiment_id: int ID of the experiment to download/read. This corresponds to section_data_set_id in the API. file_name: string File name to store the template volume. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. """ file_name = self.get_cache_path(file_name, self.DATA_MASK_KEY, experiment_id, self.resolution) if file_name is None: raise Exception("No file name to save volume.") if not os.path.exists(file_name): Manifest.safe_make_parent_dirs(file_name) self.api.download_data_mask(file_name, experiment_id, self.resolution) return nrrd.read(file_name)
def save_ephys_data(self, specimen_id, file_name): """ Save the electrophysology recordings for a cell as an NWB file. Parameters ---------- specimen_id: int ID of the specimen, from the Specimens database model in the Allen Institute API. file_name: str Path to save the NWB file. """ Manifest.safe_make_parent_dirs(file_name) criteria = '[id$eq%d],ephys_result(well_known_files(well_known_file_type[name$eq%s]))' % ( specimen_id, self.NWB_FILE_TYPE) includes = 'ephys_result(well_known_files(well_known_file_type))' results = self.model_query('Specimen', criteria=criteria, include=includes, num_rows='all') try: file_url = results[0]['ephys_result']['well_known_files'][0][ 'download_link'] except Exception as _: raise Exception("Specimen %d has no ephys data" % specimen_id) self.retrieve_file_over_http(self.api_url + file_url, file_name)
def get_template_volume(self, file_name=None): """ Read the template volume. Download it first if it doesn't exist. Parameters ---------- file_name: string File name to store the template volume. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. """ file_name = self.get_cache_path(file_name, self.TEMPLATE_KEY, self.resolution) if file_name is None: raise Exception("No save file provided for annotation volume.") if os.path.exists(file_name): annotation, info = nrrd.read(file_name) else: Manifest.safe_make_parent_dirs(file_name) annotation, info = self.api.download_template_volume(self.resolution, file_name) return annotation, info
def from_file_name(cls, file_name, cache=True, **kwargs): '''Alternative constructor using cache path file_name. Parameters ---------- file_name : string Path where storage_directories will be saved. **kwargs Keyword arguments to be supplied to __init__ Returns ------- cls : instance of GridDataApiPrerelease ''' if os.path.exists(file_name): storage_directories = json_utilities.read(file_name) else: storage_directories = _get_grid_storage_directories( cls.GRID_DATA_DIRECTORY) if cache: Manifest.safe_make_parent_dirs(file_name) json_utilities.write(file_name, storage_directories) return cls(storage_directories, **kwargs)
def get_experiments(self, dataframe=False, file_name=None, cre=None, injection_structure_ids=None): """ Read a list of experiments that match certain criteria. If caching is enabled, this will save the whole (unfiltered) list of experiments to a file. Parameters ---------- dataframe: boolean Return the list of experiments as a Pandas DataFrame. If False, return a list of dictionaries. Default False. file_name: string File name to save/read the structures table. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. cre: boolean or list If True, return only cre-positive experiments. If False, return only cre-negative experiments. If None, return all experients. If list, return all experiments with cre line names in the supplied list. Default None. injection_structure_ids: list Only return experiments that were injected in the structures provided here. If None, return all experiments. Default None. """ file_name = self.get_cache_path(file_name, self.EXPERIMENTS_KEY) if os.path.exists(file_name): experiments = json_utilities.read(file_name) else: experiments = self.api.experiment_source_search(injection_structures="root") # removing these elements because they are specific to a particular # resolution for e in experiments: del e["num-voxels"] del e["injection-volume"] del e["sum"] del e["name"] if self.cache: Manifest.safe_make_parent_dirs(file_name) json_utilities.write(file_name, experiments) # filter the read/downloaded list of experiments experiments = self.filter_experiments(experiments, cre, injection_structure_ids) if dataframe: experiments = pd.DataFrame(experiments) experiments.set_index(["id"], inplace=True, drop=False) return experiments
def get_experiments(self, dataframe=False, file_name=None, cre=None, injection_structure_ids=None, age=None, gender=None, workflow_state=None, workflows=None, project_code=None): """Read a list of experiments. If caching is enabled, this will save the whole (unfiltered) list of experiments to a file. Parameters ---------- dataframe: boolean Return the list of experiments as a Pandas DataFrame. If False, return a list of dictionaries. Default False. file_name: string File name to save/read the structures table. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. """ file_name = self.get_cache_path(file_name, self.EXPERIMENTS_PRERELEASE_KEY) if os.path.exists(file_name): experiments = json_utilities.read(file_name) else: experiments = self.api.get_experiments() if self.cache: Manifest.safe_make_parent_dirs(file_name) json_utilities.write(file_name, experiments) # filter the read/downloaded list of experiments experiments = self.filter_experiments(experiments, cre, injection_structure_ids, age, gender, workflow_state, workflows, project_code) if dataframe: experiments = pd.DataFrame(experiments) experiments.set_index(['id'], inplace=True, drop=False) return experiments
def save_ophys_experiment_data(self, ophys_experiment_id, file_name): Manifest.safe_make_parent_dirs(file_name) data = self.template_query('brain_observatory_queries', 'ophys_experiment_data', ophys_experiment_id=ophys_experiment_id) try: file_url = data[0]['download_link'] except Exception as _: raise Exception("ophys experiment %d has no data file" % ophys_experiment_id) self._log.warning( "Downloading ophys_experiment %d NWB. This can take some time." % ophys_experiment_id) self.retrieve_file_over_http(self.api_url + file_url, file_name)
def get_deformation_field(self, section_data_set_id, header_path=None, voxel_path=None): ''' Extract the local alignment parameters for this dataset. This a 3D vector image (3 components) describing a deformable local mapping from CCF voxels to this section data set's affine-aligned image stack. Parameters ---------- section_data_set_id : int Download the deformation field for this data set header_path : str, optional If supplied, the deformation field header will be downloaded to this path. voxel_path : str, optiona If supplied, the deformation field voxels will be downloaded to this path. Returns ------- numpy.ndarray : 3D X 3 component vector array (origin 0, 0, 0; 25-micron isometric resolution) defining a deformable transformation from CCF-space to affine-transformed image space. ''' if self.resolution not in self.DFMFLD_RESOLUTIONS: warnings.warn( 'deformation fields are only available at {} isometric resolutions, but this is a '\ '{}-micron cache'.format(self.DFMFLD_RESOLUTIONS, self.resolution) ) header_path = self.get_cache_path(header_path, self.DEFORMATION_FIELD_HEADER_KEY, section_data_set_id) voxel_path = self.get_cache_path(voxel_path, self.DEFORMATION_FIELD_VOXEL_KEY, section_data_set_id) if not (os.path.exists(header_path) and os.path.exists(voxel_path)): Manifest.safe_make_parent_dirs(header_path) Manifest.safe_make_parent_dirs(voxel_path) self.api.download_deformation_field(section_data_set_id, header_path=header_path, voxel_path=voxel_path) return sitk.GetArrayFromImage(sitk.ReadImage(str( header_path))) # TODO the str call here is only necessary in 2.7
def get_structure_mask(self, structure_id, file_name=None, annotation_file_name=None): """ Read a 3D numpy array shaped like the annotation volume that has non-zero values where voxels belong to a particular structure. This will take care of identifying substructures. Parameters ---------- structure_id: int ID of a structure. file_name: string File name to store the structure mask. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. annotation_file_name: string File name to store the annotation volume. If it already exists, it will be read from this file. If file_name is None, the file_name will be pulled out of the manifest. Default is None. """ file_name = self.get_cache_path(file_name, self.STRUCTURE_MASK_KEY, structure_id) if os.path.exists(file_name): return nrrd.read(file_name) else: ont = self.get_ontology() structure_ids = ont.get_descendant_ids([structure_id]) annotation, _ = self.get_annotation_volume(annotation_file_name) mask = self.make_structure_mask(structure_ids, annotation) if self.cache: Manifest.safe_make_parent_dirs(file_name) nrrd.write(file_name, mask) return mask, None
def get_deformation_field(self, section_data_set_id, header_path=None, voxel_path=None): ''' Extract the local alignment parameters for this dataset. This a 3D vector image (3 components) describing a deformable local mapping from CCF voxels to this section data set's affine-aligned image stack. Parameters ---------- section_data_set_id : int Download the deformation field for this data set header_path : str, optional If supplied, the deformation field header will be downloaded to this path. voxel_path : str, optiona If supplied, the deformation field voxels will be downloaded to this path. Returns ------- numpy.ndarray : 3D X 3 component vector array (origin 0, 0, 0; 25-micron isometric resolution) defining a deformable transformation from CCF-space to affine-transformed image space. ''' if self.resolution not in self.DFMFLD_RESOLUTIONS: warnings.warn( 'deformation fields are only available at {} isometric resolutions, but this is a '\ '{}-micron cache'.format(self.DFMFLD_RESOLUTIONS, self.resolution) ) header_path = self.get_cache_path(header_path, self.DEFORMATION_FIELD_HEADER_KEY, section_data_set_id) voxel_path = self.get_cache_path(voxel_path, self.DEFORMATION_FIELD_VOXEL_KEY, section_data_set_id) if not (os.path.exists(header_path) and os.path.exists(voxel_path)): Manifest.safe_make_parent_dirs(header_path) Manifest.safe_make_parent_dirs(voxel_path) self.api.download_deformation_field( section_data_set_id, header_path=header_path, voxel_path=voxel_path ) return sitk.GetArrayFromImage(sitk.ReadImage(str(header_path))) # TODO the str call here is only necessary in 2.7
def get_session_data(self, session_id: int, filter_by_validity: bool = True, **unit_filter_kwargs): """ Obtain an EcephysSession object containing detailed data for a single session """ path = self.get_cache_path(None, self.SESSION_NWB_KEY, session_id, session_id) def read(_path): session_api = self._build_nwb_api_for_session( _path, session_id, filter_by_validity, **unit_filter_kwargs) return EcephysSession(api=session_api, test=True) Manifest.safe_make_parent_dirs(path) return one_file_call_caching(path, partial(self.s3fs.get, self._get_s3_path(path), path), lambda *a, **k: None, read, num_tries=self.fetch_tries)
def test_get_grid_storage_directories(storage_dirs, query_result, fn_temp_dir): # ------------------------------------------------------------------------ # test dirs only have grid/ subdirectory with mock.patch('allensdk.internal.core.lims_utilities.query', new=lambda a: query_result): obtained = _get_grid_storage_directories( GridDataApiPrerelease.GRID_DATA_DIRECTORY) assert not obtained # ------------------------------------------------------------------------ # test returns storage_dirs for path in storage_dirs.values(): Manifest.safe_make_parent_dirs(os.path.join(path, 'grid')) with mock.patch('allensdk.internal.core.lims_utilities.query', new=lambda a: query_result): obtained = _get_grid_storage_directories( GridDataApiPrerelease.GRID_DATA_DIRECTORY) for key, value in obtained: assert storage_dirs[key] == value
def download_section(savepath, section_id, downsample): # Downloading all of the images from a section data set image_api = ImageDownloadApi() input_directory = str(section_id) + '_input' output_directory = str(section_id) + '_output' format_str = '.jpg' section_images = image_api.section_image_query(section_id) section_image_ids = [si['id'] for si in section_images] # You have probably noticed that the AllenSDK has a logger which notifies you of file downloads. # Since we are downloading ~300 images, we don't want to see messages for each one. # The following line will temporarily disable the download logger.(optional) logging.getLogger( 'allensdk.api.api.retrieve_file_over_http').disabled = True for section_image_id in section_image_ids: file_name = str(section_image_id) + format_str input_file_path = os.path.join(savepath, input_directory, file_name) output_file_path = os.path.join(savepath, output_directory, file_name) Manifest.safe_make_parent_dirs(input_file_path) image_api.download_section_image(section_image_id, file_path=input_file_path, downsample=downsample, expression=0) Manifest.safe_make_parent_dirs(output_file_path) image_api.download_section_image(section_image_id, file_path=output_file_path, downsample=downsample, expression=1) # re-enable the logger (optional) logging.getLogger( 'allensdk.api.api.retrieve_file_over_http').disabled = False file_names = os.listdir(os.path.join(savepath, input_directory)) print(len(file_names))
def copy_file_entry(source, dest, use_rsync, make_parent_dirs, chmod=None): leftmost = None if make_parent_dirs: leftmost = Manifest.safe_make_parent_dirs(dest) if use_rsync: sp.check_call(['rsync', '-a', source, dest]) else: if Path(source).is_dir(): shutil.copytree(source, dest) else: shutil.copy(source, dest) if chmod is not None: chmod_target = leftmost if leftmost is not None else dest apply_permissions = lambda path: path.chmod(int(f"0o{chmod}", 0)) walk_fs_tree(chmod_target, apply_permissions) logging.info(f"copied from {source} to {dest}")
def get_experiment_structure_unionizes(self, experiment_id, file_name=None, is_injection=None, structure_ids=None, include_descendants=False, hemisphere_ids=None): """ Retrieve the structure unionize data for a specific experiment. Filter by structure, injection status, and hemisphere. Parameters ---------- experiment_id: int ID of the experiment of interest. Corresponds to section_data_set_id in the API. file_name: string File name to save/read the experiments list. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. is_injection: boolean If True, only return unionize records that disregard non-injection pixels. If False, only return unionize records that disregard injection pixels. If None, return all records. Default None. structure_ids: list Only return unionize records for a specific set of structures. If None, return all records. Default None. include_descendants: boolean Include all descendant records for specified structures. Default False. hemisphere_ids: list Only return unionize records that disregard pixels outside of a hemisphere. or set of hemispheres. Left = 1, Right = 2, Both = 3. If None, include all records [1, 2, 3]. Default None. """ file_name = self.get_cache_path(file_name, self.STRUCTURE_UNIONIZES_KEY, experiment_id) if os.path.exists(file_name): unionizes = pd.DataFrame.from_csv(file_name) else: unionizes = self.api.get_structure_unionizes([experiment_id]) unionizes = pd.DataFrame(unionizes) # rename section_data_set_id column to experiment_id unionizes.columns = [ 'experiment_id' if c == 'section_data_set_id' else c for c in unionizes.columns ] if self.cache: Manifest.safe_make_parent_dirs(file_name) unionizes.to_csv(file_name) return self.filter_structure_unionizes(unionizes, is_injection, structure_ids, include_descendants, hemisphere_ids)
def download_brain_slice(df): # create an image download API image_api = ImageDownloadApi() format_str = ".jpg" # You have probably noticed that the AllenSDK has a logger which notifies you of file downloads. # Since we are downloading ~300 images, we don't want to see messages for each one. # The following line will temporarily disable the download logger. logging.getLogger("allensdk.api.api.retrieve_file_over_http").disabled = True # get parameters path, downsample, indices = ask_parameters_for_downloading(df) print( "Downloads initiated", end="...", file=sys.stderr, flush=True, ) for index in indices: # from indices, get experiment id and gene symbol from df exp_id = df["Experiment"][index] # set the dirname as the gene symbol dirname = df["Gene Symbol"][index] plane = df["Plane"][index] section_data_set_id = exp_id section_image_directory = os.path.join(path, dirname) # get the image ids for all of the images in this data set section_images = image_api.section_image_query( section_data_set_id ) # Should be a dicionary of the features of section images section_image_ids = [ si["id"] for si in section_images ] # Take value of 'id' from the dictionary # Create a progress bar pbar_image = tqdm(total=len(section_image_ids), desc=dirname + " " + plane) for section_image_id in section_image_ids: file_name = str(section_image_id) + format_str file_path = os.path.join(section_image_directory, file_name) Manifest.safe_make_parent_dirs(file_path) # Check if the file is already downloaded, which happens if the downloads have been interrupted. saved_file_names = os.listdir(section_image_directory) if file_name in saved_file_names: pass else: image_api.download_section_image( section_image_id, file_path=file_path, downsample=downsample ) pbar_image.update() pbar_image.close() # re-enable the logger logging.getLogger("allensdk.api.api.retrieve_file_over_http").disabled = False print( "Downloads completed.", file=sys.stderr, flush=True, )
def write_ecephys_nwb(output_path, session_id, session_start_time, stimulus_table_path, invalid_epochs, probes, running_speed_path, session_sync_path, eye_tracking_rig_geometry, eye_dlc_ellipses_path, eye_gaze_mapping_path, pool_size, optotagging_table_path=None, session_metadata=None, **kwargs): nwbfile = pynwb.NWBFile( session_description='Data and metadata for an Ecephys session', identifier=f"{session_id}", session_id=f"{session_id}", session_start_time=session_start_time, institution="Allen Institute for Brain Science") if session_metadata is not None: nwbfile = add_metadata_to_nwbfile(nwbfile, session_metadata) stimulus_columns_to_drop = [ "colorSpace", "depth", "interpolate", "pos", "rgbPedestal", "tex", "texRes", "flipHoriz", "flipVert", "rgb", "signalDots" ] stimulus_table = read_stimulus_table( stimulus_table_path, columns_to_drop=stimulus_columns_to_drop) nwbfile = add_stimulus_timestamps( nwbfile, stimulus_table['start_time'].values ) # TODO: patch until full timestamps are output by stim table module nwbfile = add_stimulus_presentations(nwbfile, stimulus_table) nwbfile = add_invalid_times(nwbfile, invalid_epochs) if optotagging_table_path is not None: optotagging_table = pd.read_csv(optotagging_table_path) nwbfile = add_optotagging_table_to_nwbfile(nwbfile, optotagging_table) nwbfile = add_probewise_data_to_nwbfile(nwbfile, probes) running_speed, raw_running_data = read_running_speed(running_speed_path) add_running_speed_to_nwbfile(nwbfile, running_speed) add_raw_running_data_to_nwbfile(nwbfile, raw_running_data) add_eye_tracking_rig_geometry_data_to_nwbfile(nwbfile, eye_tracking_rig_geometry) # Collect eye tracking/gaze mapping data from files eye_tracking_frame_times = su.get_synchronized_frame_times( session_sync_file=session_sync_path, sync_line_label_keys=Dataset.EYE_TRACKING_KEYS) eye_dlc_tracking_data = read_eye_dlc_tracking_ellipses( Path(eye_dlc_ellipses_path)) if eye_gaze_mapping_path: eye_gaze_data = read_eye_gaze_mappings(Path(eye_gaze_mapping_path)) else: eye_gaze_data = None add_eye_tracking_data_to_nwbfile(nwbfile, eye_tracking_frame_times, eye_dlc_tracking_data, eye_gaze_data) Manifest.safe_make_parent_dirs(output_path) with pynwb.NWBHDF5IO(output_path, mode='w') as io: logging.info(f"writing session nwb file to {output_path}") io.write(nwbfile, cache_spec=True) probes_with_lfp = [p for p in probes if p["lfp"] is not None] probe_outputs = write_probewise_lfp_files(probes_with_lfp, session_id, session_metadata, session_start_time, pool_size=pool_size) return {'nwb_path': output_path, "probe_outputs": probe_outputs}
def get_experiment_structure_unionizes( self, experiment_id, file_name=None, is_injection=None, structure_ids=None, include_descendants=False, hemisphere_ids=None, ): """ Retrieve the structure unionize data for a specific experiment. Filter by structure, injection status, and hemisphere. Parameters ---------- experiment_id: int ID of the experiment of interest. Corresponds to section_data_set_id in the API. file_name: string File name to save/read the experiments list. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. is_injection: boolean If True, only return unionize records that disregard non-injection pixels. If False, only return unionize records that disregard injection pixels. If None, return all records. Default None. structure_ids: list Only return unionize records for a specific set of structures. If None, return all records. Default None. include_descendants: boolean Include all descendant records for specified structures. Default False. hemisphere_ids: list Only return unionize records that disregard pixels outside of a hemisphere. or set of hemispheres. Left = 1, Right = 2, Both = 3. If None, include all records [1, 2, 3]. Default None. """ file_name = self.get_cache_path(file_name, self.STRUCTURE_UNIONIZES_KEY, experiment_id) if os.path.exists(file_name): unionizes = pd.DataFrame.from_csv(file_name) else: unionizes = self.api.get_structure_unionizes([experiment_id]) unionizes = pd.DataFrame(unionizes) # rename section_data_set_id column to experiment_id unionizes.columns = ["experiment_id" if c == "section_data_set_id" else c for c in unionizes.columns] if self.cache: Manifest.safe_make_parent_dirs(file_name) unionizes.to_csv(file_name) return self.filter_structure_unionizes( unionizes, is_injection, structure_ids, include_descendants, hemisphere_ids )
def cacher(fn, *args, **kwargs): '''make an rma query, save it and return the dataframe. Parameters ---------- fn : function reference makes the actual query using kwargs. path : string where to save the data strategy : string or None, optional 'create' always generates the data, 'file' loads from disk, 'lazy' queries the server if no file exists, None generates the data and bypasses all caching behavior pre : function df|json->df|json, takes one data argument and returns filtered version, None for pass-through post : function df|json->?, takes one data argument and returns Object reader : function, optional path -> data, default NOP writer : function, optional path, data -> None, default NOP kwargs : objects passed through to the query function Returns ------- Object or None data type depends on fn, reader and/or post methods. ''' path = kwargs.pop('path', None) strategy = kwargs.pop('strategy', None) pre = kwargs.pop('pre', lambda d: d) post = kwargs.pop('post', None) reader = kwargs.pop('reader', None) writer = kwargs.pop('writer', None) if strategy is None: if writer or path: strategy = 'lazy' else: strategy = 'pass_through' if not strategy in ['lazy', 'pass_through', 'file', 'create']: raise ValueError("Unknown query strategy: {}.".format(strategy)) if 'lazy' == strategy: if os.path.exists(path): strategy = 'file' else: strategy = 'create' if strategy == 'pass_through': data = fn(*args, **kwargs) elif strategy in ['create']: Manifest.safe_make_parent_dirs(path) if writer: data = fn(*args, **kwargs) data = pre(data) writer(path, data) else: data = fn(*args, **kwargs) if reader: data = reader(path) # Note: don't provide post if fn or reader doesn't return data if post: data = post(data) return data try: data return data except: pass return
section_images = image_api.section_image_query(section_data_set_id) section_image_ids = [si['id'] for si in section_images] print(len(section_image_ids)) # You have probably noticed that the AllenSDK has a logger which notifies you of file downloads. # Since we are downloading ~300 images, we don't want to see messages for each one. # The following line will temporarily disable the download logger.(optional) logging.getLogger('allensdk.api.api.retrieve_file_over_http').disabled = True for section_image_id in section_image_ids: file_name = str(section_image_id) + format_str file_path = os.path.join(section_image_directory, file_name) Manifest.safe_make_parent_dirs(file_path) image_api.download_section_image(section_image_id, file_path=file_path, downsample=downsample,expression = expression, view='expression', colormap='expression') # re-enable the logger (optional) logging.getLogger('allensdk.api.api.retrieve_file_over_http').disabled = False file_names = os.listdir(section_image_directory) print(len(file_names))
def get_experiments(self, dataframe=False, file_name=None, cre=None, injection_structure_ids=None): """ Read a list of experiments that match certain criteria. If caching is enabled, this will save the whole (unfiltered) list of experiments to a file. Parameters ---------- dataframe: boolean Return the list of experiments as a Pandas DataFrame. If False, return a list of dictionaries. Default False. file_name: string File name to save/read the structures table. If file_name is None, the file_name will be pulled out of the manifest. If caching is disabled, no file will be saved. Default is None. cre: boolean or list If True, return only cre-positive experiments. If False, return only cre-negative experiments. If None, return all experients. If list, return all experiments with cre line names in the supplied list. Default None. injection_structure_ids: list Only return experiments that were injected in the structures provided here. If None, return all experiments. Default None. """ file_name = self.get_cache_path(file_name, self.EXPERIMENTS_KEY) if os.path.exists(file_name): experiments = json_utilities.read(file_name) else: experiments = self.api.experiment_source_search( injection_structures='root') # removing these elements because they are specific to a particular # resolution for e in experiments: del e['num-voxels'] del e['injection-volume'] del e['sum'] del e['name'] if self.cache: Manifest.safe_make_parent_dirs(file_name) json_utilities.write(file_name, experiments) # filter the read/downloaded list of experiments experiments = self.filter_experiments(experiments, cre, injection_structure_ids) if dataframe: experiments = pd.DataFrame(experiments) experiments.set_index(['id'], inplace=True, drop=False) return experiments
def write_ecephys_nwb(output_path, session_id, session_start_time, stimulus_table_path, invalid_epochs, probes, running_speed_path, session_sync_path, eye_tracking_rig_geometry, eye_dlc_ellipses_path, eye_gaze_mapping_path, pool_size, optotagging_table_path=None, session_metadata=None, **kwargs): nwbfile = pynwb.NWBFile(session_description='EcephysSession', identifier='{}'.format(session_id), session_start_time=session_start_time) if session_metadata is not None: nwbfile = add_metadata_to_nwbfile(nwbfile, session_metadata) stimulus_table = read_stimulus_table(stimulus_table_path) nwbfile = add_stimulus_timestamps( nwbfile, stimulus_table['start_time'].values ) # TODO: patch until full timestamps are output by stim table module nwbfile = add_stimulus_presentations(nwbfile, stimulus_table) nwbfile = add_invalid_times(nwbfile, invalid_epochs) if optotagging_table_path is not None: optotagging_table = pd.read_csv(optotagging_table_path) nwbfile = add_optotagging_table_to_nwbfile(nwbfile, optotagging_table) nwbfile = add_probewise_data_to_nwbfile(nwbfile, probes) running_speed, raw_running_data = read_running_speed(running_speed_path) add_running_speed_to_nwbfile(nwbfile, running_speed) add_raw_running_data_to_nwbfile(nwbfile, raw_running_data) # --- Add eye tracking ellipse fits to nwb file --- eye_tracking_frame_times = su.get_synchronized_frame_times( session_sync_file=session_sync_path, sync_line_label_keys=Dataset.EYE_TRACKING_KEYS) eye_dlc_tracking_data = read_eye_dlc_tracking_ellipses( Path(eye_dlc_ellipses_path)) if eye_tracking_data_is_valid(eye_dlc_tracking_data=eye_dlc_tracking_data, synced_timestamps=eye_tracking_frame_times): add_eye_tracking_ellipse_fit_data_to_nwbfile( nwbfile, eye_dlc_tracking_data=eye_dlc_tracking_data, synced_timestamps=eye_tracking_frame_times) # --- Append eye tracking rig geometry info to nwb file (with eye tracking) --- append_eye_tracking_rig_geometry_data_to_nwbfile( nwbfile, eye_tracking_rig_geometry=eye_tracking_rig_geometry) # --- Add gaze mapped positions to nwb file --- if eye_gaze_mapping_path: eye_gaze_data = read_eye_gaze_mappings(Path(eye_gaze_mapping_path)) add_eye_gaze_mapping_data_to_nwbfile(nwbfile, eye_gaze_data=eye_gaze_data) Manifest.safe_make_parent_dirs(output_path) io = pynwb.NWBHDF5IO(output_path, mode='w') logging.info(f"writing session nwb file to {output_path}") io.write(nwbfile) io.close() probes_with_lfp = [p for p in probes if p["lfp"] is not None] probe_outputs = write_probewise_lfp_files(probes_with_lfp, session_start_time, pool_size=pool_size) return {'nwb_path': output_path, "probe_outputs": probe_outputs}
def safe_write(data: Q): Manifest.safe_make_parent_dirs(path) write(path, data)
def cacher(fn, *args, **kwargs): '''make an rma query, save it and return the dataframe. Parameters ---------- fn : function reference makes the actual query using kwargs. path : string where to save the data strategy : string or None, optional 'create' always generates the data, 'file' loads from disk, 'lazy' queries the server if no file exists, None generates the data and bypasses all caching behavior pre : function df|json->df|json, takes one data argument and returns filtered version, None for pass-through post : function df|json->?, takes one data argument and returns Object reader : function, optional path -> data, default NOP writer : function, optional path, data -> None, default NOP kwargs : objects passed through to the query function Returns ------- Object or None data type depends on fn, reader and/or post methods. ''' path = kwargs.pop('path', None) strategy = kwargs.pop('strategy', None) pre = kwargs.pop('pre', lambda d: d) post = kwargs.pop('post', None) reader = kwargs.pop('reader', None) writer = kwargs.pop('writer', None) if strategy is None: if writer or path: strategy = 'lazy' else: strategy = 'pass_through' if strategy not in ['lazy', 'pass_through', 'file', 'create']: raise ValueError("Unknown query strategy: {}.".format(strategy)) if 'lazy' == strategy: if os.path.exists(path): strategy = 'file' else: strategy = 'create' if strategy == 'pass_through': data = fn(*args, **kwargs) elif strategy in ['create']: Manifest.safe_make_parent_dirs(path) if writer: data = fn(*args, **kwargs) data = pre(data) writer(path, data) else: data = fn(*args, **kwargs) if reader: data = reader(path) # Note: don't provide post if fn or reader doesn't return data if post: data = post(data) return data try: data return data except Exception: pass return