def test_data_file_attributes(manifest_with_data): """ Test that Manifest.data_file_attributes returns the correct CacheFileAttributes object and raises the correct error when you ask for a data file that does not exist """ mfest = Manifest('/my/cache/dir', manifest_with_data) a_obj = mfest.data_file_attributes('a') assert a_obj.url == 'http://my.url.com/myproject/path/to/a.nwb' assert a_obj.version_id == '12345' assert a_obj.file_hash == 'abcde' expected = safe_system_path('/my/cache/dir/myproject-0/path/to/a.nwb') assert a_obj.local_path == pathlib.Path(expected).resolve() b_obj = mfest.data_file_attributes('b') assert b_obj.url == 'http://my.other.url.com/different/path/b.nwb' assert b_obj.version_id == '67890' assert b_obj.file_hash == 'fghijk' expected = safe_system_path('/my/cache/dir/myproject-0/path/b.nwb') assert b_obj.local_path == pathlib.Path(expected).resolve() with pytest.raises(ValueError) as context: _ = mfest.data_file_attributes('c') msg = "file_id: c\nIs not a data file listed in manifest:" assert msg in context.value.args[0]
def test_metadata_file_attributes(manifest_for_metadata): """ Test that Manifest.metadata_file_attributes returns the correct CacheFileAttributes object and raises the correct error when you ask for a metadata file that does not exist """ mfest = Manifest('/my/cache/dir/', manifest_for_metadata) a_obj = mfest.metadata_file_attributes('a.txt') assert a_obj.url == 'http://my.url.com/path/to/a.txt' assert a_obj.version_id == '12345' assert a_obj.file_hash == 'abcde' expected = safe_system_path('/my/cache/dir/some-project-000/to/a.txt') expected = pathlib.Path(expected).resolve() assert a_obj.local_path == expected b_obj = mfest.metadata_file_attributes('b.txt') assert b_obj.url == 'http://my.other.url.com/different/path/to/b.txt' assert b_obj.version_id == '67890' assert b_obj.file_hash == 'fghijk' expected = safe_system_path('/my/cache/dir/some-project-000/path/to/b.txt') expected = pathlib.Path(expected).resolve() assert b_obj.local_path == expected # test that the correct error is raised when you ask # for a metadata file that does not exist with pytest.raises(ValueError) as context: _ = mfest.metadata_file_attributes('c.txt') msg = "c.txt\nis not in self.metadata_file_names" assert msg in context.value.args[0]
def get_ophys_experiment_dir(self): query = ''' SELECT oe.storage_directory FROM ophys_experiments oe WHERE oe.id = {}; '''.format(self.get_ophys_experiment_id()) return safe_system_path(self.fetchone(query, strict=True))
def get_behavior_stimulus_file(self): query = ''' SELECT stim.storage_directory || stim.filename AS stim_file FROM behavior_sessions bs LEFT JOIN well_known_files stim ON stim.attachable_id=bs.id AND stim.attachable_type = 'BehaviorSession' AND stim.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'StimulusPickle') WHERE bs.id= {}; '''.format(self.get_behavior_experiment_id()) return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_ophys_experiment_dir(self) -> str: """Get the storage directory associated with the ophys experiment""" query = """ SELECT oe.storage_directory FROM ophys_experiments oe WHERE oe.id = {}; """.format(self.get_ophys_experiment_id()) return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_dff_file(self): query = ''' SELECT dff.storage_directory || dff.filename AS dff_file FROM ophys_experiments oe LEFT JOIN well_known_files dff ON dff.attachable_id=oe.id AND dff.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'OphysDffTraceFile') WHERE oe.id= {}; '''.format(self.get_ophys_experiment_id()) return safe_system_path(self.fetchone(query, strict=True))
def get_rigid_motion_transform_file(self): query = ''' SELECT tra.storage_directory || tra.filename AS transform_file FROM ophys_experiments oe LEFT JOIN well_known_files tra ON tra.attachable_id=oe.id AND tra.attachable_type = 'OphysExperiment' AND tra.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'OphysMotionXyOffsetData') WHERE oe.id= {}; '''.format(self.get_ophys_experiment_id()) return safe_system_path(self.fetchone(query, strict=True))
def get_nwb_filepath(self): query = ''' SELECT wkf.storage_directory || wkf.filename AS nwb_file FROM ophys_experiments oe LEFT JOIN well_known_files wkf ON wkf.attachable_id=oe.id AND wkf.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'NWBOphys') WHERE oe.id = {}; '''.format(self.get_ophys_experiment_id()) return safe_system_path(self.fetchone(query, strict=True))
def get_demix_file(self): query = ''' SELECT wkf.storage_directory || wkf.filename AS demix_file FROM ophys_experiments oe LEFT JOIN well_known_files wkf ON wkf.attachable_id=oe.id AND wkf.attachable_type = 'OphysExperiment' AND wkf.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'DemixedTracesFile') WHERE oe.id= {}; '''.format(self.get_ophys_experiment_id()) return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_eye_tracking_filepath(self): query = '''SELECT wkf.storage_directory || wkf.filename AS eye_tracking_file FROM ophys_experiments oe LEFT JOIN well_known_files wkf ON wkf.attachable_id=oe.ophys_session_id AND wkf.attachable_type = 'OphysSession' AND wkf.well_known_file_type_id=(SELECT id FROM well_known_file_types WHERE name = 'EyeTracking Ellipses') WHERE oe.id={}; '''.format(self.get_ophys_experiment_id()) return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_max_projection_file(self): query = ''' SELECT wkf.storage_directory || wkf.filename AS maxint_file FROM ophys_experiments oe LEFT JOIN ophys_cell_segmentation_runs ocsr ON ocsr.ophys_experiment_id = oe.id AND ocsr.current = 't' LEFT JOIN well_known_files wkf ON wkf.attachable_id=ocsr.id AND wkf.attachable_type = 'OphysCellSegmentationRun' AND wkf.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'OphysMaxIntImage') WHERE oe.id= {}; '''.format(self.get_ophys_experiment_id()) return safe_system_path(self.fetchone(query, strict=True))
def get_segmentation_mask_image_file(self): query = ''' SELECT obj.storage_directory || obj.filename AS OphysSegmentationMaskImage_filename FROM ophys_experiments oe LEFT JOIN ophys_cell_segmentation_runs ocsr ON ocsr.ophys_experiment_id = oe.id AND ocsr.current = 't' LEFT JOIN well_known_files obj ON obj.attachable_id=ocsr.id AND obj.attachable_type = 'OphysCellSegmentationRun' AND obj.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'OphysSegmentationMaskImage') WHERE oe.id= {}; '''.format(self.get_ophys_experiment_id()) return safe_system_path(self.fetchone(query, strict=True))
def get_motion_corrected_image_stack_file(self): query = f""" select wkf.storage_directory || wkf.filename from well_known_files wkf join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id where wkft.name = 'MotionCorrectedImageStack' and wkf.attachable_id = {self.get_ophys_experiment_id()} """ return safe_system_path(self.fetchone(query, strict=True))
def get_average_intensity_projection_image_file(self): query = ''' SELECT avg.storage_directory || avg.filename AS avgint_file FROM ophys_experiments oe LEFT JOIN ophys_cell_segmentation_runs ocsr ON ocsr.ophys_experiment_id = oe.id AND ocsr.current = 't' LEFT JOIN well_known_files avg ON avg.attachable_id=ocsr.id AND avg.attachable_type = 'OphysCellSegmentationRun' AND avg.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'OphysAverageIntensityProjectionImage') WHERE oe.id = {}; '''.format(self.get_ophys_experiment_id()) return safe_system_path(self.fetchone(query, strict=True))
def get_sync_file(self, ophys_experiment_id=None): query = ''' SELECT sync.storage_directory || sync.filename AS sync_file FROM ophys_experiments oe JOIN ophys_sessions os ON oe.ophys_session_id = os.id LEFT JOIN well_known_files sync ON sync.attachable_id=os.id AND sync.attachable_type = 'OphysSession' AND sync.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'OphysRigSync') WHERE oe.id= {}; '''.format(self.get_ophys_experiment_id()) return safe_system_path(self.fetchone(query, strict=True))
def from_stimulus_file( cls, stimulus_file: StimulusFile, limit_to_images: Optional[List] = None) -> "Templates": """Get stimulus templates (movies, scenes) for behavior session.""" # TODO: Eventually the `grating_images_dict` should be provided by the # BehaviorLimsExtractor/BehaviorJsonExtractor classes. # - NJM 2021/2/23 gratings_dir = "/allen/programs/braintv/production/visualbehavior" gratings_dir = os.path.join(gratings_dir, "prod5/project_VisualBehavior") grating_images_dict = { "gratings_0.0": { "warped": np.asarray(imageio.imread( safe_system_path(os.path.join(gratings_dir, "warped_grating_0.png")))), "unwarped": np.asarray(imageio.imread( safe_system_path(os.path.join( gratings_dir, "masked_unwarped_grating_0.png")))) }, "gratings_90.0": { "warped": np.asarray(imageio.imread( safe_system_path(os.path.join(gratings_dir, "warped_grating_90.png")))), "unwarped": np.asarray(imageio.imread( safe_system_path(os.path.join( gratings_dir, "masked_unwarped_grating_90.png")))) }, "gratings_180.0": { "warped": np.asarray(imageio.imread( safe_system_path(os.path.join(gratings_dir, "warped_grating_180.png")))), "unwarped": np.asarray(imageio.imread( safe_system_path(os.path.join( gratings_dir, "masked_unwarped_grating_180.png")))) }, "gratings_270.0": { "warped": np.asarray(imageio.imread( safe_system_path(os.path.join(gratings_dir, "warped_grating_270.png")))), "unwarped": np.asarray(imageio.imread( safe_system_path(os.path.join( gratings_dir, "masked_unwarped_grating_270.png")))) } } pkl = stimulus_file.data t = get_stimulus_templates(pkl=pkl, grating_images_dict=grating_images_dict, limit_to_images=limit_to_images) return Templates(templates=t)
def get_event_detection_filepath(self) -> str: """Gets the filepath to the event detection data""" query = f''' SELECT wkf.storage_directory || wkf.filename AS event_detection_filepath FROM ophys_experiments oe LEFT JOIN well_known_files wkf ON wkf.attachable_id = oe.id JOIN well_known_file_types wkft ON wkf.well_known_file_type_id = wkft.id WHERE wkft.name = 'OphysEventTraceFile' AND oe.id = {self.get_ophys_experiment_id()}; ''' # noqa E501 return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_motion_corrected_image_stack_file(self) -> str: """Get the filepath for the motion corrected image stack associated with a an ophys experiment""" query = """ SELECT wkf.storage_directory || wkf.filename FROM well_known_files wkf JOIN well_known_file_types wkft ON wkft.id = wkf.well_known_file_type_id WHERE wkft.name = 'MotionCorrectedImageStack' AND wkf.attachable_id = {}; """.format(self.get_ophys_experiment_id()) return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_eye_tracking_filepath(self) -> str: """Get the filepath of the eye tracking file (*.h5) associated with the ophys experiment""" query = f""" SELECT wkf.storage_directory || wkf.filename AS eye_tracking_file FROM ophys_experiments oe LEFT JOIN well_known_files wkf ON wkf.attachable_id = oe.ophys_session_id JOIN well_known_file_types wkft ON wkf.well_known_file_type_id = wkft.id WHERE wkf.attachable_type = 'OphysSession' AND wkft.name = 'EyeTracking Ellipses' AND oe.id = {self.get_ophys_experiment_id()}; """ # noqa E501 return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_nwb_filepath(self) -> str: """Get the filepath of the nwb file associated with the ophys experiment""" query = """ SELECT wkf.storage_directory || wkf.filename AS nwb_file FROM ophys_experiments oe JOIN well_known_files wkf ON wkf.attachable_id = oe.id JOIN well_known_file_types wkft ON wkf.well_known_file_type_id = wkft.id WHERE wkft.name ='BehaviorOphysNwb' AND oe.id = {}; """.format(self.get_ophys_experiment_id()) return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_demix_file(self) -> str: """Get the filepath of the demixed traces file associated with an ophys experiment""" query = """ SELECT wkf.storage_directory || wkf.filename AS demix_file FROM ophys_experiments oe JOIN well_known_files wkf ON wkf.attachable_id = oe.id JOIN well_known_file_types wkft ON wkft.id = wkf.well_known_file_type_id WHERE wkf.attachable_type = 'OphysExperiment' AND wkft.name = 'DemixedTracesFile' AND oe.id = {}; """.format(self.get_ophys_experiment_id()) return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_rigid_motion_transform_file(self) -> str: """Get the filepath for the motion transform file (.csv) associated with an ophys experiment""" query = """ SELECT wkf.storage_directory || wkf.filename AS transform_file FROM ophys_experiments oe JOIN well_known_files wkf ON wkf.attachable_id = oe.id JOIN well_known_file_types wkft ON wkft.id = wkf.well_known_file_type_id WHERE wkf.attachable_type = 'OphysExperiment' AND wkft.name = 'OphysMotionXyOffsetData' AND oe.id = {}; """.format(self.get_ophys_experiment_id()) return safe_system_path(self.lims_db.fetchone(query, strict=True))
def from_lims( cls, db: PostgresQueryMixin, ophys_experiment_id: Union[int, str] ) -> "EventDetectionFile": query = f''' SELECT wkf.storage_directory || wkf.filename AS event_detection_filepath FROM ophys_experiments oe LEFT JOIN well_known_files wkf ON wkf.attachable_id = oe.id JOIN well_known_file_types wkft ON wkf.well_known_file_type_id = wkft.id WHERE wkft.name = 'OphysEventTraceFile' AND oe.id = {ophys_experiment_id}; ''' # noqa E501 filepath = safe_system_path(db.fetchone(query, strict=True)) return cls(filepath=filepath)
def get_sync_file(self, ophys_experiment_id=None) -> str: """Get the filepath of the sync timing file associated with the ophys experiment""" query = """ SELECT wkf.storage_directory || wkf.filename AS sync_file FROM ophys_experiments oe JOIN ophys_sessions os ON oe.ophys_session_id = os.id JOIN well_known_files wkf ON wkf.attachable_id = os.id JOIN well_known_file_types wkft ON wkft.id = wkf.well_known_file_type_id WHERE wkf.attachable_type = 'OphysSession' AND wkft.name = 'OphysRigSync' AND oe.id = {}; """.format(self.get_ophys_experiment_id()) return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_behavior_stimulus_file(self) -> str: """Get the filepath to the StimulusPickle file for the session associated with the ophys experiment id used to initialize the API""" query = """ SELECT wkf.storage_directory || wkf.filename AS stim_file FROM ophys_experiments oe JOIN ophys_sessions os ON oe.ophys_session_id = os.id JOIN behavior_sessions bs ON bs.ophys_session_id = os.id LEFT JOIN well_known_files wkf ON wkf.attachable_id = bs.id JOIN well_known_file_types wkft ON wkf.well_known_file_type_id = wkft.id WHERE wkf.attachable_type = 'BehaviorSession' AND wkft.name = 'StimulusPickle' AND oe.id = {}; """.format(self.get_ophys_experiment_id()) return safe_system_path(self.lims_db.fetchone(query, strict=True))
def from_lims( cls, db: PostgresQueryMixin, ophys_experiment_id: Union[int, str] ) -> "RigidMotionTransformFile": query = """ SELECT wkf.storage_directory || wkf.filename AS transform_file FROM ophys_experiments oe JOIN well_known_files wkf ON wkf.attachable_id = oe.id JOIN well_known_file_types wkft ON wkft.id = wkf.well_known_file_type_id WHERE wkf.attachable_type = 'OphysExperiment' AND wkft.name = 'OphysMotionXyOffsetData' AND oe.id = {}; """.format(ophys_experiment_id) filepath = safe_system_path(db.fetchone(query, strict=True)) return cls(filepath=filepath)
def get_average_intensity_projection_image_file(self) -> str: """Get the avg intensity project image filepath associated with an ophys experiment""" query = """ SELECT wkf.storage_directory || wkf.filename AS avgint_file FROM ophys_experiments oe JOIN ophys_cell_segmentation_runs ocsr ON ocsr.ophys_experiment_id = oe.id JOIN well_known_files wkf ON wkf.attachable_id=ocsr.id JOIN well_known_file_types wkft ON wkft.id = wkf.well_known_file_type_id WHERE ocsr.current = 't' AND wkf.attachable_type = 'OphysCellSegmentationRun' AND wkft.name = 'OphysAverageIntensityProjectionImage' AND oe.id = {}; """.format(self.get_ophys_experiment_id()) return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_behavior_stimulus_file(self) -> str: """Return the path to the StimulusPickle file for a session. :rtype: str """ query = f""" SELECT stim.storage_directory || stim.filename AS stim_file FROM well_known_files stim WHERE stim.attachable_id = {self.behavior_session_id} AND stim.attachable_type = 'BehaviorSession' AND stim.well_known_file_type_id IN ( SELECT id FROM well_known_file_types WHERE name = 'StimulusPickle'); """ return safe_system_path(self.lims_db.fetchone(query, strict=True))
def make_cell_html(image_files, ephys_roi_result, file_name, relative_sweep_link): html = "<html><body>" specimen = ephys_roi_result['specimens'][0] html += "<h3>Specimen %d: %s</h3>" % (specimen['id'], specimen['name']) html += "<p>page created at: %s</p>" % get_time_string() if relative_sweep_link: html += "<p><a href='sweep.html' target='_blank'> Sweep QC Figures </a></p>" else: sweep_qc_link = '/'.join([ ephys_roi_result['storage_directory'], 'qc_figures', 'sweep.html' ]) sweep_qc_link = lims_utilities.safe_system_path(sweep_qc_link) html += "<p><a href='%s' target='_blank'> Sweep QC Figures </a></p>" % sweep_qc_link fields_to_show = [ 'electrode_0_pa', 'seal_gohm', 'initial_access_resistance_mohm', 'input_resistance_mohm' ] html += "<table>" for field in fields_to_show: html += "<tr><td>%s</td><td>%s</td></tr>" % ( field, ephys_roi_result.get(field, None)) html += "</table>" for image_file_set_name in image_files: html += "<h3>%s</h3>" % image_file_set_name image_set_files = image_files[image_file_set_name] for small_img, large_img in zip(image_set_files['small'], image_set_files['large']): html += "<a href='%s' target='_blank'><img src='%s'></img></a>" % ( os.path.basename(large_img), os.path.basename(small_img)) html += ("</body></html>") with open(file_name, 'w') as f: f.write(html)
def get_objectlist_file(self) -> str: """Get the objectlist.txt filepath associated with an ophys experiment NOTE: Although this will be deprecated for visual behavior it will still be valid for visual coding. """ query = """ SELECT wkf.storage_directory || wkf.filename AS obj_file FROM ophys_experiments oe LEFT JOIN ophys_cell_segmentation_runs ocsr ON ocsr.ophys_experiment_id = oe.id JOIN well_known_files wkf ON wkf.attachable_id = ocsr.id JOIN well_known_file_types wkft ON wkft.id = wkf.well_known_file_type_id WHERE wkft.name = 'OphysSegmentationObjects' AND wkf.attachable_type = 'OphysCellSegmentationRun' AND ocsr.current = 't' AND oe.id = {}; """.format(self.get_ophys_experiment_id()) return safe_system_path(self.lims_db.fetchone(query, strict=True))