def test_data_file_attributes(manifest_with_data):
"""
Test that Manifest.data_file_attributes returns the correct
CacheFileAttributes object and raises the correct error when
you ask for a data file that does not exist
"""
mfest = Manifest('/my/cache/dir', manifest_with_data)
a_obj = mfest.data_file_attributes('a')
assert a_obj.url == 'http://my.url.com/myproject/path/to/a.nwb'
assert a_obj.version_id == '12345'
assert a_obj.file_hash == 'abcde'
expected = safe_system_path('/my/cache/dir/myproject-0/path/to/a.nwb')
assert a_obj.local_path == pathlib.Path(expected).resolve()
b_obj = mfest.data_file_attributes('b')
assert b_obj.url == 'http://my.other.url.com/different/path/b.nwb'
assert b_obj.version_id == '67890'
assert b_obj.file_hash == 'fghijk'
expected = safe_system_path('/my/cache/dir/myproject-0/path/b.nwb')
assert b_obj.local_path == pathlib.Path(expected).resolve()
with pytest.raises(ValueError) as context:
_ = mfest.data_file_attributes('c')
msg = "file_id: c\nIs not a data file listed in manifest:"
assert msg in context.value.args[0]
def test_metadata_file_attributes(manifest_for_metadata):
"""
Test that Manifest.metadata_file_attributes returns the
correct CacheFileAttributes object and raises the correct
error when you ask for a metadata file that does not exist
"""
mfest = Manifest('/my/cache/dir/', manifest_for_metadata)
a_obj = mfest.metadata_file_attributes('a.txt')
assert a_obj.url == 'http://my.url.com/path/to/a.txt'
assert a_obj.version_id == '12345'
assert a_obj.file_hash == 'abcde'
expected = safe_system_path('/my/cache/dir/some-project-000/to/a.txt')
expected = pathlib.Path(expected).resolve()
assert a_obj.local_path == expected
b_obj = mfest.metadata_file_attributes('b.txt')
assert b_obj.url == 'http://my.other.url.com/different/path/to/b.txt'
assert b_obj.version_id == '67890'
assert b_obj.file_hash == 'fghijk'
expected = safe_system_path('/my/cache/dir/some-project-000/path/to/b.txt')
expected = pathlib.Path(expected).resolve()
assert b_obj.local_path == expected
# test that the correct error is raised when you ask
# for a metadata file that does not exist
with pytest.raises(ValueError) as context:
_ = mfest.metadata_file_attributes('c.txt')
msg = "c.txt\nis not in self.metadata_file_names"
assert msg in context.value.args[0]
def get_ophys_experiment_dir(self):
query = '''
SELECT oe.storage_directory
FROM ophys_experiments oe
WHERE oe.id = {};
'''.format(self.get_ophys_experiment_id())
return safe_system_path(self.fetchone(query, strict=True))
def get_behavior_stimulus_file(self):
query = '''
SELECT stim.storage_directory || stim.filename AS stim_file
FROM behavior_sessions bs
LEFT JOIN well_known_files stim ON stim.attachable_id=bs.id AND stim.attachable_type = 'BehaviorSession' AND stim.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'StimulusPickle')
WHERE bs.id= {};
'''.format(self.get_behavior_experiment_id())
return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_ophys_experiment_dir(self) -> str:
"""Get the storage directory associated with the ophys experiment"""
query = """
SELECT oe.storage_directory
FROM ophys_experiments oe
WHERE oe.id = {};
""".format(self.get_ophys_experiment_id())
return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_dff_file(self):
query = '''
SELECT dff.storage_directory || dff.filename AS dff_file
FROM ophys_experiments oe
LEFT JOIN well_known_files dff ON dff.attachable_id=oe.id AND dff.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'OphysDffTraceFile')
WHERE oe.id= {};
'''.format(self.get_ophys_experiment_id())
return safe_system_path(self.fetchone(query, strict=True))
def get_rigid_motion_transform_file(self):
query = '''
SELECT tra.storage_directory || tra.filename AS transform_file
FROM ophys_experiments oe
LEFT JOIN well_known_files tra ON tra.attachable_id=oe.id AND tra.attachable_type = 'OphysExperiment' AND tra.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'OphysMotionXyOffsetData')
WHERE oe.id= {};
'''.format(self.get_ophys_experiment_id())
return safe_system_path(self.fetchone(query, strict=True))
def get_nwb_filepath(self):
query = '''
SELECT wkf.storage_directory || wkf.filename AS nwb_file
FROM ophys_experiments oe
LEFT JOIN well_known_files wkf ON wkf.attachable_id=oe.id AND wkf.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'NWBOphys')
WHERE oe.id = {};
'''.format(self.get_ophys_experiment_id())
return safe_system_path(self.fetchone(query, strict=True))
def get_demix_file(self):
query = '''
SELECT wkf.storage_directory || wkf.filename AS demix_file
FROM ophys_experiments oe
LEFT JOIN well_known_files wkf ON wkf.attachable_id=oe.id AND wkf.attachable_type = 'OphysExperiment' AND wkf.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'DemixedTracesFile')
WHERE oe.id= {};
'''.format(self.get_ophys_experiment_id())
return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_eye_tracking_filepath(self):
query = '''SELECT wkf.storage_directory || wkf.filename AS eye_tracking_file
FROM ophys_experiments oe
LEFT JOIN well_known_files wkf ON wkf.attachable_id=oe.ophys_session_id
AND wkf.attachable_type = 'OphysSession'
AND wkf.well_known_file_type_id=(SELECT id FROM well_known_file_types WHERE name = 'EyeTracking Ellipses')
WHERE oe.id={};
'''.format(self.get_ophys_experiment_id())
return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_max_projection_file(self):
query = '''
SELECT wkf.storage_directory || wkf.filename AS maxint_file
FROM ophys_experiments oe
LEFT JOIN ophys_cell_segmentation_runs ocsr ON ocsr.ophys_experiment_id = oe.id AND ocsr.current = 't'
LEFT JOIN well_known_files wkf ON wkf.attachable_id=ocsr.id AND wkf.attachable_type = 'OphysCellSegmentationRun' AND wkf.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'OphysMaxIntImage')
WHERE oe.id= {};
'''.format(self.get_ophys_experiment_id())
return safe_system_path(self.fetchone(query, strict=True))
def get_segmentation_mask_image_file(self):
query = '''
SELECT obj.storage_directory || obj.filename AS OphysSegmentationMaskImage_filename
FROM ophys_experiments oe
LEFT JOIN ophys_cell_segmentation_runs ocsr ON ocsr.ophys_experiment_id = oe.id AND ocsr.current = 't'
LEFT JOIN well_known_files obj ON obj.attachable_id=ocsr.id AND obj.attachable_type = 'OphysCellSegmentationRun' AND obj.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'OphysSegmentationMaskImage')
WHERE oe.id= {};
'''.format(self.get_ophys_experiment_id())
return safe_system_path(self.fetchone(query, strict=True))
def get_motion_corrected_image_stack_file(self):
query = f"""
select wkf.storage_directory || wkf.filename
from well_known_files wkf
join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id
where wkft.name = 'MotionCorrectedImageStack'
and wkf.attachable_id = {self.get_ophys_experiment_id()}
"""
return safe_system_path(self.fetchone(query, strict=True))
def get_average_intensity_projection_image_file(self):
query = '''
SELECT avg.storage_directory || avg.filename AS avgint_file
FROM ophys_experiments oe
LEFT JOIN ophys_cell_segmentation_runs ocsr ON ocsr.ophys_experiment_id = oe.id AND ocsr.current = 't'
LEFT JOIN well_known_files avg ON avg.attachable_id=ocsr.id AND avg.attachable_type = 'OphysCellSegmentationRun' AND avg.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'OphysAverageIntensityProjectionImage')
WHERE oe.id = {};
'''.format(self.get_ophys_experiment_id())
return safe_system_path(self.fetchone(query, strict=True))
def get_sync_file(self, ophys_experiment_id=None):
query = '''
SELECT sync.storage_directory || sync.filename AS sync_file
FROM ophys_experiments oe
JOIN ophys_sessions os ON oe.ophys_session_id = os.id
LEFT JOIN well_known_files sync ON sync.attachable_id=os.id AND sync.attachable_type = 'OphysSession' AND sync.well_known_file_type_id IN (SELECT id FROM well_known_file_types WHERE name = 'OphysRigSync')
WHERE oe.id= {};
'''.format(self.get_ophys_experiment_id())
return safe_system_path(self.fetchone(query, strict=True))
def from_stimulus_file(
cls, stimulus_file: StimulusFile,
limit_to_images: Optional[List] = None) -> "Templates":
"""Get stimulus templates (movies, scenes) for behavior session."""
# TODO: Eventually the `grating_images_dict` should be provided by the
# BehaviorLimsExtractor/BehaviorJsonExtractor classes.
# - NJM 2021/2/23
gratings_dir = "/allen/programs/braintv/production/visualbehavior"
gratings_dir = os.path.join(gratings_dir,
"prod5/project_VisualBehavior")
grating_images_dict = {
"gratings_0.0": {
"warped": np.asarray(imageio.imread(
safe_system_path(os.path.join(gratings_dir,
"warped_grating_0.png")))),
"unwarped": np.asarray(imageio.imread(
safe_system_path(os.path.join(
gratings_dir, "masked_unwarped_grating_0.png"))))
},
"gratings_90.0": {
"warped": np.asarray(imageio.imread(
safe_system_path(os.path.join(gratings_dir,
"warped_grating_90.png")))),
"unwarped": np.asarray(imageio.imread(
safe_system_path(os.path.join(
gratings_dir, "masked_unwarped_grating_90.png"))))
},
"gratings_180.0": {
"warped": np.asarray(imageio.imread(
safe_system_path(os.path.join(gratings_dir,
"warped_grating_180.png")))),
"unwarped": np.asarray(imageio.imread(
safe_system_path(os.path.join(
gratings_dir, "masked_unwarped_grating_180.png"))))
},
"gratings_270.0": {
"warped": np.asarray(imageio.imread(
safe_system_path(os.path.join(gratings_dir,
"warped_grating_270.png")))),
"unwarped": np.asarray(imageio.imread(
safe_system_path(os.path.join(
gratings_dir, "masked_unwarped_grating_270.png"))))
}
}
pkl = stimulus_file.data
t = get_stimulus_templates(pkl=pkl,
grating_images_dict=grating_images_dict,
limit_to_images=limit_to_images)
return Templates(templates=t)
def get_event_detection_filepath(self) -> str:
"""Gets the filepath to the event detection data"""
query = f'''
SELECT wkf.storage_directory || wkf.filename AS event_detection_filepath
FROM ophys_experiments oe
LEFT JOIN well_known_files wkf ON wkf.attachable_id = oe.id
JOIN well_known_file_types wkft ON wkf.well_known_file_type_id = wkft.id
WHERE wkft.name = 'OphysEventTraceFile'
AND oe.id = {self.get_ophys_experiment_id()};
''' # noqa E501
return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_motion_corrected_image_stack_file(self) -> str:
"""Get the filepath for the motion corrected image stack associated
with a an ophys experiment"""
query = """
SELECT wkf.storage_directory || wkf.filename
FROM well_known_files wkf
JOIN well_known_file_types wkft
ON wkft.id = wkf.well_known_file_type_id
WHERE wkft.name = 'MotionCorrectedImageStack'
AND wkf.attachable_id = {};
""".format(self.get_ophys_experiment_id())
return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_eye_tracking_filepath(self) -> str:
"""Get the filepath of the eye tracking file (*.h5) associated with the
ophys experiment"""
query = f"""
SELECT wkf.storage_directory || wkf.filename AS eye_tracking_file
FROM ophys_experiments oe
LEFT JOIN well_known_files wkf ON wkf.attachable_id = oe.ophys_session_id
JOIN well_known_file_types wkft ON wkf.well_known_file_type_id = wkft.id
WHERE wkf.attachable_type = 'OphysSession'
AND wkft.name = 'EyeTracking Ellipses'
AND oe.id = {self.get_ophys_experiment_id()};
""" # noqa E501
return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_nwb_filepath(self) -> str:
"""Get the filepath of the nwb file associated with the ophys
experiment"""
query = """
SELECT wkf.storage_directory || wkf.filename AS nwb_file
FROM ophys_experiments oe
JOIN well_known_files wkf ON wkf.attachable_id = oe.id
JOIN well_known_file_types wkft
ON wkf.well_known_file_type_id = wkft.id
WHERE wkft.name ='BehaviorOphysNwb'
AND oe.id = {};
""".format(self.get_ophys_experiment_id())
return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_demix_file(self) -> str:
"""Get the filepath of the demixed traces file associated with an
ophys experiment"""
query = """
SELECT wkf.storage_directory || wkf.filename AS demix_file
FROM ophys_experiments oe
JOIN well_known_files wkf ON wkf.attachable_id = oe.id
JOIN well_known_file_types wkft
ON wkft.id = wkf.well_known_file_type_id
WHERE wkf.attachable_type = 'OphysExperiment'
AND wkft.name = 'DemixedTracesFile'
AND oe.id = {};
""".format(self.get_ophys_experiment_id())
return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_rigid_motion_transform_file(self) -> str:
"""Get the filepath for the motion transform file (.csv) associated
with an ophys experiment"""
query = """
SELECT wkf.storage_directory || wkf.filename AS transform_file
FROM ophys_experiments oe
JOIN well_known_files wkf ON wkf.attachable_id = oe.id
JOIN well_known_file_types wkft
ON wkft.id = wkf.well_known_file_type_id
WHERE wkf.attachable_type = 'OphysExperiment'
AND wkft.name = 'OphysMotionXyOffsetData'
AND oe.id = {};
""".format(self.get_ophys_experiment_id())
return safe_system_path(self.lims_db.fetchone(query, strict=True))
def from_lims(
cls, db: PostgresQueryMixin,
ophys_experiment_id: Union[int, str]
) -> "EventDetectionFile":
query = f'''
SELECT wkf.storage_directory || wkf.filename AS event_detection_filepath
FROM ophys_experiments oe
LEFT JOIN well_known_files wkf ON wkf.attachable_id = oe.id
JOIN well_known_file_types wkft ON wkf.well_known_file_type_id = wkft.id
WHERE wkft.name = 'OphysEventTraceFile'
AND oe.id = {ophys_experiment_id};
''' # noqa E501
filepath = safe_system_path(db.fetchone(query, strict=True))
return cls(filepath=filepath)
def get_sync_file(self, ophys_experiment_id=None) -> str:
"""Get the filepath of the sync timing file associated with the
ophys experiment"""
query = """
SELECT wkf.storage_directory || wkf.filename AS sync_file
FROM ophys_experiments oe
JOIN ophys_sessions os ON oe.ophys_session_id = os.id
JOIN well_known_files wkf ON wkf.attachable_id = os.id
JOIN well_known_file_types wkft
ON wkft.id = wkf.well_known_file_type_id
WHERE wkf.attachable_type = 'OphysSession'
AND wkft.name = 'OphysRigSync'
AND oe.id = {};
""".format(self.get_ophys_experiment_id())
return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_behavior_stimulus_file(self) -> str:
"""Get the filepath to the StimulusPickle file for the session
associated with the ophys experiment id used to initialize the API"""
query = """
SELECT wkf.storage_directory || wkf.filename AS stim_file
FROM ophys_experiments oe
JOIN ophys_sessions os ON oe.ophys_session_id = os.id
JOIN behavior_sessions bs ON bs.ophys_session_id = os.id
LEFT JOIN well_known_files wkf ON wkf.attachable_id = bs.id
JOIN well_known_file_types wkft
ON wkf.well_known_file_type_id = wkft.id
WHERE wkf.attachable_type = 'BehaviorSession'
AND wkft.name = 'StimulusPickle'
AND oe.id = {};
""".format(self.get_ophys_experiment_id())
return safe_system_path(self.lims_db.fetchone(query, strict=True))
def from_lims(
cls, db: PostgresQueryMixin,
ophys_experiment_id: Union[int, str]
) -> "RigidMotionTransformFile":
query = """
SELECT wkf.storage_directory || wkf.filename AS transform_file
FROM ophys_experiments oe
JOIN well_known_files wkf ON wkf.attachable_id = oe.id
JOIN well_known_file_types wkft
ON wkft.id = wkf.well_known_file_type_id
WHERE wkf.attachable_type = 'OphysExperiment'
AND wkft.name = 'OphysMotionXyOffsetData'
AND oe.id = {};
""".format(ophys_experiment_id)
filepath = safe_system_path(db.fetchone(query, strict=True))
return cls(filepath=filepath)
def get_average_intensity_projection_image_file(self) -> str:
"""Get the avg intensity project image filepath associated with an
ophys experiment"""
query = """
SELECT wkf.storage_directory || wkf.filename AS avgint_file
FROM ophys_experiments oe
JOIN ophys_cell_segmentation_runs ocsr
ON ocsr.ophys_experiment_id = oe.id
JOIN well_known_files wkf ON wkf.attachable_id=ocsr.id
JOIN well_known_file_types wkft
ON wkft.id = wkf.well_known_file_type_id
WHERE ocsr.current = 't'
AND wkf.attachable_type = 'OphysCellSegmentationRun'
AND wkft.name = 'OphysAverageIntensityProjectionImage'
AND oe.id = {};
""".format(self.get_ophys_experiment_id())
return safe_system_path(self.lims_db.fetchone(query, strict=True))
def get_behavior_stimulus_file(self) -> str:
"""Return the path to the StimulusPickle file for a session.
:rtype: str
"""
query = f"""
SELECT
stim.storage_directory || stim.filename AS stim_file
FROM
well_known_files stim
WHERE
stim.attachable_id = {self.behavior_session_id}
AND stim.attachable_type = 'BehaviorSession'
AND stim.well_known_file_type_id IN (
SELECT id
FROM well_known_file_types
WHERE name = 'StimulusPickle');
"""
return safe_system_path(self.lims_db.fetchone(query, strict=True))
def make_cell_html(image_files, ephys_roi_result, file_name,
relative_sweep_link):
html = "<html><body>"
specimen = ephys_roi_result['specimens'][0]
html += "<h3>Specimen %d: %s</h3>" % (specimen['id'], specimen['name'])
html += "<p>page created at: %s</p>" % get_time_string()
if relative_sweep_link:
html += "<p><a href='sweep.html' target='_blank'> Sweep QC Figures </a></p>"
else:
sweep_qc_link = '/'.join([
ephys_roi_result['storage_directory'], 'qc_figures', 'sweep.html'
])
sweep_qc_link = lims_utilities.safe_system_path(sweep_qc_link)
html += "<p><a href='%s' target='_blank'> Sweep QC Figures </a></p>" % sweep_qc_link
fields_to_show = [
'electrode_0_pa', 'seal_gohm', 'initial_access_resistance_mohm',
'input_resistance_mohm'
]
html += "<table>"
for field in fields_to_show:
html += "<tr><td>%s</td><td>%s</td></tr>" % (
field, ephys_roi_result.get(field, None))
html += "</table>"
for image_file_set_name in image_files:
html += "<h3>%s</h3>" % image_file_set_name
image_set_files = image_files[image_file_set_name]
for small_img, large_img in zip(image_set_files['small'],
image_set_files['large']):
html += "<a href='%s' target='_blank'><img src='%s'></img></a>" % (
os.path.basename(large_img), os.path.basename(small_img))
html += ("</body></html>")
with open(file_name, 'w') as f:
f.write(html)
def get_objectlist_file(self) -> str:
"""Get the objectlist.txt filepath associated with an ophys experiment
NOTE: Although this will be deprecated for visual behavior it will
still be valid for visual coding.
"""
query = """
SELECT wkf.storage_directory || wkf.filename AS obj_file
FROM ophys_experiments oe
LEFT JOIN ophys_cell_segmentation_runs ocsr
ON ocsr.ophys_experiment_id = oe.id
JOIN well_known_files wkf ON wkf.attachable_id = ocsr.id
JOIN well_known_file_types wkft
ON wkft.id = wkf.well_known_file_type_id
WHERE wkft.name = 'OphysSegmentationObjects'
AND wkf.attachable_type = 'OphysCellSegmentationRun'
AND ocsr.current = 't'
AND oe.id = {};
""".format(self.get_ophys_experiment_id())
return safe_system_path(self.lims_db.fetchone(query, strict=True))
