def fetch_cell(specimen_id):
import allensdk.internal.core.lims_utilities as lu
query = """
select a3d.* from specimens sp
join alignment3ds a3d on sp.alignment3d_id = a3d.id
where sp.id = %d
"""
a3d = lu.query(query % specimen_id)[0]
m = np.array([[a3d['tvr_00'], a3d['tvr_01'], a3d['tvr_02'], a3d['tvr_09']],
[a3d['tvr_03'], a3d['tvr_04'], a3d['tvr_05'], a3d['tvr_10']],
[a3d['tvr_06'], a3d['tvr_07'], a3d['tvr_08'], a3d['tvr_11']],
[0, 0, 0, 1]])
query = """
select wkf.storage_directory||wkf.filename as swc_file from well_known_files wkf
join neuron_reconstructions nr on wkf.attachable_id = nr.id
where nr.specimen_id = %d
and nr.superseded = false
and nr.manual = true
and wkf.well_known_file_type_id = 303941301
"""
swc_file = lu.query(query % specimen_id)[0]['swc_file']
return swc_file, m
def get_container_info(container_id):
res = lu.query("""
select * from ophys_experiments oe
where experiment_container_id = %d
and oe.workflow_state != 'failed'
""" % container_id)
return res
def get_ophys_experiments(self):
query_result = lu.query(experiment_pre_release_query)
experiment_list = []
for q in query_result:
c = collections.defaultdict(collections.defaultdict)
# # For development: print key/val pairs generated from LIMS query:
# for key, val in sorted(q.items(), key=lambda x: x[0]):
# print(key, val)
# raise
c['id'] = q['o_id']
c['imaging_depth'] = q['depth']
c['targeted_structure']['acronym'] = q['acronym']
c['specimen']['donor'] = collections.defaultdict(collections.defaultdict)
c['specimen']['donor']['external_donor_name'] = q['acronym']
c['specimen']['donor']['transgenic_lines'] = [collections.defaultdict(collections.defaultdict), collections.defaultdict(collections.defaultdict)]
c['specimen']['donor']['transgenic_lines'][0]['transgenic_line_type_name'] = 'driver'
c['specimen']['donor']['transgenic_lines'][0]['name'] = q['driver']
c['specimen']['donor']['transgenic_lines'][1]['transgenic_line_type_name'] = 'reporter'
c['specimen']['donor']['transgenic_lines'][1]['name'] = q['reporter']
c['date_of_acquisition'] = q['date_of_acquisition']
c['specimen']['donor']['date_of_birth'] = q['date_of_birth']
c['experiment_container_id'] = q['ec_id']
c['stimulus_name'] = q['stimulus_name']
c['specimen']['donor']['external_donor_name'] = q['external_donor_name']
c['specimen']['name'] = q['specimen']
c['fail_eye_tracking'] = q['fail_eye_tracking']
experiment_list.append(c)
return experiment_list
def debug(experiment_id, local=False):
OUTPUT_DIRECTORY = "/data/informatics/CAM/neuropil"
SDK_PATH = "/data/informatics/CAM/neuropil/allensdk"
SCRIPT = "/data/informatics/CAM/neuropil/allensdk/allensdk/internal/pipeline_modules/run_neuropil_correction.py"
exp = lu.query("select * from ophys_experiments where id = %d" %
experiment_id)[0]
sd = exp["storage_directory"]
test_file = "/data/informatics/CAM/demix/%d/demixed_traces.h5" % experiment_id
if os.path.exists(test_file):
roi_trace_file = test_file
else:
roi_trace_file = os.path.join(sd, "demix",
"%d_demixed_traces.h5" % experiment_id)
exp_dir = os.path.join(OUTPUT_DIRECTORY, str(experiment_id))
input_data = dict(roi_trace_file=roi_trace_file,
neuropil_trace_file=os.path.join(sd, "processed",
"neuropil_traces.h5"),
storage_directory=exp_dir)
run_module(SCRIPT,
input_data,
exp_dir,
sdk_path=SDK_PATH,
pbs=dict(vmem=160,
job_name="np_%d" % experiment_id,
walltime="36:00:00"),
local=local,
optional_args=['--log-level', 'DEBUG'])
def get_motion_filepath(experiment_id):
return lu.query("""
select CONCAT(wkf.storage_directory, wkf.filename) as path
from well_known_files wkf
join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id
join ophys_experiments oe on oe.id = wkf.attachable_id
where oe.id = {} and
wkft.name like 'OphysMotionXyOffsetData'""".format(experiment_id))[0]["path"]
def get_wkf(wkf_type, experiment_id):
wkf = lu.query("""
select CONCAT(wkf.storage_directory, wkf.filename) as path
from well_known_files wkf
join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id
where wkft.name LIKE '{}' and wkf.attachable_id = {}
""".format(wkf_type, experiment_id))[0]["path"]
return wkf
def get_experiment_nwb_file(experiment_id):
res = lu.query("""
select * from well_known_files wkf
join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id
where attachable_id = %d
and wkft.name = 'NWBOphys'
""" % experiment_id)
return os.path.join(res[0]['storage_directory'], res[0]['filename'])
def get_segmentation_filepath(experiment_id, file_type):
return lu.query("""
select CONCAT(wkf.storage_directory, wkf.filename) as path
from well_known_files wkf
join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id
join ophys_cell_segmentation_runs ocsr on ocsr.id = wkf.attachable_id
join ophys_experiments oe on oe.id = ocsr.ophys_experiment_id
where oe.id = {} and wkft.name like '{}'
and ocsr.current = 't'""".format(experiment_id, file_type))[0]["path"]
def get_model_info(experiment_id):
res = lu.query("""
select CONCAT(wkf.storage_directory, wkf.filename) as path, wkf.id
from ophys_experiments oe
join ophys_sessions os on os.id = oe.ophys_session_id
join projects p on p.id = os.project_id
join well_known_files wkf on wkf.attachable_id = p.id
join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id
where oe.id = {} and
wkft.name = 'RoiLabelModel'""".format(experiment_id))[0]
return res["path"], res["id"]
def create_input_data(experiment_id):
data = {}
data["log_0"] = get_motion_filepath(experiment_id)
model, model_id = get_model_info(experiment_id)
data["roi_label_model"] = model
data["roi_label_model_id"] = model_id
data["targeted_structure_id"] = lu.query("""
select targeted_structure_id
from ophys_experiments oe
where oe.id = {}""".format(experiment_id))[0]["targeted_structure_id"]
data["imaging_depth"] = lu.query("""
select calculated_depth
from ophys_experiments oe
where oe.id = {}""".format(experiment_id))[0]["calculated_depth"]
data["drivers"] = get_genotype_info(experiment_id, "D")
data["reporters"] = get_genotype_info(experiment_id, "R")
data["max_int_file"] = get_segmentation_filepath(
experiment_id, "OphysSegmentationMaskData")
data["object_list"] = get_segmentation_filepath(
experiment_id, "OphysSegmentationObjects")
return data
def get_genotype_info(experiment_id, code):
res = lu.query("""
select g.name
from ophys_experiments oe
join ophys_sessions os on os.id = oe.ophys_session_id
join specimens s on s.id = os.specimen_id
join donors d on d.id = s.donor_id
join donors_genotypes dg on dg.donor_id = d.id
join genotypes g on g.id = dg.genotype_id
join genotype_types gt on gt.id = g.genotype_type_id
where oe.id = {} and gt.code like '{}'""".format(experiment_id, code))
output = set()
for line in res:
output.add(line["name"])
return list(output)
def get_experiment_containers(self):
query_result = lu.query(container_pre_release_query)
container_list = []
for q in query_result:
# # For development: print key/val pairs generated from LIMS query:
# for key, val in sorted(q.items(), key=lambda x: x[0]):
# print(key, val)
# raise
c = collections.defaultdict(collections.defaultdict)
c['id'] = q['ec_id']
c['targeted_structure']['acronym'] = q['acronym']
c['specimen']['donor'] = collections.defaultdict(
collections.defaultdict)
c['specimen']['donor']['external_donor_name'] = q[
'external_donor_name']
c['specimen']['donor']['transgenic_lines'] = [
collections.defaultdict(collections.defaultdict),
collections.defaultdict(collections.defaultdict)
]
c['specimen']['donor']['transgenic_lines'][0][
'transgenic_line_type_name'] = 'driver'
c['specimen']['donor']['transgenic_lines'][0]['name'] = q['driver']
c['specimen']['donor']['transgenic_lines'][1][
'transgenic_line_type_name'] = 'reporter'
c['specimen']['donor']['transgenic_lines'][1]['name'] = q[
'reporter']
c['specimen']['name'] = q['specimen']
c['imaging_depth'] = q['depth']
c['failed'] = q['oa_state'] == 'failed'
if q['donor_tags'] == u'Epileptiform Events':
c['specimen']['donor']['conditions'] = [
collections.defaultdict(collections.defaultdict)
]
c['specimen']['donor']['conditions'][0][
'name'] = u'Epileptiform Events'
elif q['donor_tags'] == u'':
pass
else:
raise
container_list.append(c)
return container_list
def get_experiment_info(experiment_id):
logging.info("Downloading paths/metadata for experiment ID: %d",
experiment_id)
query = "select storage_directory, id from ophys_sessions where id = " + str(
experiment_id)
storage_directory = lu.query(query)[0]['storage_directory']
logging.info("\tStorage directory: %s", storage_directory)
movie_file = glob.glob(storage_directory + '*video-1.avi')[0]
metadata_file = glob.glob(storage_directory + '*video-1.h5')[0]
cr_points = get_polygon(experiment_id, 'Corneal Reflection Bounding Box')
pupil_points = get_polygon(experiment_id, 'Pupil Bounding Box')
logging.info("\tmovie file: %s", movie_file)
logging.info("\tmetadata file: %s", metadata_file)
return dict(movie_file=movie_file,
metadata_file=metadata_file,
corneal_reflection_points=cr_points,
pupil_points=pupil_points)
def get_polygon(experiment_id, group_name):
query = """
select ago.* from avg_graphic_objects ago
join avg_graphic_objects pago on pago.id = ago.parent_id
join avg_group_labels agl on pago.group_label_id = agl.id
join sub_images si on si.id = pago.sub_image_id
join specimens sp on sp.id = si.specimen_id
join ophys_sessions os on os.specimen_id = sp.id
where agl.name = '%s'
and os.id = %d
""" % (group_name, experiment_id)
try:
path = np.array(
[int(v) for v in lu.query(query)[0]['path'].split(',')])
except KeyError as e:
return []
except IndexError as e:
return []
points = path.reshape((len(path) / 2, 2))
return points
def debug(experiment_id, local=False):
OUTPUT_DIRECTORY = "/data/informatics/CAM/demix"
SDK_PATH = "/data/informatics/CAM/analysis/allensdk"
SCRIPT = "/data/informatics/CAM/analysis/allensdk/allensdk/internal/pipeline_modules/run_demixing.py"
sd = lu.query(
"select storage_directory from ophys_experiments where id = %d" %
experiment_id)[0]['storage_directory']
rois = lu.query("select * from cell_rois where ophys_experiment_id = %d" %
experiment_id)
exc_labels = lu.query("""
select cr.id, rel.name as exclusion_label from cell_rois cr
join cell_rois_roi_exclusion_labels crrel on crrel.cell_roi_id = cr.id
join roi_exclusion_labels rel on crrel.roi_exclusion_label_id = rel.id
where cr.ophys_experiment_id = %d
""" % experiment_id)
nrois = {
roi['id']: dict(width=roi['width'],
height=roi['height'],
x=roi['x'],
y=roi['y'],
id=roi['id'],
valid=roi['valid_roi'],
mask=roi['mask_matrix'],
exclusion_labels=[])
for roi in rois
}
for exc_label in exc_labels:
nrois[exc_label['id']]['exclusion_labels'].append(
exc_label['exclusion_label'])
movie_path_response = lu.query('''
select wkf.filename, wkf.storage_directory from well_known_files wkf
join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id
where wkf.attachable_id = {} and wkf.attachable_type = 'OphysExperiment'
and wkft.name = 'MotionCorrectedImageStack'
'''.format(experiment_id))
movie_h5_path = os.path.join(movie_path_response[0]['storage_directory'],
movie_path_response[0]['filename'])
exp_dir = os.path.join(OUTPUT_DIRECTORY, str(experiment_id))
input_data = {
"movie_h5": movie_h5_path,
"traces_h5": os.path.join(sd, "processed", "roi_traces.h5"),
"roi_masks": nrois.values(),
"output_file": os.path.join(exp_dir, "demixed_traces.h5")
}
run_module(SCRIPT,
input_data,
exp_dir,
sdk_path=SDK_PATH,
pbs=dict(vmem=160,
job_name="demix_%d" % experiment_id,
walltime="36:00:00"),
local=local,
optional_args=['--log-level', 'DEBUG'])
def debug(experiment_id, local=False):
OUTPUT_DIRECTORY = "/data/informatics/CAM/eye_calibration"
SDK_PATH = "/data/informatics/CAM/eye_calibration/allensdk"
SCRIPT = ("/data/informatics/CAM/eye_calibration/allensdk/allensdk"
"/internal/pipeline_modules/run_ophys_eye_calibration.py")
frame_width = 640
frame_height = 480
cr_file = get_wkf("EyeTracking Corneal Reflection", experiment_id)
pupil_file = get_wkf("EyeTracking Pupil", experiment_id)
exp_info = lu.query("""
select *
from ophys_sessions os
where os.id = {}
""".format(experiment_id))[0]
exp_dir = os.path.join(OUTPUT_DIRECTORY, str(experiment_id))
# clear out missing values to let us get defaults
for key, value in list(exp_info.items()):
if value is None:
del exp_info[key]
input_data = {
"cr_params_file": cr_file,
"pupil_params_file": pupil_file,
"frame_width": frame_width,
"frame_height": frame_height,
"output_file": os.path.join(exp_dir,
"eye_tracking_to_screen_mapping.h5"),
"monitor_position_x_mm": exp_info.get(
"screen_center_x_mm", eye_calibration.MONITOR_POSITION_OLD[0]*10),
"monitor_position_y_mm": exp_info.get(
"screen_center_y_mm", eye_calibration.MONITOR_POSITION_OLD[1]*10),
"monitor_position_z_mm": exp_info.get(
"screen_center_z_mm", eye_calibration.MONITOR_POSITION_OLD[2]*10),
"monitor_rotation_x_deg": exp_info.get("screen_rotation_x_deg", 0),
"monitor_rotation_y_deg": exp_info.get("screen_rotation_y_deg", 0),
"monitor_rotation_z_deg": exp_info.get("screen_rotation_z_deg", 0),
"camera_position_x_mm": exp_info.get(
"camera_center_x_mm", eye_calibration.CAMERA_POSITION_OLD[0]*10),
"camera_position_y_mm": exp_info.get(
"camera_center_y_mm", eye_calibration.CAMERA_POSITION_OLD[1]*10),
"camera_position_z_mm": exp_info.get(
"camera_center_z_mm", eye_calibration.CAMERA_POSITION_OLD[2]*10),
"camera_rotation_x_deg": exp_info.get(
"camera_rotation_x_deg",
eye_calibration.CAMERA_ROTATIONS_OLD[0]*180/np.pi),
"camera_rotation_y_deg": exp_info.get(
"camera_rotation_y_deg",
eye_calibration.CAMERA_ROTATIONS_OLD[1]*180/np.pi),
"camera_rotation_z_deg": exp_info.get(
"camera_rotation_z_deg",
eye_calibration.CAMERA_ROTATIONS_OLD[2]*180/np.pi),
"led_position_x_mm": exp_info.get(
"led_center_x_mm", eye_calibration.LED_POSITION_ORIGINAL[0]*10),
"led_position_y_mm": exp_info.get(
"led_center_y_mm", eye_calibration.LED_POSITION_ORIGINAL[1]*10),
"led_position_z_mm": exp_info.get(
"led_center_z_mm", eye_calibration.LED_POSITION_ORIGINAL[2]*10)
}
# TEMPORARY HACKS TO DEAL WITH BAD DATA IN LIMS
# TODO: REMOVE WHEN DATAFIXES DONE
if input_data["monitor_position_x_mm"] == -86.2:
input_data["monitor_position_x_mm"] = \
eye_calibration.MONITOR_POSITION_NEW[0]*10
input_data["monitor_position_y_mm"] = \
eye_calibration.MONITOR_POSITION_NEW[1]*10
input_data["monitor_position_z_mm"] = \
eye_calibration.MONITOR_POSITION_NEW[2]*10
run_module(SCRIPT,
input_data,
exp_dir,
sdk_path=SDK_PATH,
local=local)
def get_experiment_session(experiment_id):
return lu.query("""
select stimulus_name from ophys_sessions os
join ophys_experiments oe on oe.ophys_session_id = os.id
where oe.id = %d
""" % experiment_id)[0]['stimulus_name']
def get_cell_metrics(self):
query_result = lu.query(cell_specimens_pre_release_query)
mappings = self.get_stimulus_mappings()
thumbnails = [
m['item'] for m in mappings
if m['item_type'] == 'T' and m['level'] == 'R'
]
cell_list = []
for q in query_result:
c = collections.defaultdict(collections.defaultdict)
c['all_stim'] = q['a_valid'] and q['b_valid'] and q['c_valid']
for key in [
'cell_specimen_id', 'area', 'donor_full_genotype',
'experiment_container_id', 'imaging_depth', 'specimen_id',
'tld1_id', 'tld1_name', 'tld2_id', 'tld2_name', 'tlr1_id',
'tlr1_name'
]:
c[key] = q[key]
if q['failed_experiment_container'] == 't':
c['failed_experiment_container'] = True
elif q['failed_experiment_container'] == 'f':
c['failed_experiment_container'] = False
else:
raise RuntimeError(
'Unexpected value: {} not in ("t", "f")'.format(
q['failed_experiment_container']))
# Session A metrics:
for key in [
'dsi_dg', 'g_dsi_dg', 'g_osi_dg', 'osi_dg', 'p_dg',
'p_run_mod_dg', 'peak_dff_dg', 'pref_dir_dg', 'pref_tf_dg',
'reliability_dg', 'reliability_nm3', 'run_mod_dg',
'tfdi_dg', 'tfdi_dg'
]:
if q['crara_data'] is None:
c[key] = None
else:
c[key] = q['crara_data']['roi_cell_metrics'].get(key, None)
# Session B metrics:
for key in [
'g_osi_sg', 'image_sel_ns', 'osi_sg', 'p_ns',
'p_run_mod_ns', 'p_run_mod_sg', 'p_sg', 'peak_dff_ns',
'peak_dff_sg', 'pref_image_ns', 'pref_ori_sg',
'pref_phase_sg', 'pref_sf_sg', 'pref_image_ns',
'pref_ori_sg', 'reliability_ns', 'reliability_sg',
'run_mod_ns', 'run_mod_sg', 'sfdi_sg', 'time_to_peak_ns',
'time_to_peak_sg'
]:
if q['crarb_data'] is None:
c[key] = None
else:
c[key] = q['crarb_data']['roi_cell_metrics'].get(key, None)
# Session C metrics:
for key in [
'reliability_nm2',
'rf_area_off_lsn',
'rf_area_on_lsn',
'rf_center_off_x_lsn',
'rf_center_off_y_lsn',
'rf_center_on_x_lsn',
'rf_center_on_y_lsn',
'rf_chi2_lsn',
'rf_distance_lsn',
'rf_overlap_index_lsn',
]:
if q['crarc_data'] is None:
c[key] = None
else:
c[key] = q['crarc_data']['roi_cell_metrics'].get(key, None)
for suffix in ['a', 'b', 'c']:
if not q['crar%s_data' % suffix] is None:
c['reliability_nm1_%s' %
suffix] = q['crar%s_data' %
suffix]['roi_cell_metrics'].get(
'reliability_nm1', None)
else:
c['reliability_nm1_%s' % suffix] = None
# Fake in thumbnail images:
for t in thumbnails:
c[t] = None
# # For development: print key/val pairs generated from LIMS query:
# for key, val in sorted(q.items(), key=lambda x: x[0]):
# if key == 'crarb_data':
# print('crarb_data[roi_cell_metrics]')
# for key2, val2 in sorted(val['roi_cell_metrics'].items(), key=lambda x: x[0]):
# print(' ', key2, val2)
# else:
# print(key, val)
# raise
cell_list.append(c)
return cell_list
