def fetch_cell(specimen_id): import allensdk.internal.core.lims_utilities as lu query = """ select a3d.* from specimens sp join alignment3ds a3d on sp.alignment3d_id = a3d.id where sp.id = %d """ a3d = lu.query(query % specimen_id)[0] m = np.array([[a3d['tvr_00'], a3d['tvr_01'], a3d['tvr_02'], a3d['tvr_09']], [a3d['tvr_03'], a3d['tvr_04'], a3d['tvr_05'], a3d['tvr_10']], [a3d['tvr_06'], a3d['tvr_07'], a3d['tvr_08'], a3d['tvr_11']], [0, 0, 0, 1]]) query = """ select wkf.storage_directory||wkf.filename as swc_file from well_known_files wkf join neuron_reconstructions nr on wkf.attachable_id = nr.id where nr.specimen_id = %d and nr.superseded = false and nr.manual = true and wkf.well_known_file_type_id = 303941301 """ swc_file = lu.query(query % specimen_id)[0]['swc_file'] return swc_file, m
def get_container_info(container_id): res = lu.query(""" select * from ophys_experiments oe where experiment_container_id = %d and oe.workflow_state != 'failed' """ % container_id) return res
def get_ophys_experiments(self): query_result = lu.query(experiment_pre_release_query) experiment_list = [] for q in query_result: c = collections.defaultdict(collections.defaultdict) # # For development: print key/val pairs generated from LIMS query: # for key, val in sorted(q.items(), key=lambda x: x[0]): # print(key, val) # raise c['id'] = q['o_id'] c['imaging_depth'] = q['depth'] c['targeted_structure']['acronym'] = q['acronym'] c['specimen']['donor'] = collections.defaultdict(collections.defaultdict) c['specimen']['donor']['external_donor_name'] = q['acronym'] c['specimen']['donor']['transgenic_lines'] = [collections.defaultdict(collections.defaultdict), collections.defaultdict(collections.defaultdict)] c['specimen']['donor']['transgenic_lines'][0]['transgenic_line_type_name'] = 'driver' c['specimen']['donor']['transgenic_lines'][0]['name'] = q['driver'] c['specimen']['donor']['transgenic_lines'][1]['transgenic_line_type_name'] = 'reporter' c['specimen']['donor']['transgenic_lines'][1]['name'] = q['reporter'] c['date_of_acquisition'] = q['date_of_acquisition'] c['specimen']['donor']['date_of_birth'] = q['date_of_birth'] c['experiment_container_id'] = q['ec_id'] c['stimulus_name'] = q['stimulus_name'] c['specimen']['donor']['external_donor_name'] = q['external_donor_name'] c['specimen']['name'] = q['specimen'] c['fail_eye_tracking'] = q['fail_eye_tracking'] experiment_list.append(c) return experiment_list
def debug(experiment_id, local=False): OUTPUT_DIRECTORY = "/data/informatics/CAM/neuropil" SDK_PATH = "/data/informatics/CAM/neuropil/allensdk" SCRIPT = "/data/informatics/CAM/neuropil/allensdk/allensdk/internal/pipeline_modules/run_neuropil_correction.py" exp = lu.query("select * from ophys_experiments where id = %d" % experiment_id)[0] sd = exp["storage_directory"] test_file = "/data/informatics/CAM/demix/%d/demixed_traces.h5" % experiment_id if os.path.exists(test_file): roi_trace_file = test_file else: roi_trace_file = os.path.join(sd, "demix", "%d_demixed_traces.h5" % experiment_id) exp_dir = os.path.join(OUTPUT_DIRECTORY, str(experiment_id)) input_data = dict(roi_trace_file=roi_trace_file, neuropil_trace_file=os.path.join(sd, "processed", "neuropil_traces.h5"), storage_directory=exp_dir) run_module(SCRIPT, input_data, exp_dir, sdk_path=SDK_PATH, pbs=dict(vmem=160, job_name="np_%d" % experiment_id, walltime="36:00:00"), local=local, optional_args=['--log-level', 'DEBUG'])
def get_motion_filepath(experiment_id): return lu.query(""" select CONCAT(wkf.storage_directory, wkf.filename) as path from well_known_files wkf join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id join ophys_experiments oe on oe.id = wkf.attachable_id where oe.id = {} and wkft.name like 'OphysMotionXyOffsetData'""".format(experiment_id))[0]["path"]
def get_wkf(wkf_type, experiment_id): wkf = lu.query(""" select CONCAT(wkf.storage_directory, wkf.filename) as path from well_known_files wkf join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id where wkft.name LIKE '{}' and wkf.attachable_id = {} """.format(wkf_type, experiment_id))[0]["path"] return wkf
def get_experiment_nwb_file(experiment_id): res = lu.query(""" select * from well_known_files wkf join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id where attachable_id = %d and wkft.name = 'NWBOphys' """ % experiment_id) return os.path.join(res[0]['storage_directory'], res[0]['filename'])
def get_segmentation_filepath(experiment_id, file_type): return lu.query(""" select CONCAT(wkf.storage_directory, wkf.filename) as path from well_known_files wkf join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id join ophys_cell_segmentation_runs ocsr on ocsr.id = wkf.attachable_id join ophys_experiments oe on oe.id = ocsr.ophys_experiment_id where oe.id = {} and wkft.name like '{}' and ocsr.current = 't'""".format(experiment_id, file_type))[0]["path"]
def get_model_info(experiment_id): res = lu.query(""" select CONCAT(wkf.storage_directory, wkf.filename) as path, wkf.id from ophys_experiments oe join ophys_sessions os on os.id = oe.ophys_session_id join projects p on p.id = os.project_id join well_known_files wkf on wkf.attachable_id = p.id join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id where oe.id = {} and wkft.name = 'RoiLabelModel'""".format(experiment_id))[0] return res["path"], res["id"]
def create_input_data(experiment_id): data = {} data["log_0"] = get_motion_filepath(experiment_id) model, model_id = get_model_info(experiment_id) data["roi_label_model"] = model data["roi_label_model_id"] = model_id data["targeted_structure_id"] = lu.query(""" select targeted_structure_id from ophys_experiments oe where oe.id = {}""".format(experiment_id))[0]["targeted_structure_id"] data["imaging_depth"] = lu.query(""" select calculated_depth from ophys_experiments oe where oe.id = {}""".format(experiment_id))[0]["calculated_depth"] data["drivers"] = get_genotype_info(experiment_id, "D") data["reporters"] = get_genotype_info(experiment_id, "R") data["max_int_file"] = get_segmentation_filepath( experiment_id, "OphysSegmentationMaskData") data["object_list"] = get_segmentation_filepath( experiment_id, "OphysSegmentationObjects") return data
def get_genotype_info(experiment_id, code): res = lu.query(""" select g.name from ophys_experiments oe join ophys_sessions os on os.id = oe.ophys_session_id join specimens s on s.id = os.specimen_id join donors d on d.id = s.donor_id join donors_genotypes dg on dg.donor_id = d.id join genotypes g on g.id = dg.genotype_id join genotype_types gt on gt.id = g.genotype_type_id where oe.id = {} and gt.code like '{}'""".format(experiment_id, code)) output = set() for line in res: output.add(line["name"]) return list(output)
def get_experiment_containers(self): query_result = lu.query(container_pre_release_query) container_list = [] for q in query_result: # # For development: print key/val pairs generated from LIMS query: # for key, val in sorted(q.items(), key=lambda x: x[0]): # print(key, val) # raise c = collections.defaultdict(collections.defaultdict) c['id'] = q['ec_id'] c['targeted_structure']['acronym'] = q['acronym'] c['specimen']['donor'] = collections.defaultdict( collections.defaultdict) c['specimen']['donor']['external_donor_name'] = q[ 'external_donor_name'] c['specimen']['donor']['transgenic_lines'] = [ collections.defaultdict(collections.defaultdict), collections.defaultdict(collections.defaultdict) ] c['specimen']['donor']['transgenic_lines'][0][ 'transgenic_line_type_name'] = 'driver' c['specimen']['donor']['transgenic_lines'][0]['name'] = q['driver'] c['specimen']['donor']['transgenic_lines'][1][ 'transgenic_line_type_name'] = 'reporter' c['specimen']['donor']['transgenic_lines'][1]['name'] = q[ 'reporter'] c['specimen']['name'] = q['specimen'] c['imaging_depth'] = q['depth'] c['failed'] = q['oa_state'] == 'failed' if q['donor_tags'] == u'Epileptiform Events': c['specimen']['donor']['conditions'] = [ collections.defaultdict(collections.defaultdict) ] c['specimen']['donor']['conditions'][0][ 'name'] = u'Epileptiform Events' elif q['donor_tags'] == u'': pass else: raise container_list.append(c) return container_list
def get_experiment_info(experiment_id): logging.info("Downloading paths/metadata for experiment ID: %d", experiment_id) query = "select storage_directory, id from ophys_sessions where id = " + str( experiment_id) storage_directory = lu.query(query)[0]['storage_directory'] logging.info("\tStorage directory: %s", storage_directory) movie_file = glob.glob(storage_directory + '*video-1.avi')[0] metadata_file = glob.glob(storage_directory + '*video-1.h5')[0] cr_points = get_polygon(experiment_id, 'Corneal Reflection Bounding Box') pupil_points = get_polygon(experiment_id, 'Pupil Bounding Box') logging.info("\tmovie file: %s", movie_file) logging.info("\tmetadata file: %s", metadata_file) return dict(movie_file=movie_file, metadata_file=metadata_file, corneal_reflection_points=cr_points, pupil_points=pupil_points)
def get_polygon(experiment_id, group_name): query = """ select ago.* from avg_graphic_objects ago join avg_graphic_objects pago on pago.id = ago.parent_id join avg_group_labels agl on pago.group_label_id = agl.id join sub_images si on si.id = pago.sub_image_id join specimens sp on sp.id = si.specimen_id join ophys_sessions os on os.specimen_id = sp.id where agl.name = '%s' and os.id = %d """ % (group_name, experiment_id) try: path = np.array( [int(v) for v in lu.query(query)[0]['path'].split(',')]) except KeyError as e: return [] except IndexError as e: return [] points = path.reshape((len(path) / 2, 2)) return points
def debug(experiment_id, local=False): OUTPUT_DIRECTORY = "/data/informatics/CAM/demix" SDK_PATH = "/data/informatics/CAM/analysis/allensdk" SCRIPT = "/data/informatics/CAM/analysis/allensdk/allensdk/internal/pipeline_modules/run_demixing.py" sd = lu.query( "select storage_directory from ophys_experiments where id = %d" % experiment_id)[0]['storage_directory'] rois = lu.query("select * from cell_rois where ophys_experiment_id = %d" % experiment_id) exc_labels = lu.query(""" select cr.id, rel.name as exclusion_label from cell_rois cr join cell_rois_roi_exclusion_labels crrel on crrel.cell_roi_id = cr.id join roi_exclusion_labels rel on crrel.roi_exclusion_label_id = rel.id where cr.ophys_experiment_id = %d """ % experiment_id) nrois = { roi['id']: dict(width=roi['width'], height=roi['height'], x=roi['x'], y=roi['y'], id=roi['id'], valid=roi['valid_roi'], mask=roi['mask_matrix'], exclusion_labels=[]) for roi in rois } for exc_label in exc_labels: nrois[exc_label['id']]['exclusion_labels'].append( exc_label['exclusion_label']) movie_path_response = lu.query(''' select wkf.filename, wkf.storage_directory from well_known_files wkf join well_known_file_types wkft on wkft.id = wkf.well_known_file_type_id where wkf.attachable_id = {} and wkf.attachable_type = 'OphysExperiment' and wkft.name = 'MotionCorrectedImageStack' '''.format(experiment_id)) movie_h5_path = os.path.join(movie_path_response[0]['storage_directory'], movie_path_response[0]['filename']) exp_dir = os.path.join(OUTPUT_DIRECTORY, str(experiment_id)) input_data = { "movie_h5": movie_h5_path, "traces_h5": os.path.join(sd, "processed", "roi_traces.h5"), "roi_masks": nrois.values(), "output_file": os.path.join(exp_dir, "demixed_traces.h5") } run_module(SCRIPT, input_data, exp_dir, sdk_path=SDK_PATH, pbs=dict(vmem=160, job_name="demix_%d" % experiment_id, walltime="36:00:00"), local=local, optional_args=['--log-level', 'DEBUG'])
def debug(experiment_id, local=False): OUTPUT_DIRECTORY = "/data/informatics/CAM/eye_calibration" SDK_PATH = "/data/informatics/CAM/eye_calibration/allensdk" SCRIPT = ("/data/informatics/CAM/eye_calibration/allensdk/allensdk" "/internal/pipeline_modules/run_ophys_eye_calibration.py") frame_width = 640 frame_height = 480 cr_file = get_wkf("EyeTracking Corneal Reflection", experiment_id) pupil_file = get_wkf("EyeTracking Pupil", experiment_id) exp_info = lu.query(""" select * from ophys_sessions os where os.id = {} """.format(experiment_id))[0] exp_dir = os.path.join(OUTPUT_DIRECTORY, str(experiment_id)) # clear out missing values to let us get defaults for key, value in list(exp_info.items()): if value is None: del exp_info[key] input_data = { "cr_params_file": cr_file, "pupil_params_file": pupil_file, "frame_width": frame_width, "frame_height": frame_height, "output_file": os.path.join(exp_dir, "eye_tracking_to_screen_mapping.h5"), "monitor_position_x_mm": exp_info.get( "screen_center_x_mm", eye_calibration.MONITOR_POSITION_OLD[0]*10), "monitor_position_y_mm": exp_info.get( "screen_center_y_mm", eye_calibration.MONITOR_POSITION_OLD[1]*10), "monitor_position_z_mm": exp_info.get( "screen_center_z_mm", eye_calibration.MONITOR_POSITION_OLD[2]*10), "monitor_rotation_x_deg": exp_info.get("screen_rotation_x_deg", 0), "monitor_rotation_y_deg": exp_info.get("screen_rotation_y_deg", 0), "monitor_rotation_z_deg": exp_info.get("screen_rotation_z_deg", 0), "camera_position_x_mm": exp_info.get( "camera_center_x_mm", eye_calibration.CAMERA_POSITION_OLD[0]*10), "camera_position_y_mm": exp_info.get( "camera_center_y_mm", eye_calibration.CAMERA_POSITION_OLD[1]*10), "camera_position_z_mm": exp_info.get( "camera_center_z_mm", eye_calibration.CAMERA_POSITION_OLD[2]*10), "camera_rotation_x_deg": exp_info.get( "camera_rotation_x_deg", eye_calibration.CAMERA_ROTATIONS_OLD[0]*180/np.pi), "camera_rotation_y_deg": exp_info.get( "camera_rotation_y_deg", eye_calibration.CAMERA_ROTATIONS_OLD[1]*180/np.pi), "camera_rotation_z_deg": exp_info.get( "camera_rotation_z_deg", eye_calibration.CAMERA_ROTATIONS_OLD[2]*180/np.pi), "led_position_x_mm": exp_info.get( "led_center_x_mm", eye_calibration.LED_POSITION_ORIGINAL[0]*10), "led_position_y_mm": exp_info.get( "led_center_y_mm", eye_calibration.LED_POSITION_ORIGINAL[1]*10), "led_position_z_mm": exp_info.get( "led_center_z_mm", eye_calibration.LED_POSITION_ORIGINAL[2]*10) } # TEMPORARY HACKS TO DEAL WITH BAD DATA IN LIMS # TODO: REMOVE WHEN DATAFIXES DONE if input_data["monitor_position_x_mm"] == -86.2: input_data["monitor_position_x_mm"] = \ eye_calibration.MONITOR_POSITION_NEW[0]*10 input_data["monitor_position_y_mm"] = \ eye_calibration.MONITOR_POSITION_NEW[1]*10 input_data["monitor_position_z_mm"] = \ eye_calibration.MONITOR_POSITION_NEW[2]*10 run_module(SCRIPT, input_data, exp_dir, sdk_path=SDK_PATH, local=local)
def get_experiment_session(experiment_id): return lu.query(""" select stimulus_name from ophys_sessions os join ophys_experiments oe on oe.ophys_session_id = os.id where oe.id = %d """ % experiment_id)[0]['stimulus_name']
def get_cell_metrics(self): query_result = lu.query(cell_specimens_pre_release_query) mappings = self.get_stimulus_mappings() thumbnails = [ m['item'] for m in mappings if m['item_type'] == 'T' and m['level'] == 'R' ] cell_list = [] for q in query_result: c = collections.defaultdict(collections.defaultdict) c['all_stim'] = q['a_valid'] and q['b_valid'] and q['c_valid'] for key in [ 'cell_specimen_id', 'area', 'donor_full_genotype', 'experiment_container_id', 'imaging_depth', 'specimen_id', 'tld1_id', 'tld1_name', 'tld2_id', 'tld2_name', 'tlr1_id', 'tlr1_name' ]: c[key] = q[key] if q['failed_experiment_container'] == 't': c['failed_experiment_container'] = True elif q['failed_experiment_container'] == 'f': c['failed_experiment_container'] = False else: raise RuntimeError( 'Unexpected value: {} not in ("t", "f")'.format( q['failed_experiment_container'])) # Session A metrics: for key in [ 'dsi_dg', 'g_dsi_dg', 'g_osi_dg', 'osi_dg', 'p_dg', 'p_run_mod_dg', 'peak_dff_dg', 'pref_dir_dg', 'pref_tf_dg', 'reliability_dg', 'reliability_nm3', 'run_mod_dg', 'tfdi_dg', 'tfdi_dg' ]: if q['crara_data'] is None: c[key] = None else: c[key] = q['crara_data']['roi_cell_metrics'].get(key, None) # Session B metrics: for key in [ 'g_osi_sg', 'image_sel_ns', 'osi_sg', 'p_ns', 'p_run_mod_ns', 'p_run_mod_sg', 'p_sg', 'peak_dff_ns', 'peak_dff_sg', 'pref_image_ns', 'pref_ori_sg', 'pref_phase_sg', 'pref_sf_sg', 'pref_image_ns', 'pref_ori_sg', 'reliability_ns', 'reliability_sg', 'run_mod_ns', 'run_mod_sg', 'sfdi_sg', 'time_to_peak_ns', 'time_to_peak_sg' ]: if q['crarb_data'] is None: c[key] = None else: c[key] = q['crarb_data']['roi_cell_metrics'].get(key, None) # Session C metrics: for key in [ 'reliability_nm2', 'rf_area_off_lsn', 'rf_area_on_lsn', 'rf_center_off_x_lsn', 'rf_center_off_y_lsn', 'rf_center_on_x_lsn', 'rf_center_on_y_lsn', 'rf_chi2_lsn', 'rf_distance_lsn', 'rf_overlap_index_lsn', ]: if q['crarc_data'] is None: c[key] = None else: c[key] = q['crarc_data']['roi_cell_metrics'].get(key, None) for suffix in ['a', 'b', 'c']: if not q['crar%s_data' % suffix] is None: c['reliability_nm1_%s' % suffix] = q['crar%s_data' % suffix]['roi_cell_metrics'].get( 'reliability_nm1', None) else: c['reliability_nm1_%s' % suffix] = None # Fake in thumbnail images: for t in thumbnails: c[t] = None # # For development: print key/val pairs generated from LIMS query: # for key, val in sorted(q.items(), key=lambda x: x[0]): # if key == 'crarb_data': # print('crarb_data[roi_cell_metrics]') # for key2, val2 in sorted(val['roi_cell_metrics'].items(), key=lambda x: x[0]): # print(' ', key2, val2) # else: # print(key, val) # raise cell_list.append(c) return cell_list