def infer_output_dir(cls, scan_key, relative=False, mkdir=False): image_locators = { "NIS": get_nd2_files, "ScanImage": get_scan_image_files, "Scanbox": get_scan_box_files, } image_locator = image_locators[(scan.Scan & scan_key).fetch1("acq_software")] scan_dir = find_full_path(get_imaging_root_data_dir(), image_locator(scan_key)[0]).parent root_dir = find_root_directory(get_imaging_root_data_dir(), scan_dir) paramset_key = ProcessingParamSet.fetch1() processed_dir = pathlib.Path(get_processed_root_data_dir()) output_dir = ( processed_dir / scan_dir.relative_to(root_dir) / f'{paramset_key["processing_method"]}_{paramset_key["paramset_idx"]}' ) if mkdir: output_dir.mkdir(parents=True, exist_ok=True) return output_dir.relative_to( processed_dir) if relative else output_dir
def get_loader_result(key, table): """ Retrieve the loaded processed imaging results from the loader (e.g. suite2p, caiman, etc.) :param key: the `key` to one entry of ProcessingTask or Curation :param table: the class defining the table to retrieve the loaded results from (e.g. ProcessingTask, Curation) :return: a loader object of the loaded results (e.g. suite2p.Suite2p, caiman.CaImAn, etc.) """ method, output_dir = (ProcessingParamSet * table & key).fetch1( "processing_method", _table_attribute_mapper[table.__name__]) output_path = find_full_path(get_imaging_root_data_dir(), output_dir) if method == "suite2p": from element_interface import suite2p_loader loaded_dataset = suite2p_loader.Suite2p(output_path) elif method == "caiman": from element_interface import caiman_loader loaded_dataset = caiman_loader.CaImAn(output_path) else: raise NotImplementedError( "Unknown/unimplemented method: {}".format(method)) return method, loaded_dataset
def make(self, key): task_mode, output_dir = (PreProcessTask & key).fetch1( "task_mode", "preprocess_output_dir") preprocess_output_dir = find_full_path(get_imaging_root_data_dir(), output_dir) if task_mode == "none": pass elif task_mode in ["load", "trigger"]: raise NotImplementedError( "Pre-processing steps are not implemented." "Please overwrite this `make` function with" "desired pre-processing steps.") else: raise ValueError(f"Unknown task mode: {task_mode}") self.insert1({**key})
def test_find_valid_full_path(pipeline, sessions_csv): from element_interface.utils import find_full_path get_imaging_root_data_dir = pipeline['get_imaging_root_data_dir'] # add more options for root directories if sys.platform == 'win32': ephys_root_data_dir = [get_imaging_root_data_dir(), 'J:/', 'M:/'] else: ephys_root_data_dir = [get_imaging_root_data_dir(), 'mnt/j', 'mnt/m'] # test: providing relative-path: correctly search for the full-path sessions, _ = sessions_csv sess = sessions.iloc[0] session_full_path = pathlib.Path(sess.session_dir) rel_path = pathlib.Path(session_full_path).relative_to( pathlib.Path(get_imaging_root_data_dir())) full_path = find_full_path(ephys_root_data_dir, rel_path) assert full_path == session_full_path
def ingest_sessions(session_csv_path="./user_data/sessions.csv", skip_duplicates=True, verbose=True): root_data_dir = get_imaging_root_data_dir() # ---------- Insert new "Session" and "Scan" --------- with open(session_csv_path, newline="") as f: input_sessions = list(csv.DictReader(f, delimiter=",")) # Folder structure: root / subject / session / .tif (raw) session_list, session_dir_list, scan_list, scanner_list = [], [], [], [] for sess in input_sessions: sess_dir = find_full_path(root_data_dir, Path(sess["session_dir"])) # search for either ScanImage or Scanbox files (in that order) for scan_pattern, scan_type, glob_func in zip( ["*.tif", "*.sbx"], ["ScanImage", "Scanbox"], [sess_dir.glob, sess_dir.rglob], ): scan_filepaths = [fp.as_posix() for fp in glob_func(scan_pattern)] if len(scan_filepaths): acq_software = scan_type break else: raise FileNotFoundError( "Unable to identify scan files from the supported " + "acquisition softwares (ScanImage, Scanbox) at: " + f"{sess_dir}") if acq_software == "ScanImage": import scanreader from element_interface import scanimage_utils try: # attempt to read .tif as a scanimage file loaded_scan = scanreader.read_scan(scan_filepaths) recording_time = scanimage_utils.get_scanimage_acq_time( loaded_scan) header = scanimage_utils.parse_scanimage_header(loaded_scan) scanner = header["SI_imagingSystem"].strip("'") except Exception as e: print(f"ScanImage loading error: {scan_filepaths}\n{str(e)}") continue elif acq_software == "Scanbox": import sbxreader try: # attempt to load Scanbox sbx_fp = pathlib.Path(scan_filepaths[0]) sbx_meta = sbxreader.sbx_get_metadata(sbx_fp) # read from file when Scanbox support this recording_time = datetime.fromtimestamp(sbx_fp.stat().st_ctime) scanner = sbx_meta.get("imaging_system", "Scanbox") except Exception as e: print(f"Scanbox loading error: {scan_filepaths}\n{str(e)}") continue else: raise NotImplementedError( "Processing scan from acquisition software of " + f"type {acq_software} is not yet implemented") session_key = { "subject": sess["subject"], "session_datetime": recording_time } if session_key not in session.Session(): scanner_list.append({"scanner": scanner}) session_list.append(session_key) scan_list.append({ **session_key, "scan_id": 0, "scanner": scanner, "acq_software": acq_software, }) session_dir_list.append({ **session_key, "session_dir": sess_dir.relative_to(root_data_dir).as_posix(), }) new_equipment = set(val for dic in scanner_list for val in dic.values()) if verbose: print(f"\n---- Insert {len(new_equipment)} entry(s) into " + "experiment.Equipment ----") Equipment.insert(scanner_list, skip_duplicates=True) if verbose: print( f"\n---- Insert {len(session_list)} entry(s) into session.Session ----" ) session.Session.insert(session_list) session.SessionDirectory.insert(session_dir_list) if verbose: print(f"\n---- Insert {len(scan_list)} entry(s) into scan.Scan ----") scan.Scan.insert(scan_list, skip_duplicates=skip_duplicates) if verbose: print("\n---- Successfully completed ingest_sessions ----")
def make(self, key): task_mode = (ProcessingTask & key).fetch1("task_mode") output_dir = (ProcessingTask & key).fetch1("processing_output_dir") output_dir = find_full_path(get_miniscope_root_data_dir(), output_dir) if task_mode == "load": method, loaded_result = get_loader_result(key, ProcessingTask) if method == "caiman": loaded_caiman = loaded_result key = {**key, "processing_time": loaded_caiman.creation_time} else: raise NotImplementedError( f"Loading of {method} data is not yet" f"supported") elif task_mode == "trigger": method = (ProcessingTask * ProcessingParamSet * ProcessingMethod * Recording & key).fetch1("processing_method") if method == "caiman": import caiman from element_interface.run_caiman import run_caiman avi_files = (Recording * RecordingInfo * RecordingInfo.File & key).fetch("file_path") avi_files = [ find_full_path(get_miniscope_root_data_dir(), avi_file).as_posix() for avi_file in avi_files ] params = (ProcessingTask * ProcessingParamSet & key).fetch1("params") sampling_rate = (ProcessingTask * Recording * RecordingInfo & key).fetch1("fps") input_hash = dict_to_uuid(dict(**key, **params)) input_hash_fp = output_dir / f".{input_hash }.json" if not input_hash_fp.exists(): start_time = datetime.utcnow() run_caiman( file_paths=avi_files, parameters=params, sampling_rate=sampling_rate, output_dir=output_dir.as_posix(), is3D=False, ) completion_time = datetime.utcnow() with open(input_hash_fp, "w") as f: json.dump( { "start_time": start_time, "completion_time": completion_time, "duration": (completion_time - start_time).total_seconds(), }, f, default=str, ) _, imaging_dataset = get_loader_result(key, ProcessingTask) caiman_dataset = imaging_dataset key["processing_time"] = caiman_dataset.creation_time key["package_version"] = caiman.__version__ else: raise NotImplementedError( f"Automatic triggering of {method} analysis" f" is not yet supported") else: raise ValueError(f"Unknown task mode: {task_mode}") self.insert1(key)
def make(self, key): # Search recording directory for miniscope raw files acquisition_software, recording_directory = (Recording & key).fetch1( "acquisition_software", "recording_directory") recording_path = find_full_path(get_miniscope_root_data_dir(), recording_directory) recording_filepaths = [ file_path.as_posix() for file_path in recording_path.glob("*.avi") ] if not recording_filepaths: raise FileNotFoundError(f"No .avi files found in " f"{recording_directory}") if acquisition_software == "Miniscope-DAQ-V3": recording_timestamps = recording_path / "timestamp.dat" if not recording_timestamps.exists(): raise FileNotFoundError(f"No timestamp file found in " f"{recording_directory}") nchannels = 1 # Assumes a single channel # Parse number of frames from timestamp.dat file with open(recording_timestamps) as f: next(f) nframes = sum(1 for line in f if int(line[0]) == 0) # Parse image dimension and frame rate video = cv2.VideoCapture(recording_filepaths[0]) _, frame = video.read() frame_size = np.shape(frame) px_height = frame_size[0] px_width = frame_size[1] fps = video.get(cv2.CAP_PROP_FPS) elif acquisition_software == "Miniscope-DAQ-V4": recording_metadata = list(recording_path.glob("metaData.json"))[0] recording_timestamps = list( recording_path.glob("timeStamps.csv"))[0] if not recording_metadata.exists(): raise FileNotFoundError(f"No .json file found in " f"{recording_directory}") if not recording_timestamps.exists(): raise FileNotFoundError(f"No timestamp (*.csv) file found in " f"{recording_directory}") with open(recording_metadata.as_posix()) as f: metadata = json.loads(f.read()) with open(recording_timestamps, newline="") as f: time_stamps = list(csv.reader(f, delimiter=",")) nchannels = 1 # Assumes a single channel nframes = len(time_stamps) - 1 px_height = metadata["ROI"]["height"] px_width = metadata["ROI"]["width"] fps = int(metadata["frameRate"].replace("FPS", "")) gain = metadata["gain"] spatial_downsample = 1 # Assumes no spatial downsampling led_power = metadata["led0"] time_stamps = np.array([ list(map(int, time_stamps[i])) for i in range(1, len(time_stamps)) ]) else: raise NotImplementedError( f"Loading routine not implemented for {acquisition_software}" " acquisition software") # Insert in RecordingInfo self.insert1( dict( key, nchannels=nchannels, nframes=nframes, px_height=px_height, px_width=px_width, fps=fps, gain=gain, spatial_downsample=spatial_downsample, led_power=led_power, time_stamps=time_stamps, recording_duration=nframes / fps, )) # Insert file(s) recording_files = [ pathlib.Path(f).relative_to( find_root_directory(get_miniscope_root_data_dir(), f)).as_posix() for f in recording_filepaths ] self.File.insert([{ **key, "file_id": i, "file_path": f } for i, f in enumerate(recording_files)])
def make(self, key): task_mode, output_dir = (ProcessingTask & key).fetch1( "task_mode", "processing_output_dir") output_dir = find_full_path(get_imaging_root_data_dir(), output_dir).as_posix() if not output_dir: output_dir = ProcessingTask.infer_output_dir(key, relative=True, mkdir=True) # update processing_output_dir ProcessingTask.update1({ **key, "processing_output_dir": output_dir.as_posix() }) if task_mode == "load": method, imaging_dataset = get_loader_result(key, ProcessingTask) if method == "suite2p": if (scan.ScanInfo & key).fetch1("nrois") > 0: raise NotImplementedError( f"Suite2p ingestion error - Unable to handle" f" ScanImage multi-ROI scanning mode yet") suite2p_dataset = imaging_dataset key = {**key, "processing_time": suite2p_dataset.creation_time} elif method == "caiman": caiman_dataset = imaging_dataset key = {**key, "processing_time": caiman_dataset.creation_time} else: raise NotImplementedError("Unknown method: {}".format(method)) elif task_mode == "trigger": method = (ProcessingTask * ProcessingParamSet * ProcessingMethod * scan.Scan & key).fetch1("processing_method") preprocess_paramsets = ( PreProcessParamSteps.Step() & dict( preprocess_param_steps_id=key["preprocess_param_steps_id"]) ).fetch("paramset_idx") if len(preprocess_paramsets) == 0: # No pre-processing steps were performed on the acquired dataset, so process the acquired files. image_files = (ProcessingTask * scan.Scan * scan.ScanInfo * scan.ScanInfo.ScanFile & key).fetch("file_path") image_files = [ find_full_path(get_imaging_root_data_dir(), image_file) for image_file in image_files ] else: preprocess_output_dir = (PreProcessTask & key).fetch1("preprocess_output_dir") preprocess_output_dir = find_full_path( get_imaging_root_data_dir(), preprocess_output_dir) if not preprocess_output_dir.exists(): raise FileNotFoundError( f"Pre-processed output directory not found ({preprocess_output_dir})" ) image_files = [ fp for fp in preprocess_output_dir.glob("*.tif") ] if method == "suite2p": import suite2p suite2p_params = (ProcessingTask * ProcessingParamSet & key).fetch1("params") suite2p_params["save_path0"] = output_dir suite2p_params["fs"] = ( ProcessingTask * scan.Scan * scan.ScanInfo & key).fetch1("fps") input_format = pathlib.Path(image_files[0]).suffix suite2p_params["input_format"] = input_format[1:] suite2p_paths = { "data_path": [image_files[0].parent.as_posix()], "tiff_list": [f.as_posix() for f in image_files], } suite2p.run_s2p(ops=suite2p_params, db=suite2p_paths) # Run suite2p _, imaging_dataset = get_loader_result(key, ProcessingTask) suite2p_dataset = imaging_dataset key = {**key, "processing_time": suite2p_dataset.creation_time} elif method == "caiman": from element_interface.run_caiman import run_caiman params = (ProcessingTask * ProcessingParamSet & key).fetch1("params") sampling_rate = (ProcessingTask * scan.Scan * scan.ScanInfo & key).fetch1("fps") ndepths = (ProcessingTask * scan.Scan * scan.ScanInfo & key).fetch1("ndepths") is3D = bool(ndepths > 1) if is3D: raise NotImplementedError( "Caiman pipeline is not capable of analyzing 3D scans at the moment." ) run_caiman( file_paths=[f.as_posix() for f in image_files], parameters=params, sampling_rate=sampling_rate, output_dir=output_dir, is3D=is3D, ) _, imaging_dataset = get_loader_result(key, ProcessingTask) caiman_dataset = imaging_dataset key["processing_time"] = caiman_dataset.creation_time else: raise ValueError(f"Unknown task mode: {task_mode}") self.insert1(key)