def infer_output_dir(cls, scan_key, relative=False, mkdir=False):
image_locators = {
"NIS": get_nd2_files,
"ScanImage": get_scan_image_files,
"Scanbox": get_scan_box_files,
}
image_locator = image_locators[(scan.Scan
& scan_key).fetch1("acq_software")]
scan_dir = find_full_path(get_imaging_root_data_dir(),
image_locator(scan_key)[0]).parent
root_dir = find_root_directory(get_imaging_root_data_dir(), scan_dir)
paramset_key = ProcessingParamSet.fetch1()
processed_dir = pathlib.Path(get_processed_root_data_dir())
output_dir = (
processed_dir / scan_dir.relative_to(root_dir) /
f'{paramset_key["processing_method"]}_{paramset_key["paramset_idx"]}'
)
if mkdir:
output_dir.mkdir(parents=True, exist_ok=True)
return output_dir.relative_to(
processed_dir) if relative else output_dir
def get_loader_result(key, table):
"""
Retrieve the loaded processed imaging results from the loader (e.g. suite2p, caiman, etc.)
:param key: the `key` to one entry of ProcessingTask or Curation
:param table: the class defining the table to retrieve
the loaded results from (e.g. ProcessingTask, Curation)
:return: a loader object of the loaded results
(e.g. suite2p.Suite2p, caiman.CaImAn, etc.)
"""
method, output_dir = (ProcessingParamSet * table & key).fetch1(
"processing_method", _table_attribute_mapper[table.__name__])
output_path = find_full_path(get_imaging_root_data_dir(), output_dir)
if method == "suite2p":
from element_interface import suite2p_loader
loaded_dataset = suite2p_loader.Suite2p(output_path)
elif method == "caiman":
from element_interface import caiman_loader
loaded_dataset = caiman_loader.CaImAn(output_path)
else:
raise NotImplementedError(
"Unknown/unimplemented method: {}".format(method))
return method, loaded_dataset
def make(self, key):
task_mode, output_dir = (PreProcessTask & key).fetch1(
"task_mode", "preprocess_output_dir")
preprocess_output_dir = find_full_path(get_imaging_root_data_dir(),
output_dir)
if task_mode == "none":
pass
elif task_mode in ["load", "trigger"]:
raise NotImplementedError(
"Pre-processing steps are not implemented."
"Please overwrite this `make` function with"
"desired pre-processing steps.")
else:
raise ValueError(f"Unknown task mode: {task_mode}")
self.insert1({**key})
def test_find_valid_full_path(pipeline, sessions_csv):
from element_interface.utils import find_full_path
get_imaging_root_data_dir = pipeline['get_imaging_root_data_dir']
# add more options for root directories
if sys.platform == 'win32':
ephys_root_data_dir = [get_imaging_root_data_dir(), 'J:/', 'M:/']
else:
ephys_root_data_dir = [get_imaging_root_data_dir(), 'mnt/j', 'mnt/m']
# test: providing relative-path: correctly search for the full-path
sessions, _ = sessions_csv
sess = sessions.iloc[0]
session_full_path = pathlib.Path(sess.session_dir)
rel_path = pathlib.Path(session_full_path).relative_to(
pathlib.Path(get_imaging_root_data_dir()))
full_path = find_full_path(ephys_root_data_dir, rel_path)
assert full_path == session_full_path
def ingest_sessions(session_csv_path="./user_data/sessions.csv",
skip_duplicates=True,
verbose=True):
root_data_dir = get_imaging_root_data_dir()
# ---------- Insert new "Session" and "Scan" ---------
with open(session_csv_path, newline="") as f:
input_sessions = list(csv.DictReader(f, delimiter=","))
# Folder structure: root / subject / session / .tif (raw)
session_list, session_dir_list, scan_list, scanner_list = [], [], [], []
for sess in input_sessions:
sess_dir = find_full_path(root_data_dir, Path(sess["session_dir"]))
# search for either ScanImage or Scanbox files (in that order)
for scan_pattern, scan_type, glob_func in zip(
["*.tif", "*.sbx"],
["ScanImage", "Scanbox"],
[sess_dir.glob, sess_dir.rglob],
):
scan_filepaths = [fp.as_posix() for fp in glob_func(scan_pattern)]
if len(scan_filepaths):
acq_software = scan_type
break
else:
raise FileNotFoundError(
"Unable to identify scan files from the supported " +
"acquisition softwares (ScanImage, Scanbox) at: " +
f"{sess_dir}")
if acq_software == "ScanImage":
import scanreader
from element_interface import scanimage_utils
try: # attempt to read .tif as a scanimage file
loaded_scan = scanreader.read_scan(scan_filepaths)
recording_time = scanimage_utils.get_scanimage_acq_time(
loaded_scan)
header = scanimage_utils.parse_scanimage_header(loaded_scan)
scanner = header["SI_imagingSystem"].strip("'")
except Exception as e:
print(f"ScanImage loading error: {scan_filepaths}\n{str(e)}")
continue
elif acq_software == "Scanbox":
import sbxreader
try: # attempt to load Scanbox
sbx_fp = pathlib.Path(scan_filepaths[0])
sbx_meta = sbxreader.sbx_get_metadata(sbx_fp)
# read from file when Scanbox support this
recording_time = datetime.fromtimestamp(sbx_fp.stat().st_ctime)
scanner = sbx_meta.get("imaging_system", "Scanbox")
except Exception as e:
print(f"Scanbox loading error: {scan_filepaths}\n{str(e)}")
continue
else:
raise NotImplementedError(
"Processing scan from acquisition software of " +
f"type {acq_software} is not yet implemented")
session_key = {
"subject": sess["subject"],
"session_datetime": recording_time
}
if session_key not in session.Session():
scanner_list.append({"scanner": scanner})
session_list.append(session_key)
scan_list.append({
**session_key,
"scan_id": 0,
"scanner": scanner,
"acq_software": acq_software,
})
session_dir_list.append({
**session_key,
"session_dir":
sess_dir.relative_to(root_data_dir).as_posix(),
})
new_equipment = set(val for dic in scanner_list for val in dic.values())
if verbose:
print(f"\n---- Insert {len(new_equipment)} entry(s) into " +
"experiment.Equipment ----")
Equipment.insert(scanner_list, skip_duplicates=True)
if verbose:
print(
f"\n---- Insert {len(session_list)} entry(s) into session.Session ----"
)
session.Session.insert(session_list)
session.SessionDirectory.insert(session_dir_list)
if verbose:
print(f"\n---- Insert {len(scan_list)} entry(s) into scan.Scan ----")
scan.Scan.insert(scan_list, skip_duplicates=skip_duplicates)
if verbose:
print("\n---- Successfully completed ingest_sessions ----")
def make(self, key):
task_mode = (ProcessingTask & key).fetch1("task_mode")
output_dir = (ProcessingTask & key).fetch1("processing_output_dir")
output_dir = find_full_path(get_miniscope_root_data_dir(), output_dir)
if task_mode == "load":
method, loaded_result = get_loader_result(key, ProcessingTask)
if method == "caiman":
loaded_caiman = loaded_result
key = {**key, "processing_time": loaded_caiman.creation_time}
else:
raise NotImplementedError(
f"Loading of {method} data is not yet"
f"supported")
elif task_mode == "trigger":
method = (ProcessingTask * ProcessingParamSet * ProcessingMethod *
Recording & key).fetch1("processing_method")
if method == "caiman":
import caiman
from element_interface.run_caiman import run_caiman
avi_files = (Recording * RecordingInfo * RecordingInfo.File
& key).fetch("file_path")
avi_files = [
find_full_path(get_miniscope_root_data_dir(),
avi_file).as_posix()
for avi_file in avi_files
]
params = (ProcessingTask * ProcessingParamSet
& key).fetch1("params")
sampling_rate = (ProcessingTask * Recording * RecordingInfo
& key).fetch1("fps")
input_hash = dict_to_uuid(dict(**key, **params))
input_hash_fp = output_dir / f".{input_hash }.json"
if not input_hash_fp.exists():
start_time = datetime.utcnow()
run_caiman(
file_paths=avi_files,
parameters=params,
sampling_rate=sampling_rate,
output_dir=output_dir.as_posix(),
is3D=False,
)
completion_time = datetime.utcnow()
with open(input_hash_fp, "w") as f:
json.dump(
{
"start_time":
start_time,
"completion_time":
completion_time,
"duration":
(completion_time - start_time).total_seconds(),
},
f,
default=str,
)
_, imaging_dataset = get_loader_result(key, ProcessingTask)
caiman_dataset = imaging_dataset
key["processing_time"] = caiman_dataset.creation_time
key["package_version"] = caiman.__version__
else:
raise NotImplementedError(
f"Automatic triggering of {method} analysis"
f" is not yet supported")
else:
raise ValueError(f"Unknown task mode: {task_mode}")
self.insert1(key)
def make(self, key):
# Search recording directory for miniscope raw files
acquisition_software, recording_directory = (Recording & key).fetch1(
"acquisition_software", "recording_directory")
recording_path = find_full_path(get_miniscope_root_data_dir(),
recording_directory)
recording_filepaths = [
file_path.as_posix() for file_path in recording_path.glob("*.avi")
]
if not recording_filepaths:
raise FileNotFoundError(f"No .avi files found in "
f"{recording_directory}")
if acquisition_software == "Miniscope-DAQ-V3":
recording_timestamps = recording_path / "timestamp.dat"
if not recording_timestamps.exists():
raise FileNotFoundError(f"No timestamp file found in "
f"{recording_directory}")
nchannels = 1 # Assumes a single channel
# Parse number of frames from timestamp.dat file
with open(recording_timestamps) as f:
next(f)
nframes = sum(1 for line in f if int(line[0]) == 0)
# Parse image dimension and frame rate
video = cv2.VideoCapture(recording_filepaths[0])
_, frame = video.read()
frame_size = np.shape(frame)
px_height = frame_size[0]
px_width = frame_size[1]
fps = video.get(cv2.CAP_PROP_FPS)
elif acquisition_software == "Miniscope-DAQ-V4":
recording_metadata = list(recording_path.glob("metaData.json"))[0]
recording_timestamps = list(
recording_path.glob("timeStamps.csv"))[0]
if not recording_metadata.exists():
raise FileNotFoundError(f"No .json file found in "
f"{recording_directory}")
if not recording_timestamps.exists():
raise FileNotFoundError(f"No timestamp (*.csv) file found in "
f"{recording_directory}")
with open(recording_metadata.as_posix()) as f:
metadata = json.loads(f.read())
with open(recording_timestamps, newline="") as f:
time_stamps = list(csv.reader(f, delimiter=","))
nchannels = 1 # Assumes a single channel
nframes = len(time_stamps) - 1
px_height = metadata["ROI"]["height"]
px_width = metadata["ROI"]["width"]
fps = int(metadata["frameRate"].replace("FPS", ""))
gain = metadata["gain"]
spatial_downsample = 1 # Assumes no spatial downsampling
led_power = metadata["led0"]
time_stamps = np.array([
list(map(int, time_stamps[i]))
for i in range(1, len(time_stamps))
])
else:
raise NotImplementedError(
f"Loading routine not implemented for {acquisition_software}"
" acquisition software")
# Insert in RecordingInfo
self.insert1(
dict(
key,
nchannels=nchannels,
nframes=nframes,
px_height=px_height,
px_width=px_width,
fps=fps,
gain=gain,
spatial_downsample=spatial_downsample,
led_power=led_power,
time_stamps=time_stamps,
recording_duration=nframes / fps,
))
# Insert file(s)
recording_files = [
pathlib.Path(f).relative_to(
find_root_directory(get_miniscope_root_data_dir(),
f)).as_posix() for f in recording_filepaths
]
self.File.insert([{
**key, "file_id": i,
"file_path": f
} for i, f in enumerate(recording_files)])
def make(self, key):
task_mode, output_dir = (ProcessingTask & key).fetch1(
"task_mode", "processing_output_dir")
output_dir = find_full_path(get_imaging_root_data_dir(),
output_dir).as_posix()
if not output_dir:
output_dir = ProcessingTask.infer_output_dir(key,
relative=True,
mkdir=True)
# update processing_output_dir
ProcessingTask.update1({
**key, "processing_output_dir":
output_dir.as_posix()
})
if task_mode == "load":
method, imaging_dataset = get_loader_result(key, ProcessingTask)
if method == "suite2p":
if (scan.ScanInfo & key).fetch1("nrois") > 0:
raise NotImplementedError(
f"Suite2p ingestion error - Unable to handle"
f" ScanImage multi-ROI scanning mode yet")
suite2p_dataset = imaging_dataset
key = {**key, "processing_time": suite2p_dataset.creation_time}
elif method == "caiman":
caiman_dataset = imaging_dataset
key = {**key, "processing_time": caiman_dataset.creation_time}
else:
raise NotImplementedError("Unknown method: {}".format(method))
elif task_mode == "trigger":
method = (ProcessingTask * ProcessingParamSet * ProcessingMethod *
scan.Scan & key).fetch1("processing_method")
preprocess_paramsets = (
PreProcessParamSteps.Step()
& dict(
preprocess_param_steps_id=key["preprocess_param_steps_id"])
).fetch("paramset_idx")
if len(preprocess_paramsets) == 0:
# No pre-processing steps were performed on the acquired dataset, so process the acquired files.
image_files = (ProcessingTask * scan.Scan * scan.ScanInfo *
scan.ScanInfo.ScanFile
& key).fetch("file_path")
image_files = [
find_full_path(get_imaging_root_data_dir(), image_file)
for image_file in image_files
]
else:
preprocess_output_dir = (PreProcessTask
& key).fetch1("preprocess_output_dir")
preprocess_output_dir = find_full_path(
get_imaging_root_data_dir(), preprocess_output_dir)
if not preprocess_output_dir.exists():
raise FileNotFoundError(
f"Pre-processed output directory not found ({preprocess_output_dir})"
)
image_files = [
fp for fp in preprocess_output_dir.glob("*.tif")
]
if method == "suite2p":
import suite2p
suite2p_params = (ProcessingTask * ProcessingParamSet
& key).fetch1("params")
suite2p_params["save_path0"] = output_dir
suite2p_params["fs"] = (
ProcessingTask * scan.Scan * scan.ScanInfo
& key).fetch1("fps")
input_format = pathlib.Path(image_files[0]).suffix
suite2p_params["input_format"] = input_format[1:]
suite2p_paths = {
"data_path": [image_files[0].parent.as_posix()],
"tiff_list": [f.as_posix() for f in image_files],
}
suite2p.run_s2p(ops=suite2p_params,
db=suite2p_paths) # Run suite2p
_, imaging_dataset = get_loader_result(key, ProcessingTask)
suite2p_dataset = imaging_dataset
key = {**key, "processing_time": suite2p_dataset.creation_time}
elif method == "caiman":
from element_interface.run_caiman import run_caiman
params = (ProcessingTask * ProcessingParamSet
& key).fetch1("params")
sampling_rate = (ProcessingTask * scan.Scan * scan.ScanInfo
& key).fetch1("fps")
ndepths = (ProcessingTask * scan.Scan * scan.ScanInfo
& key).fetch1("ndepths")
is3D = bool(ndepths > 1)
if is3D:
raise NotImplementedError(
"Caiman pipeline is not capable of analyzing 3D scans at the moment."
)
run_caiman(
file_paths=[f.as_posix() for f in image_files],
parameters=params,
sampling_rate=sampling_rate,
output_dir=output_dir,
is3D=is3D,
)
_, imaging_dataset = get_loader_result(key, ProcessingTask)
caiman_dataset = imaging_dataset
key["processing_time"] = caiman_dataset.creation_time
else:
raise ValueError(f"Unknown task mode: {task_mode}")
self.insert1(key)