def make(self, key):
method = (ProcessingParamSet * ProcessingTask
& key).fetch1('processing_method')
if method == 'suite2p':
if key['extraction_method'] == 'suite2p_deconvolution':
data_dir = pathlib.Path(Processing._get_suite2p_dir(key))
s2p_loader = suite2p.Suite2p(data_dir)
self.insert1(key)
# ---- iterate through all s2p plane outputs ----
spikes = []
for s2p in s2p_loader.planes.values():
mask_count = len(
spikes) # increment mask id from all "plane"
for mask_idx, spks in enumerate(s2p.spks):
spikes.append({
**key, 'mask': mask_idx + mask_count,
'fluo_channel': 0,
'activity_trace': spks
})
self.Trace.insert(spikes)
else:
raise NotImplementedError(
'Unknown/unimplemented method: {}'.format(method))
def make(self, key):
method = (ProcessingMethod * ProcessingTask
& key).fetch1('processing_method')
if method == 'suite2p':
data_dir = pathlib.Path(Processing._get_suite2p_dir(key))
s2p_loader = suite2p.Suite2p(data_dir)
# ---- build motion correction key
align_chn = s2p_loader.planes[0].alignment_channel
self.insert1({**key, 'mc_channel': align_chn})
# ---- iterate through all s2p plane outputs ----
for plane, s2p in s2p_loader.planes.items():
mc_key = (ScanInfo.Field * ProcessingTask & key & {
'plane': plane
}).fetch1('KEY')
# -- rigid motion correction --
rigid_mc = {
'y_shifts': s2p.ops['yoff'],
'x_shifts': s2p.ops['xoff'],
'y_std': np.nanstd(s2p.ops['yoff']),
'x_std': np.nanstd(s2p.ops['xoff']),
'outlier_frames': s2p.ops['badframes']
}
self.RigidMotionCorrection.insert1({**mc_key, **rigid_mc})
# -- non-rigid motion correction --
if s2p.ops['nonrigid']:
nonrigid_mc = {
'block_height': s2p.ops['block_size'][0],
'block_width': s2p.ops['block_size'][1],
'block_count_y': s2p.ops['nblocks'][0],
'block_count_x': s2p.ops['nblocks'][1],
'outlier_frames': s2p.ops['badframes']
}
nr_blocks = [{
**mc_key, 'block_id': b_id,
'block_y': b_y,
'block_x': b_x,
'y_shifts': bshift_y,
'x_shifts': bshift_x,
'y_std': np.nanstd(bshift_y),
'x_std': np.nanstd(bshift_x)
} for b_id, (b_y, b_x, bshift_y, bshift_x) in enumerate(
zip(s2p.ops['xblock'], s2p.ops['yblock'],
s2p.ops['yoff1'].T, s2p.ops['xoff1'].T))]
self.NonRigidMotionCorrection.insert1({
**mc_key,
**nonrigid_mc
})
self.Block.insert(nr_blocks)
else:
raise NotImplementedError(
'Unknown/unimplemented method: {}'.format(method))
def make(self, key):
method = (ProcessingParamSet * ProcessingTask
& key).fetch1('processing_method')
if method == 'suite2p':
data_dir = pathlib.Path(Processing._get_suite2p_dir(key))
s2p_loader = suite2p.Suite2p(data_dir)
field_keys = (ScanInfo.Field & key).fetch('KEY',
order_by='field_z')
# ---- iterate through all s2p plane outputs ----
masks, cells = [], []
for plane, s2p in s2p_loader.planes.items():
seg_key = (ScanInfo.Field * ProcessingTask & key
& field_keys[plane]).fetch1('KEY')
mask_count = len(masks) # increment mask id from all "plane"
for mask_idx, (is_cell, cell_prob, mask_stat) in enumerate(
zip(s2p.iscell, s2p.cell_prob, s2p.stat)):
masks.append({
**seg_key, 'mask': mask_idx + mask_count,
'seg_channel': s2p.segmentation_channel,
'mask_npix': mask_stat['npix'],
'mask_center_x': mask_stat['med'][1],
'mask_center_y': mask_stat['med'][0],
'mask_xpix': mask_stat['xpix'],
'mask_ypix': mask_stat['ypix'],
'mask_weights': mask_stat['lam']
})
if is_cell:
cells.append({
**seg_key, 'mask_classification_method':
'suite2p_default_classifier',
'mask': mask_idx + mask_count,
'mask_type': 'soma',
'confidence': cell_prob
})
self.insert1(key)
self.Mask.insert(masks, ignore_extra_fields=True)
if cells:
MaskClassification.insert1(
{
**key, 'mask_classification_method':
'suite2p_default_classifier'
},
allow_direct_insert=True)
MaskClassification.MaskType.insert(cells,
ignore_extra_fields=True,
allow_direct_insert=True)
else:
raise NotImplementedError(
'Unknown/unimplemented method: {}'.format(method))
def make(self, key):
# ----
# trigger suite2p or caiman here
# ----
method = (ProcessingParamSet * ProcessingTask
& key).fetch1('processing_method')
if method == 'suite2p':
if (ScanInfo & key).fetch1('nrois') > 0:
raise NotImplementedError(
f'Suite2p ingestion error - Unable to handle ScanImage multi-ROI scanning mode yet'
)
data_dir = pathlib.Path(Processing._get_suite2p_dir(key))
if data_dir.exists():
s2p_loader = suite2p.Suite2p(data_dir)
key = {
**key, 'proc_completion_time': s2p_loader.creation_time,
'proc_curation_time': s2p_loader.curation_time
}
self.insert1(key)
# Insert file(s)
root = pathlib.Path(PhysicalFile._get_root_data_dir())
files = data_dir.glob(
'*'
) # works for Suite2p, maybe something more file-specific for CaImAn
files = [
pathlib.Path(f).relative_to(root).as_posix() for f in files
if f.is_file()
]
PhysicalFile.insert(zip(files), skip_duplicates=True)
self.ProcessingOutputFile.insert([{
**key, 'file_path': f
} for f in files],
ignore_extra_fields=True)
else:
start_time = datetime.now()
# trigger Suite2p here
# wait for completion, then insert with "completion_time", "start_time", no "curation_time"
return
else:
raise NotImplementedError('Unknown method: {}'.format(method))
def make(self, key):
method = (ProcessingParamSet * ProcessingTask
& key).fetch1('processing_method')
if method == 'suite2p':
data_dir = pathlib.Path(Processing._get_suite2p_dir(key))
s2p_loader = suite2p.Suite2p(data_dir)
self.insert1(key)
# ---- iterate through all s2p plane outputs ----
fluo_traces, fluo_chn2_traces = [], []
for s2p in s2p_loader.planes.values():
mask_count = len(
fluo_traces) # increment mask id from all "plane"
for mask_idx, (f, fneu) in enumerate(zip(s2p.F, s2p.Fneu)):
fluo_traces.append({
**key, 'mask': mask_idx + mask_count,
'fluo_channel': 0,
'fluorescence': f,
'neuropil_fluorescence': fneu
})
if len(s2p.F_chan2):
mask_chn2_count = len(
fluo_chn2_traces) # increment mask id from all "plane"
for mask_idx, (f2, fneu2) in enumerate(
zip(s2p.F_chan2, s2p.Fneu_chan2)):
fluo_chn2_traces.append({
**key, 'mask': mask_idx + mask_chn2_count,
'fluo_channel': 1,
'fluorescence': f2,
'neuropil_fluorescence': fneu2
})
self.Trace.insert(fluo_traces + fluo_chn2_traces)
else:
raise NotImplementedError(
'Unknown/unimplemented method: {}'.format(method))
def make(self, key):
method = (ProcessingMethod * ProcessingTask
& key).fetch1('processing_method')
if method == 'suite2p':
data_dir = pathlib.Path(Processing._get_suite2p_dir(key))
s2p_loader = suite2p.Suite2p(data_dir)
# ---- build segmentation key
seg_channel = s2p_loader.planes[0].segmentation_channel
self.insert1({**key, 'seg_channel': seg_channel})
# ---- iterate through all s2p plane outputs ----
masks = []
for plane, s2p in s2p_loader.planes.items():
seg_key = (ScanInfo.Field * ProcessingTask & key & {
'plane': plane
}).fetch1('KEY')
mask_count = len(masks) # increment mask id from all "plane"
for mask_idx, (is_cell, cell_prob, mask_stat) in enumerate(
zip(s2p.iscell, s2p.cell_prob, s2p.stat)):
mask = {
**seg_key, 'mask': mask_idx + mask_count,
'is_cell': bool(is_cell),
'cell_prob': cell_prob,
'npix': mask_stat['npix'],
'center_x': mask_stat['med'][1],
'center_y': mask_stat['med'][0],
'xpix': mask_stat['xpix'],
'ypix': mask_stat['ypix'],
'weights': mask_stat['lam']
}
masks.append(mask)
self.Mask.insert(masks, ignore_extra_fields=True)
self.Cell.insert(masks, ignore_extra_fields=True)
else:
raise NotImplementedError(
'Unknown/unimplemented method: {}'.format(method))
def make(self, key):
method = (ProcessingMethod * ProcessingTask
& key).fetch1('processing_method')
if method == 'suite2p':
data_dir = pathlib.Path(Processing._get_suite2p_dir(key))
s2p_loader = suite2p.Suite2p(data_dir)
# ---- iterate through all s2p plane outputs ----
for plane, s2p in s2p_loader.items():
mc_key = (ScanInfo.Field * ProcessingTask & key & {
'plane': plane
}).fetch1('KEY')
img_dict = {
'ref_image': s2p.ref_image,
'average_image': s2p.mean_image,
'correlation_image': s2p.correlation_map,
'max_proj_image': s2p.max_proj_image
}
self.insert1({**mc_key, **img_dict})
else:
raise NotImplementedError(
'Unknown/unimplemented method: {}'.format(method))