def load(self, key):
from pipeline.utils import galvo_corrections
# load
print("Loading scan", flush=True)
reader = scanreader.read_scan(
(experiment.Scan() & key).local_filenames_as_wildcard)
scan = reader[key["field"] - 1, :, :,
key["channel"] - 1].astype(np.float32)
# raster correction
print("Raster correction", flush=True)
pipe = (fuse.MotionCorrection() & key).module
raster_phase = (pipe.RasterCorrection() & key).fetch1("raster_phase")
fill_fraction = (pipe.ScanInfo() & key).fetch1("fill_fraction")
scan = galvo_corrections.correct_raster(scan, raster_phase,
fill_fraction)
# motion correction
print("Motion correction", flush=True)
x_shifts, y_shifts = (pipe.MotionCorrection() & key).fetch1(
"x_shifts", "y_shifts")
scan = galvo_corrections.correct_motion(scan, x_shifts, y_shifts)
return scan, reader.num_scanning_depths
def make(self, key):
# get movie
scan, scale = (AstroCaMovie & key).fetch1("corrected_scan",
"scale_factor")
scan = scan.astype(np.float32) * scale
# get regressors
X = (tune.OriDesign() & key).fetch1("regressors")
pipe = (fuse.MotionCorrection() & key).module
nfields_name = ("nfields/nrois"
if "nrois" in pipe.ScanInfo.heading else "nfields")
nfields = int((pipe.ScanInfo & key).proj(n=nfields_name).fetch1("n"))
X = X[key["field"] - 1::nfields, :]
if abs(X.shape[0] - scan.shape[2]) > 1:
raise PipelineException(
"The sync frames do not match scan frames.")
else:
# truncate scan if X is shorter
if X.shape[0] < scan.shape[2]:
warn("Scan is longer than design matrix")
scan = scan[:, :, :X.shape[0]]
# truncate design matrix if scan is shorter
if scan.shape[2] < X.shape[0]:
warn("Scan is shorter than design matrix")
X = X[:scan.shape[2], :]
# limit the analysis to times when X is non-zero
ix = (X**2).sum(axis=1) > 1e-4 * (X**2).sum(axis=1).max()
X = X[ix, :]
scan = scan[:, :, ix]
# normalize regressors
X = X - X.mean(axis=0)
X /= np.sqrt((X**2).sum(axis=0, keepdims=True))
# normalize movie
scan -= scan.mean(axis=2, keepdims=True)
key["activity_map"] = np.sqrt((scan**2).sum(axis=2))
scan /= key["activity_map"][:, :, None] + 1e-6
# compute response
key["response_map"] = np.tensordot(scan,
np.linalg.pinv(X),
axes=(2, 1))
self.insert1(key)
def make(self, key):
from pipeline.utils import performance
# Read scan
print("Reading scan...")
scan_filename = (experiment.Scan & key).local_filenames_as_wildcard
scan = scanreader.read_scan(scan_filename)
# Get some params
pipe = (fuse.MotionCorrection() & key).module
# Map: Correct scan in parallel
f = performance.parallel_correct_scan # function to map
raster_phase = (pipe.RasterCorrection() & key).fetch1("raster_phase")
fill_fraction = (pipe.ScanInfo() & key).fetch1("fill_fraction")
y_shifts, x_shifts = (pipe.MotionCorrection() & key).fetch1(
"y_shifts", "x_shifts")
kwargs = {
"raster_phase": raster_phase,
"fill_fraction": fill_fraction,
"y_shifts": y_shifts,
"x_shifts": x_shifts,
}
results = performance.map_frames(
f,
scan,
field_id=key["field"] - 1,
channel=key["channel"] - 1,
kwargs=kwargs,
)
# Reduce: Rescale and save as int16
height, width, _ = results[0][1].shape
corrected_scan = np.zeros([height, width, scan.num_frames],
dtype=np.int16)
max_abs_intensity = max(np.abs(c).max() for f, c in results)
scale = max_abs_intensity / (2**15 - 1)
for frames, chunk in results:
corrected_scan[:, :, frames] = (chunk / scale).astype(np.int16)
# Insert
self.insert1({
**key, "scale_factor": scale,
"corrected_scan": corrected_scan
})
stack.CorrectedStack().populate(reserve_jobs=True, suppress_errors=True)
# reso/meso (up to SummaryImages)
for pipe in [reso, meso]:
pipe.ScanInfo().populate(next_scans, reserve_jobs=True, suppress_errors=True)
pipe.Quality().populate(next_scans, reserve_jobs=True, suppress_errors=True)
pipe.RasterCorrection().populate(next_scans, reserve_jobs=True, suppress_errors=True)
pipe.MotionCorrection().populate(next_scans, reserve_jobs=True, suppress_errors=True)
pipe.SummaryImages().populate(next_scans, reserve_jobs=True, suppress_errors=True)
# Field Registration
stack.InitialRegistration().populate(next_scans, reserve_jobs=True, suppress_errors=True)
stack.FieldRegistration().populate(next_scans, reserve_jobs=True, suppress_errors=True)
# fuse
fuse.MotionCorrection().populate(next_scans, reserve_jobs=True, suppress_errors=True)
fuse.ScanSet().populate(next_scans, reserve_jobs=True, suppress_errors=True)
fuse.Activity().populate(next_scans, reserve_jobs=True, suppress_errors=True)
fuse.ScanDone().populate(next_scans, reserve_jobs=True, suppress_errors=True)
# tune (these are memory intensive)
if POPULATE_TUNE:
tune_scans = next_scans & (experiment.Scan() & 'scan_ts > "2017-12-00 00:00:00"')
#stimulus.Sync needs to be run from Matlab
tune.STA().populate(tune_scans, reserve_jobs=True, suppress_errors=True)
tune.STAQual().populate(tune_scans, reserve_jobs=True, suppress_errors=True)
tune.STAExtent().populate(tune_scans, reserve_jobs=True)
tune.CaMovie().populate(tune_scans, reserve_jobs=True, suppress_errors=True)
tune.Drift().populate(tune_scans, reserve_jobs=True, suppress_errors=True)
def key_source(self):
rel2 = (stimulus.Clip() * fuse.MotionCorrection() * stimulus.Movie()
& 'movie_class in ("cinema", "youtube", "unreal")').aggr(
stimulus.Trial(), repeats="count(movie_name)")
return AstrocyteScans.Field & stimulus.Sync() & (
rel2 & "repeats > 2").proj()