def load(self, key):
from pipeline.utils import galvo_corrections
# load
print("Loading scan", flush=True)
reader = scanreader.read_scan(
(experiment.Scan() & key).local_filenames_as_wildcard)
scan = reader[key["field"] - 1, :, :,
key["channel"] - 1].astype(np.float32)
# raster correction
print("Raster correction", flush=True)
pipe = (fuse.MotionCorrection() & key).module
raster_phase = (pipe.RasterCorrection() & key).fetch1("raster_phase")
fill_fraction = (pipe.ScanInfo() & key).fetch1("fill_fraction")
scan = galvo_corrections.correct_raster(scan, raster_phase,
fill_fraction)
# motion correction
print("Motion correction", flush=True)
x_shifts, y_shifts = (pipe.MotionCorrection() & key).fetch1(
"x_shifts", "y_shifts")
scan = galvo_corrections.correct_motion(scan, x_shifts, y_shifts)
return scan, reader.num_scanning_depths
def _get_scan_sample(key,
sample_length=15,
sample_size=(-1, -1),
sample_fps=5):
""" Load and correct the scan, get some frames from the middle, resize them and
interpolate to 5 Hz.
Arguments:
key: Dictionary with scan keys including slice and channel.
length: Length (in minutes) of the sample.
size: (height, width) Spatial dimensions for the sample.
fps: Desired frames per second of the sample.
"""
import scanreader
from scipy.interpolate import interp1d
# Read the scan
scan_filename = (experiment.Scan() & key).local_filenames_as_wildcard
scan = scanreader.read_scan(scan_filename, dtype=np.float32)
# Load scan
half_length = round(sample_length / 2 * 60 *
scan.fps) # 7.5 minutes of recording
if (scan.num_frames < half_length * 2):
raise ValueError('Scan {} is too short (< {} min long).'.format(
key, sample_length))
middle_frame = int(np.floor(scan.num_frames / 2))
frames = slice(middle_frame - half_length, middle_frame + half_length)
sample = scan[key['slice'] - 1, :, :, key['channel'] - 1, frames]
num_frames = sample.shape[-1]
# Correct the scan
correct_raster = (reso.RasterCorrection() & key).get_correct_raster()
correct_motion = (reso.MotionCorrection() & key).get_correct_motion()
corrected_sample = correct_motion(correct_raster(sample), frames)
# Resize
resized_sample = np.empty([*sample_size, num_frames], dtype=np.float32)
for i in range(num_frames):
resized_sample[:, :, i] = misc.imresize(corrected_sample[:, :, i],
sample_size,
interp='lanczos',
mode='F')
resized_sample = corrected_sample
# Interpolate to desired frame rate (if memory is a constrain, run per pixel)
num_output_frames = round(sample_length * 60 * sample_fps)
f = interp1d(np.linspace(0, 1, num_frames),
resized_sample,
kind='cubic',
copy=False)
output_sample = f(np.linspace(0, 1, num_output_frames))
return output_sample
def _make_tuples(self, key):
# Load the scan
import scanreader
scan_filename = (experiment.Scan() & key).local_filenames_as_wildcard
scan = scanreader.read_scan(scan_filename)
scan = (scan[key['slice']-1, :, :, 0, :]).astype(np.float32, copy=False)
# Correct the scan
correct_motion = (preprocess.Prepare.GalvoMotion() & key).get_correct_motion()
correct_raster = (preprocess.Prepare.Galvo() & key).get_correct_raster()
scan = correct_motion(correct_raster(scan))
design, cov = (OriDesignMatrix() & key).fetch1['design_matrix', 'regressor_cov']
height, width, nslices = (preprocess.Prepare.Galvo() & key).fetch1('px_height', 'px_width', 'nslices')
design = design[key['slice'] - 1::nslices, :]
if scan.shape[2] == 2*design.shape[0]:
scan = (scan[:,:,::2] + scan[:,:,1::2])/2 # this is a hack for mesoscope scanner -- needs fixing
assert design.shape[0] == scan.shape[2]
height, width = scan.shape[0:2] # hack for mesoscope -- needs fixing
assert (height, width) == scan.shape[0:2]
# remove periods where the design matrix has any nans
ix = np.logical_not(np.isnan(design).any(axis=1))
design = design[ix, :]
design = design - design.mean()
nregressors = design.shape[1]
# normalize scan
m = scan.mean(axis=-1, keepdims=True)
scan -= m
scan /= m
v = (scan**2).sum(axis=-1)
# estimate degrees of freedom per pixel
spectrum = np.abs(np.fft.fft(scan, axis=-1))
dof = (spectrum.sum(axis=-1)**2/(spectrum**2).sum(axis=-1)).astype(np.int32)
# solve
scan = scan[:, :, ix].reshape((-1, design.shape[0])).T
x, r2, rank, sv = linalg.lstsq(design, scan, overwrite_a=True, overwrite_b=True, check_finite=False)
del scan, design
assert rank == nregressors
x = x.T.reshape((height, width, -1))
r2 = 1-r2.reshape((height, width))/v
self.insert1(dict(key, regr_coef_maps=x, r2_map=r2, dof_map=dof))
def session_plot(self):
""" Do a plot of how temperature progress through a session"""
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
# Check that plot is restricted to a single session
session_key = self.fetch('KEY', limit=1)[0]
session_key.pop('scan_idx')
if len(self & session_key) != len(self):
raise PipelineException(
'Plot can only be generated for one session at a '
'time')
# Get times and timestamps, scan_ts
scan_indices, ts, temperatures = self.fetch('scan_idx',
'temp_time',
'temperatures',
order_by='scan_idx')
session_ts = (experiment.Session() & self).fetch1('session_ts')
scan_ts = (experiment.Scan() & self).fetch('scan_ts',
order_by='scan_idx')
abs_ts = [(sts - session_ts).seconds + (t - t[0])
for sts, t in zip(scan_ts, ts)]
# Plot
fig = plt.figure(figsize=(10, 5))
for abs_ts_, temp_, scan_idx in zip(abs_ts, temperatures,
scan_indices):
plt.plot(abs_ts_ / 3600, temp_,
label='Scan {}'.format(scan_idx)) # in hours
plt.title(
'Temperature for {animal_id}-{session} starting at {session_ts}'.
format(session_ts=session_ts, **session_key))
plt.ylabel('Temperature (Celsius)')
plt.xlabel('Hour')
plt.legend()
# Plot formatting
plt.gca().yaxis.set_major_locator(ticker.MultipleLocator(0.5))
plt.grid(linestyle='--', alpha=0.8)
return fig
pipe.MotionCorrection().populate(next_scans, reserve_jobs=True, suppress_errors=True)
pipe.SummaryImages().populate(next_scans, reserve_jobs=True, suppress_errors=True)
# Field Registration
stack.InitialRegistration().populate(next_scans, reserve_jobs=True, suppress_errors=True)
stack.FieldRegistration().populate(next_scans, reserve_jobs=True, suppress_errors=True)
# fuse
fuse.MotionCorrection().populate(next_scans, reserve_jobs=True, suppress_errors=True)
fuse.ScanSet().populate(next_scans, reserve_jobs=True, suppress_errors=True)
fuse.Activity().populate(next_scans, reserve_jobs=True, suppress_errors=True)
fuse.ScanDone().populate(next_scans, reserve_jobs=True, suppress_errors=True)
# tune (these are memory intensive)
if POPULATE_TUNE:
tune_scans = next_scans & (experiment.Scan() & 'scan_ts > "2017-12-00 00:00:00"')
#stimulus.Sync needs to be run from Matlab
tune.STA().populate(tune_scans, reserve_jobs=True, suppress_errors=True)
tune.STAQual().populate(tune_scans, reserve_jobs=True, suppress_errors=True)
tune.STAExtent().populate(tune_scans, reserve_jobs=True)
tune.CaMovie().populate(tune_scans, reserve_jobs=True, suppress_errors=True)
tune.Drift().populate(tune_scans, reserve_jobs=True, suppress_errors=True)
tune.OriDesign().populate(tune_scans, reserve_jobs=True, suppress_errors=True)
tune.OriMap().populate(tune_scans, reserve_jobs=True, suppress_errors=True)
tune.Cos2Map().populate(tune_scans, reserve_jobs=True, suppress_errors=True)
tune.OriMapQuality().populate(tune_scans, reserve_jobs=True, suppress_errors=True)
tune.OracleMap().populate(tune_scans, reserve_jobs=True, suppress_errors=True)
tune.MovieOracle().populate(tune_scans, reserve_jobs=True, suppress_errors=True)
#!/usr/local/bin/python3
from pipeline import experiment, reso, meso, fuse, stack, pupil, treadmill, posture
from stimulus import stimulus
from stimline import tune
# # Scans
# for priority in range(120, -130, -10): # highest to lowest priority
# next_scans = (experiment.AutoProcessing() & 'priority > {}'.format(priority) &
# (experiment.Scan() & 'scan_ts > "2019-01-01 00:00:00"'))
next_scans = (experiment.AutoProcessing() &
(experiment.Scan() & 'scan_ts > "2019-01-01 00:00:00"'))
# stimulus
stimulus.Sync().populate(next_scans, reserve_jobs=True, suppress_errors=True)
stimulus.BehaviorSync().populate(next_scans,
reserve_jobs=True,
suppress_errors=True)
# treadmill, pupil, posture
treadmill.Treadmill().populate(next_scans,
reserve_jobs=True,
suppress_errors=True)
pupil.Eye().populate(next_scans, reserve_jobs=True, suppress_errors=True)
pupil.FittedPupil().populate(next_scans,
reserve_jobs=True,
suppress_errors=True)
posture.Posture().populate(next_scans, reserve_jobs=True, suppress_errors=True)
# stack
stack.StackInfo().populate(stack.CorrectionChannel(),
def key_source(self):
return experiment.Scan() & experiment.Scan.BehaviorFile().proj()