def load(self, key): from pipeline.utils import galvo_corrections # load print("Loading scan", flush=True) reader = scanreader.read_scan( (experiment.Scan() & key).local_filenames_as_wildcard) scan = reader[key["field"] - 1, :, :, key["channel"] - 1].astype(np.float32) # raster correction print("Raster correction", flush=True) pipe = (fuse.MotionCorrection() & key).module raster_phase = (pipe.RasterCorrection() & key).fetch1("raster_phase") fill_fraction = (pipe.ScanInfo() & key).fetch1("fill_fraction") scan = galvo_corrections.correct_raster(scan, raster_phase, fill_fraction) # motion correction print("Motion correction", flush=True) x_shifts, y_shifts = (pipe.MotionCorrection() & key).fetch1( "x_shifts", "y_shifts") scan = galvo_corrections.correct_motion(scan, x_shifts, y_shifts) return scan, reader.num_scanning_depths
def _get_scan_sample(key, sample_length=15, sample_size=(-1, -1), sample_fps=5): """ Load and correct the scan, get some frames from the middle, resize them and interpolate to 5 Hz. Arguments: key: Dictionary with scan keys including slice and channel. length: Length (in minutes) of the sample. size: (height, width) Spatial dimensions for the sample. fps: Desired frames per second of the sample. """ import scanreader from scipy.interpolate import interp1d # Read the scan scan_filename = (experiment.Scan() & key).local_filenames_as_wildcard scan = scanreader.read_scan(scan_filename, dtype=np.float32) # Load scan half_length = round(sample_length / 2 * 60 * scan.fps) # 7.5 minutes of recording if (scan.num_frames < half_length * 2): raise ValueError('Scan {} is too short (< {} min long).'.format( key, sample_length)) middle_frame = int(np.floor(scan.num_frames / 2)) frames = slice(middle_frame - half_length, middle_frame + half_length) sample = scan[key['slice'] - 1, :, :, key['channel'] - 1, frames] num_frames = sample.shape[-1] # Correct the scan correct_raster = (reso.RasterCorrection() & key).get_correct_raster() correct_motion = (reso.MotionCorrection() & key).get_correct_motion() corrected_sample = correct_motion(correct_raster(sample), frames) # Resize resized_sample = np.empty([*sample_size, num_frames], dtype=np.float32) for i in range(num_frames): resized_sample[:, :, i] = misc.imresize(corrected_sample[:, :, i], sample_size, interp='lanczos', mode='F') resized_sample = corrected_sample # Interpolate to desired frame rate (if memory is a constrain, run per pixel) num_output_frames = round(sample_length * 60 * sample_fps) f = interp1d(np.linspace(0, 1, num_frames), resized_sample, kind='cubic', copy=False) output_sample = f(np.linspace(0, 1, num_output_frames)) return output_sample
def _make_tuples(self, key): # Load the scan import scanreader scan_filename = (experiment.Scan() & key).local_filenames_as_wildcard scan = scanreader.read_scan(scan_filename) scan = (scan[key['slice']-1, :, :, 0, :]).astype(np.float32, copy=False) # Correct the scan correct_motion = (preprocess.Prepare.GalvoMotion() & key).get_correct_motion() correct_raster = (preprocess.Prepare.Galvo() & key).get_correct_raster() scan = correct_motion(correct_raster(scan)) design, cov = (OriDesignMatrix() & key).fetch1['design_matrix', 'regressor_cov'] height, width, nslices = (preprocess.Prepare.Galvo() & key).fetch1('px_height', 'px_width', 'nslices') design = design[key['slice'] - 1::nslices, :] if scan.shape[2] == 2*design.shape[0]: scan = (scan[:,:,::2] + scan[:,:,1::2])/2 # this is a hack for mesoscope scanner -- needs fixing assert design.shape[0] == scan.shape[2] height, width = scan.shape[0:2] # hack for mesoscope -- needs fixing assert (height, width) == scan.shape[0:2] # remove periods where the design matrix has any nans ix = np.logical_not(np.isnan(design).any(axis=1)) design = design[ix, :] design = design - design.mean() nregressors = design.shape[1] # normalize scan m = scan.mean(axis=-1, keepdims=True) scan -= m scan /= m v = (scan**2).sum(axis=-1) # estimate degrees of freedom per pixel spectrum = np.abs(np.fft.fft(scan, axis=-1)) dof = (spectrum.sum(axis=-1)**2/(spectrum**2).sum(axis=-1)).astype(np.int32) # solve scan = scan[:, :, ix].reshape((-1, design.shape[0])).T x, r2, rank, sv = linalg.lstsq(design, scan, overwrite_a=True, overwrite_b=True, check_finite=False) del scan, design assert rank == nregressors x = x.T.reshape((height, width, -1)) r2 = 1-r2.reshape((height, width))/v self.insert1(dict(key, regr_coef_maps=x, r2_map=r2, dof_map=dof))
def session_plot(self): """ Do a plot of how temperature progress through a session""" import matplotlib.pyplot as plt import matplotlib.ticker as ticker # Check that plot is restricted to a single session session_key = self.fetch('KEY', limit=1)[0] session_key.pop('scan_idx') if len(self & session_key) != len(self): raise PipelineException( 'Plot can only be generated for one session at a ' 'time') # Get times and timestamps, scan_ts scan_indices, ts, temperatures = self.fetch('scan_idx', 'temp_time', 'temperatures', order_by='scan_idx') session_ts = (experiment.Session() & self).fetch1('session_ts') scan_ts = (experiment.Scan() & self).fetch('scan_ts', order_by='scan_idx') abs_ts = [(sts - session_ts).seconds + (t - t[0]) for sts, t in zip(scan_ts, ts)] # Plot fig = plt.figure(figsize=(10, 5)) for abs_ts_, temp_, scan_idx in zip(abs_ts, temperatures, scan_indices): plt.plot(abs_ts_ / 3600, temp_, label='Scan {}'.format(scan_idx)) # in hours plt.title( 'Temperature for {animal_id}-{session} starting at {session_ts}'. format(session_ts=session_ts, **session_key)) plt.ylabel('Temperature (Celsius)') plt.xlabel('Hour') plt.legend() # Plot formatting plt.gca().yaxis.set_major_locator(ticker.MultipleLocator(0.5)) plt.grid(linestyle='--', alpha=0.8) return fig
pipe.MotionCorrection().populate(next_scans, reserve_jobs=True, suppress_errors=True) pipe.SummaryImages().populate(next_scans, reserve_jobs=True, suppress_errors=True) # Field Registration stack.InitialRegistration().populate(next_scans, reserve_jobs=True, suppress_errors=True) stack.FieldRegistration().populate(next_scans, reserve_jobs=True, suppress_errors=True) # fuse fuse.MotionCorrection().populate(next_scans, reserve_jobs=True, suppress_errors=True) fuse.ScanSet().populate(next_scans, reserve_jobs=True, suppress_errors=True) fuse.Activity().populate(next_scans, reserve_jobs=True, suppress_errors=True) fuse.ScanDone().populate(next_scans, reserve_jobs=True, suppress_errors=True) # tune (these are memory intensive) if POPULATE_TUNE: tune_scans = next_scans & (experiment.Scan() & 'scan_ts > "2017-12-00 00:00:00"') #stimulus.Sync needs to be run from Matlab tune.STA().populate(tune_scans, reserve_jobs=True, suppress_errors=True) tune.STAQual().populate(tune_scans, reserve_jobs=True, suppress_errors=True) tune.STAExtent().populate(tune_scans, reserve_jobs=True) tune.CaMovie().populate(tune_scans, reserve_jobs=True, suppress_errors=True) tune.Drift().populate(tune_scans, reserve_jobs=True, suppress_errors=True) tune.OriDesign().populate(tune_scans, reserve_jobs=True, suppress_errors=True) tune.OriMap().populate(tune_scans, reserve_jobs=True, suppress_errors=True) tune.Cos2Map().populate(tune_scans, reserve_jobs=True, suppress_errors=True) tune.OriMapQuality().populate(tune_scans, reserve_jobs=True, suppress_errors=True) tune.OracleMap().populate(tune_scans, reserve_jobs=True, suppress_errors=True) tune.MovieOracle().populate(tune_scans, reserve_jobs=True, suppress_errors=True)
#!/usr/local/bin/python3 from pipeline import experiment, reso, meso, fuse, stack, pupil, treadmill, posture from stimulus import stimulus from stimline import tune # # Scans # for priority in range(120, -130, -10): # highest to lowest priority # next_scans = (experiment.AutoProcessing() & 'priority > {}'.format(priority) & # (experiment.Scan() & 'scan_ts > "2019-01-01 00:00:00"')) next_scans = (experiment.AutoProcessing() & (experiment.Scan() & 'scan_ts > "2019-01-01 00:00:00"')) # stimulus stimulus.Sync().populate(next_scans, reserve_jobs=True, suppress_errors=True) stimulus.BehaviorSync().populate(next_scans, reserve_jobs=True, suppress_errors=True) # treadmill, pupil, posture treadmill.Treadmill().populate(next_scans, reserve_jobs=True, suppress_errors=True) pupil.Eye().populate(next_scans, reserve_jobs=True, suppress_errors=True) pupil.FittedPupil().populate(next_scans, reserve_jobs=True, suppress_errors=True) posture.Posture().populate(next_scans, reserve_jobs=True, suppress_errors=True) # stack stack.StackInfo().populate(stack.CorrectionChannel(),
def key_source(self): return experiment.Scan() & experiment.Scan.BehaviorFile().proj()