def upgrade_sqlite(path): """ for io in ScanboxIO.iter_every_io(): print io.db_path upgrade_sqlite(io.db_path) """ import shutil path = Path(path) Session = open_sqlite(path.str) meta = schema.SQLite3Base.metadata bind = Session.kw.get('bind') shutil.copy2(path.str, path.with_suffix('.backup.sqlite3').str) schema.fix_incremental(meta, bind) return Session
class Scanbox3IO(object): """ locates directory dir/ meta.mat mmap.npy? """ # has mat # has sbx # others def __init__(self, path): self.path = Path(path) # without_suffixes '.mat', '.sbx' def __repr__(self): return '{}({!r})'.format(type(self).__name__, self.path.str) @property def matpath(self): return self.path.with_suffix('.mat') @property def sbxpath(self): return self.path.with_suffix('.sbx') @memoized_property def mat(self): return ZeroDimensionArrayView.from_mat(self.matpath.str) @memoized_property def sbx(self): return @memoized_property def mmap(self): pass
class ScanboxEphysView(object): def __init__(self, io, path): self.io = io self.path = Path(path).with_suffix('.txt') self.npy = self.path.with_suffix('.io').joinpath('ephys.npy') @property def trace(self): return np.load(self.npy.str).tolist() if self.npy.is_file() else [] def toDict(self): return dict(trace=self.trace) def reimport(self): print 'Importing ephys data. This may take a few minutes.' np.save(self.npy.str, import_ephys(self.path)) print 'Import Success!' return self.toDict() def remove(self): self.npy.unlink() return self.toDict()
class ScanimageIO(object): session_name = 'main' r_value = 0.7 def __init__(self, path): # wself.path = Path(str(path) + '.imported') if '.imported' in str(path): self.path = Path(path) else: self.path = Path(str(path) + '.imported') @classmethod def get_record(cls, rec_path): return ScanimageRecord(rec_path) @property def exists(self): return self.path.is_dir() @memoized_property def tiff(self): tiffpath = self.path.with_suffix('.tif') print 'Import from {}...Please allow a few minutes.'.format( tiffpath.name) return tifffile.imread(tiffpath.str) def import_raw(self): if self.exists: return False nchan = util.infer_nchannels(self.tiff) if nchan: self.create_package_path() self.convert_channels(nchan) else: print 'Unable to import data.' print 'Import done!' return self.toDict() def create_package_path(self): self.path.mkdir() print 'Path `{}` created.'.format(self.path.str) def remove_package(self): shutil.rmtree(self.path.str) return self.toDict() def convert_channels(self, nchan): for index in range(nchan): self.convert_channel(nchan, index) def convert_channel(self, nchan, chan): tiff = self.tiff[chan::nchan] print 'Converting channel {}...({} frames.)'.format(chan, len(tiff)) return getattr(self, 'ch{}'.format(chan)).import_raw(tiff) def toDict(self): return dict(exists=self.exists, path=self.path.str, sessions=self.sessions) @memoized_property def ch0(self): return ScanimageChannel(self.path.joinpath('ch0')) @memoized_property def ch1(self): return ScanimageChannel(self.path.joinpath('ch1')) @memoized_property def channel(self): chan = self.session.opt.setdefault('channel', 0) return getattr(self, 'ch{}'.format(chan)) @channel.invalidator def set_channel(self, channel): self.session.opt['channel'] = channel return self @property def nchannel(self): return len(list(self.path.glob('ch?.mmap.npy'))) @property def channel_numbers(self): return list(range(self.nchannel)) @property def channel_number(self): return self.session.opt['channel'] colormap_index = 0 colormaps = ['jet', 'gray', 'gist_heat', 'afmhot', 'bone'] @property def has_blank_and_flicker(self): if self.db: rec = self.db.rec return [rec.blankOn, rec.flickerOn] else: return [None, None] @property def sfrequency(self): return self.db.locator.sfrequencies.current if self.db else '' # return self.session.opt.setdefault(# 'sfrequency', ) @property def sfrequency_index(self): return self.sfrequencies.index( self.sfrequency) if self.sfrequency else 0 @property def sfrequencies(self): return self.db.locator.sfrequencies if self.db else [] @property def orientations(self): return self.db.orientations if self.db else [] # @sfrequency.invalidator def set_sfrequency_index(self, sfreq_index): self.sfrequencies.set_cursor(sfreq_index) self.session.opt['sfrequency'] = self.sfrequencies.current self.session.opt.save() return dict(index=sfreq_index, value=self.sfrequency) @memoized_property def db(self): return ScanimageDBAdaptor(self.session.ed) if self.session.ed else None @property def main_response(self): return MainResponse.from_adaptor(self.channel.stat.MEAN, self.db) @property def sessions(self): return map(ScanimageSession, sorted(self.path.ls('*.session'))) @memoized_property def session(self): return ScanimageSession( self.path.joinpath(self.session_name).with_suffix('.session')) @session.invalidator def set_session(self, session_name): self.session_name = session_name return self def upsert_roi(self, roi_kwargs): gp = validate_guess_params(roi_kwargs.pop('guessParams', {})) return self.session.roi.upsert(ROI(guess_params=gp, **roi_kwargs)) def invalidate_rois(self): for roi in self.session.roi.values(): roi.invalidated = True roi.blank = None roi.flicker = None roi.responses = {} def export_sfreqfit_data_as_mat(self, rid): roi = self.session.roi[rid] sio = StringIO() value = roi.sfreqfit.toDict() io.savemat(sio, value) return sio.getvalue() def update_responses(self, id, heavy=False): roi = self.session.roi[id] trace = self.make_trace(roi) if self.session.ed: with self.session.roi.bulk_on: try: if self.db.locator.override_blank(True): roi.blank = Orientation.from_adaptor( 'blank', trace, self.db) if self.db.locator.override_flicker(True): roi.flicker = Orientation.from_adaptor( 'flicker', trace, self.db) finally: self.db.locator.override() for sf in self.db.locator.sfrequencies.loop(): response = ROIResponse.from_adaptor(roi, trace, self.db) roi.responses[self.sfrequency] = response roi.update_with_adaptor(self.db) for sf, resp in roi.sorted_responses: gp = roi.guess_params.get(sf) or resp.sog_initial_guess print 'SoG custom guess for {}: {}'.format(sf, gp) resp.update_fit_and_decay(roi, self.db, gp, heavy) p_value = roi.anova_all.get('p') if heavy: if p_value < 0.01: print('Computing bootstrap for preferred SF') # '{} conditions...').format( # len(self.db.orientations) * len(self.db.sfrequencies)) roi.update_bootstrap_for_sf(self.db) peaksf = roi.sfreqfit.peak_sfreq.x peak_resp = roi.responses[peaksf] print 'Determine peak spatial frequency: {}'.format( peaksf) peak_resp.bootstrap = BootstrapResponse.from_adaptor( peak_resp, self.db) else: print( 'P value is not less than 0.01. ({}) ' 'Skip bootstrap.').format(p_value) roi.invalidated = False print 'Done updating response for ROI ' + str(roi.id) print ' ' print ' ' return self.session.roi.upsert(roi) else: response = ROIResponse.from_scanbox(roi, trace) roi.responses[self.sfrequency] = response roi.update_with_adaptor(self.db) roi.invalidated = False return self.session.roi.upsert(roi) sog_initial_guess = ((0, 1), (0, 1), (15, 60), (0, 0.01)) def make_delta_trace(self, roi, trace, dx=0, dy=0): extras = self.session.roi.values() extras.remove(roi) main_trace, main_mask = roi.trace(trace, dx, dy) neur_trace, neur_mask = roi.neuropil_trace(trace, extras, dx, dy) return main_trace - neur_trace * self.r_value def make_trace(self, roi, old=False): # checked same function if old: # print 'no centroid yet...perform old' extras = self.session.roi.values() extras.remove(roi) main_trace, main_mask = roi.trace(self.channel.mmap) neur_trace, neur_mask = roi.neuropil_trace(self.channel.mmap, extras) # neuropil_mask, roi_mask = roi.trim_bounding_mask(neur_mask, main_mask) # roi.masks = dict( # neuripil = neuropil_mask.tolist(), # roi = roi_mask.tolist()) return main_trace - neur_trace * self.r_value else: # print 'has centroid...perform new' vecs = roi.split_by_vectors(len(self.channel.mmap)) traces = np.split(self.channel.mmap, vecs.index[1:]) return np.concatenate([ self.make_delta_trace(roi, trace, dx, dy) for dx, dy, trace in zip(vecs.x, vecs.y, traces) ]) def export_plots(self, id): roi = self.session.roi[id] return roi.export_plots_as_zip_for_download()
class TrajectoryIO(object): session_name = 'main' def __init__(self, path): self.path = Path(path).merge_or_join_suffix('.imported', on='.tif') @property def exists(self): return self.path.is_dir() @property def as_record(self): return dict(name=self.tiffpath.name, user='******', time=self.datetime.strftime('%H:%M:%S'), host='IOS', desc=self.desc, mouse='Mouse 3', package=self) @property def desc(self): return '{}'.format(tifffile.format_size(self.tiffpath.stat().st_size)) @property def datetime(self): return datetime.fromtimestamp(float(self.tiffpath.stem)) @property def tiffpath(self): return self.path.with_suffix('.tif') def toDict(self): return dict(exists=self.exists, path=self.path.str, sessions=self.sessions) @memoized_property def tiff(self): tiffpath = self.path.with_suffix('.tif') file_size = tiffpath.lstat().st_size print 'Import from {}...'.format(tiffpath) print 'File size {:,} bytes.'.format(file_size) return tifffile.imread(tiffpath.str) def import_raw(self): if self.exists: return False self.create_package_path() print 'Looking for matching VR log file...' log = TrajectoryLog.query(self.datetime) # it can fail but never reports. print 'Converting channel and timestamps...' tss = np.fromfile(self.path.with_suffix('.csv').str, sep='\n', dtype='float64') TrajectoryChannel(self.path.joinpath('channel')).import_raw( self.tiff, tss, log) print 'Import done!' return self.toDict() def create_package_path(self): self.path.mkdir() print 'Path `{}` created.'.format(self.path.str) def remove_package(self): shutil.rmtree(self.path.str) return self.toDict() @memoized_property def channel(self): tfilter = self.session.opt.get('filter') if tfilter: if tfilter._indices is not None: return TrajectoryChannel( self.path.joinpath('channel')).set_indices( tfilter._indices) return TrajectoryChannel(self.path.joinpath('channel')) @property def sessions(self): return map(TrajectorySession, sorted(self.path.ls('*.session'))) @memoized_property def session(self): return TrajectorySession( self.path.joinpath(self.session_name).with_suffix('.session')) @session.invalidator def set_session(self, session_name): self.session_name = session_name return self @memoized_property def alog(self): # aligned log in JSON format return [ dict(e=e, x=x, y=y, v=v) for e, x, y, v in self.channel.alog[['E', 'X', 'Y', 'V']] ] @property def main_response(self): return TrajectoryResponse(self.channel.stat.MEAN) def upsert_roi(self, roi_kwargs): return self.session.roi.upsert(ROI(**roi_kwargs)) def upsert_filter(self, filter_kwargs): tfilter = TrajectoryFilter(**filter_kwargs) tfilter._indices = tfilter.make_indices(self.channel.original_velocity) rv = self.session.opt.upsert(tfilter) for roi in self.session.roi.values(): roi.invalidated = True self.session.roi.save() return rv def make_response(self, id): roi = self.session.roi[id] extras = self.session.roi.values() extras.remove(roi) main_trace, main_mask = roi.trace(self.channel.mmap) neur_trace, neur_mask = roi.neuropil_trace(self.channel.mmap, extras) trace = main_trace - neur_trace * 0.7 roi.invalidated = False roi.response = TrajectoryResponse(trace) return self.session.roi.upsert(roi) @memoized_property def velocity_stat(self): return dict( max=self.channel.alog.V.max(), mean=self.channel.alog.V.mean(), min=self.channel.alog.V.min(), )
class TrajectoryChannel(object): indices = slice(None) cmap_name = 'jet' def __init__(self, path): self.path = Path(path) def set_indices(self, indices): self.indices = indices return self @property def alogpath(self): return self.path.with_suffix('.alog.npy') @property def mmappath(self): return self.path.with_suffix('.mmap.npy') @property def timepath(self): return self.path.with_suffix('.time.npy') @property def statpath(self): return self.path.with_suffix('.stat.npy') @property def metapath(self): return self.path.with_suffix('.meta.json') @memoized_property def cmap8bit(self): norm = Normalize(vmin=self.stat.MIN.min() / 256, vmax=self.stat.MAX.max() / 256) return ScalarMappable(norm=norm, cmap=plt.get_cmap(self.cmap_name)) @cmap8bit.invalidator def set_cmap(self, which=0): name = ['jet', 'gray', 'hot', 'hsv'][int(which)] self.cmap_name = name @memoized_property def mmap(self): # indices meta = TrajectoryChannelMeta.load(self.metapath.str) shape = (meta.z, meta.y, meta.x) return np.memmap(self.mmappath.str, mode='r', dtype=self.meta.dtype, shape=shape)[self.indices] @memoized_property def mmap8bit(self): return self.mmap.view('uint8')[..., 1::2] @memoized_property def meta(self): # indices meta = TrajectoryChannelMeta.load(self.metapath.str) if not isinstance(self.indices, slice): meta.z = self.indices.sum() return meta @memoized_property def stat(self): # indices stat = np.load(self.statpath.str) return np.rec.fromrecords(stat, dtype=stat.dtype)[self.indices] @memoized_property def time(self): # indices return np.load(self.timepath.str)[self.indices] @property def duration(self): return self.time[-1] - self.time[0] @property def framerate(self): return len(self.mmap) / self.duration @memoized_property def alog(self): # indices alog = np.load(self.alogpath.str) return np.rec.fromrecords(alog, dtype=alog.dtype)[self.indices] @memoized_property def original_velocity(self): alog = np.load(self.alogpath.str) return np.rec.fromrecords(alog, dtype=alog.dtype).V def import_raw(self, tiff, timestamps, log): print 'Align log and timestamp...' log_aligned = log.align(timestamps) print 'Extracting metadata...' meta = TrajectoryChannelMeta(tiff.dtype.name, *tiff.shape) print 'Calculating basic statistics...' max = tiff.max(axis=(1, 2)) min = tiff.min(axis=(1, 2)) mean = tiff.mean(axis=(1, 2)) stat = np.rec.fromarrays([max, min, mean], names='MAX, MIN, MEAN') mmap = np.memmap(self.mmappath.str, mode='w+', dtype=tiff.dtype, shape=tiff.shape) print 'Write to disk...' mmap[:] = tiff[:] meta.save(self.metapath.str) np.save(self.statpath.str, stat) np.save(self.timepath.str, timestamps) np.save(self.alogpath.str, log_aligned) print 'Converting done!' return self def toDict(self): z, y, x = self.mmap.shape return dict(depth=z, height=y, width=x) def request_frame(self, index): return self.cmap8bit.to_rgba(self.mmap8bit[index], bytes=True).tostring()
class ScanboxMatView(ZeroDimensionArrayView): def __init__(self, path): self.path = Path(path).ensure_suffix('.mat') self.bound_sbx_path = self.path.with_suffix('.sbx') array = io.loadmat(self.path.str, squeeze_me=True).get('info') super(ScanboxMatView, self).__init__(array) @property def is_aligned(self): return self.path.name.startswith('Aligned') @property def sbxsize(self): return self.path.with_suffix('.sbx').size @property def sbxtime(self): return self.path.with_suffix('.sbx').created_at @property def sbxpath(self): return self.path.with_suffix('.sbx').relative_to(opt.scanbox_root) @property def iopath(self): return self.sbxpath.with_suffix('.io') @property def shape(self): return tuple(reversed((self.nframes, self.channels) + self.dimension)) @property def dimension(self): return Dimension(*self.sz) @property def nframes(self): nframes = int(self.sbxsize / self.recordsPerBuffer / self.dimension.width / 2 / self.channels) return nframes * (1 if self.scanmode else 2) @property def framerate(self): # recordsPerBuffer = self.originalRecordsPerBuffer \ # if self.is_aligned else self.recordsPerBuffer if self.is_aligned: try: recordsPerBuffer = self.originalRecordsPerBuffer except: recordsPerBuffer = self.recordsPerBuffer else: recordsPerBuffer = self.recordsPerBuffer rate = self.resfreq / recordsPerBuffer return rate if self.scanmode else rate * 2 @property def nchannels(self): return 2 if self.channels == 1 else 1 @property def factor(self): return 1 if self.channels == 1 else 2 @property def scanmodestr(self): return 'uni' if self.scanmode == 1 else 'bi' def get_max_idx(self, size): return int(size / self.recordsPerBuffer / self.sz[1] * self.factor / 4 - 1) def get_shape(self, size): return self.get_max_idx(size) + 1, self.recordsPerBuffer, self.sz[1] @property def recordsPerBuffer(self): rpb = self._dict.get('recordsPerBuffer') return rpb * 2 if self.scanmode is 0 else rpb # def __dir__(self): # quick and dirty: need to use descriptor set # return super(ScanboxInfoView, self).__dir__() + \ # 'path nchan factor framerate recordsPerBuffer sz'.split() def toDict(self): data = self.items() data['iopath'] = str(self.iopath) data['framerate'] = self.framerate data['frameratestr'] = str(self.framerate) + ' fps' data['sbxsize'] = self.sbxsize.str data['sbxtime'] = time.mktime(self.sbxtime.timetuple()) data['sbxpath'] = self.sbxpath.str data['nchannels'] = self.nchannels data['nframes'] = self.nframes data['nframesstr'] = str(self.nframes) + ' frames' data['scanmodestr'] = self.scanmodestr data['focal_pane_args'] = self.focal_pane_args return data @property def duration(self): return self.nframes / self.framerate # return '{s.nframes} frames at {s.framerate} fps is 00:01:14:01'.format(s=self) # duration = frame_count / frame_rate # @property # def originalRecordsPerBuffer(self): # obuf = self._dict.get('originalRecordsPerBuffer') # buf = self.recordsPerBuffer # print 'original: {}, normal: buf'.format(obuf, buf) # return obuf or buf @property def focal_pane_args(self): try: if self.volscan: _, _, n = map(int, self.otparam) else: n = 1 except: n = 1 try: if self.volscan: waves = list(map(int, self.otwave)) else: waves = [0] except: waves = [0] return dict(waves=waves, n=n) @property def memmap(self): # first channel shape = self.get_shape(self.bound_sbx_path.size) chans = np.memmap(self.bound_sbx_path.str, dtype='uint16', mode='r', shape=shape) return chans[0::mat.nchannels] @property def opened(self): return self.bound_sbx_path.open('rb')
class ScanimageChannel(object): cmap_name = 'jet' def __init__(self, path): self.path = Path(path) self.dcmap = DistortedColormap('jet', xmid=0.5, ymid=0.5) @property def mmappath(self): return self.path.with_suffix('.mmap.npy') @memoized_property def norm(self): return Normalize( vmin=self.stat.MIN.min()/256, vmax=self.stat.MAX.max()/256) @memoized_property def cmap8bit(self): return ScalarMappable(norm=self.norm, cmap=self.dcmap.distorted) @cmap8bit.invalidator def update_colormap(self, name, xmid, ymid): x = float(xmid) / 100 y = float(ymid) / 100 self.dcmap = DistortedColormap(name, xmid=x, ymid=y) @memoized_property def mmap(self): shape = (self.meta.z, self.meta.y, self.meta.x) return np.memmap(self.mmappath.str, mode='r', dtype=self.meta.dtype, shape=shape) @memoized_property def mmap8bit(self): return self.mmap.view('uint8')[..., 1::2] @property def metapath(self): return self.path.with_suffix('.meta.json') @memoized_property def meta(self): return ScanimageChannelMeta.load(self.metapath.str) @property def statpath(self): return self.path.with_suffix('.stat.npy') @memoized_property def stat(self): stat = np.load(self.statpath.str) return np.rec.fromrecords(stat, dtype=stat.dtype) def import_raw(self, tiff): print 'Drift correct...each step may take a few minutes...' drift = driftcorrect.getdrift3(tiff) print 'Got drift data...try to correct.' corr = driftcorrect.driftcorrect2(tiff, drift) print 'Extracting metadata...' meta = ScanimageChannelMeta(corr.dtype.name, *corr.shape) print 'Calculating basic statistics...' max = corr.max(axis=(1,2)) min = corr.min(axis=(1,2)) mean = corr.mean(axis=(1,2)) stat = np.rec.fromarrays([max, min, mean], names='MAX, MIN, MEAN') mmap = np.memmap(self.mmappath.str, mode='w+', dtype=corr.dtype, shape=corr.shape) print 'Write to disk...' mmap[:] = corr[:] meta.save(self.metapath.str) np.save(self.statpath.str, stat) print 'Converting done!' return self def toDict(self): z, y, x = self.mmap.shape return dict(depth=z, height=y, width=x) def request_frame(self, index): # frame = self.mmap8bit[index] # self.cmap8bit.set_clim(frame.min(), frame.max()) # return self.cmap8bit.to_rgba(frame, bytes=True).tostring() return self.cmap8bit.to_rgba(self.mmap8bit[index], bytes=True).tostring()
class ScanboxIO(object): session = None channel = None def __init__(self, path): self.path = Path(path).ensure_suffix('.io') self.session = SessionBoundNamespace(self.db_session_factory(), db.Workspace, db.ROI, db.Datatag, db.Condition, db.Trial) @property def is_there(self): return self.path.exists() @property def db_path(self): return self.path.joinpath('db.sqlite3').absolute() @property def db_session_factory(self): maker = db.get_sessionmaker(self.db_path) # sessionmaker can configure without bind(engine) # so setup event first. it's doable. # event.remove(maker, 'before_attach', db.SQLite3Base.before_attach) event.listen(maker, 'before_flush', db.before_flush) event.listen(maker, 'after_commit', db.after_commit) return maker def set_workspace(self, id): self.workspace_id = id return self @property def workspace(self): return self.session.Workspace.one_or_none(id=self.workspace_id) @property def mat(self): return ScanboxMatView(self.path.with_suffix('.mat')) @property def sbx(self): return ScanboxSBXView(self.path.with_suffix('.sbx')) @property def ephys(self): return ScanboxEphysView(self, self.path.with_suffix('.txt')) @property def attributes(self): error = None try: workspaces = self.session.Workspace.all() except Exception as e: notable = 'no such table' in str(e) nocolumn = 'no such column' in str(e) workspaces = [] error = dict(notable=notable, nocolumn=nocolumn, detail=str(e), type=type(e).__name__) return dict(hops=self.path.relative_to(opt.scanbox_root).parts, path=self.path.str, is_there=self.is_there, mat=self.mat, sbx=self.sbx, error=error, mode='uni' if self.mat.scanmode else 'bi', workspaces=workspaces) def get_channel(self, number): return ScanboxChannel(self.path.joinpath('{}.chan'.format(number))) def set_channel(self, number): self.channel = self.get_channel(number) return self def remove_io(self): self.path.rmtree() self.session = SessionBoundNamespace( # unnecessary implementation self.db_session_factory(), db.Workspace, db.ROI, db.Datatag, db.Condition, db.Trial) return self.attributes def import_raw(self): self.path.mkdir_if_none() db.recreate(self.db_path) self.session = SessionBoundNamespace(self.db_session_factory(), db.Workspace, db.ROI, db.Datatag, db.Condition, db.Trial) self.import_exp_as_condition() for chan in range(self.mat.nchannels): self.get_channel(chan).import_with_io(self) return self.attributes def import_exp_as_condition(self): s = glab() entity = s.query(ExperimentV1).filter_by( keyword=self.path.stem).one_or_none() if entity: print 'There is matching condition data in ED for {!r}'.format( self.path.stem) try: session = self.session._session with session.begin(): condition = db.Condition.from_expv1(entity) condition.trials.extend([ db.Trial.init_and_update(**trial) for trial in entity ]) session.add(condition) except Exception as e: print 'Condition import failed with reason below,', str(e) else: print 'There is no matching condition data in ED for {!r}'.format( self.path.stem) def upgrade_db_schema(self): db.upgrade(db.SQLite3Base.metadata, self.db_session_factory.kw.get('bind')) return self.attributes