def get_sessionmaker(dbpath, echo=True, autocommit=True, **kw): from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker path = Path(dbpath) engine = create_engine('sqlite:///{}'.format(dbpath), echo=echo, **kw) if path.is_file() else recreate('', echo=echo) return sessionmaker(engine, autocommit=autocommit)
def __init__(self, path, n_focal_pane=1, c_focal_pane=0): self.n_focal_pane = n_focal_pane self.c_focal_pane = c_focal_pane self.path = Path(path).ensure_suffix('.chan') self.channel = int(self.path.stem) self.maxppath = self.path.join_suffixes('.maxp.npy') self.mmappath = self.path.join_suffixes('.mmap.npy') self.statpath = self.path.join_suffixes('.stat.npy') self.metapath = self.path.join_suffixes('.meta.json') self.cmap = colormaps['Jet'] self.min_val = 0 self.max_val = 255
def upgrade_sqlite(path): """ for io in ScanboxIO.iter_every_io(): print io.db_path upgrade_sqlite(io.db_path) """ import shutil path = Path(path) Session = open_sqlite(path.str) meta = schema.SQLite3Base.metadata bind = Session.kw.get('bind') shutil.copy2(path.str, path.with_suffix('.backup.sqlite3').str) schema.fix_incremental(meta, bind) return Session
def make_path(self): now = time.strftime('%Y-%m-%dT%H-%M-%S', time.localtime()) self.path = Path(self.basedir, now) try: os.makedirs(self.path.str) except Exception as e: raise Exception('Failed creating a base directory: ' + str(e))
class DataStoreLocator(object): def __init__(self, path, glob='*', delta=timedelta(180)): self.path = Path(path).resolve() self.glob = glob self.delta = delta self.now = datetime.now() def item_is_within(self, item): return self.now - datetime.fromtimestamp( item.stat().st_ctime) < self.delta @property def items_within(self): return (item for item in self.path.glob(self.glob) if self.item_is_within(item)) @property def rendered(self): sbxs, dirs = [], [] for item in self.items_within: if item.is_dir() and not item.suffix == '.io': dirs.append(self.render_dir(item)) elif item.is_file() and item.suffix == '.sbx' and item.with_suffix( '.mat').is_file(): sbxs.append(self.render_sbx(item)) return dict(sbxs=sbxs, dirs=dirs) def render_dir(self, item): return dict(name=item.name, ctime=item.stat().st_ctime) def render_sbx(self, item): return dict(name=item.stem, ctime=item.stat().st_ctime, path=item.str, size=item.size.str)
class ScanimageRecord(object): """ For `compatible_path`, the path should be formed like below. For example, '/Volumes/Data/Recordings/2P1/Dario/2015.11.20/x.151101.4/ref_p19_005.tif', This will be considered as, {whatever_basepath}/{experimenter}/{date}/{mousename}/{imagename} So the directory structure always will matter. """ def __init__(self, compatible_path): self.tiff_path = Path(compatible_path).with_suffix('.tif') self.package_path = Path(compatible_path).with_suffix('.imported') self.mouse, self.date, self.user = self.tiff_path.parts[::-1][1:4] def toDict(self): return dict(user=self.user, mouse=self.mouse, date=self.date, desc=self.desc, name=self.tiff_path.stem, package=self.package) @memoized_property def package(self): return ScanimageIO(self.package_path) @property def desc(self): return '{}'.format( SizeUnit(self.tiff_path.stat().st_size) # tifffile.format_size(self.tiff_path.stat().st_size) )
class TrajectorySession(object): roi = None opt = None __repr__ = repr.auto_strict def __init__(self, path): self.path = Path(path).with_suffix('.session') self.roi = HybridNamespace.from_path(self.path.joinpath('roi')) self.opt = HybridNamespace.from_path(self.path.joinpath('opt')) def toDict(self): return dict(name=self.path.stem, path=self.path.str) @property def exists(self): return self.path.is_dir() def create(self): self.path.mkdir() def remove(self): shutil.rmtree(self.path.str)
def ephemeral(profile): from pacu.util import identity from pacu.core.model import fixture l = manager.instance('log') resource = identity.formattempfile('%s-db-engine-ephemeral.db') engine = _create_sqlite_engine(profile.uri + resource, profile.echo.bool) engine.__pacu_protect__ = profile.PROTECT.bool s = get_scoped(engine) if not Path(resource).is_file(): fixture.base.setup(s) l.info('Tables of ephemeral db has initialized.') return s
def import_ephys(filepath): """ filepath = '/Volumes/Users/ht/dev/current/pacu/tmp/Jack/jzg1/day1/day1_000_007.txt' risings = import_ephys(filepath) """ path = Path(filepath) if not path.is_file(): raise NoRawDataError('Can not find raw data {!r}'.format(path.name)) try: raw = np.genfromtxt(path.str, delimiter=' ', names=['TTL', 'ON'], dtype='b') first_frame = find_nth_rising(raw['TTL']) data = raw[first_frame:] data['ON'][data['ON'] == 60] = 1 edges = np.diff(data['TTL']) > 0 ed_indices = np.where(edges)[0] + 1 on_chunks = np.split(data['ON'], ed_indices) return np.array([chunk.any() for chunk in on_chunks]) except Exception as e: raise RawDataParseError(e)
class Scanbox3IO(object): """ locates directory dir/ meta.mat mmap.npy? """ # has mat # has sbx # others def __init__(self, path): self.path = Path(path) # without_suffixes '.mat', '.sbx' def __repr__(self): return '{}({!r})'.format(type(self).__name__, self.path.str) @property def matpath(self): return self.path.with_suffix('.mat') @property def sbxpath(self): return self.path.with_suffix('.sbx') @memoized_property def mat(self): return ZeroDimensionArrayView.from_mat(self.matpath.str) @memoized_property def sbx(self): return @memoized_property def mmap(self): pass
class ScanboxSBXView(object): def __init__(self, path): self.path = Path(path) # def toDict(self): # return dict( # name = self.path.stem, # user = '******', # time = str(self.path.created_at), # host = '2P', # desc = '', # mouse = self.path.size.str, # package = 'package', # ) def toDict(self): return dict( path=self.path.relative_to(opt.scanbox_root).str, time=str(self.path.created_at), size=self.path.size.str, )
class ScanboxEphysView(object): def __init__(self, io, path): self.io = io self.path = Path(path).with_suffix('.txt') self.npy = self.path.with_suffix('.io').joinpath('ephys.npy') @property def trace(self): return np.load(self.npy.str).tolist() if self.npy.is_file() else [] def toDict(self): return dict(trace=self.trace) def reimport(self): print 'Importing ephys data. This may take a few minutes.' np.save(self.npy.str, import_ephys(self.path)) print 'Import Success!' return self.toDict() def remove(self): self.npy.unlink() return self.toDict()
from pacu.util.path import Path import ujson as json raw_json = Path.here('all.json').read() features = json.loads(raw_json)
import re import os import time from tifffile import TiffWriter from pacu.util.path import Path from pacu.core.svc.andor.handler.base import BaseHandler from pacu.util.compat import str re_filename = re.compile(r'^\w+$') users_desktop = Path(os.path.expanduser('~'), 'Desktop') ip1_datapath = Path('D:', 'data') class WriterHandler(BaseHandler): def sync_name(self, member, filedir, filename): path = ip1_datapath.joinpath(member, filedir) if not path.is_dir(): os.makedirs(path.str) self.tifpath = path.joinpath(filename).with_suffix('.tif') self.csvpath = path.joinpath(filename).with_suffix('.csv') return True # self.ready() def check(self, filename): pass def ready(self): if self.tifpath.is_file() or self.csvpath.is_file(): raise Exception( 'Filename already exists. Please provide new one...')
def from_metadata_filename(cls, filename, meta): sbxpath = Path(filename).with_suffix('.sbx') if sbxpath.is_file(): return cls(sbxpath.str, meta)
class TrajectoryChannel(object): indices = slice(None) cmap_name = 'jet' def __init__(self, path): self.path = Path(path) def set_indices(self, indices): self.indices = indices return self @property def alogpath(self): return self.path.with_suffix('.alog.npy') @property def mmappath(self): return self.path.with_suffix('.mmap.npy') @property def timepath(self): return self.path.with_suffix('.time.npy') @property def statpath(self): return self.path.with_suffix('.stat.npy') @property def metapath(self): return self.path.with_suffix('.meta.json') @memoized_property def cmap8bit(self): norm = Normalize(vmin=self.stat.MIN.min() / 256, vmax=self.stat.MAX.max() / 256) return ScalarMappable(norm=norm, cmap=plt.get_cmap(self.cmap_name)) @cmap8bit.invalidator def set_cmap(self, which=0): name = ['jet', 'gray', 'hot', 'hsv'][int(which)] self.cmap_name = name @memoized_property def mmap(self): # indices meta = TrajectoryChannelMeta.load(self.metapath.str) shape = (meta.z, meta.y, meta.x) return np.memmap(self.mmappath.str, mode='r', dtype=self.meta.dtype, shape=shape)[self.indices] @memoized_property def mmap8bit(self): return self.mmap.view('uint8')[..., 1::2] @memoized_property def meta(self): # indices meta = TrajectoryChannelMeta.load(self.metapath.str) if not isinstance(self.indices, slice): meta.z = self.indices.sum() return meta @memoized_property def stat(self): # indices stat = np.load(self.statpath.str) return np.rec.fromrecords(stat, dtype=stat.dtype)[self.indices] @memoized_property def time(self): # indices return np.load(self.timepath.str)[self.indices] @property def duration(self): return self.time[-1] - self.time[0] @property def framerate(self): return len(self.mmap) / self.duration @memoized_property def alog(self): # indices alog = np.load(self.alogpath.str) return np.rec.fromrecords(alog, dtype=alog.dtype)[self.indices] @memoized_property def original_velocity(self): alog = np.load(self.alogpath.str) return np.rec.fromrecords(alog, dtype=alog.dtype).V def import_raw(self, tiff, timestamps, log): print 'Align log and timestamp...' log_aligned = log.align(timestamps) print 'Extracting metadata...' meta = TrajectoryChannelMeta(tiff.dtype.name, *tiff.shape) print 'Calculating basic statistics...' max = tiff.max(axis=(1, 2)) min = tiff.min(axis=(1, 2)) mean = tiff.mean(axis=(1, 2)) stat = np.rec.fromarrays([max, min, mean], names='MAX, MIN, MEAN') mmap = np.memmap(self.mmappath.str, mode='w+', dtype=tiff.dtype, shape=tiff.shape) print 'Write to disk...' mmap[:] = tiff[:] meta.save(self.metapath.str) np.save(self.statpath.str, stat) np.save(self.timepath.str, timestamps) np.save(self.alogpath.str, log_aligned) print 'Converting done!' return self def toDict(self): z, y, x = self.mmap.shape return dict(depth=z, height=y, width=x) def request_frame(self, index): return self.cmap8bit.to_rgba(self.mmap8bit[index], bytes=True).tostring()
def __init__(self, path): self.path = Path(path).with_suffix('.session') self.roi = HybridNamespace.from_path(self.path.joinpath('roi')) self.opt = HybridNamespace.from_path(self.path.joinpath('opt'))
def __init__(self, path): # wself.path = Path(str(path) + '.imported') if '.imported' in str(path): self.path = Path(path) else: self.path = Path(str(path) + '.imported')
from pacu.util.path import Path from pacu.core.scanbox import adapter localpath = Path('/Volumes/Users/ht/tmp/pysbx-data') localsbx = [] for path in localpath.rglob('*.mat'): try: localsbx.append(adapter.get_meta(path.str)) except Exception as e: print 'Unable to read {}'.format(path.str) else: print 'Data loaded: {}'.format(path.str) def get(req): data = [ dict(uid=index, text=Path(sbx.raw.filename).stem) for index, sbx in enumerate(localsbx) ] return dict(data=data)
def __init__(self, io, path): self.io = io self.path = Path(path).with_suffix('.txt') self.npy = self.path.with_suffix('.io').joinpath('ephys.npy')
def __init__(self, path): self.path = Path(path).merge_or_join_suffix('.imported', on='.tif')
class TrajectoryIO(object): session_name = 'main' def __init__(self, path): self.path = Path(path).merge_or_join_suffix('.imported', on='.tif') @property def exists(self): return self.path.is_dir() @property def as_record(self): return dict(name=self.tiffpath.name, user='******', time=self.datetime.strftime('%H:%M:%S'), host='IOS', desc=self.desc, mouse='Mouse 3', package=self) @property def desc(self): return '{}'.format(tifffile.format_size(self.tiffpath.stat().st_size)) @property def datetime(self): return datetime.fromtimestamp(float(self.tiffpath.stem)) @property def tiffpath(self): return self.path.with_suffix('.tif') def toDict(self): return dict(exists=self.exists, path=self.path.str, sessions=self.sessions) @memoized_property def tiff(self): tiffpath = self.path.with_suffix('.tif') file_size = tiffpath.lstat().st_size print 'Import from {}...'.format(tiffpath) print 'File size {:,} bytes.'.format(file_size) return tifffile.imread(tiffpath.str) def import_raw(self): if self.exists: return False self.create_package_path() print 'Looking for matching VR log file...' log = TrajectoryLog.query(self.datetime) # it can fail but never reports. print 'Converting channel and timestamps...' tss = np.fromfile(self.path.with_suffix('.csv').str, sep='\n', dtype='float64') TrajectoryChannel(self.path.joinpath('channel')).import_raw( self.tiff, tss, log) print 'Import done!' return self.toDict() def create_package_path(self): self.path.mkdir() print 'Path `{}` created.'.format(self.path.str) def remove_package(self): shutil.rmtree(self.path.str) return self.toDict() @memoized_property def channel(self): tfilter = self.session.opt.get('filter') if tfilter: if tfilter._indices is not None: return TrajectoryChannel( self.path.joinpath('channel')).set_indices( tfilter._indices) return TrajectoryChannel(self.path.joinpath('channel')) @property def sessions(self): return map(TrajectorySession, sorted(self.path.ls('*.session'))) @memoized_property def session(self): return TrajectorySession( self.path.joinpath(self.session_name).with_suffix('.session')) @session.invalidator def set_session(self, session_name): self.session_name = session_name return self @memoized_property def alog(self): # aligned log in JSON format return [ dict(e=e, x=x, y=y, v=v) for e, x, y, v in self.channel.alog[['E', 'X', 'Y', 'V']] ] @property def main_response(self): return TrajectoryResponse(self.channel.stat.MEAN) def upsert_roi(self, roi_kwargs): return self.session.roi.upsert(ROI(**roi_kwargs)) def upsert_filter(self, filter_kwargs): tfilter = TrajectoryFilter(**filter_kwargs) tfilter._indices = tfilter.make_indices(self.channel.original_velocity) rv = self.session.opt.upsert(tfilter) for roi in self.session.roi.values(): roi.invalidated = True self.session.roi.save() return rv def make_response(self, id): roi = self.session.roi[id] extras = self.session.roi.values() extras.remove(roi) main_trace, main_mask = roi.trace(self.channel.mmap) neur_trace, neur_mask = roi.neuropil_trace(self.channel.mmap, extras) trace = main_trace - neur_trace * 0.7 roi.invalidated = False roi.response = TrajectoryResponse(trace) return self.session.roi.upsert(roi) @memoized_property def velocity_stat(self): return dict( max=self.channel.alog.V.max(), mean=self.channel.alog.V.mean(), min=self.channel.alog.V.min(), )
def TrajectoryIOFetcher(recording, trial, session): root = manager.instance('opt').trajectory_root path = Path(root).joinpath(recording, trial) return TrajectoryIO(path).set_session(session)
def __init__(self, path): self.path = Path(path)
def __init__(self, path): self.path = Path(path) # without_suffixes '.mat', '.sbx'
def __init__(self, compatible_path): self.tiff_path = Path(compatible_path).with_suffix('.tif') self.package_path = Path(compatible_path).with_suffix('.imported') self.mouse, self.date, self.user = self.tiff_path.parts[::-1][1:4]
from __future__ import absolute_import import atexit from pacu.util.path import Path from pacu.util.newtype.incremental_hash import IncrementalHash try: from ctypes import wintypes as ctypes from ctypes import windll as cdll lib_basepath = Path.here('win') except: import ctypes from ctypes import cdll lib_basepath = Path.here('linux') class CDLLDescriptor(IncrementalHash): dll = None def __get__(self, inst, type): return self.dll if inst else self def __key_set__(self, key): try: self.dll = cdll.LoadLibrary(lib_basepath.joinpath(key).str) except OSError as e: print 'CDLL OSError:', e self.dll = NotImplemented return key
class ScanimageIO(object): session_name = 'main' r_value = 0.7 def __init__(self, path): # wself.path = Path(str(path) + '.imported') if '.imported' in str(path): self.path = Path(path) else: self.path = Path(str(path) + '.imported') @classmethod def get_record(cls, rec_path): return ScanimageRecord(rec_path) @property def exists(self): return self.path.is_dir() @memoized_property def tiff(self): tiffpath = self.path.with_suffix('.tif') print 'Import from {}...Please allow a few minutes.'.format( tiffpath.name) return tifffile.imread(tiffpath.str) def import_raw(self): if self.exists: return False nchan = util.infer_nchannels(self.tiff) if nchan: self.create_package_path() self.convert_channels(nchan) else: print 'Unable to import data.' print 'Import done!' return self.toDict() def create_package_path(self): self.path.mkdir() print 'Path `{}` created.'.format(self.path.str) def remove_package(self): shutil.rmtree(self.path.str) return self.toDict() def convert_channels(self, nchan): for index in range(nchan): self.convert_channel(nchan, index) def convert_channel(self, nchan, chan): tiff = self.tiff[chan::nchan] print 'Converting channel {}...({} frames.)'.format(chan, len(tiff)) return getattr(self, 'ch{}'.format(chan)).import_raw(tiff) def toDict(self): return dict(exists=self.exists, path=self.path.str, sessions=self.sessions) @memoized_property def ch0(self): return ScanimageChannel(self.path.joinpath('ch0')) @memoized_property def ch1(self): return ScanimageChannel(self.path.joinpath('ch1')) @memoized_property def channel(self): chan = self.session.opt.setdefault('channel', 0) return getattr(self, 'ch{}'.format(chan)) @channel.invalidator def set_channel(self, channel): self.session.opt['channel'] = channel return self @property def nchannel(self): return len(list(self.path.glob('ch?.mmap.npy'))) @property def channel_numbers(self): return list(range(self.nchannel)) @property def channel_number(self): return self.session.opt['channel'] colormap_index = 0 colormaps = ['jet', 'gray', 'gist_heat', 'afmhot', 'bone'] @property def has_blank_and_flicker(self): if self.db: rec = self.db.rec return [rec.blankOn, rec.flickerOn] else: return [None, None] @property def sfrequency(self): return self.db.locator.sfrequencies.current if self.db else '' # return self.session.opt.setdefault(# 'sfrequency', ) @property def sfrequency_index(self): return self.sfrequencies.index( self.sfrequency) if self.sfrequency else 0 @property def sfrequencies(self): return self.db.locator.sfrequencies if self.db else [] @property def orientations(self): return self.db.orientations if self.db else [] # @sfrequency.invalidator def set_sfrequency_index(self, sfreq_index): self.sfrequencies.set_cursor(sfreq_index) self.session.opt['sfrequency'] = self.sfrequencies.current self.session.opt.save() return dict(index=sfreq_index, value=self.sfrequency) @memoized_property def db(self): return ScanimageDBAdaptor(self.session.ed) if self.session.ed else None @property def main_response(self): return MainResponse.from_adaptor(self.channel.stat.MEAN, self.db) @property def sessions(self): return map(ScanimageSession, sorted(self.path.ls('*.session'))) @memoized_property def session(self): return ScanimageSession( self.path.joinpath(self.session_name).with_suffix('.session')) @session.invalidator def set_session(self, session_name): self.session_name = session_name return self def upsert_roi(self, roi_kwargs): gp = validate_guess_params(roi_kwargs.pop('guessParams', {})) return self.session.roi.upsert(ROI(guess_params=gp, **roi_kwargs)) def invalidate_rois(self): for roi in self.session.roi.values(): roi.invalidated = True roi.blank = None roi.flicker = None roi.responses = {} def export_sfreqfit_data_as_mat(self, rid): roi = self.session.roi[rid] sio = StringIO() value = roi.sfreqfit.toDict() io.savemat(sio, value) return sio.getvalue() def update_responses(self, id, heavy=False): roi = self.session.roi[id] trace = self.make_trace(roi) if self.session.ed: with self.session.roi.bulk_on: try: if self.db.locator.override_blank(True): roi.blank = Orientation.from_adaptor( 'blank', trace, self.db) if self.db.locator.override_flicker(True): roi.flicker = Orientation.from_adaptor( 'flicker', trace, self.db) finally: self.db.locator.override() for sf in self.db.locator.sfrequencies.loop(): response = ROIResponse.from_adaptor(roi, trace, self.db) roi.responses[self.sfrequency] = response roi.update_with_adaptor(self.db) for sf, resp in roi.sorted_responses: gp = roi.guess_params.get(sf) or resp.sog_initial_guess print 'SoG custom guess for {}: {}'.format(sf, gp) resp.update_fit_and_decay(roi, self.db, gp, heavy) p_value = roi.anova_all.get('p') if heavy: if p_value < 0.01: print('Computing bootstrap for preferred SF') # '{} conditions...').format( # len(self.db.orientations) * len(self.db.sfrequencies)) roi.update_bootstrap_for_sf(self.db) peaksf = roi.sfreqfit.peak_sfreq.x peak_resp = roi.responses[peaksf] print 'Determine peak spatial frequency: {}'.format( peaksf) peak_resp.bootstrap = BootstrapResponse.from_adaptor( peak_resp, self.db) else: print( 'P value is not less than 0.01. ({}) ' 'Skip bootstrap.').format(p_value) roi.invalidated = False print 'Done updating response for ROI ' + str(roi.id) print ' ' print ' ' return self.session.roi.upsert(roi) else: response = ROIResponse.from_scanbox(roi, trace) roi.responses[self.sfrequency] = response roi.update_with_adaptor(self.db) roi.invalidated = False return self.session.roi.upsert(roi) sog_initial_guess = ((0, 1), (0, 1), (15, 60), (0, 0.01)) def make_delta_trace(self, roi, trace, dx=0, dy=0): extras = self.session.roi.values() extras.remove(roi) main_trace, main_mask = roi.trace(trace, dx, dy) neur_trace, neur_mask = roi.neuropil_trace(trace, extras, dx, dy) return main_trace - neur_trace * self.r_value def make_trace(self, roi, old=False): # checked same function if old: # print 'no centroid yet...perform old' extras = self.session.roi.values() extras.remove(roi) main_trace, main_mask = roi.trace(self.channel.mmap) neur_trace, neur_mask = roi.neuropil_trace(self.channel.mmap, extras) # neuropil_mask, roi_mask = roi.trim_bounding_mask(neur_mask, main_mask) # roi.masks = dict( # neuripil = neuropil_mask.tolist(), # roi = roi_mask.tolist()) return main_trace - neur_trace * self.r_value else: # print 'has centroid...perform new' vecs = roi.split_by_vectors(len(self.channel.mmap)) traces = np.split(self.channel.mmap, vecs.index[1:]) return np.concatenate([ self.make_delta_trace(roi, trace, dx, dy) for dx, dy, trace in zip(vecs.x, vecs.y, traces) ]) def export_plots(self, id): roi = self.session.roi[id] return roi.export_plots_as_zip_for_download()
def get(req): data = [ dict(uid=index, text=Path(sbx.raw.filename).stem) for index, sbx in enumerate(localsbx) ] return dict(data=data)
def ScanimageIOFetcher(year, month, day, mouse, image, session): root = manager.instance('opt').scanimage_root date = '{}.{:2}.{:2}'.format(year, month, day) path = Path(root).joinpath(date, mouse, image) return ScanimageIO(path).set_session(session)