class TrajectorySession(object):
roi = None
opt = None
__repr__ = repr.auto_strict
def __init__(self, path):
self.path = Path(path).with_suffix('.session')
self.roi = HybridNamespace.from_path(self.path.joinpath('roi'))
self.opt = HybridNamespace.from_path(self.path.joinpath('opt'))
def toDict(self):
return dict(name=self.path.stem, path=self.path.str)
@property
def exists(self):
return self.path.is_dir()
def create(self):
self.path.mkdir()
def remove(self):
shutil.rmtree(self.path.str)
class ScanimageIO(object):
session_name = 'main'
r_value = 0.7
def __init__(self, path):
# wself.path = Path(str(path) + '.imported')
if '.imported' in str(path):
self.path = Path(path)
else:
self.path = Path(str(path) + '.imported')
@classmethod
def get_record(cls, rec_path):
return ScanimageRecord(rec_path)
@property
def exists(self):
return self.path.is_dir()
@memoized_property
def tiff(self):
tiffpath = self.path.with_suffix('.tif')
print 'Import from {}...Please allow a few minutes.'.format(
tiffpath.name)
return tifffile.imread(tiffpath.str)
def import_raw(self):
if self.exists:
return False
nchan = util.infer_nchannels(self.tiff)
if nchan:
self.create_package_path()
self.convert_channels(nchan)
else:
print 'Unable to import data.'
print 'Import done!'
return self.toDict()
def create_package_path(self):
self.path.mkdir()
print 'Path `{}` created.'.format(self.path.str)
def remove_package(self):
shutil.rmtree(self.path.str)
return self.toDict()
def convert_channels(self, nchan):
for index in range(nchan):
self.convert_channel(nchan, index)
def convert_channel(self, nchan, chan):
tiff = self.tiff[chan::nchan]
print 'Converting channel {}...({} frames.)'.format(chan, len(tiff))
return getattr(self, 'ch{}'.format(chan)).import_raw(tiff)
def toDict(self):
return dict(exists=self.exists,
path=self.path.str,
sessions=self.sessions)
@memoized_property
def ch0(self):
return ScanimageChannel(self.path.joinpath('ch0'))
@memoized_property
def ch1(self):
return ScanimageChannel(self.path.joinpath('ch1'))
@memoized_property
def channel(self):
chan = self.session.opt.setdefault('channel', 0)
return getattr(self, 'ch{}'.format(chan))
@channel.invalidator
def set_channel(self, channel):
self.session.opt['channel'] = channel
return self
@property
def nchannel(self):
return len(list(self.path.glob('ch?.mmap.npy')))
@property
def channel_numbers(self):
return list(range(self.nchannel))
@property
def channel_number(self):
return self.session.opt['channel']
colormap_index = 0
colormaps = ['jet', 'gray', 'gist_heat', 'afmhot', 'bone']
@property
def has_blank_and_flicker(self):
if self.db:
rec = self.db.rec
return [rec.blankOn, rec.flickerOn]
else:
return [None, None]
@property
def sfrequency(self):
return self.db.locator.sfrequencies.current if self.db else ''
# return self.session.opt.setdefault(# 'sfrequency', )
@property
def sfrequency_index(self):
return self.sfrequencies.index(
self.sfrequency) if self.sfrequency else 0
@property
def sfrequencies(self):
return self.db.locator.sfrequencies if self.db else []
@property
def orientations(self):
return self.db.orientations if self.db else []
# @sfrequency.invalidator
def set_sfrequency_index(self, sfreq_index):
self.sfrequencies.set_cursor(sfreq_index)
self.session.opt['sfrequency'] = self.sfrequencies.current
self.session.opt.save()
return dict(index=sfreq_index, value=self.sfrequency)
@memoized_property
def db(self):
return ScanimageDBAdaptor(self.session.ed) if self.session.ed else None
@property
def main_response(self):
return MainResponse.from_adaptor(self.channel.stat.MEAN, self.db)
@property
def sessions(self):
return map(ScanimageSession, sorted(self.path.ls('*.session')))
@memoized_property
def session(self):
return ScanimageSession(
self.path.joinpath(self.session_name).with_suffix('.session'))
@session.invalidator
def set_session(self, session_name):
self.session_name = session_name
return self
def upsert_roi(self, roi_kwargs):
gp = validate_guess_params(roi_kwargs.pop('guessParams', {}))
return self.session.roi.upsert(ROI(guess_params=gp, **roi_kwargs))
def invalidate_rois(self):
for roi in self.session.roi.values():
roi.invalidated = True
roi.blank = None
roi.flicker = None
roi.responses = {}
def export_sfreqfit_data_as_mat(self, rid):
roi = self.session.roi[rid]
sio = StringIO()
value = roi.sfreqfit.toDict()
io.savemat(sio, value)
return sio.getvalue()
def update_responses(self, id, heavy=False):
roi = self.session.roi[id]
trace = self.make_trace(roi)
if self.session.ed:
with self.session.roi.bulk_on:
try:
if self.db.locator.override_blank(True):
roi.blank = Orientation.from_adaptor(
'blank', trace, self.db)
if self.db.locator.override_flicker(True):
roi.flicker = Orientation.from_adaptor(
'flicker', trace, self.db)
finally:
self.db.locator.override()
for sf in self.db.locator.sfrequencies.loop():
response = ROIResponse.from_adaptor(roi, trace, self.db)
roi.responses[self.sfrequency] = response
roi.update_with_adaptor(self.db)
for sf, resp in roi.sorted_responses:
gp = roi.guess_params.get(sf) or resp.sog_initial_guess
print 'SoG custom guess for {}: {}'.format(sf, gp)
resp.update_fit_and_decay(roi, self.db, gp, heavy)
p_value = roi.anova_all.get('p')
if heavy:
if p_value < 0.01:
print('Computing bootstrap for preferred SF')
# '{} conditions...').format(
# len(self.db.orientations) * len(self.db.sfrequencies))
roi.update_bootstrap_for_sf(self.db)
peaksf = roi.sfreqfit.peak_sfreq.x
peak_resp = roi.responses[peaksf]
print 'Determine peak spatial frequency: {}'.format(
peaksf)
peak_resp.bootstrap = BootstrapResponse.from_adaptor(
peak_resp, self.db)
else:
print(
'P value is not less than 0.01. ({}) '
'Skip bootstrap.').format(p_value)
roi.invalidated = False
print 'Done updating response for ROI ' + str(roi.id)
print ' '
print ' '
return self.session.roi.upsert(roi)
else:
response = ROIResponse.from_scanbox(roi, trace)
roi.responses[self.sfrequency] = response
roi.update_with_adaptor(self.db)
roi.invalidated = False
return self.session.roi.upsert(roi)
sog_initial_guess = ((0, 1), (0, 1), (15, 60), (0, 0.01))
def make_delta_trace(self, roi, trace, dx=0, dy=0):
extras = self.session.roi.values()
extras.remove(roi)
main_trace, main_mask = roi.trace(trace, dx, dy)
neur_trace, neur_mask = roi.neuropil_trace(trace, extras, dx, dy)
return main_trace - neur_trace * self.r_value
def make_trace(self, roi, old=False): # checked same function
if old:
# print 'no centroid yet...perform old'
extras = self.session.roi.values()
extras.remove(roi)
main_trace, main_mask = roi.trace(self.channel.mmap)
neur_trace, neur_mask = roi.neuropil_trace(self.channel.mmap,
extras)
# neuropil_mask, roi_mask = roi.trim_bounding_mask(neur_mask, main_mask)
# roi.masks = dict(
# neuripil = neuropil_mask.tolist(),
# roi = roi_mask.tolist())
return main_trace - neur_trace * self.r_value
else:
# print 'has centroid...perform new'
vecs = roi.split_by_vectors(len(self.channel.mmap))
traces = np.split(self.channel.mmap, vecs.index[1:])
return np.concatenate([
self.make_delta_trace(roi, trace, dx, dy)
for dx, dy, trace in zip(vecs.x, vecs.y, traces)
])
def export_plots(self, id):
roi = self.session.roi[id]
return roi.export_plots_as_zip_for_download()
class ScanboxIO(object):
session = None
channel = None
def __init__(self, path):
self.path = Path(path).ensure_suffix('.io')
self.session = SessionBoundNamespace(self.db_session_factory(),
db.Workspace, db.ROI, db.Datatag,
db.Condition, db.Trial)
@property
def is_there(self):
return self.path.exists()
@property
def db_path(self):
return self.path.joinpath('db.sqlite3').absolute()
@property
def db_session_factory(self):
maker = db.get_sessionmaker(self.db_path)
# sessionmaker can configure without bind(engine)
# so setup event first. it's doable.
# event.remove(maker, 'before_attach', db.SQLite3Base.before_attach)
event.listen(maker, 'before_flush', db.before_flush)
event.listen(maker, 'after_commit', db.after_commit)
return maker
def set_workspace(self, id):
self.workspace_id = id
return self
@property
def workspace(self):
return self.session.Workspace.one_or_none(id=self.workspace_id)
@property
def mat(self):
return ScanboxMatView(self.path.with_suffix('.mat'))
@property
def sbx(self):
return ScanboxSBXView(self.path.with_suffix('.sbx'))
@property
def ephys(self):
return ScanboxEphysView(self, self.path.with_suffix('.txt'))
@property
def attributes(self):
error = None
try:
workspaces = self.session.Workspace.all()
except Exception as e:
notable = 'no such table' in str(e)
nocolumn = 'no such column' in str(e)
workspaces = []
error = dict(notable=notable,
nocolumn=nocolumn,
detail=str(e),
type=type(e).__name__)
return dict(hops=self.path.relative_to(opt.scanbox_root).parts,
path=self.path.str,
is_there=self.is_there,
mat=self.mat,
sbx=self.sbx,
error=error,
mode='uni' if self.mat.scanmode else 'bi',
workspaces=workspaces)
def get_channel(self, number):
return ScanboxChannel(self.path.joinpath('{}.chan'.format(number)))
def set_channel(self, number):
self.channel = self.get_channel(number)
return self
def remove_io(self):
self.path.rmtree()
self.session = SessionBoundNamespace( # unnecessary implementation
self.db_session_factory(), db.Workspace, db.ROI, db.Datatag,
db.Condition, db.Trial)
return self.attributes
def import_raw(self):
self.path.mkdir_if_none()
db.recreate(self.db_path)
self.session = SessionBoundNamespace(self.db_session_factory(),
db.Workspace, db.ROI, db.Datatag,
db.Condition, db.Trial)
self.import_exp_as_condition()
for chan in range(self.mat.nchannels):
self.get_channel(chan).import_with_io(self)
return self.attributes
def import_exp_as_condition(self):
s = glab()
entity = s.query(ExperimentV1).filter_by(
keyword=self.path.stem).one_or_none()
if entity:
print 'There is matching condition data in ED for {!r}'.format(
self.path.stem)
try:
session = self.session._session
with session.begin():
condition = db.Condition.from_expv1(entity)
condition.trials.extend([
db.Trial.init_and_update(**trial) for trial in entity
])
session.add(condition)
except Exception as e:
print 'Condition import failed with reason below,', str(e)
else:
print 'There is no matching condition data in ED for {!r}'.format(
self.path.stem)
def upgrade_db_schema(self):
db.upgrade(db.SQLite3Base.metadata,
self.db_session_factory.kw.get('bind'))
return self.attributes
class TrajectoryIO(object):
session_name = 'main'
def __init__(self, path):
self.path = Path(path).merge_or_join_suffix('.imported', on='.tif')
@property
def exists(self):
return self.path.is_dir()
@property
def as_record(self):
return dict(name=self.tiffpath.name,
user='******',
time=self.datetime.strftime('%H:%M:%S'),
host='IOS',
desc=self.desc,
mouse='Mouse 3',
package=self)
@property
def desc(self):
return '{}'.format(tifffile.format_size(self.tiffpath.stat().st_size))
@property
def datetime(self):
return datetime.fromtimestamp(float(self.tiffpath.stem))
@property
def tiffpath(self):
return self.path.with_suffix('.tif')
def toDict(self):
return dict(exists=self.exists,
path=self.path.str,
sessions=self.sessions)
@memoized_property
def tiff(self):
tiffpath = self.path.with_suffix('.tif')
file_size = tiffpath.lstat().st_size
print 'Import from {}...'.format(tiffpath)
print 'File size {:,} bytes.'.format(file_size)
return tifffile.imread(tiffpath.str)
def import_raw(self):
if self.exists:
return False
self.create_package_path()
print 'Looking for matching VR log file...'
log = TrajectoryLog.query(self.datetime)
# it can fail but never reports.
print 'Converting channel and timestamps...'
tss = np.fromfile(self.path.with_suffix('.csv').str,
sep='\n',
dtype='float64')
TrajectoryChannel(self.path.joinpath('channel')).import_raw(
self.tiff, tss, log)
print 'Import done!'
return self.toDict()
def create_package_path(self):
self.path.mkdir()
print 'Path `{}` created.'.format(self.path.str)
def remove_package(self):
shutil.rmtree(self.path.str)
return self.toDict()
@memoized_property
def channel(self):
tfilter = self.session.opt.get('filter')
if tfilter:
if tfilter._indices is not None:
return TrajectoryChannel(
self.path.joinpath('channel')).set_indices(
tfilter._indices)
return TrajectoryChannel(self.path.joinpath('channel'))
@property
def sessions(self):
return map(TrajectorySession, sorted(self.path.ls('*.session')))
@memoized_property
def session(self):
return TrajectorySession(
self.path.joinpath(self.session_name).with_suffix('.session'))
@session.invalidator
def set_session(self, session_name):
self.session_name = session_name
return self
@memoized_property
def alog(self): # aligned log in JSON format
return [
dict(e=e, x=x, y=y, v=v)
for e, x, y, v in self.channel.alog[['E', 'X', 'Y', 'V']]
]
@property
def main_response(self):
return TrajectoryResponse(self.channel.stat.MEAN)
def upsert_roi(self, roi_kwargs):
return self.session.roi.upsert(ROI(**roi_kwargs))
def upsert_filter(self, filter_kwargs):
tfilter = TrajectoryFilter(**filter_kwargs)
tfilter._indices = tfilter.make_indices(self.channel.original_velocity)
rv = self.session.opt.upsert(tfilter)
for roi in self.session.roi.values():
roi.invalidated = True
self.session.roi.save()
return rv
def make_response(self, id):
roi = self.session.roi[id]
extras = self.session.roi.values()
extras.remove(roi)
main_trace, main_mask = roi.trace(self.channel.mmap)
neur_trace, neur_mask = roi.neuropil_trace(self.channel.mmap, extras)
trace = main_trace - neur_trace * 0.7
roi.invalidated = False
roi.response = TrajectoryResponse(trace)
return self.session.roi.upsert(roi)
@memoized_property
def velocity_stat(self):
return dict(
max=self.channel.alog.V.max(),
mean=self.channel.alog.V.mean(),
min=self.channel.alog.V.min(),
)
class ScanimageSession(object):
roi = None
opt = None
__repr__ = repr.auto_strict
def __init__(self, path):
self.path = Path(path).with_suffix('.session')
self.package, self.mouse, self.date, self.user = self.path.parts[::-1][
1:5]
self.roi = HybridNamespace.from_path(self.path.joinpath('roi'))
self.opt = HybridNamespace.from_path(self.path.joinpath('opt'))
def toDict(self):
return dict(name=self.path.stem, path=self.path.str)
@property
def glams_session(self):
return manager.get('db').section('glams')()
@property
def ed_session(self):
return manager.get('db').section('ed')()
def query_experiment_db(self):
mouse = self.glams_session.query(Mice).filter_by(name=self.mouse).one()
return self.ed_session.query(VisStim2P).filter_by(
mouse_id=mouse.id,
date=self.datetime.date(),
filename=self.package.rstrip('.imported'))
@property
def datetime(self):
return datetime.strptime(self.date, '%Y.%m.%d')
@memoized_property
def ed(self):
# return Path('tmp/Dario/2016.01.27/r.151117.3/DM9_RbV1_Contra004004.pickle').load_pickle()
# return Path('ed.2015.12.02.bV1_Contra_004.pickle').load_pickle()
try:
return self.query_experiment_db().one()
except:
print 'NONE DB ENTITY'
return None
@property
def has_ed(self):
return bool(self.query_experiment_db().count())
@property
def exists(self):
return self.path.is_dir()
def create(self):
print 'create session, mkdir', self.path
self.path.mkdir()
def remove(self):
print 'remove session, rmtree', self.path
shutil.rmtree(self.path.str)
def get_rois_json_safe(self):
rois = []
total = len(self.roi)
for index, rv in enumerate(self.roi.values()):
print 'Check {}/{} ROI...'.format(index + 1, total)
try:
json.loads(json.dumps(rv)) # verify
except Exception as e:
print 'ROI #{} has problem, "{}", SKIP LOADING!'.format(
rv.id, e)
rv.error = str(e)
rv.blank = None
rv.flicker = None
rv.responses = {}
rois.append(rv)
else:
rois.append(rv)
return rois