def gen_trial_matrix2(generator, iwhisker=0, ifeatures=[5, 6]):
nfeat = len(ifeatures)
filecount = len(list(generator))
timecount = 0
for n in generator:
table = traj.MeasurementsTable(n)
data = table.asarray()
timecount = data[:, 1].max() + 1 + 10 #adding a fudge factor
break
im = nan * zeros((nfeat, filecount, timecount))
for i, n in enumerate(generator):
try:
print "[%5d of %5d] %s" % (i, filecount, n)
table = traj.MeasurementsTable(n)
data = table.asarray()
mask = data[:, 0] == iwhisker
# select whisker
t = data[mask, 1].astype(uint)
# get times
for j, ifeat in enumerate(ifeatures):
im[j, i, t] = data[mask, ifeat]
# get feature
except:
pass
return im
def load(vidname):
import reader, trace, traj
vname, wname, mname = fnames(vidname)
v = reader.Reader(vname, adjuststipple=0)
w = trace.Load_Whiskers(wname)
try:
m = traj.MeasurementsTable(mname)
except:
traceback.print_exc(file=sys.stdout)
sys.stdout.flush()
m = None
return v, w, m
def gen_trial_matrix_all_whiskers(n_whiskers=3, ifeatures=[5, 6]):
a = annotate_trials(r"F:\CuratedSessions",
filename=r"F:\CuratedSessions\all.trialtypes.pickle")
out = a.by_type_session()
n_trials = len(a.data)
n_trial_type = len(out.keys())
n_sessions = len(out.values()[0].keys())
def get_n_timepoints():
table = traj.MeasurementsTable(
list(seq2measurements([a.data.keys()[0]]))[0]) # -_-;
data = table.asarray()
return data[:, 1].max() + 10 # add a fudge factor
# each row: trial type id, session id, whisker id
index = zeros((n_trials * n_whiskers, 3), dtype=uint8)
im = nan * zeros(
(len(ifeatures), n_trials * n_whiskers, get_n_timepoints()))
session_index = {}
for i, k in enumerate(out.values()[0].iterkeys()):
session_index[k] = i
row = 0
for kTrialType, vTrialType in out.iteritems():
for kSession, vSession in vTrialType.iteritems():
for filename in seq2measurements(vSession):
print "[%5d of %5d] %s" % (row, im.shape[1], filename)
try:
data = traj.MeasurementsTable(filename).asarray()
for iWhisker in xrange(n_whiskers):
index[row, :] = [
int(kTrialType), session_index[kSession], iWhisker
]
mask = data[:, 0] == iWhisker
for i, ifeat in enumerate(ifeatures):
im[i, row, data[mask,
1].astype(uint32)] = data[mask,
ifeat]
row += 1
except IOError:
pass
return index, im
def build_histograms(fname, min_ms, max_ms, norm=True, dth_deg=2.5):
# Tried memoizing here. AFAICT, no speed up.
table = traj.MeasurementsTable(fname)
data = table.asarray()
out = {}
nframes = data[:, 1].max()
dt = frame_delta_ms(nframes)
wmask = (data[:, 0] == 0) # select whisker 0
tmask = (data[:,1]*dt >=min_ms) \
* (data[:,1]*dt <=max_ms) # select epoch
# angle of most posterior whisker
mask = wmask * tmask
nn, xx = histogram(awrap(data[mask, 5]),
arange(-180.0, 180.0, dth_deg),
density=norm)
out['angle'] = (xx[1:], nn)
# avg angle change between whiskers in the same frame
nwhiskers = int(data[:, 0].max()) + 1
nframes = int(data[:, 1].max()) + 1
angles = nan * zeros((nframes, nwhiskers))
for i in xrange(nwhiskers):
mask = (data[:, 0] == i) * tmask
angles[data[mask, 1].astype(int), i] = awrap(data[mask, 5])
dth = diff(angles, axis=1).mean(axis=1)
nn, xx = histogram(dth, arange(-180.0, 180.0, dth_deg), density=norm)
out['spread'] = (xx[1:], nn)
#all angles
mask = tmask * (data[:, 0] >= 0) * (
data[:, 0] <= 2) # actually only accept most posterior 3
nn, xx = histogram(awrap(data[mask, 5]), arange(-180.0, 180.0, dth_deg))
out['all'] = (xx[1:], nn)
return out
def proc(root):
import fnmatch
import os
def gen_names(root):
for r, dirnames, filenames in os.walk(root):
for filename in fnmatch.filter(filenames, '*.measurements'):
yield os.path.join(r, filename)
filecount = len(list(gen_names(root)))
count = 0.0
recall = 0.0
precision = 0.0
training_max = 0.0
training_min = 1.0
training_mean = 0.0
for i, n in enumerate(gen_names(root)):
if 1: #i < 5:
try:
print "[%5d of %5d] %s" % (i, filecount, n)
table = traj.MeasurementsTable(n)
recall += heuristic_recall(table)
precision += heuristic_precision(table)
t = training_set_fraction(table)
training_max = max(training_max, t)
training_min = min(training_min, t)
training_mean += t
count += 1.0
except:
pass
recall /= count
precision /= count
training_mean /= count
return locals()
wrowcmp = lambda a, b: wcmp(a[1], b[1])
for fid, wv in wvd.iteritems():
#print fid
seq = wid_sequence_from_frame(wv) #ordered wid's
labels, p, vp = model.viterbi_by_lookup(fid, seq)
tids = map(statemap.get, labels)
logp[fid] = p
vlogp[fid] = vp
for tid, wid in zip(tids, seq):
if not tid is None:
traj.setdefault(tid, {})[fid] = wid
return traj, logp, vlogp
if __name__ == '__main__':
import sys, traj, trace
name, whisker_src, measurements_src, dest = sys.argv
wvd = trace.Load_Whiskers(whisker_src)
table = traj.MeasurementsTable(measurements_src)
data = table.asarray()
traj = table.get_trajectories()
model = LeftRightModel()
model.train(wvd, traj, data=data)
traj_hmm, logp, vlogp = apply_model(wvd, model)
table.commit_trajectories(traj)
table.save(dest)
def itertrials(meta):
for fname in gen_measurements(meta):
table = traj.MeasurementsTable(fname)
yield fname, table.asarray()
def countof_frames(fname):
v = traj.MeasurementsTable(fname).asarray()[:, 1].max()
print fname, v
return int(v)
1] #know the order b.c. already sorted by file extenstion
tfile = args[0]
w = trace.Load_Whiskers(src)
data = numpy.array(list(summary.features(w, options.face)))
if tfile:
t, tid = load_trajectories(tfile)
summary.commit_traj_to_data_table(t, data)
elif all(
map(
lambda f: os.path.splitext(f)[-1] in
['.trajectories', '.measurements'], args)):
sources = dict(map(lambda f: (os.path.splitext(f)[-1], f), args))
try:
data = traj.MeasurementsTable(sources['.measurements']).asarray()
except KeyError:
raise UserException, "A .measurements file must be provided as one of the source files."
try:
t, tid = load_trajectories(sources['.trajectories'])
summary.commit_traj_to_data_table(t, data)
except KeyError:
pass
else:
raise parser.error(
"An unexpected combination of source file types was provided.")
table = traj.MeasurementsTable(data).update_velocities()
if os.path.splitext(dst)[-1] == '.mat':
table.save_to_matlab_file(dst)
def get_n_timepoints():
table = traj.MeasurementsTable(
list(seq2measurements([a.data.keys()[0]]))[0]) # -_-;
data = table.asarray()
return data[:, 1].max() + 10 # add a fudge factor
def gen_trial_matrix_summary(trialtypes=None, sessions=None):
a = annotate_trials(r"F:\CuratedSessions",
filename=r"F:\CuratedSessions\all.trialtypes.pickle")
out = a.by_type_session()
if trialtypes is None:
trialtypes = out.keys()
if sessions is None:
sessions = out.values()[0].keys()
def gen_trials():
for t in trialtypes:
for s in sessions:
for e in out.get(str(t), {}).get(s, []):
yield e
def count_trials():
return len(list(gen_trials()))
def get_n_timepoints():
table = traj.MeasurementsTable(
list(seq2measurements([a.data.keys()[0]]))[0]) # -_-;
data = table.asarray()
return data[:, 1].max() + 10 # add a fudge factor
ntrials = count_trials()
ntime = get_n_timepoints()
nfeat = 4 # average angle, angle spread, projected mean follicle position,
# projected follicle position spread
index = zeros((ntrials, 2), dtype=uint8)
im = nan * zeros((nfeat, ntrials, ntime))
session_index = {}
for i, k in enumerate(out.values()[0].iterkeys()):
session_index[k] = i
def gen_trial_index():
for t in trialtypes:
for s in sessions:
for e in out.get(str(t), {}).get(s, []):
yield int(t), session_index[s]
row = 0
for i, (filename, code) in enumerate(
zip(seq2measurements(gen_trials()), gen_trial_index())):
print "[%5d of %5d] %s" % (i, ntrials, filename)
try:
data = traj.MeasurementsTable(filename).asarray()
index[row, :] = code
nwhiskers = max(data[:, 0].astype(int)) + 1
count = zeros((nwhiskers, ntime))
working = zeros((nwhiskers, ntime))
mask = data[:, 0] >= 0
count[data[mask, 0].astype(int),
data[mask, 1].astype(int)] = 1 # [whisker,time] = 1
#angles
for iwhisker in xrange(nwhiskers):
mask = data[:, 0] == iwhisker
working[iwhisker, data[mask, 1].astype(int)] = data[mask, 5]
#mean angle
im[0, row, :] = working.sum(0) / count.sum(0)
#angle spread
im[1, row, :] = working.ptp(0)
#follicle position
#For leo's data, just use the xposition (column 7)
for iwhisker in xrange(nwhiskers):
mask = data[:, 0] == iwhisker
working[iwhisker, data[mask, 1].astype(int)] = data[mask, 7]
#mean follicle
im[2, row, :] = working.sum(0) / count.sum(0)
#angle follcile
im[3, row, :] = working.ptp(0)
row += 1
except IOError:
pass
return index, im
import matplotlib.pyplot as plt
import numpy as np, os.path, pandas
# Data location
session = '0509A_cropped_truncated_4'
side = 'top'
#~ session = '0527_cropped_truncated_5'; side = 'left'
whisk_rootdir = os.path.expanduser('~/mnt/bruno-nix/whisker_video/processed')
whisk_file = os.path.join(whisk_rootdir, session, session + '.whiskers')
measure_file = os.path.join(whisk_rootdir, session, session + '.measurements')
# Load the traces
frame2segment_id2whisker_seg = trace.Load_Whiskers(whisk_file)
# Load the correspondence between traces and identified whiskers
tmt = traj.MeasurementsTable(measure_file)
whisker_id2frame2segment_id = tmt.get_trajectories()
# Identify any missing frames
frames_l = frame2segment_id2whisker_seg.keys()
sorted_frames = np.sort(frames_l)
# Iterate over frames
rec_l = []
# Iterate over whiskers
# It looks like it numbers them from Bottom to Top for side == 'left'
# whiski colors them R, G, B
for wid, frame2segment_id in whisker_id2frame2segment_id.items():
# Iterate over frames
for frame, segment_id in frame2segment_id.items():
def save_measurements( name, *args ):
whiskers, trajectories, facehint = args[0:3]
traj.MeasurementsTable( {'whiskers':whiskers, 'facehint':facehint} ).commit_trajectories(trajectories).save(name)
def load_measurements( name ):
try:
return [traj.MeasurementsTable(name).get_trajectories(),0]
except IOError:
print "Couldn't open file. Creating: ", name
return {},0
warnings.simplefilter("default", UserWarning)
if __name__ == '__main__':
parser = optparse.OptionParser(usage="Usage: %prog source destination")
options, args = parser.parse_args()
if len(args) != 2:
parser.error(
"Expected source and destination file names but recieved %d arguments."
% len(args))
src, dst = args
if not os.path.exists(os.path.split(
dst)[0]): # check path, do this before doing long calculation
raise IOError, "Could not find destination file %s" % dst
#
# Get data from source
#
ext = os.path.splitext(src)[1]
if ext == '.whiskers':
w = trace.Load_Whiskers(src)
data = array(list(summary.features(w)))
elif ext == '.measurements':
data = traj.MeasurementsTable(src).asarray()
else:
raise IOError, "Source file extension not recognized. Got: %s" % ext
traj = summary._simpletraj(data)
save_trajectories(dst, traj)
