def writeFile(f,fname='TestMe',sps=None,start=0):
ndoc = nmpml.blankDocument()
if sps:
tf = mdat.newData(f, {'Name':fname, 'SampleType':'timeseries', 'SamplesPerSecond':sps, "StartTime":start})
else:
tf = mdat.newData(f, {'Name':fname, 'SampleType':'function'})
ndoc.newElement(tf)
io.write(ndoc, fname+".mdat")
def tffmax(ds, usamp=True):
ftt.scanForTFValues(ds, window=.2, minFreq=4.9, maxFreq = 250.0)
tfv = ds.getSubData('/ftvals')
tf = _cleanupTFV(tfv.getData())
if tf.shape[0]==0:
print("No values in fmax tf")
return None
if usamp:
sv = tf[:,0].min()
tf = uniformsample(tf, 1.0)
dat = mdat.newData(tf, {'SampleType':'timeseries', 'SamplesPerSecond':1.0, "StartTime":sv})
else:
dat = mdat.newData(tf, {'SampleType':'function'})
return dat
def getSinTF(dname):
dp = getDataPairs(dname)
for exp in dp:
dpl = dp[exp]
print "Analyzing Sin Data for experiment %s" % exp
dpl = [p for p in dpl if p[0][10]=='S' and not p[0][11]=='T']
ds = assemble(dpl, dname)
dat = ds.getData()
dat[:,1] = cal._applyfilter(dat[:,1], MFF)
# dat[:,0] = smooth(dat[:,0]) #This now occurs in assemble BEFORE resampling
ds.datinit(dat, {"SampleType":"timeseries", "StartTime":0.0,
"Labels":["HairPosition","MicroFlownVelocity"], "SamplesPerSecond":SIGFS})
nfn = '%s_compositSinData.mdat' % exp
io.write(ds, nfn, newdoc=True)
tfds = ctf.tffmax(ds, False)
if not tfds:
continue
tfds.data[:,2]+=pi
tfds.data[:,2] = smoothphase(tfds.data[:,2])
io.write(tfds, "%s_SinTF_Function.mdat" % exp, newdoc=True)
tf = tfds.getData()
tf = row_stack( [array([[0,0,0]]), tf, array([[250,tf[-1,1],tf[-1,2]]])])
tf = ctf.uniformsample(tf, 1.0)
tfds = miendata.newData(tf, {'Name':exp+"SinTF", 'SampleType':'timeseries', 'SamplesPerSecond':1.0, "StartTime":0})
io.write(tfds, "%s_SinTF_ResampledTimeseries.mdat" % exp, newdoc=True)
def assemble(filepairs, dname):
dat_all = []
for fp in filepairs:
bfn = os.path.join(dname, fp[0])
afn = os.path.join(dname, fp[1])
dbin = io.read(bfn).getElements("Data")[0]
daos = io.read(afn).getElements("Data")[0]
trig= dbin.getData()[:,TRIGCHAN]
ind = argmax(trig[1:] - trig[:-1])+1
mfd = dbin.getData()[ind:,MFCHAN]
dbin.datinit(mfd, {"SampleType":"timeseries", "StartTime":0.0,
"Labels":["MicroFlownVoltage"], "SamplesPerSecond":dbin.fs()})
tsd = smooth(daos.getData(),daos.attrib("SamplesPerSecond")) #remove clicks BEFORE resampling
daos.datinit(tsd, {"SampleType":"timeseries", "StartTime":0.0,
"Labels":["HairPosition"], "SamplesPerSecond":daos.fs()})
resample(dbin, SIGFS)
resample(daos, SIGFS)
dat2 = dbin.getData()
dat1 = daos.getData()
if dat1.shape[0] < dat2.shape[0]:
dat2 = dat2[:dat1.shape[0]]
elif dat1.shape[0] > dat2.shape[0]:
dat1 = dat1[:dat2.shape[0]]
dat1 -= dat1.mean()
dat2 -= dat2.mean()
dd = column_stack([dat1, dat2])
dat_all.append(dd)
dat = row_stack(dat_all)
ds = miendata.newData(dat, {'SampleType':'timeseries', 'SamplesPerSecond':SIGFS,
'StartTime':0.0, "Labels":['HairPosition', 'MicroFlownVoltage']})
return ds
def makeMasks(self, event):
#updated
c = self.cell
points = reshape(c.get_drawing_coords(), (-1, 8))
diams = (points[:,3]+points[:,7])/2
#filt = diams < 8.5
#get abstract model data file
d = self.gui.askParam([{"Name":"Density Model File",
"Value":'classModelsOnly.mien',
"Browser":FileBrowse}])
if not d:
return
md = io.read(d[0])
masks = md.getElements("AbstractModel", depth=1)
#for each model, construct masks
for mask in masks:
gmm = mask.getElements('MienBlock', {'Function':'ccbcv.gmm.gmm'})
if not gmm:
continue
gmm = gmm[0]
dat = maskPoints(mask, points)
#dat = dat*filt[:,newaxis]*mask.attrib('total_weight')
dat = dat*mask.attrib('total_weight')
a= {'Name':mask.name().rstrip('gm'),'SampleType':'mask'}
for atr in mask.attributes:
if atr.startswith('meta_'):
a[atr] = mask.attrib(atr)
d = newData(dat, a)
c.newElement(d)
self.gui.update_all(object=c, event="Rebuild")
self.gui.report("finished masking")
def procFile(fn, rot, nfn):
f = open(fn, 'rb')
h = getInfo(f)
nframes=h['nframes']
print "converting %s (%i frames)" % (fn, nframes)
if nframes<1000:
frames = arange(nframes)
else:
frames = None
dat = chunkOFrames(fn, frames)
me = dat.mean(3)[...,0]
out = []
for i in range(nframes):
frame = readFrameN(f, h, i) - me
if rot!=None:
frame, junk = imrotate(frame, 1, rot)
x = frame.sum(1)
if options['w']>1:
x=convolve(ones(options['w']), x, mode='same')
out.append(argmax(x))
if not i%500:
print "... frame %i" % i
out = array(out, float32)
dat = mdat.newData(out, {'SampleType':'timeseries', 'SamplesPerSecond':1.0/h['timescale']})
doc = nmpml.blankDocument()
doc.newElement(dat)
io.write(doc, nfn)
def sum_stack(fn,sw):
'''Measures relative illumination between frames. Sums the values of each frame, divides by the number of pixels, and removes the mean.'''
dat, fs = retrieveData(fn[0],sw)
nframes = dat.shape[3]
print('processing image stack: %ix%i, %i frames ...' % (dat.shape[0], dat.shape[1], nframes))
s = zeros(nframes)
for k in arange(nframes):
s[k]=sum(dat[:,:,0,k])
numpix = float32(dat.shape[0]*dat.shape[1])
s = s/numpix
s -= s.mean()
if len(s.shape)==2:
s=s.reshape(-1,)
h = nmpdat.newHeader(fs=fs)
nd = nmpdat.newData(s, h)
doc = nmp.blankDocument()
doc.newElement(nd)
if sw.has_key('dir'):
a = io.write(doc,sw['dir'] + '/' + sw['sum_file'])
else:
a = io.write(doc,sw['sum_file'])
if a:
print "%s successfully written." % sw['sum_file']
else:
print "%s failed to write." % sw['sum_file']
return s, fs
def knit(dname):
if os.path.isfile(os.path.join(dname, 'concat_ex.mdat')):
os.unlink(os.path.join(dname, 'concat_ex.mdat'))
if os.path.isfile(os.path.join(dname, 'concat.mdat')):
os.unlink(os.path.join(dname, 'concat.mdat'))
print("=== processing directory %s ===" % dname)
dat_all = []
dat_ex = []
mf_allch = []
mdat = [f for f in os.listdir(dname) if f.endswith("_ts.mdat")]
date = mdat[0][:10]
if all([f[:10] == date for f in mdat]):
if os.path.isfile(os.path.join(dname, date+'concat_ex.mdat')):
os.unlink(os.path.join(dname, date+'concat_ex.mdat'))
if os.path.isfile(os.path.join(dname, date+'concat.mdat')):
os.unlink(os.path.join(dname, date+'concat.mdat'))
else:
print "Multiple experiments present -- aborting. Put separate experiments in different folders."
return None
bin = [f for f in os.listdir(dname) if f.endswith(".bin")]
for f in mdat:
ff1=os.path.join(dname, f)
f2 = getMatch(f[:-8], bin)
if not f2:
print("can't match %s" % (f,))
continue
ff2 = os.path.join(dname, f2)
print("adding file pair %s, %s" % (f, f2))
dat1 = io.read(ff1).getElements("Data")[0]
dat2 = io.read(ff2).getElements("Data")[0]
dat2 = cropMicroflown(dat2)
# crpd = dat2.getData()
# ds = miendata.newData(crpd, {'SampleType':'timeseries', 'SamplesPerSecond':10000})
# doc = nmpml.blankDocument()
# doc.newElement(ds)
# io.write(doc, os.path.join(dname, 'crpd.mdat'))
resample(dat1, 1000)
resample(dat2, 1000)
dat1 = dat1.getData()
dat2 = dat2.getData()[:,2]
if dat1.shape[0] < dat2.shape[0]:
dat2 = dat2[:dat1.shape[0]]
elif dat1.shape[0] > dat2.shape[0]:
dat1 = dat1[:dat2.shape[0]]
dat1 -= dat1.mean()
dat2 -= dat2.mean()
dd = column_stack([dat1, dat2])
dat_all.append(dd)
if not any([q in f.lower() for q in EXCLUDE]):
dat_ex.append(dd)
dat = row_stack(dat_all)
ds = miendata.newData(dat, {'SampleType':'timeseries', 'SamplesPerSecond':1000})
doc = nmpml.blankDocument()
doc.newElement(ds)
io.write(doc, os.path.join(dname, date+'concat.mdat'))
if len(dat_ex) < len(dat_all):
dat = row_stack(dat_ex)
ds.datinit(dat)
io.write(doc, os.path.join(dname, date+'concat_ex.mdat'))
def findDataElement(self):
de = self.getElements("Data")
if de:
self.data = de[0]
else:
from mien.nmpml.data import newData
self.data = newData(zeros((0,2)),{"SampleType":"labeledevents", "SamplesPerSecond":20000})
self.newElement(self.data)
def setCellData(self, a, fs):
d = self.getElements("Data")
if d:
d = d[0]
d.datinit(a, {"SampleType": "timeseries", "SamplesPerSecond": fs})
else:
d = newData(a, {"Name": "activation_data", "SampleType": "timeseries", "SamplesPerSecond": fs})
self.newElement(d)
def maskPoints(am, points):
pts = (points[:,0:3]+points[:,4:7]) / 2
rads = (points[:,3]+points[:,7]) / 4
ds = newData(pts, {"Name":'points', "SampleType":"generic"})
am.run(ds)
vals = ds.getSubData('gmmout').getData()
vals = vals * rads[:,newaxis]
return vals
def plot(self, a, h):
if not self.dv.data or self.dv.data.__tag__!="Data":
dat=newData(None, {'SampleType':'group'})
self.dv.document.newElement(dat)
self.dv.report('Auto-generating Data element')
self.dv.data=dat
self.dv.onNewData()
self.dv.data.datinit(a, h)
self.dv.update_all(object=self.dv.data)
def writeFile(f, sps, fname='CubicFunctions', start=0):
ndoc = nmpml.blankDocument()
tf = mdat.newData(
f, {
'Name': fname,
'SampleType': 'timeseries',
'SamplesPerSecond': sps,
"StartTime": start
})
ndoc.newElement(tf)
io.write(ndoc, fname + ".mdat")
def getData(self):
de = self.getElements("Data")
if de:
return de[0]
else:
print "Can't find a data element. Making an empty one"
from mien.nmpml.data import newData
attrs = {"Type":"sfield", "Edge":self.attrib("Edge"), "Origin":self.attrib("Origin")}
a=array([[[]]])
d = newData(a, attrs)
self.newElement(d)
return d
def saveView(cv):
vg=viewGroup(cv)
pv = vg.getElements('Data', {'SampleType':'CVViewpoint'}, depth=1)
vpn = "View%03d" % len(pv)
d=cv.askParam([{'Name':'Name This View', 'Value':vpn}])
if not d:
return
n=d[0]
nd = newData(None, {'SampleType':'CVViewpoint'})
vg.newElement(nd)
nd.setName(n)
view=getCurrentVP(cv)
nd.datinit(view)
cv.report('view added')
def findDataElement(self):
datr = ["Type", "Labels", "Columns", "Url", "DataType", "SamplesPerSecond"]
attrs = {"Type":"Auto"}
for k in datr:
if self.attributes.has_key(k):
attrs[k]=self.attrib(k)
del(self.attributes[k])
de = self.getElements("Data")
if de:
self.data = de[0]
else:
print "Can't find a data element. Making an empty one"
from mien.nmpml.data import newData
self.data = newData(zeros((0,0), float32), {'SampleType':'Locus'})
self.newElement(self.data)
def pts2data(arg):
"""pts2data fname [fname2 [...]] - extract points from sphere fiducials to a text data file
This function will load the contents of all the named files (provided they are readable by MIEN),
select all the Fiducial elements that have Style == spheres, extract the point data from these
elements, concatenate the data into a single data element, and print that data to standard output
in text data file format. You can use shell redirection (e.g. "pts2data fname > data.txt") to
write the points to a file.
"""
doc = io.readall(arg)
fid = doc.getElements("Fiducial", {"Style": "spheres"})
pts = [f.getPoints() for f in fid]
pts = row_stack(pts)
d = md.newData(pts, {"SampleType": "generic"})
f = io.match_extension("txt")[0]
io.write(d, sys.stdout, format=f, newdoc=True)
def spikeDistanceTest(st, rt):
m, id1, id2=getAllDist(st, rt)
print m
d=makeEquiv(m, id1, id2)
print d
de=newData(zeros(0), {'SampleType':'group'})
for i in range(d.shape[0]):
rst=rt.getData()
print rst.shape
rst=take(rst[:,0], nonzero(rst[:,1]==d[i,1])[0], 0)
est=st.getData()
est=take(est[:,0], nonzero(est[:,1]==d[i,0])[0], 0)
cst=zeros((rst.shape[0]+est.shape[0],2))
cst[:rst.shape[0],0]=rst
cst[rst.shape[0]:,0]=est
cst[rst.shape[0]:,1]=1
cde=de.createSubData("/unit%i" % (int(d[i,1]),), cst, {'SampleType':'labeledevents', 'SamplesPerSecond':rt.fs()})
io.write(de, "check_results.mdat", format=".mdat", newdoc=True)
def allTFsRs(fnames):
ndoc = nmpml.blankDocument()
for fname in fnames:
doc = io.read(fname)
ds = doc.getElements("Data", depth=1)[0]
bn=os.path.splitext(fname)[0]
tf = tffmax(ds, False)
if not tf:
continue
tf = tf.getData()
if fname.startswith("2009") or fname.startswith("2010"):
tf[:,2]+=pi
tf[:,2] = tf[:,2] - ( 2*pi*(tf[:,2]>pi))
tf = row_stack( [array([[0,0,0]]), tf, array([[250,tf[-1,1],tf[-1,2]]])])
tf = uniformsample(tf, 1.0)
tf = mdat.newData(tf, {'Name':bn, 'SampleType':'timeseries', 'SamplesPerSecond':1.0, "StartTime":0})
ndoc.newElement(tf)
io.write(ndoc, "allTFsResamp.mdat")
def _getBiasData(self, rerun=False):
bd=self.document.getElements('Data', 'BiasField', depth=1)
if bd and not rerun:
return bd[0]
if bd:
bd=bd[0]
bd.datinit(None, {'SampleType':'group'})
else:
bd=newData(None, {'Name':'BiasField', 'SampleType':'group'})
self.document.newElement(bd)
els=[0,1]
if self.biasmode=='conditioned':
els.append(2)
try:
bd=self.alg.run(bd, els)
except:
raise
bd=self.alg.run(bd, [0])
return bd
def saveWaves(spi, bn, t, dat):
fn = bn + "_%s.mdat" % (t.name(),)
tem = t.getElements("Data", "template", depth=1)[0]
lead = tem.attrib("Lead")
length = tem.getData().shape[0]
# ignore spikes too close to the ends of the data file
spi = spi[where(spi > lead)]
spi = spi[where(spi < (dat.shape()[0] - (length - lead)))]
if spi.shape[0] < 1:
return
waves = takewindows(dat.getData(), spi, lead, length)
head = {"SampleType": "ensemble", "Lead": lead, "Reps": spi.shape[0], "SamplesPerSecond": dat.fs()}
waves = newData(waves, head)
h = waves.createSubData("/hidden")
h.newElement(t.clone(False))
h.newElement(tem.clone(False))
io.write(waves, fn, newdoc=True, format=".mdat")
def makeUniformMask(self, event):
d=self.askParam([{"Name":"Density",
"Value":.1},
{"Name":"Name",
"Value":"AfferentClassXX"},
{"Name":"Which Sections",
"Type":"List",
"Value":["All", "Only selected sections",
"Only non-selected sections"]}])
if not d:
return
points = self.cell.get_drawing_coords()
nc=reshape(points, (-1, 8)).shape[0]
maskdat=ones(nc, float32)*d[0]
if d[2]!="All":
m=self.sectionMask(("non" in d[2]))
if m==None:
return
maskdat=maskdat*m
d = newData(reshape(maskdat, (-1,1)), {'Name':d[1],'SampleType':'mask'})
self.cell.newElement(d)
self.gui.update_all(object=d, event="Create")
def makeMasks(self, event):
c = self.cell
points = reshape(c.get_drawing_coords(), (-1, 8))
diams = (points[:,3]+points[:,7])/2
of = open('modeltemplate.txt', 'w')
for p in points:
of.write("%.3f %.3f %.3f %.3f %.3f %.3f %.3f %.3f\n" % tuple(p))
of.close()
check=os.system("matlab -nojvm -nosplash -r \"maskalldata('modeltemplate.txt'); quit\"")
if check:
self.report("system call to matlab maskalldata failed")
return
lines=open("masked_modeltemplate.txt").readlines()
tags = array(map(lambda l:map(float, l.split()),lines))
filt = diams < 8.5
tags = tags*filt[:,NewAxis]
for i in range(tags.shape[1]):
n = "AfferentClass%02d" % (i+1,)
d = newData(tags[:,i:i+1], {'Name':n,'SampleType':'mask'})
c.newElement(d)
self.gui.update_all(object=c, event="Rebuild")
def newFile(s,d=()): d=dict(d) nd = nmpdat.newData(s, d) doc = nmp.blankDocument() doc.newElement(nd) return doc
def comp_series(fn,sw):
'''Compares illumination of movie images to stimulus generator record running the LED.'''
if 'n' in sw.keys():
s, fs = retrieveData(fn[0],sw)
else:
s, fs = sum_stack(fn,sw)
if len(s.shape)==2 and s.shape[1] == 1:
s=s.reshape(-1,)
b, fsb = retrieveData(fn[1],sw)
if 'e' in sw.keys():
es, esind = near_event_finder(s,fs,stdnum=5,numback=6)
eb, ebind = near_event_finder(b[:,0],fsb,thresh=0.1)
elif 'f' in sw.keys():
es, esind = subfs_event_finder(s,fs)
eb, ebind = subfs_event_finder(b[:,0],fsb,5)
else:
es, esind = event_finder(s,fs)
eb, ebind = event_finder(b[:,0],fsb,thresh=1)
df = len(eb) - len(es)
if df:
z=zeros(df,s.dtype)
es=hstack([es,z])
tdiff = es-eb
#Start: code to ensure all events are detected
bb = b[:,0]
z = zeros_like(bb)
z[ebind] = 1
btot = vstack([bb,z]).T
hb = nmpdat.newHeader(fs=fsb, l=['RawStimulus','Events'])
ndb = nmpdat.newData(btot, hb)
docb = nmp.blankDocument()
docb.newElement(ndb)
if sw.has_key('dir'):
a = io.write(docb,sw['dir'] + '/' + sw['stim_file'])
else:
a = io.write(docb,sw['stim_file'])
if a:
print "%s successfully written." % sw['stim_file']
else:
print "%s failed to write." % sw['stim_file']
zs = zeros_like(s)
zs[esind] = 1
stot = vstack([s,zs]).T
hs = nmpdat.newHeader(fs=fs, l=['CameraSum','Events'])
nds = nmpdat.newData(stot, hs)
docs = nmp.blankDocument()
docs.newElement(nds)
if sw.has_key('dir'):
a = io.write(docs,sw['dir'] + '/' + sw['cam_file'])
else:
a = io.write(docs,sw['cam_file'])
if a:
print "%s successfully written." % sw['cam_file']
else:
print "%s failed to write." % sw['cam_file']
#End: code to ensure all events are detected
tot = vstack([es,eb,tdiff]).T
h = nmpdat.newHeader(fs=1.0, l=['VideoEvents','StimulusEvents','TimeDifference'])
nd = nmpdat.newData(tot, h)
doc = nmp.blankDocument()
doc.newElement(nd)
if sw.has_key('dir'):
a = io.write(doc,sw['dir'] + '/' + sw['comp_file'])
else:
a = io.write(doc,sw['comp_file'])
if a:
print "%s successfully written." % sw['comp_file']
else:
print "%s failed to write." % sw['comp_file']
ang = options['a']
crap, index = imrotate(dat, ang)
else:
ang, index = guessAngle(dat, options['t'])
print "guessing angle to be %i" % ang
print "processing %i frames ..." % nframes
out = []
#nframes = min(600, nframes)
for i in range(nframes):
frame = readFrameN(f, h, i)
if options['m']:
frame = frame - mean
frame, junk = imrotate(frame, ang, index)
x = frame.sum(1)
if options['w']>1:
x=convolve(ones(options['w']), x, mode='same')
out.append(argmin(x))
if not i%500:
print i
out = array(out, float32)
if L:
L=L*AOS_PixPerDiv/NIKON_PixPerDiv
out-=out.mean()
out = out/L
dat = mdat.newData(out, {'SampleType':'timeseries', 'SamplesPerSecond':1.0/h['timescale']})
doc = nmpml.blankDocument()
doc.newElement(dat)
io.write(doc, fnameout)
