def cldispatch(functs, shortopts='', longopts=[]):
'''
This function supports "command dispatch", for producing a command that
takes, as its first argument, a subcommand (for example, git works in this
way).
functs may be either a list of functions, or a dictionary of
{str:function}. In the former case, the list is converted to a dict using
f.__name__ as the key for f.
'''
if type(functs) != dict:
functs = dict([(f.__name__, f) for f in functs])
usage = clusage.replace("SCMD", "\n".join(functs))
usage = usage.replace("CLO", shortopts + " " + " ".join(longopts))
try:
opts, args = swparse(shortopts, usage, longopts)
cmd = args[0]
if cmd.lower() == "help":
f = functs[args[1]]
report(f.__doc__)
sys.exit()
else:
f = functs[cmd]
except SystemExit:
raise
except:
print(sys.exc_info())
report(usage)
sys.exit()
args = args[1:]
return f(opts, args)
def readMD(s):
f = StringIO.StringIO(s)
dats = {}
cat = f.read(16)
while len(cat) == 16:
pl, hl, dl = struct.unpack("<IIQ", cat)
path = f.read(pl)
head = eval(f.read(hl))
dat = None
if dl:
ct = f.read(1)
if ct in ['<', '>', '|', "{"]:
ct = ct + f.read(6)
dti, nd = struct.unpack("<3sI", ct)
if dti.startswith("{"):
dti = dti[1:]
dti = np.dtype(dti)
else:
report("warning, old mdat. May not be platform portable")
nd = struct.unpack("<I", f.read(4))[0]
dti = dtype("<" + ct)
sh = struct.unpack("<" + "Q" * nd, f.read(8 * nd))
dat = f.read(dl)
dat = np.fromstring(dat, dti)
dti = dti.str
if DTS != dti[0]:
dtil = np.dtype(DTS + dti[1:])
dat = dat.astype(dtil)
dat = np.reshape(dat, sh)
dats[path] = (dat, head)
cat = f.read(16)
return dats
def getTests(dsheet=None, bdir=basedir):
files = getDsheet(dsheet)
doc = gd.Doc()
testk = re.compile('test(\d+)')
for f in files:
ns = "track%s" % f
mouse = numOrRan(f)[0]
dfpath = os.path.join(bdir, pathprefix + str(mouse), pathprefix + f)
pstpath = os.path.join(dfpath, pathprefix + f + '.pst')
if not os.path.isfile(pstpath):
report("No data for %s" % f)
continue
else:
report("found data for %s" % f)
ld = gio.read(pstpath)
tids = {}
for tests in files[f]:
ids = numOrRan(tests[testLab])
for i in ids:
tids[i] = tests
found = set()
for k in ld:
m = testk.match(k)
if m:
i = int(m.groups()[0])
if i in tids:
doc[ns + '.' + k] = ld[k]
found.add(i)
for i in found:
n = ns + '.test%i.' % i
for k in tids[i]:
doc[n + k] = tids[i][k]
return doc
def deserialize(s):
l = struct.unpack('<I', s[:4])[0]
doc = zlib.decompress(s[4:l + 4])
doc = readString(doc, False)
s = s[l + 4:]
try:
if s:
try:
f = readMD(s)
except:
s = zlib.decompress(s)
f = readMD(s)
des = [k[:-4] for k in doc.find('Data') if k.endswith('.tag')]
#return doc, f, des
for de in des:
try:
up = upath(doc, de)
try:
d, h = f[up]
except KeyError:
d, h = f[up + '/']
setdat(doc, de, d, h)
except:
report("can't find data for element %s" % (de,))
raise
except:
raise
report("cant load data")
return doc
def getcell(cellid, dname='bicIC', sheet='fulldatasheet.ods', tab='sheet1.table', cellidlab='cell',
mouseid='data', testn='test', cond='condition', pstdirp='Mouse ', redundantdirs=True, condgroup=int,
stimchan='stim.ch0', shift=True):
'''
Return a data document for a single cell. Parameters are the same as for condition, except for cellid, which
is an integer indicating which cell to get. As long as "fulldatasheet.ods" is present in the named "dname", and
lists all available cells in Zach/Graham format, then no parameters other than cellid and dname should need to be
changed.
'''
dname = os.path.join(BASEDIR, dname)
ds = gio.read(os.path.join(dname, sheet))[tab]
l = list(ds[0, :])
try:
cellcol = [i for (i, s) in enumerate(l) if s.lower().startswith(cellidlab)][0]
mousecol = [i for (i, s) in enumerate(l) if s.lower().startswith(mouseid)][0]
testcol = [i for (i, s) in enumerate(l) if s.lower().startswith(testn)][0]
condcol = [i for (i, s) in enumerate(l) if s.lower().startswith(cond)][0]
except IndexError:
raise KeyError('Data sheet doesnt contain the specified columns')
clines = [(ds[i, mousecol], ds[i, testcol], ds[i, condcol]) for i in range(ds.shape[0]) if
ds[i, cellcol] == str(cellid)]
if not clines:
raise KeyError('No tests in this data sheet for this cell')
if not all([clines[i][0] == clines[0][0] for i in range(1, len(clines))]):
report('WARNING: single cell reported in multiple recording tracks !!??')
sd = gd.Doc()
for mn in set([clines[i][0] for i in range(len(clines))]):
pstn = pstdirp + mn + '.pst'
if redundantdirs:
pstn = os.path.join(pstdirp + mn, pstn)
pstn = os.path.join(dname, pstn)
sd = sd.fuse(gio.read(pstn))
else:
pstn = pstdirp + clines[0][0] + '.pst'
if redundantdirs:
pstn = os.path.join(pstdirp + clines[0][0], pstn)
pstn = os.path.join(dname, pstn)
sd = gio.read(pstn)
d = gd.Doc()
for l in clines:
tests = _numOrRan(l[1])
for it in tests:
otn = 'test%i' % it
trd = traces(sd, otn, stimchan, shift)
for trn in trd:
tr = trd[trn]
tr['condition'] = condgroup(l[2])
tr['cell'] = cellid
nn = "te%itr%i" % (it, trn)
d.set(nn, tr)
return d
def lockcall(self, method, args):
'''
Attempt to acquire self.lock, execute the indicated method with the
indicated arguments, and release the lock. The method call is wrapped
in a general try/except, gauranteeing that the lock is always released
(but making failed method calls hard to debug)
'''
self.lock.acquire()
try:
apply(method, args)
except:
raise
report("Call to %s%s failed" % (str(method), str(args)))
self.lock.release()
def row2list(r):
l = []
mel = 0
largest_real = 0
for e in r['_els']:
if not e['_tag'] == 'table:table-cell':
report('warning: unexpected ods table element %s' % (str(e),))
continue
v = tableCellContents(e)
nreps = int(e.get('table:number-columns-repeated', 1))
for i in range(nreps):
l.append(v)
if v:
mel = max(mel, len(v))
largest_real = len(l)
return (l[:largest_real], mel)
def send_request(self, verb, path, payload=None, ctype=None, headers=None):
if not headers:
headers = {}
if not 'Accept' in headers:
headers['Accept'] = self.ctype
if ctype:
if verb == "get":
headers['Accept'] = ctype
else:
headers['Content-Type'] = ctype
req = self.build_request(verb, path, headers, payload)
try:
s = self.opener.open(req).read()
except urllib2.HTTPError, e:
s = "ERROR %i:" % e.code + e.read()
report(s)
if 'www-authenticate' in e.hdrs:
report('Authentication required: %s' % e.hdrs['www-authenticate'])
def comp2s(stim1, evts1, stim2, evts2, length=LENGTH, lead=LEAD, compress='no', clevel=0, testprop=TESTPROP,
bootstrap=BOOTSTRAP, report=None):
'''
Like compare, but considers the case where there are two stimuli in
addition to two response sets
'''
rc1 = ece(stim1, evts1, length, lead)
rc2 = ece(stim2, evts2, length, lead)
if compress in DRMODES:
uc = np.column_stack([ucse(stim1, 10000, length),
ucse(stim2, 10000, length)])
ce = eqrc((rc1, rc2))
cspace = DRMODES[compress](ce, uc, clevel)
if report:
report('Using %i components' % cspace.shape[0])
rc1 = np.dot(cspace, rc1)
rc2 = np.dot(cspace, rc2)
return lltest(rc1, rc2, testprop, bootstrap)
def addtests(d, l=200, match='cond1', q=10000, mods=(AC, TGC), sp={}):
td = gd.Doc()
iod = d[match]
arate = float(len(flat(iod['evts']))) / len(iod['evts'])
arate = arate / (l / 1e6)
report(arate)
stims = iod['stims']
nstims = np.unique(stims).shape[0]
l = int(l)
td[match] = iod
for k in d:
if k.startswith('cond') and k != match:
td[k] = d[k]
for M in mods:
kw = sp.get(M.name, {})
mi = M(l, arate, q, nstims, **kw)
td['cond' + M.name] = {'stims': stims, 'evts': [mi(st) for st in stims]}
td['docsource'] = {'cell': d['docsource.cell'] + '+tests',
'pars': d['docsource.pars'] + (l, match, q)}
return td
def insert(self, fit, ps):
'''
Add a unit to the population and store it in self.store
'''
if len(self.units) >= self.pars['size']:
self.alg.kill(self.units)
self.units[ps] = fit
self.store.record(fit, ps)
self.nunits += 1
if fit > self.best[0]:
sps = ''
for p in ps:
sps += "%.4g, " % p
sps = sps[:-2]
report('new best %.4f (%s)' % (fit, sps))
self.best = (fit, ps)
if self.model.perfect != None and fit == self.model.perfect:
report(
"You hear the singing of Angels and a chorus of trumpets, hailing the birth of a perfect unit. All further competition is futile. Exiting")
self.abort = "perfected"
def search(self):
'''
This is the main loop called by run (sometimes in a thread). It calls
self.alg.next(self.units, self.range.nd) to get a paremeter set, calls
self.model.eval to evaluate it, and calls self.insert to store it.
'''
if len(self.units) < self.pars['size']:
ps = self.alg.initcond()
else:
ps = self.alg.next(self.units)
try:
(fit, ec) = self.model.eval(ps)
except Exception:
e = sys.exc_info()
error(e, None, rep="GA Eval error")
fit = "model.eval threw exception"
if type(fit) in [str, unicode]:
report('Eval error, %s -> %s' % (str(ps), fit))
else:
self.lockcall(self.insert, (fit, ps + ec))
def prep(self):
'''
Set up the optimization run.
'''
self.abort = False
self.best = (None, None)
self.nunits = self.store.size()
self.threads = []
if self.nunits:
self.units = self.store.get(self.pars['size'])
z = self.units.items()
snd = len(z[0][0])
if snd < self.range.nd:
raise StandardError(
'Attempt to resume %i-d optimizer from a storage of %i-d parameters' % (self.range.nd, snd))
elif snd > self.range.nd:
report(
'resuming an %i-d optimizer from a store of %i-d parameter sets. Hopefully this is intentional (eg. there are eval-conditions in the model). If not, dont run this optimizer' % (
self.range.nd, snd))
f = np.array([i[1] for i in z])
bi = f.argmax()
self.best = (f[bi], z[bi][0])
#FIXME - sanity check for the stored components should go here
else:
self.store.saveNodes(self.describe())
self.units = {}
report("prep complete")
self.lock = threading.Lock()
if self.units:
report("Resuming. %i units (%s)" % (self.nunits, self.fitstats(True)))
def bigFig(m1, m2, compress='istac', clevel=.85):
stim = gwn.getstim('bl')
uc = ucse(stim)
rcse = evalSystem(m1, True, cid=0)
rcse2 = evalSystem(m2, True, cid=1)
rc = eqrc((rcse, rcse2))
iss = DRMODES[compress](rc, uc, clevel)
report('Using %i components' % iss.shape[0])
plt.figure(3)
plt.clf()
ll = lltest(rcse, rcse2)
_prow(rcse, rcse2, 0, 3, True, ll)
rc1 = np.dot(iss, rcse)
rc2 = np.dot(iss, rcse2)
ll = lltest(rc1, rc2)
_prow(rc1, rc2, 1, 3, True, ll)
rcb1 = np.dot(iss.transpose(), rc1)
rcb2 = np.dot(iss.transpose(), rc2)
#ll = lltest(rcb1, rcb2)
_prow(rcb1, rcb2, 2, 3, True, ll)
f = plt.figure(3)
f.subplots_adjust(left=.04, right=.99, bottom=.05, top=.99, wspace=.05, hspace=.05)
f.canvas.draw()
def swparse(shortopts='', usage=bogousage, longopts=()):
'''
Parse command line arguments using getopt.gnu_getopt, but at a slightly
higher level. shortopts and longopts are the string and list arguments
to getopt (defaulting to '' and () respectively). Usage is a documentation
string. If parsing fails, or if the option "-h" is passed, usage is printed
and SysExit is raised. Otherwise, the return value is (options
<{str:str}>, files <[str]>), The dictionary options has keys containing
any command line options, with the leading dashes stripped off, associated
to their values (if any). Files is a list of the remaining command line
arguments.
'''
try:
options, files = getopt.gnu_getopt(sys.argv[1:], shortopts, longopts)
except getopt.error:
report(usage)
sys.exit()
switches = {}
for o in options:
switches[o[0].lstrip('-')] = o[1]
if 'h' in switches:
report(usage)
sys.exit()
return switches, files
def run(self, background=False):
'''
Run the optimizer. If background is True, run in a thread.
and return the thread.
'''
if background:
t = threading.Thread(target=self.run, args=(False,))
t.start()
return t
self.start = time.time()
self.abort = False
report("Starting Run (pid=%s)" % (str(os.getpid(), )))
while not self.done():
try:
self.threadcall(self.search, ())
if self.nunits and not self.nunits % 200:
report("Evaluate %i. %s" % (self.nunits, self.fitstats(True)))
except KeyboardInterrupt:
report('Manual abort. Shutting down.')
self.abort = True
report("Run Completed")
def resolvelink(self, d, k='', value=True):
'''
return the value of a link or slice dictionary d, located at key k
(if k is None, relative links are not allowed)
'''
k = k.split('.')
t = d['_link']
if t.startswith('.'):
nt = t.rstrip('.')
nl = len(t) - len(nt)
if nl > len(k):
report('WARNING: relative link %s encountered at %s. Not enough nesting levels to specify it.' % (
t, '.'.join(k)))
return None
else:
t = '.'.join(k[:-nl]) + "." + nt
if not value:
return t
if not t.strip():
return self
nv = self.get(t)
if nv == None:
report('WARNING: link at %s to %s is broken' % ('.'.join(k), t))
return None
if d['_slice'] != None:
nvt = search.classify(nv)
sl = d['_slice']
if nvt == '#':
if not type(sl[0]) == tuple:
sl = (sl,)
nv = nv[[apply(slice, t) for t in sl]]
elif nvt == '[':
if type(sl[0]) == tuple:
sl = sl[0]
nv = nv[apply(slice, sl)]
else:
report('WARNING: slice at %s of %s tries to slice a non-sequence' % ('.'.join(k), t))
nv = None
return nv
def condition(dname='bicIC', sheet='datasheet.ods', tab='sheet1.table', cellid='cell', mouseid='data',
testn='test', cond='condition', pstdirp='Mouse ', redundantdirs=True, writefile='conditioned.gic',
condgroup=int, stimchan='stim.ch0'):
'''
Return a data document containing every test specified in the data sheet
"sheet". "dname" indicates a directory (subdirectory of BASEDIR) to search for
the data sheet and data files (or "mouse" directories).
tab, cellid, mouseid, testn, and cond specify the layout of the sheet (which
table is used and what the column labels for the relevant metadata are. These
can be left as default values if using the sort of sheet Zach Mayko and I
(Graham Cummins) have typically been using. Pstdirp and redundantdirs specify
the layout of the data directory structure, and again can remain at default if
your structure looks like Christine Portfor's typical layout. Writefile
specifies a file name to store the new document in (or omit it to not save the
doc).
condgroup is a function that is applied to the value of the "condition" metadata
The default (cast to int) is OK if condition labels are integers, and all distinctly
labeled conditions should be treated differently (otherwise, use a function that
makes similarity classes in some way, using the raw values of "condition").
Stimchan specifies where the relevant stimulus data is stored by Batlab.
This code assumes a single channel stimulus on channel 0. Although this
parameter can be changed to deal with single channels on some other
channel, any significant change to the stimulus will probably break
most code in this module.
'''
dname = os.path.join(BASEDIR, dname)
ds = gio.read(os.path.join(dname, sheet))[tab]
l = list(ds[0, :])
try:
cellcol = [i for (i, s) in enumerate(l) if s.lower().startswith(cellid)][0]
mousecol = [i for (i, s) in enumerate(l) if s.lower().startswith(mouseid)][0]
testcol = [i for (i, s) in enumerate(l) if s.lower().startswith(testn)][0]
condcol = [i for (i, s) in enumerate(l) if s.lower().startswith(cond)][0]
except IndexError:
raise KeyError('Data sheet doesnt contain the specified columns')
d = gd.Doc()
dfiles = {}
for i in range(1, ds.shape[0]):
l = ds[i, :]
try:
cid = int(l[cellcol])
mid = l[mousecol]
cond = l[condcol]
if condgroup:
cond = condgroup(cond)
tests = _numOrRan(l[testcol])
except:
report('failed to parse data sheet line %i' % i)
continue
if not mid in dfiles:
pstn = pstdirp + mid + '.pst'
if redundantdirs:
pstn = os.path.join(pstdirp + mid, pstn)
pstn = os.path.join(dname, pstn)
try:
dfiles[mid] = gio.read(pstn)
report('loading %s' % pstn)
except:
report('failed to open data file for %s (tried path %s)' % (mid, pstn))
dfiles[mid] = 'BAD'
if dfiles[mid] == 'BAD':
continue
metas = {'animal': mid, 'condition': cond, 'cell': cid}
for it in tests:
otn = 'test%i' % it
try:
trd = traces(dfiles[mid], otn, stimchan)
except:
report("Cant find a test %s for penetration %s" % (otn, mid))
continue
for trn in trd:
tr = trd[trn]
tr.update(metas)
nn = "r%s_te%itr%i" % (mid, it, trn)
d.set(nn, tr)
if writefile:
gio.write(d, os.path.join(dname, writefile))
return d
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place, Suite 330, Boston, MA 02111-1307 USA
#
from __future__ import print_function, unicode_literals
from gicdat.control import report
import numpy as np
from grouping import idents
from gicdat.util import maxdiag
import cext.portforsgic as pfce #@UnresolvedImport
try:
from pyentropy import DiscreteSystem
except:
report("Warning, no pyentropy. Better stick to the direct method!")
def _jprob(x, y, a):
return float(np.logical_and(a[:, 0] == x, a[:, 1] == y).sum()) / a.shape[0]
def h_direct(s):
s = np.array(s)
prob = np.bincount(s) / float(s.shape[0])
h = -1.0 * np.array([prob[x] * np.log(prob[x]) for x in np.unique(s)]).sum() / np.log(2)
return h
def minf_c_direct(cond):
return pfce.mi_direct(cond['stims'], cond['evts'])
"""
#Copied a bunch of content to deprecated_select on 3/2/12. refactoring.
from __future__ import print_function, unicode_literals
import numpy as np
import gicdat.doc as gd
from gicdat.util import infdiag
from mixmod import mmcall
try:
from Pycluster import treecluster, kmedoids, kcluster, somcluster, Tree, Node
except:
from gicdat.control import report
report("Pycluster is not found. Several methods in the clust module will fail")
try:
from zss import compare as zscmp
from zss.test_tree import Node as ZSNode
except:
from gicdat.control import report
report("Zhang-shasha is not found. pct2zst and zssdist in clust will fail")
def dt_first(dm, t):
'''
dm: DistMat(N), t: x -> ids: Partition1(N)
A naive grouping function which groups dm using a threshold, t. It depends
on the ordering of dm, since it assigns each item to the same group as the
def tr(l=100000, r=1.0):
a = AC(l, r)
evts = [a(0) for i in range(1000)]
report(len(flat(evts)) / 1000.0)
