def test_EM__evidence_repeatable_and_monotonic(self):
if 'EM' not in self.learnAlgs:
raise SkipTest
kwargs = self.get_kwargs()
hmodel1, LP1, Info1 = run(self.Data, self.allocModelName,
self.obsModelName, 'EM', **kwargs)
hmodel2, LP2, Info2 = run(self.Data, self.allocModelName,
self.obsModelName, 'EM', **kwargs)
assert len(Info1['evTrace']) == len(Info2['evTrace'])
assert np.allclose( Info1['evTrace'], Info2['evTrace'])
assert self.verify_monotonic(Info1['evTrace'])
def test_movb_repeatable_across_diff_num_batches(self):
if 'moVB' not in self.learnAlgs:
raise SkipTest
for nBatch in [1, 2, 3]:
kwargs = self.get_kwargs()
kwargs['nBatch'] = nBatch
hmodel1, LP1, Info1 = run(self.Data, self.allocModelName,
self.obsModelName, 'moVB', **kwargs)
hmodel2, LP2, Info2 = run(self.Data, self.allocModelName,
self.obsModelName, 'moVB', **kwargs)
assert len(Info1['evTrace']) == len(Info2['evTrace'])
assert np.allclose( Info1['evTrace'], Info2['evTrace'])
def test_EM__evidence_repeatable_and_monotonic(self):
if 'EM' not in self.learnAlgs:
raise SkipTest
kwargs = self.get_kwargs()
hmodel1, LP1, Info1 = run(self.Data, self.allocModelName,
self.obsModelName, 'EM', **kwargs)
hmodel2, LP2, Info2 = run(self.Data, self.allocModelName,
self.obsModelName, 'EM', **kwargs)
assert len(Info1['evTrace']) == len(Info2['evTrace'])
assert np.allclose(Info1['evTrace'], Info2['evTrace'])
assert self.verify_monotonic(Info1['evTrace'])
def test_movb_repeatable_across_diff_num_batches(self):
if 'moVB' not in self.learnAlgs:
raise SkipTest
for nBatch in [1, 2, 3]:
kwargs = self.get_kwargs()
kwargs['nBatch'] = nBatch
hmodel1, LP1, Info1 = run(self.Data, self.allocModelName,
self.obsModelName, 'moVB', **kwargs)
hmodel2, LP2, Info2 = run(self.Data, self.allocModelName,
self.obsModelName, 'moVB', **kwargs)
assert len(Info1['evTrace']) == len(Info2['evTrace'])
assert np.allclose(Info1['evTrace'], Info2['evTrace'])
def test_vb_repeatable_and_monotonic(self):
''' Verify VB runs with same seed produce exact same output, monotonic ELBO.
'''
if 'VB' not in self.learnAlgs:
raise SkipTest
kwargs = self.get_kwargs()
hmodel1, LP1, Info1 = run(self.Data, self.allocModelName,
self.obsModelName, 'VB', **kwargs)
hmodel2, LP2, Info2 = run(self.Data, self.allocModelName,
self.obsModelName, 'VB', **kwargs)
assert len(Info1['evTrace']) == len(Info2['evTrace'])
assert np.allclose(Info1['evTrace'], Info2['evTrace'])
assert self.verify_monotonic(Info1['evTrace'])
def test_vb_repeatable_and_monotonic(self):
''' Verify VB runs with same seed produce exact same output, monotonic ELBO.
'''
if 'VB' not in self.learnAlgs:
raise SkipTest
kwargs = self.get_kwargs()
hmodel1, LP1, Info1 = run(self.Data, self.allocModelName,
self.obsModelName, 'VB', **kwargs)
hmodel2, LP2, Info2 = run(self.Data, self.allocModelName,
self.obsModelName, 'VB', **kwargs)
assert len(Info1['evTrace']) == len(Info2['evTrace'])
assert np.allclose( Info1['evTrace'], Info2['evTrace'])
assert self.verify_monotonic(Info1['evTrace'])
def test_movb_with_one_batch_equivalent_to_vb(self):
if 'moVB' not in self.learnAlgs:
raise SkipTest
kwargs = self.get_kwargs()
kwargs['nBatch'] = 1
__, __, vbInfo = run(self.Data, self.allocModelName, self.obsModelName,
'VB', **kwargs)
__, __, movbInfo = run(self.Data, self.allocModelName,
self.obsModelName, 'moVB', **kwargs)
vbEv = vbInfo['evTrace'][:-1]
movbEv = movbInfo['evTrace']
print vbEv
print movbEv
assert len(vbEv) == len(movbEv)
for ii in range(len(vbEv)):
assert closeAtMSigFigs(vbEv[ii], movbEv[ii], M=8)
def test_movb_with_one_batch_equivalent_to_vb(self):
if 'moVB' not in self.learnAlgs:
raise SkipTest
kwargs = self.get_kwargs()
kwargs['nBatch'] = 1
__, __, vbInfo = run(self.Data, self.allocModelName,
self.obsModelName, 'VB', **kwargs)
__, __, movbInfo = run(self.Data, self.allocModelName,
self.obsModelName, 'moVB', **kwargs)
vbEv = vbInfo['evTrace'][:-1]
movbEv = movbInfo['evTrace']
print vbEv
print movbEv
assert len(vbEv) == len(movbEv)
for ii in range(len(vbEv)):
assert closeAtMSigFigs(vbEv[ii], movbEv[ii], M=8)
def test_sovb_with_one_batch_equivalent_to_vb(self):
if 'soVB' not in self.learnAlgs:
raise SkipTest
kwargs = self.get_kwargs()
kwargs['nBatch'] = 1
kwargs['rhoexp'] = 0
__, __, vbInfo = run(self.Data, self.allocModelName, self.obsModelName,
'VB', **kwargs)
__, __, sovbInfo = run(self.Data, self.allocModelName,
self.obsModelName, 'soVB', **kwargs)
vbEv = vbInfo['evTrace'][:-1]
sovbEv = sovbInfo['evTrace']
for ii in range(len(vbEv)):
if hasattr(self, 'mustRetainLPAcrossLapsForGuarantees'):
print vbEv[ii], sovbEv[ii]
assert closeAtMSigFigs(vbEv[ii], sovbEv[ii], M=2)
else:
assert closeAtMSigFigs(vbEv[ii], sovbEv[ii], M=8)
def test_sovb_with_one_batch_equivalent_to_vb(self):
if 'soVB' not in self.learnAlgs:
raise SkipTest
kwargs = self.get_kwargs()
kwargs['nBatch'] = 1
kwargs['rhoexp'] = 0
__, __, vbInfo = run(self.Data, self.allocModelName,
self.obsModelName, 'VB', **kwargs)
__, __, sovbInfo = run(self.Data, self.allocModelName,
self.obsModelName, 'soVB', **kwargs)
vbEv = vbInfo['evTrace'][:-1]
sovbEv = sovbInfo['evTrace']
for ii in range(len(vbEv)):
if hasattr(self, 'mustRetainLPAcrossLapsForGuarantees'):
print vbEv[ii], sovbEv[ii]
assert closeAtMSigFigs(vbEv[ii], sovbEv[ii], M=2)
else:
assert closeAtMSigFigs(vbEv[ii], sovbEv[ii], M=8)
def test_vb_sovb_and_movb_all_estimate_evBound_in_same_ballpark(self):
if 'moVB' not in self.learnAlgs:
raise SkipTest
kwargs = self.get_kwargs()
kwargs['nBatch'] = 5
__, __, vbInfo = run(self.Data, self.allocModelName, self.obsModelName,
'VB', **kwargs)
__, __, movbInfo = run(self.Data, self.allocModelName,
self.obsModelName, 'moVB', **kwargs)
__, __, sovbInfo = run(self.Data, self.allocModelName,
self.obsModelName, 'soVB', **kwargs)
vbEv = vbInfo['evBound']
movbEv = movbInfo['evBound']
sovbEv = np.mean(sovbInfo['evTrace'][-10:])
print vbEv
print movbEv
print sovbEv
assert closeAtMSigFigs(vbEv, movbEv, M=2)
assert closeAtMSigFigs(vbEv, sovbEv, M=2)
def test_vb_sovb_and_movb_all_estimate_evBound_in_same_ballpark(self):
if 'moVB' not in self.learnAlgs:
raise SkipTest
kwargs = self.get_kwargs()
kwargs['nBatch'] = 5
__, __, vbInfo = run(self.Data, self.allocModelName,
self.obsModelName, 'VB', **kwargs)
__, __, movbInfo = run(self.Data, self.allocModelName,
self.obsModelName, 'moVB', **kwargs)
__, __, sovbInfo = run(self.Data, self.allocModelName,
self.obsModelName, 'soVB', **kwargs)
vbEv = vbInfo['evBound']
movbEv = movbInfo['evBound']
sovbEv = np.mean(sovbInfo['evTrace'][-10:])
print vbEv
print movbEv
print sovbEv
assert closeAtMSigFigs(vbEv, movbEv, M=2)
assert closeAtMSigFigs(vbEv, sovbEv, M=2)
def run_EM__fromScratch(self, taskid=0):
''' Returns True/False for whether single run of EM finds ideal params.
'''
print '' # new line for nosetests
print '============================================== task %d' % (
taskid)
kwargs = self.get_kwargs()
kwargs.update(self.fromScratchArgs)
model, LP, Info = run(self.Data,
self.allocModelName,
self.obsModelName,
'EM',
taskid=taskid,
**kwargs)
assert self.verify_monotonic(Info['evTrace'])
# First, find perm of estimated comps to true comps
# if one exists, by using obsmodel params
permIDs = None
allocKeyList = []
for key in self.TrueParams:
if key == 'K':
continue
elif hasattr(model.obsModel.comp[0], key):
arrTrue = self.TrueParams[key]
arrEst = Util.buildArrForObsModelParams(
model.obsModel.comp, key)
if permIDs is None:
isG, permIDs = self.verify_close_under_some_perm(
arrTrue, arrEst, key)
if not isG:
print ' FAILED TO FIND IDEAL PARAMS'
argstring = ' '.join(sys.argv[1:])
if 'nocapture' in argstring:
from matplotlib import pylab
bnpy.viz.GaussViz.plotGauss2DFromHModel(model)
pylab.show()
return False
arrEst = arrEst[permIDs]
assert self.verify_close(arrTrue, arrEst, key)
else:
allocKeyList.append(key)
assert permIDs is not None
for key in allocKeyList:
print '--------------- %s' % (key)
arrTrue = self.TrueParams[key]
arrEst = getattr(model.allocModel, key)[permIDs]
Util.pprint(arrTrue, 'true')
Util.pprint(arrEst, 'est')
assert self.verify_close(arrTrue, arrEst, key)
return True
def run_EM__fromScratch(self, taskid=0):
''' Returns True/False for whether single run of EM finds ideal params.
'''
print '' # new line for nosetests
print '============================================== task %d' % (taskid)
kwargs = self.get_kwargs()
kwargs.update(self.fromScratchArgs)
model, LP, Info = run(self.Data, self.allocModelName, self.obsModelName,
'EM', taskid=taskid, **kwargs)
assert self.verify_monotonic(Info['evTrace'])
# First, find perm of estimated comps to true comps
# if one exists, by using obsmodel params
permIDs = None
allocKeyList = []
for key in self.TrueParams:
if key == 'K':
continue
elif hasattr(model.obsModel.comp[0], key):
arrTrue = self.TrueParams[key]
arrEst = Util.buildArrForObsModelParams(model.obsModel.comp, key)
if permIDs is None:
isG, permIDs = self.verify_close_under_some_perm(arrTrue, arrEst, key)
if not isG:
print ' FAILED TO FIND IDEAL PARAMS'
argstring = ' '.join(sys.argv[1:])
if 'nocapture' in argstring:
from matplotlib import pylab
bnpy.viz.GaussViz.plotGauss2DFromHModel(model)
pylab.show()
return False
arrEst = arrEst[permIDs]
assert self.verify_close(arrTrue, arrEst, key)
else:
allocKeyList.append(key)
assert permIDs is not None
for key in allocKeyList:
print '--------------- %s' % (key)
arrTrue = self.TrueParams[key]
arrEst = getattr(model.allocModel, key)[permIDs]
Util.pprint(arrTrue, 'true')
Util.pprint(arrEst, 'est')
assert self.verify_close(arrTrue, arrEst, key)
return True
def test_vb_repeatable_when_continued(self):
if 'VB' not in self.learnAlgs:
raise SkipTest
kwargs = self.get_kwargs_do_save_disk()
kwargs['nLap'] = 10
hmodel1, LP1, Info1 = run(self.Data, self.allocModelName,
self.obsModelName, 'VB', **kwargs)
kwargs['nLap'] = 5
kwargs['startLap'] = 5
hmodel2, LP2, Info2 = continueRun(self.Data, self.allocModelName,
self.obsModelName, 'VB', **kwargs)
if hasattr(self, 'mustRetainLPAcrossLapsForGuarantees'):
print Info1['evTrace'][-1]
print Info2['evTrace'][-1]
assert closeAtMSigFigs(Info1['evTrace'][-1], Info2['evTrace'][-1], M=2)
else:
assert Info1['evTrace'][-1] == Info2['evTrace'][-1]
def test_vb_repeatable_when_continued(self):
if 'VB' not in self.learnAlgs:
raise SkipTest
kwargs = self.get_kwargs_do_save_disk()
kwargs['nLap'] = 10
hmodel1, LP1, Info1 = run(self.Data, self.allocModelName,
self.obsModelName, 'VB', **kwargs)
kwargs['nLap'] = 5
kwargs['startLap'] = 5
hmodel2, LP2, Info2 = continueRun(self.Data, self.allocModelName,
self.obsModelName, 'VB', **kwargs)
if hasattr(self, 'mustRetainLPAcrossLapsForGuarantees'):
print Info1['evTrace'][-1]
print Info2['evTrace'][-1]
assert closeAtMSigFigs(Info1['evTrace'][-1],
Info2['evTrace'][-1],
M=2)
else:
assert Info1['evTrace'][-1] == Info2['evTrace'][-1]
def test_EM__fromTruth(self):
''' Verify EM alg will not drastically alter model init'd to ideal params.
Performs one run of EM on provided dataset, using 'trueparams' init.
'''
print '' # new line for nosetests
if 'EM' not in self.learnAlgs or not hasattr(self, 'TrueParams'):
raise SkipTest
# Create keyword args for "trueparams" initialization
kwargs = self.get_kwargs(initname='trueparams')
self.Data.TrueParams = self.TrueParams
self.Data.TrueParams['K'] = self.K
# Run EM from a "trueparams" init
model, LP, Info = run(self.Data, self.allocModelName,
self.obsModelName, 'EM', **kwargs)
assert self.verify_monotonic(Info['evTrace'])
for key in self.TrueParams:
if key == 'K':
continue
elif hasattr(model.allocModel, key):
print '--------------- %s' % (key)
arrEst = getattr(model.allocModel, key)
arrTrue = self.TrueParams[key]
Util.pprint(arrTrue, 'true')
Util.pprint(arrEst, 'est')
assert self.verify_close(arrTrue, arrEst, key)
elif hasattr(model.obsModel.comp[0], key):
for k in range(self.K):
arrTrue = self.TrueParams[key][k]
arrEst = getattr(model.obsModel.comp[k], key)
print '--------------- %d/%d %s' % (k + 1, self.K, key)
Util.pprint(arrTrue, 'true')
Util.pprint(arrEst, 'est')
assert self.verify_close(arrTrue, arrEst, key)
def test_EM__fromTruth(self):
''' Verify EM alg will not drastically alter model init'd to ideal params.
Performs one run of EM on provided dataset, using 'trueparams' init.
'''
print '' # new line for nosetests
if 'EM' not in self.learnAlgs or not hasattr(self, 'TrueParams'):
raise SkipTest
# Create keyword args for "trueparams" initialization
kwargs = self.get_kwargs(initname='trueparams')
self.Data.TrueParams = self.TrueParams
self.Data.TrueParams['K'] = self.K
# Run EM from a "trueparams" init
model, LP, Info = run(self.Data, self.allocModelName, self.obsModelName,
'EM', **kwargs)
assert self.verify_monotonic(Info['evTrace'])
for key in self.TrueParams:
if key == 'K':
continue
elif hasattr(model.allocModel, key):
print '--------------- %s' % (key)
arrEst = getattr(model.allocModel, key)
arrTrue = self.TrueParams[key]
Util.pprint(arrTrue, 'true')
Util.pprint(arrEst, 'est')
assert self.verify_close(arrTrue, arrEst, key)
elif hasattr(model.obsModel.comp[0], key):
for k in range(self.K):
arrTrue = self.TrueParams[key][k]
arrEst = getattr(model.obsModel.comp[k], key)
print '--------------- %d/%d %s' % (k+1, self.K, key)
Util.pprint(arrTrue, 'true')
Util.pprint(arrEst, 'est')
assert self.verify_close(arrTrue, arrEst, key)
