def testAlgos(dim=3):
# generate a dataset
f = StochQuad(noiseLevel=0.2)
fw = FunctionWrapper(dim, f, record_samples=True)
[fw.nextSamples(1) for _ in range(100)]
ds = fw._seen
dw = DataFunctionWrapper(ds, f, shuffling=False)
x0 = ones(dim)
for algoclass in [SGD, SGD, OracleSGD, Almeida, Amari, RMSProp, AdaGrad, MomentumSGD, AveragingSGD]:
dw.reset()
print algoclass.__name__
algo = algoclass(dw, x0, callback=printy)
algo.run(16)
def testAlgos(dim=3):
# generate a dataset
f = StochQuad(noiseLevel=0.2)
fw = FunctionWrapper(dim, f, record_samples=True)
[fw.nextSamples(1) for _ in range(100)]
ds = fw._seen
dw = DataFunctionWrapper(ds, f, shuffling=False)
x0 = ones(dim)
for algoclass in [SGD, SGD, OracleSGD, Almeida, Amari, RMSProp, AdaGrad,
MomentumSGD, AveragingSGD]:
dw.reset()
print algoclass.__name__
algo = algoclass(dw, x0, callback=printy)
algo.run(16)
def testPlot4(trials=40, maxsteps=512):
fun = StochQuad(noiseLevel=100., curvature=1)
fwrap = FunctionWrapper(trials, fun, record_samples=True)
fwrap.nextSamples(100000)
fwrap = DataFunctionWrapper(fwrap._seen, fun, shuffling=False)
for i, (aclass, aparams) in enumerate([
(vSGD, {
'batch_size': 1
}),
(vSGDfd, {
'batch_size': 1
}),
]):
pylab.subplot(2, 1, 2)
fwrap.reset()
ls = lossTraces(fwrap=fwrap,
aclass=aclass,
dim=trials,
maxsteps=maxsteps,
algoparams=aparams)
plotWithPercentiles(ls, algo_colors[aclass], aclass.__name__)
pylab.semilogy()
pylab.xlim(0, maxsteps)
pylab.legend()
pylab.subplot(2, 2, i + 1)
fwrap.reset()
plotHeatmap(fwrap, aclass, aparams, trials, maxsteps)
pylab.show()
def testMinibatch(dim=4):
f = StochQuad(noiseLevel=0.2)
fw = FunctionWrapper(dim, f, record_samples=True)
x0 = ones(dim)
for mb in [1, 3, 15, 250]:
print "minibatch", mb
algo = SGD(fw, x0, callback=printy, batch_size=mb, learning_rate=0.1)
algo.run(10)
print
[fw.nextSamples(1) for _ in range(2500)]
dw = DataFunctionWrapper(fw._seen, f, shuffling=False)
print "Fixed samples"
for mb in [1, 3, 15, 250]:
print "minibatch", mb
dw.reset()
algo = SGD(dw, x0, callback=printy, batch_size=mb, learning_rate=0.1)
algo.run(10)
print
def testMinibatch(dim=4):
f = StochQuad(noiseLevel=0.2)
fw = FunctionWrapper(dim, f, record_samples=True)
x0 = ones(dim)
for mb in [1,3,15,250]:
print 'minibatch', mb
algo = SGD(fw, x0, callback=printy, batch_size=mb, learning_rate=0.1)
algo.run(10)
print
[fw.nextSamples(1) for _ in range(2500)]
dw = DataFunctionWrapper(fw._seen, f, shuffling=False)
print 'Fixed samples'
for mb in [1,3,15,250]:
print 'minibatch', mb
dw.reset()
algo = SGD(dw, x0, callback=printy, batch_size=mb, learning_rate=0.1)
algo.run(10)
print
def plotAllCombinations(aclasses,
avariants,
fclasses,
fvariants,
trials,
maxsteps,
maxbatchsize=10):
fundic = {}
ploti = 1
rows = sum([len(avariants[ac]) for ac in aclasses]) + len(aclasses) - 1
cols = len(fvariants) * len(fclasses) + len(fclasses) - 1
f_mid = int(median(range(len(fvariants))))
for ac_id, aclass in enumerate(aclasses):
a_mid = int(median(range(len(avariants[aclass]))))
for as_id, aparams in enumerate(avariants[aclass]):
if as_id == 0 and ac_id > 0:
ploti += cols
for fc_id, fclass in enumerate(fclasses):
if fc_id not in fundic:
# shared samples across all uses of one function
fun = fclass()
fwrap = FunctionWrapper(trials, fun, record_samples=True)
fwrap.nextSamples(maxbatchsize * (maxsteps + 10))
fundic[fc_id] = fwrap._seen
data = fundic[fc_id]
for fs_id, fsettings in enumerate(fvariants):
if fs_id == 0 and fc_id > 0:
ploti += 1
fun = fclass(**fsettings)
provider = DataFunctionWrapper(data, fun, shuffling=False)
pylab.subplot(rows, cols, ploti)
ploti += 1
plotHeatmap(provider, aclass, aparams, trials, maxsteps)
if ac_id == 0 and as_id == 0 and fs_id == f_mid:
pylab.title(fclass.__name__[5:])
if fs_id == 0 and as_id == a_mid:
pylab.ylabel(aclass.__name__[:6])
pylab.subplots_adjust(left=0.1,
bottom=0.01,
right=0.99,
top=0.9,
wspace=0.05,
hspace=0.05)
