def testAlgos(dim=3): # generate a dataset f = StochQuad(noiseLevel=0.2) fw = FunctionWrapper(dim, f, record_samples=True) [fw.nextSamples(1) for _ in range(100)] ds = fw._seen dw = DataFunctionWrapper(ds, f, shuffling=False) x0 = ones(dim) for algoclass in [SGD, SGD, OracleSGD, Almeida, Amari, RMSProp, AdaGrad, MomentumSGD, AveragingSGD]: dw.reset() print algoclass.__name__ algo = algoclass(dw, x0, callback=printy) algo.run(16)
def testPlot4(trials=40, maxsteps=512): fun = StochQuad(noiseLevel=100., curvature=1) fwrap = FunctionWrapper(trials, fun, record_samples=True) fwrap.nextSamples(100000) fwrap = DataFunctionWrapper(fwrap._seen, fun, shuffling=False) for i, (aclass, aparams) in enumerate([ (vSGD, { 'batch_size': 1 }), (vSGDfd, { 'batch_size': 1 }), ]): pylab.subplot(2, 1, 2) fwrap.reset() ls = lossTraces(fwrap=fwrap, aclass=aclass, dim=trials, maxsteps=maxsteps, algoparams=aparams) plotWithPercentiles(ls, algo_colors[aclass], aclass.__name__) pylab.semilogy() pylab.xlim(0, maxsteps) pylab.legend() pylab.subplot(2, 2, i + 1) fwrap.reset() plotHeatmap(fwrap, aclass, aparams, trials, maxsteps) pylab.show()
def testMinibatch(dim=4): f = StochQuad(noiseLevel=0.2) fw = FunctionWrapper(dim, f, record_samples=True) x0 = ones(dim) for mb in [1, 3, 15, 250]: print "minibatch", mb algo = SGD(fw, x0, callback=printy, batch_size=mb, learning_rate=0.1) algo.run(10) print [fw.nextSamples(1) for _ in range(2500)] dw = DataFunctionWrapper(fw._seen, f, shuffling=False) print "Fixed samples" for mb in [1, 3, 15, 250]: print "minibatch", mb dw.reset() algo = SGD(dw, x0, callback=printy, batch_size=mb, learning_rate=0.1) algo.run(10) print
def testMinibatch(dim=4): f = StochQuad(noiseLevel=0.2) fw = FunctionWrapper(dim, f, record_samples=True) x0 = ones(dim) for mb in [1,3,15,250]: print 'minibatch', mb algo = SGD(fw, x0, callback=printy, batch_size=mb, learning_rate=0.1) algo.run(10) print [fw.nextSamples(1) for _ in range(2500)] dw = DataFunctionWrapper(fw._seen, f, shuffling=False) print 'Fixed samples' for mb in [1,3,15,250]: print 'minibatch', mb dw.reset() algo = SGD(dw, x0, callback=printy, batch_size=mb, learning_rate=0.1) algo.run(10) print
def plotAllCombinations(aclasses, avariants, fclasses, fvariants, trials, maxsteps, maxbatchsize=10): fundic = {} ploti = 1 rows = sum([len(avariants[ac]) for ac in aclasses]) + len(aclasses) - 1 cols = len(fvariants) * len(fclasses) + len(fclasses) - 1 f_mid = int(median(range(len(fvariants)))) for ac_id, aclass in enumerate(aclasses): a_mid = int(median(range(len(avariants[aclass])))) for as_id, aparams in enumerate(avariants[aclass]): if as_id == 0 and ac_id > 0: ploti += cols for fc_id, fclass in enumerate(fclasses): if fc_id not in fundic: # shared samples across all uses of one function fun = fclass() fwrap = FunctionWrapper(trials, fun, record_samples=True) fwrap.nextSamples(maxbatchsize * (maxsteps + 10)) fundic[fc_id] = fwrap._seen data = fundic[fc_id] for fs_id, fsettings in enumerate(fvariants): if fs_id == 0 and fc_id > 0: ploti += 1 fun = fclass(**fsettings) provider = DataFunctionWrapper(data, fun, shuffling=False) pylab.subplot(rows, cols, ploti) ploti += 1 plotHeatmap(provider, aclass, aparams, trials, maxsteps) if ac_id == 0 and as_id == 0 and fs_id == f_mid: pylab.title(fclass.__name__[5:]) if fs_id == 0 and as_id == a_mid: pylab.ylabel(aclass.__name__[:6]) pylab.subplots_adjust(left=0.1, bottom=0.01, right=0.99, top=0.9, wspace=0.05, hspace=0.05)