def _runsome(): trials = 50 maxsteps = 2000 fwrap = FunctionWrapper(trials, StochQuad(noiseLevel=1, curvature=1)) for aclass in [vSGD]: for aparams in algo_variants[aclass]: for fclass in fun_classes[:2]: for fsettings in fun_settings[:2]: fwrap = FunctionWrapper(trials, fclass(**fsettings)) lossTraces(fwrap, aclass, algoparams=aparams, dim=trials, maxsteps=maxsteps, storesteps=[10])
def testPlot1(trials=20): f = FunctionWrapper(trials, StochQuad(noiseLevel=0.2)) ls = lossTraces(fwrap=f, aclass=SGD, dim=trials, maxsteps=100, algoparams={'learning_rate':0.2}) pylab.plot(ls, 'b:') pylab.plot(pylab.mean(ls, axis=1), 'r-') pylab.semilogy() pylab.show()
def testPlot2(trials=51, maxsteps=5000): f = FunctionWrapper(trials, OptimumJumper(StochQuad(noiseLevel=10, curvature=1), jumptime=1000, jumpdist_std=1)) for aclass, aparams in [#(SGD, {'learning_rate':0.1}), #(SGD, {'learning_rate':0.01}), #(AveragingSGD, {'learning_rate':0.01}), #(AveragingSGD, {'learning_rate':0.01, 'fixedDecay':0.1}), #(AveragingSGD, {'learning_rate':0.01, 'fixedDecay':0.1}), #(AveragingSGD, {'learning_rate':0.1}), #(AveragingSGD, {'learning_rate':1.0}), (AveragingOracle, {}), (AveragingOracle, {"fixedDecay":0.1}), #(AveragingOracle, {"fixedDecay":0.01}), (AdaptivelyAveragingOracle, {}), #(AdaGrad, {'init_lr':0.3}), #(Amari, {'init_lr':0.1, 'time_const':100}), #(RMSProp, {'init_lr':0.1}), (OracleSGD, {}), #(vSGD, {'verbose':False}), #(vSGDfd, {}), ]: ls = lossTraces(fwrap=f, aclass=aclass, dim=trials, maxsteps=maxsteps, algoparams=aparams) plotWithPercentiles(ls, algo_colors[aclass], aclass.__name__) pylab.semilogy() pylab.xlim(0, maxsteps) pylab.legend() pylab.show()
def testPlot4(trials=40, maxsteps=512): fun = StochQuad(noiseLevel=100., curvature=1) fwrap = FunctionWrapper(trials, fun, record_samples=True) fwrap.nextSamples(100000) fwrap = DataFunctionWrapper(fwrap._seen, fun, shuffling=False) for i, (aclass, aparams) in enumerate([ (vSGD, { 'batch_size': 1 }), (vSGDfd, { 'batch_size': 1 }), ]): pylab.subplot(2, 1, 2) fwrap.reset() ls = lossTraces(fwrap=fwrap, aclass=aclass, dim=trials, maxsteps=maxsteps, algoparams=aparams) plotWithPercentiles(ls, algo_colors[aclass], aclass.__name__) pylab.semilogy() pylab.xlim(0, maxsteps) pylab.legend() pylab.subplot(2, 2, i + 1) fwrap.reset() plotHeatmap(fwrap, aclass, aparams, trials, maxsteps) pylab.show()
def testPlot1(trials=20): f = FunctionWrapper(trials, StochQuad(noiseLevel=0.2)) ls = lossTraces(fwrap=f, aclass=SGD, dim=trials, maxsteps=100, algoparams={'learning_rate': 0.2}) pylab.plot(ls, 'b:') pylab.plot(pylab.mean(ls, axis=1), 'r-') pylab.semilogy() pylab.show()
def testPlot4(trials=40, maxsteps=512): fun = StochQuad(noiseLevel=100., curvature=1) fwrap = FunctionWrapper(trials, fun, record_samples=True) fwrap.nextSamples(100000) fwrap = DataFunctionWrapper(fwrap._seen, fun, shuffling=False) for i, (aclass, aparams) in enumerate([(vSGD, {'batch_size':1}), (vSGDfd, {'batch_size':1}), ]): pylab.subplot(2, 1, 2) fwrap.reset() ls = lossTraces(fwrap=fwrap, aclass=aclass, dim=trials, maxsteps=maxsteps, algoparams=aparams) plotWithPercentiles(ls, algo_colors[aclass], aclass.__name__) pylab.semilogy() pylab.xlim(0, maxsteps) pylab.legend() pylab.subplot(2, 2, i + 1) fwrap.reset() plotHeatmap(fwrap, aclass, aparams, trials, maxsteps) pylab.show()
def testPlot2(trials=51, maxsteps=5000): f = FunctionWrapper( trials, OptimumJumper(StochQuad(noiseLevel=10, curvature=1), jumptime=1000, jumpdist_std=1)) for aclass, aparams in [ #(SGD, {'learning_rate':0.1}), #(SGD, {'learning_rate':0.01}), #(AveragingSGD, {'learning_rate':0.01}), #(AveragingSGD, {'learning_rate':0.01, 'fixedDecay':0.1}), #(AveragingSGD, {'learning_rate':0.01, 'fixedDecay':0.1}), #(AveragingSGD, {'learning_rate':0.1}), #(AveragingSGD, {'learning_rate':1.0}), (AveragingOracle, {}), (AveragingOracle, { "fixedDecay": 0.1 }), #(AveragingOracle, {"fixedDecay":0.01}), (AdaptivelyAveragingOracle, {}), #(AdaGrad, {'init_lr':0.3}), #(Amari, {'init_lr':0.1, 'time_const':100}), #(RMSProp, {'init_lr':0.1}), (OracleSGD, {}), #(vSGD, {'verbose':False}), #(vSGDfd, {}), ]: ls = lossTraces(fwrap=f, aclass=aclass, dim=trials, maxsteps=maxsteps, algoparams=aparams) plotWithPercentiles(ls, algo_colors[aclass], aclass.__name__) pylab.semilogy() pylab.xlim(0, maxsteps) pylab.legend() pylab.show()