def prepare_dbg(self,mbatch_provider,Npoint,nsteps,eval_start,save_callback):
""" Prepare the data for visualization """
print "Preparing data for visualization..."
mbatch_provider.getMiniBatch(self.cfg.batchsize, self.layers[0].act)
if eval_start == EvalStartType.trainingset:
print "Starting Eval from Trainingset"
pass
elif eval_start == EvalStartType.vnoise:
print "Starting Eval from VNoise"
cp.fill_rnd_uniform(self.layers[0].act)
cp.apply_scalar_functor(self.layers[0].act,cp.scalar_functor.MULT,0.3)
elif eval_start == EvalStartType.h1noise:
print "Starting Eval from H1Noise"
cp.fill_rnd_uniform(self.layers[1].act)
cp.apply_scalar_functor(self.layers[1].act,cp.scalar_functor.MULT,0.3)
self.downPass(1,sample=True)
self.dbg_datout = []
video = self.cfg.video
for layer_num,layer in enumerate(self.layers[0:-1]):
self.upPass(layer_num, sample=False)
uq = UpdateQ(len(self.layers))
uq.push([1]) # start with some layer in between
step = 0
while uq.minupdates([]) < nsteps:
layernum = uq.pop(firstlayer=0)
if video and layernum == 0:
self.updateLayer(layernum,sample=False)
self.save_fantasy(step, Npoint,save_callback, self.layers[0].act)
self.updateLayer(layernum,sample=True)
step+=1
while uq.minupdates([]) < nsteps+2:
layernum = uq.pop(firstlayer=0)
self.updateLayer(layernum,sample=False)
self.updateLayer(0,sample=False)
# pass up again before we save fantasies -- assures that we see bottom-up activities!
for layer_num,layer in enumerate(self.layers[0:-1]):
self.upPass(layer_num, sample=False)
self.save_fantasy(nsteps+1,Npoint,save_callback, self.layers[0].act)
self.dbg_sampleset = mbatch_provider.sampleset_[:, 0:Npoint].T
print "Pulling Layer-Activations..."
self.act = {}
self.act_info = {}
for l in xrange(1, self.cfg.num_layers):
L = self.layers[l]
if l<self.cfg.num_layers-1:
self.act_info["%d-subs"%l] = dict(px=np.sqrt(L.size), py=np.sqrt(L.size))
self.act["%d-subs"%l] = L.act.np
if self.weights[0].mat.shape[0] < 800*6:
print "Trying to pull W0..."
try:
self.W=self.weights[0].mat.np
if len(self.weights)>1:
self.W1=self.weights[1].mat.np
except MemoryError:
print("weights too big!")
print "done."
def __init__(self, cfg, weights,biases):
self.cfg=cfg
self.NumCorrect = 0
self.Errorrate=[]
self.testError=[]
self.NumberOfLayers = cfg.num_layers+1
self.preEpochHook = lambda mlp,epoch: mlp
self.Weights = weights
self.DeltaWeightsOld = []
self.WeightsLearnRate = []
self.dWeights = []
self.dBias = []
self.Bias = biases
self.DeltaBiasOld = []
self.BiasLearnRate = []
l = 0.001
self.NumberOfNeuronsPerLayer = []
for i in xrange(self.NumberOfLayers-2):
#self.Weights.append(newWeights)
dim1, dim2 = self.Weights[i].shape
self.createCopyFilled(self.DeltaWeightsOld,self.Weights[i] , 0)
self.createCopyFilled(self.WeightsLearnRate,self.Weights[i] , l)
if not self.cfg.finetune_online_learning or (self.cfg.finetune_online_learning and self.cfg.finetune_rprop):
self.createCopyFilled(self.dWeights,self.Weights[i] , 0)
self.createCopyFilled(self.dBias,self.Bias[i] , 0)
self.createFilled(self.DeltaBiasOld, dim2, 1, 0)
self.createFilled(self.BiasLearnRate, dim2, 1, l)
self.NumberOfNeuronsPerLayer.append(dim1)
# create dense matrix for last layer
dim1,dim2 = self.Weights[-1].shape[1], self.cfg.num_classes
if self.cfg.load and self.loadLastLayer(dim1,dim2):
pass
else:
self.Weights.append(cp.dev_tensor_float_cm([dim1,dim2]))
cp.fill_rnd_uniform(self.Weights[-1])
#print "Initializing weights with rnd(%2.5f)",
cp.apply_scalar_functor(self.Weights[-1],cp.scalar_functor.SUBTRACT, 0.5)
#cp.apply_scalar_functor(self.Weights[-1],cp.scalar_functor.MULT, 1./math.sqrt(self.Weights[-2].w))
cp.apply_scalar_functor(self.Weights[-1],cp.scalar_functor.MULT, 1./self.Weights[-2].shape[1])
self.createFilled(self.Bias, dim2, 1, 0)
self.createFilled(self.DeltaBiasOld, dim2, 1, 0)
self.createFilled(self.BiasLearnRate, dim2, 1, l)
self.createFilled(self.DeltaWeightsOld,dim1,dim2,0)
self.createFilled(self.WeightsLearnRate,dim1,dim2,l)
if not self.cfg.finetune_online_learning or (self.cfg.finetune_online_learning and self.cfg.finetune_rprop):
self.createCopyFilled(self.dWeights,self.Weights[-1] , 0)
self.createCopyFilled(self.dBias,self.Bias[-1] , 0)
self.NumberOfNeuronsPerLayer.append(dim1)
self.NumberOfNeuronsPerLayer.append(dim2)
self.reconstruction_error = []
def __init__(self, source_layer, target_layer):
"""Constructor
@param source_layer pointer to the previous neuron layer.
@param target_layer pointer to the next neuron layer.
"""
self.source=source_layer
self.target=target_layer
dim1 = self.target.activations.h
dim2 = self.source.activations.h
self.weight = cp.get_filled_matrix(dim1, dim2, 0.0)
cp.fill_rnd_uniform(self.weight)
cp.apply_scalar_functor(self.weight, cp.scalar_functor.SUBTRACT, 0.5)
cp.apply_scalar_functor(self.weight, cp.scalar_functor.DIV, 10)
self.bias = cp.get_filled_matrix(dim1, 1, 0)
def __init__(self, source_layer, target_layer):
"""Constructor
@param source_layer reference to previous neuron layer.
@param target_layer reference to next neuron layer.
"""
self.source = source_layer
self.target = target_layer
dim1 = self.target.activations.shape[0]
dim2 = self.source.activations.shape[0]
self.weight = cp.get_filled_matrix(dim1, dim2, 0.0)
cp.fill_rnd_uniform(self.weight)
self.weight -= 0.5
self.weight /= 10.0
self.bias = cp.dev_tensor_float(dim1)
cp.fill(self.bias, 0)
def prepare_dbg(self, mbatch_provider, Npoint, nsteps, eval_start, save_callback):
""" Prepare the data for visualization """
print "Preparing data for visualization..."
mbatch_provider.getMiniBatch(self.cfg.batchsize, self.layers[0].act)
if eval_start == EvalStartType.trainingset:
print "Starting Eval from Trainingset"
pass
elif eval_start == EvalStartType.vnoise:
print "Starting Eval from VNoise"
cp.fill_rnd_uniform(self.layers[0].act)
cp.apply_scalar_functor(self.layers[0].act, cp.scalar_functor.MULT, 0.3)
elif eval_start == EvalStartType.h1noise:
print "Starting Eval from H1Noise"
cp.fill_rnd_uniform(self.layers[1].act)
cp.apply_scalar_functor(self.layers[1].act, cp.scalar_functor.MULT, 0.3)
self.downPass(1, sample = True)
self.dbg_datout = []
video = self.cfg.video
for layer_num, layer in enumerate(self.layers[0:-1]):
self.upPass(layer_num, sample = False)
#if layer_num+2 < len(self.layers):
#assert(False)
num_meanfield = 100
for step in xrange(nsteps+num_meanfield):
sample = not step>nsteps
self.upPass(self.cfg.num_layers-2, sample = sample)
if video:
for lay_num in reversed(xrange(1, self.cfg.num_layers)):
self.downPass(lay_num, sample = False)
self.save_fantasy(step, Npoint, save_callback, self.layers[0].act)
self.downPass(self.cfg.num_layers-1, sample = sample)
for layer_num in reversed(xrange(1, self.cfg.num_layers)):
self.downPass(layer_num, sample = False)
layer = self.layers[layer_num-1]
#for bla in xrange(1):
# self.downPass(1, sample = False)
# self.upPass(0, sample = False)
# pass up again before we save fantasies -- assures that we see bottom-up activities!
for layer_num, layer in enumerate(self.layers[0:-1]):
self.upPass(layer_num, sample = False)
#if layer_num+2 < len(self.layers):
#assert(False)
self.save_fantasy(nsteps+1, Npoint, save_callback, self.layers[0].act)
self.dbg_sampleset = mbatch_provider.sampleset_[:, 0:Npoint].T
print "Pulling Layer-Activations..."
self.act = {}
self.act_info = {}
for l in xrange(1, self.cfg.num_layers):
L = self.layers[l]
if l<self.cfg.num_layers-1:
self.act_info["%d-subs"%l] = dict(px = np.sqrt(L.size), py = np.sqrt(L.size))
self.act["%d-subs"%l] = L.act.np
if self.weights[0].mat.shape[0] < 800*6:
print "Trying to pull W0..."
try:
self.W = self.weights[0].mat.np
if len(self.weights)>1:
self.W1 = self.weights[1].mat.np
except MemoryError:
print("weights too big!")
print "done."
def prepare_dbg(self, mbatch_provider, Npoint, nsteps, eval_start,
save_callback):
""" Prepare the data for visualization """
print "Preparing data for visualization..."
mbatch_provider.getMiniBatch(self.cfg.batchsize, self.layers[0].act)
if eval_start == EvalStartType.trainingset:
print "Starting Eval from Trainingset"
pass
elif eval_start == EvalStartType.vnoise:
print "Starting Eval from VNoise"
cp.fill_rnd_uniform(self.layers[0].act)
cp.apply_scalar_functor(self.layers[0].act, cp.scalar_functor.MULT,
0.3)
elif eval_start == EvalStartType.h1noise:
print "Starting Eval from H1Noise"
cp.fill_rnd_uniform(self.layers[1].act)
cp.apply_scalar_functor(self.layers[1].act, cp.scalar_functor.MULT,
0.3)
self.downPass(1, sample=True)
self.dbg_datout = []
video = self.cfg.video
for layer_num, layer in enumerate(self.layers[0:-1]):
self.upPass(layer_num, sample=False)
#if layer_num+2 < len(self.layers):
#assert(False)
num_meanfield = 100
for step in xrange(nsteps + num_meanfield):
sample = not step > nsteps
self.upPass(self.cfg.num_layers - 2, sample=sample)
if video:
for lay_num in reversed(xrange(1, self.cfg.num_layers)):
self.downPass(lay_num, sample=False)
self.save_fantasy(step, Npoint, save_callback,
self.layers[0].act)
self.downPass(self.cfg.num_layers - 1, sample=sample)
for layer_num in reversed(xrange(1, self.cfg.num_layers)):
self.downPass(layer_num, sample=False)
layer = self.layers[layer_num - 1]
#for bla in xrange(1):
# self.downPass(1, sample = False)
# self.upPass(0, sample = False)
# pass up again before we save fantasies -- assures that we see bottom-up activities!
for layer_num, layer in enumerate(self.layers[0:-1]):
self.upPass(layer_num, sample=False)
#if layer_num+2 < len(self.layers):
#assert(False)
self.save_fantasy(nsteps + 1, Npoint, save_callback,
self.layers[0].act)
self.dbg_sampleset = mbatch_provider.sampleset_[:, 0:Npoint].T
print "Pulling Layer-Activations..."
self.act = {}
self.act_info = {}
for l in xrange(1, self.cfg.num_layers):
L = self.layers[l]
if l < self.cfg.num_layers - 1:
self.act_info["%d-subs" % l] = dict(px=np.sqrt(L.size),
py=np.sqrt(L.size))
self.act["%d-subs" % l] = L.act.np
if self.weights[0].mat.shape[0] < 800 * 6:
print "Trying to pull W0..."
try:
self.W = self.weights[0].mat.np
if len(self.weights) > 1:
self.W1 = self.weights[1].mat.np
except MemoryError:
print("weights too big!")
print "done."
def __init__(self, cfg, weights, biases):
self.cfg = cfg
self.NumCorrect = 0
self.Errorrate = []
self.testError = []
self.NumberOfLayers = cfg.num_layers + 1
self.preEpochHook = lambda mlp, epoch: mlp
self.Weights = weights
self.DeltaWeightsOld = []
self.WeightsLearnRate = []
self.dWeights = []
self.dBias = []
self.Bias = biases
self.DeltaBiasOld = []
self.BiasLearnRate = []
l = 0.001
self.NumberOfNeuronsPerLayer = []
for i in xrange(self.NumberOfLayers - 2):
#self.Weights.append(newWeights)
dim1, dim2 = self.Weights[i].shape
self.createCopyFilled(self.DeltaWeightsOld, self.Weights[i], 0)
self.createCopyFilled(self.WeightsLearnRate, self.Weights[i], l)
if not self.cfg.finetune_online_learning or (
self.cfg.finetune_online_learning
and self.cfg.finetune_rprop):
self.createCopyFilled(self.dWeights, self.Weights[i], 0)
self.createCopyFilled(self.dBias, self.Bias[i], 0)
self.createFilled(self.DeltaBiasOld, dim2, 1, 0)
self.createFilled(self.BiasLearnRate, dim2, 1, l)
self.NumberOfNeuronsPerLayer.append(dim1)
# create dense matrix for last layer
dim1, dim2 = self.Weights[-1].shape[1], self.cfg.num_classes
if self.cfg.load and self.loadLastLayer(dim1, dim2):
pass
else:
self.Weights.append(cp.dev_tensor_float_cm([dim1, dim2]))
cp.fill_rnd_uniform(self.Weights[-1])
#print "Initializing weights with rnd(%2.5f)",
cp.apply_scalar_functor(self.Weights[-1],
cp.scalar_functor.SUBTRACT, 0.5)
#cp.apply_scalar_functor(self.Weights[-1],cp.scalar_functor.MULT, 1./math.sqrt(self.Weights[-2].w))
cp.apply_scalar_functor(self.Weights[-1], cp.scalar_functor.MULT,
1. / self.Weights[-2].shape[1])
self.createFilled(self.Bias, dim2, 1, 0)
self.createFilled(self.DeltaBiasOld, dim2, 1, 0)
self.createFilled(self.BiasLearnRate, dim2, 1, l)
self.createFilled(self.DeltaWeightsOld, dim1, dim2, 0)
self.createFilled(self.WeightsLearnRate, dim1, dim2, l)
if not self.cfg.finetune_online_learning or (
self.cfg.finetune_online_learning and self.cfg.finetune_rprop):
self.createCopyFilled(self.dWeights, self.Weights[-1], 0)
self.createCopyFilled(self.dBias, self.Bias[-1], 0)
self.NumberOfNeuronsPerLayer.append(dim1)
self.NumberOfNeuronsPerLayer.append(dim2)
self.reconstruction_error = []
def prepare_dbg(self, mbatch_provider, Npoint, nsteps, eval_start,
save_callback):
""" Prepare the data for visualization """
print "Preparing data for visualization..."
mbatch_provider.getMiniBatch(self.cfg.batchsize, self.layers[0].act)
if eval_start == EvalStartType.trainingset:
print "Starting Eval from Trainingset"
pass
elif eval_start == EvalStartType.vnoise:
print "Starting Eval from VNoise"
cp.fill_rnd_uniform(self.layers[0].act)
cp.apply_scalar_functor(self.layers[0].act, cp.scalar_functor.MULT,
0.3)
elif eval_start == EvalStartType.h1noise:
print "Starting Eval from H1Noise"
cp.fill_rnd_uniform(self.layers[1].act)
cp.apply_scalar_functor(self.layers[1].act, cp.scalar_functor.MULT,
0.3)
self.downPass(1, sample=True)
self.dbg_datout = []
video = self.cfg.video
for layer_num, layer in enumerate(self.layers[0:-1]):
self.upPass(layer_num, sample=False)
uq = UpdateQ(len(self.layers))
uq.push([1]) # start with some layer in between
step = 0
while uq.minupdates([]) < nsteps:
layernum = uq.pop(firstlayer=0)
if video and layernum == 0:
self.updateLayer(layernum, sample=False)
self.save_fantasy(step, Npoint, save_callback,
self.layers[0].act)
self.updateLayer(layernum, sample=True)
step += 1
while uq.minupdates([]) < nsteps + 2:
layernum = uq.pop(firstlayer=0)
self.updateLayer(layernum, sample=False)
self.updateLayer(0, sample=False)
# pass up again before we save fantasies -- assures that we see bottom-up activities!
for layer_num, layer in enumerate(self.layers[0:-1]):
self.upPass(layer_num, sample=False)
self.save_fantasy(nsteps + 1, Npoint, save_callback,
self.layers[0].act)
self.dbg_sampleset = mbatch_provider.sampleset_[:, 0:Npoint].T
print "Pulling Layer-Activations..."
self.act = {}
self.act_info = {}
for l in xrange(1, self.cfg.num_layers):
L = self.layers[l]
if l < self.cfg.num_layers - 1:
self.act_info["%d-subs" % l] = dict(px=np.sqrt(L.size),
py=np.sqrt(L.size))
self.act["%d-subs" % l] = L.act.np
if self.weights[0].mat.shape[0] < 800 * 6:
print "Trying to pull W0..."
try:
self.W = self.weights[0].mat.np
if len(self.weights) > 1:
self.W1 = self.weights[1].mat.np
except MemoryError:
print("weights too big!")
print "done."
