def init_vars(self):
self.gran[1:] = [1 for l in range(1,self.numLayers)]
self.horiz[1:] = [max((self.horiz[0]/(self.layer_sizes[l-1]/self.layer_sizes[l]/4)),1) for l in range(1,self.numLayers)]
self.vert[1:] = [max((self.vert[0]/(self.layer_sizes[l-1]/self.layer_sizes[l]/4)),1) for l in range(1,self.numLayers)]
self.scale = np.array([self.layer_sizes[l-1]/self.layer_sizes[l] for l in range(self.numLayers-1,0,-1)])
#self.scale should = 14 for layer 1
#self.gran should = 7
self.indices = np.array([np.array(range(self.layer_sizes[l])) for l in range(self.numLayers)])
self.centers = np.array([np.array([max((self.indices[l][i]*self.scale[l-1])+(np.random.randint(2)-2)*self.gran[l-1],0) for i in range(self.layer_sizes[l])]) for l in range(self.numLayers-1,0,-1)])
self.hspans = [[(np.random.randint(3)+1)*self.gran[l-1] for i in range(self.layer_sizes[l])] for l in range(self.numLayers-1,0,-1)]
self.vspans = [[(np.random.randint(8)+1) for i in range(self.layer_sizes[l])] for l in range(self.numLayers-1,0,-1)]
#self.fields = np.array([[np.array(np.hstack(receptive_field(range(self.layer_sizes[l-1]),1,self.horiz[l-1],self.vert[l-1], self.centers[l-1][i], self.vspans[l-1][i], self.hspans[l-1][i])),dtype='int') for i in range(self.layer_sizes[l])] for l in range(self.numLayers-1,0,-1)])
if self.numLayers == 3:
fields = [np.array(np.hstack(rf4(self.indices[0],1,self.horiz[0],self.vert[0], self.centers[0][i], self.vspans[0][i], self.hspans[0][i])),dtype='int') for i in range(self.layer_sizes[1])]
self.fields = np.array([np.array([np.array(np.hstack(receptive_field(self.indices[l-1],1,self.horiz[l-1],self.vert[l-1], self.centers[l-1][i], self.vspans[l-1][i], self.hspans[l-1][i])),dtype='int') for i in range(self.layer_sizes[l])]) for l in range(self.numLayers-1,0,-1)])
#3 and 8 are pretty arbitrary constants here--should turn them into variables at some point
self.x = np.array(np.zeros((self.layer_sizes[0], self.num_channels)),dtype='int')
self.h = np.array([np.zeros((self.layer_sizes[i], self.num_channels)) for i in range(1, self.numLayers)],dtype='int')
self.q = np.array([np.zeros((self.layer_sizes[i], self.num_channels)) for i in range(1, self.numLayers)])
self.q1 = self.q
self.x_gen = np.array(np.zeros(self.x.shape),dtype='int')
self.h_gen = np.array([np.zeros((self.layer_sizes[i], self.num_channels)) for i in range(1, self.numLayers)],dtype='int')
self.W = np.array([np.array([np.random.rand(self.fields[l-1][h].shape[0])*0.1-0.05 for h in range(self.layer_sizes[l])]) for l in range(self.numLayers-1,0,-1)])
self.pos_W = np.array([np.array([np.zeros(self.W[l][i].shape) for i in range(self.layer_sizes[l+1])]) for l in range(self.numLayers - 1)])
self.neg_W = np.array([np.array([np.zeros(self.W[l][i].shape) for i in range(self.layer_sizes[l+1])]) for l in range(self.numLayers - 1)])
self.W_inc = np.array([np.array([np.zeros(self.W[l][i].shape) for i in range(self.layer_sizes[l+1])]) for l in range(self.numLayers - 1)])
self.a = np.zeros(self.x.shape)
self.b = np.array([np.zeros(self.h[i].shape) for i in range(0, self.numLayers - 1)])
self.a_inc = np.zeros(self.a.shape)
self.b_inc = np.array([np.zeros(self.b[i].shape) for i in range(0, self.numLayers - 1)])
self.errHist = np.zeros((1000, self.num_channels))
def train(self, data, layer=1, datatype='songs', iterations=400, rate=0.1, moment=0, reg=0.0002, noise=0.05):
alpha = rate
momentum = moment
weightcost = reg
iters = iterations
log = np.fromfile('log.txt', dtype='int8', count=1)
if datatype == 'songs':
N = data.numSongs
typeflag = 4
cof = 0
#Assumes dimensionality of data matches network architecture
C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])),dtype='int8') + 1
C[0,:data.whole_song.shape[0]] = data.whole_song
elif datatype == 'phrases':
N = np.amax(data.Nums_u[0])
C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])),dtype='int8') + 1
C[0,:data.P1_u.shape[0]] = data.P1_u
C[1,:data.P2_u.shape[0]] = data.P2_u
C[2,:data.Wv_u.shape[0]] = data.Wv_u
C[3,:data.Ns_u.shape[0]] = data.Ns_u
output = np.zeros((self.num_channels, N, 4, 16, 8)) + 255
elif datatype == 'chains':
N = np.amax(data.Nums_u[2])
typeflag = 2
cof = 0
C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1
C[0,:data.PC_u.shape[0]] = data.PC_u
C[0,:data.P2C_u.shape[0]] = data.P2C_u
C[0,:data.WC_u.shape[0]] = data.WC_u
C[0,:data.NC_u.shape[0]] = data.NC_u
elif datatype == 'tables':
typeflag = 1
cof = 0
N = data.Nums[1][0]
C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])),dtype='int8') + 1
C[0,:data.Tb.shape[0]] = data.Tb
elif datatype == 'instr_pulse':
typeflag = 3
cof = 0
N = data.Nums_u[3][0]
C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1
C[0,:data.PI.shape[0]] = data.PI
elif datatype == 'instr_wave':
typeflag = 3
cof = 1
N = data.Nums_u[3][1]
C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1
C[0,:data.WI.shape[0]] = data.WI
elif datatype == 'instr_kit':
typeflag = 3
cof = 2
N = data.Nums_u[3][2]
C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1
C[0,:data.KI.shape[0]] = data.KI
elif datatype == 'instr_noise':
typeflag = 3
cof = 3
N = data.Nums_u[3][3]
C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1
C[0,:data.NI.shape[0]] = data.NI
k = 0
if (layer==1):
#Main loop - TRAIN LAYER 1
while k < iters:
self.total_err = np.zeros(self.num_channels)
self.err = np.zeros((N, self.num_channels))
print "epoch" + str(k) + "-1L:"
#loop through training set, choosing N examples randomly
for dim in range(self.num_channels):
i = 0
while i < data.Nums_u[typeflag][dim+cof]:
#fetch set of examples
r = np.random.randint(0,data.Nums_u[typeflag][dim+cof])
#r = i
noiseMatrix = np.random.rand(self.x.shape[0])
noiseMatrix = (noiseMatrix >= noise)/1
self.x[:,dim] = np.multiply(noiseMatrix, C[dim,r,:])
self.z = np.array([np.zeros((self.layer_sizes[lay], self.num_channels)) for lay in range(0, self.numLayers)])
#POS
#compute activation of h1:
for h in range(self.h[0].shape[0]):
#print "h: " + str(h)
sample = np.hstack(receptive_field(self.x[:,dim],1,self.horiz[0],self.vert[0], self.centers[0][h], self.vspans[0][h], self.hspans[0][h]))
#print "W: " + str(self.W[0][h])
#print "sample: " + str(sample)
#print "dot: " + str(np.dot(np.transpose(self.W[0][h]),sample))
z = np.dot(np.transpose(self.W[0][h]),sample)
#print "b: " + str(self.b[0][h,dim])
#print "q: " + str(self.q[0][h,dim])
#print "z: " + str(z)
#print "expit: " + str(expit(z + self.b[0][h,dim]))
self.q[0][h,dim] = expit(z + self.b[0][h,dim])
#print self.q[0][h,dim]
rando = np.random.rand(1)
self.h[0][h,dim] = (rando[0] <= self.q[0][h,dim])/1
#print self.h[0][h,dim]
#NEG
#generate fantasy
for h in range(self.h[0].shape[0]):
self.z[0][self.fields[0][h],dim] += np.dot(self.W[0][h],self.h[0][h,dim])
p1 = expit(self.z[0][:,dim] + self.a[:,dim])
rando = np.random.rand(self.x.shape[0])
self.x_gen[:,dim] = (rando <= p1)/1
#h1 again:
for h in range(self.h[0].shape[0]):
sample = np.hstack(receptive_field(self.x_gen[:,dim],1,self.horiz[0],self.vert[0], self.centers[0][h], self.vspans[0][h], self.hspans[0][h]))
z = np.dot(np.transpose(self.W[0][h]),sample)
self.q1[0][h,dim] = expit(z + self.b[0][h,dim])
rando = np.random.rand(1)
self.h_gen[0][h,dim] = (rando[0] <= self.q1[0][h,dim])/1
#COST
#sum errors
self.err[r,dim] = (sum(self.x[:,dim]-self.x_gen[:,dim]))**2
self.total_err[dim] = sum(self.err[:,dim])
#CD-1 learning
for h in range(self.h[0].shape[0]):
#self.pos_W[0][h] = np.outer(self.x[:,dim][self.fields[0][h]],self.q[0][h,dim])
self.pos_W[0][h] = np.multiply(self.x[:,dim][self.fields[0][h]],self.q[0][h,dim])
self.neg_W[0][h] = np.multiply(self.x_gen[:,dim][self.fields[0][h]],self.q1[0][h,dim])
self.W_inc[0][h] = (momentum * self.W_inc[0][h]) + alpha * (self.pos_W[0][h] - self.neg_W[0][h] - (self.W[0][h] * weightcost))
self.b_inc[0][:,dim] = (momentum * self.b_inc[0][:,dim]) + alpha * (self.h[0][:,dim]-self.h_gen[0][:,dim])
self.a_inc[:,dim] = (momentum * self.a_inc[:,dim]) + alpha * (self.x[:,dim]-self.x_gen[:,dim])
self.W[0][:] = self.W[0][:] + self.W_inc[0][:]
self.b[0][:,dim] = self.b[0][:,dim] + self.b_inc[0][:,dim]
self.a[:,dim] = self.a[:,dim] + self.a_inc[:,dim]
#end epoch
i += 1
print "cost-channel " + str(dim) + ":" + str(self.total_err[dim]/data.Nums_u[typeflag][dim])
self.errHist[k,dim] = self.total_err[dim]
k += 1
elif (layer == 2):
#TRAIN LAYER 2
while k < (iters):
self.total_err = np.zeros(self.num_channels)
self.err = np.zeros((N, self.num_channels))
print "epoch" + str(k) + "-2L:"
#loop through training set
for dim in range(self.num_channels):
i = 0
while i < data.Nums_u[typeflag][dim+cof]:
r = np.random.randint(0,data.Nums_u[typeflag][dim+cof])
self.x[:,dim] = C[dim,r,:]
self.z = np.array([np.zeros((self.layer_sizes[lay], self.num_channels)) for lay in range(0, self.numLayers)])
#POS
#compute activation of h1:
z = np.dot(np.transpose(self.W[0][:,:,dim]),self.x[:,dim])
q0 = expit(z + self.b[0][:,dim])
rando = np.random.rand(self.h[0].shape[0])
self.h[0][:,dim] = (rando <= q0)/1
for h in range(self.h[0].shape[0]):
sample = np.hstack(receptive_field(self.x[:,dim],1,self.horiz[0],self.vert[0], self.centers[0][h], self.vspans[0][h], self.hspans[0][h]))
z = np.dot(np.transpose(self.W[0][h]),sample)
self.q[0][h,dim] = expit(z + self.b[0][h,dim])
rando = np.random.rand(1)
self.h[0][h,dim] = (rando[0] <= self.q[0][h,dim])/1
#compute activation of h2:
for h in range(self.h[1].shape[0]):
sample = np.hstack(receptive_field(self.h[0][:,dim],1,self.horiz[1],self.vert[1], self.centers[1][h], self.vspans[1][h], self.hspans[1][h]))
z = np.dot(np.transpose(self.W[1][h]),sample)
self.q[1][h,dim] = expit(z + self.b[1][h,dim])
rando = np.random.rand(1)
self.h[1][h,dim] = (rando[0] <= self.q[1][h,dim])/1
#NEG
#generate fantasy on h1:
for h in range(self.h[1].shape[0]):
self.z[1][self.fields[1][h],dim] += np.dot(self.W[1][h],self.h[1][h,dim])
p1 = expit(self.z[1][:,dim] + self.a[:,dim])
rando = np.random.rand(self.x.shape[0])
self.h_gen[0][:,dim] = (rando <= p1)/1
#fantasize data for error measure
for h in range(self.h[0].shape[0]):
self.z[0][self.fields[0][h],dim] += np.dot(self.W[0][h],self.h_gen[0][h,dim])
p1 = expit(self.z[0][:,dim] + self.a[:,dim])
rando = np.random.rand(self.x.shape[0])
self.x_gen[:,dim] = (rando <= p1)/1
#h2 again:
for h in range(self.h[1].shape[0]):
sample = np.hstack(receptive_field(self.x_gen[:,dim],1,self.horiz[1],self.vert[1], self.centers[1][h], self.vspans[1][h], self.hspans[1][h]))
z = np.dot(np.transpose(self.W[1][h]),sample)
self.q1[1][h,dim] = expit(z + self.b[1][h,dim])
rando = np.random.rand(1)
self.h_gen[1][h,dim] = (rando[0] <= self.q1[1][h,dim])/1
#COST
#sum errors
self.err[r,dim] = (sum(self.x[:,dim]-self.x_gen[:,dim]))**2
self.total_err[dim] += sum(self.err[:,dim])
# {save examples code}
#CD-1 learning
for h in range(self.h[1].shape[0]):
self.pos_W[1][h] = np.multiply(self.x[:,dim][self.fields[1][h]],self.q[1][h,dim])
self.neg_W[1][h] = np.multiply(self.x_gen[:,dim][self.fields[1][h]],self.q1[1][h,dim])
self.W_inc[1][h] = (momentum * self.W_inc[1][h]) + alpha * (self.pos_W[1][h] - self.neg_W[1][h] - (self.W[1][h] * weightcost))
self.b_inc[1][:,dim] = (momentum * self.b_inc[1][:,dim]) + alpha * (self.h[1][:,dim]-self.h_gen[1][:,dim])
self.W[1][:,:,dim] = self.W[1][:,:,dim] + self.W_inc[1][:,:,dim]
self.b[1][:,dim] = self.b[1][:,dim] + self.b_inc[1][:,dim]
#end epoch
i += 1
print "cost-channel " + str(dim) + ":" + str(self.total_err[dim]/data.Nums_u[typeflag][dim])
self.errHist[k,dim] = self.total_err[dim]
k += 1
"""I've only coded up to layer 2 for CRBM training."""
elif (layer == 3):
#TRAIN LAYER 3
while k < (iters):
self.total_err = np.zeros(self.num_channels)
self.err = np.zeros((N, self.num_channels))
print "epoch" + str(k) + "-3L:"
#loop through training set
for dim in range(self.num_channels):
i = 0
while i < data.Nums_u[typeflag][dim+cof]:
r = np.random.randint(0,data.Nums_u[typeflag][dim+cof])
self.x[:,dim] = C[dim,r,:]
#POS
#compute activation of h1:
z = np.dot(np.transpose(self.W[0][:,:,dim]),self.x[:,dim])
q = expit(z + self.b[0][:,dim])
rando = np.random.rand(self.h[0].shape[0])
self.h[0][:,dim] = (rando <= q)/1
#compute activation of h2:
z = np.dot(np.transpose(self.W[1][:,:,dim]),self.h[0][:,dim])
q = expit(z + self.b[1][:,dim])
rando = np.random.rand(self.h[1].shape[0])
self.h[1][:,dim] = (rando <= q)/1
#compute activation of h3:
z = np.dot(np.transpose(self.W[2][:,:,dim]),self.h[1][:,dim])
q = expit(z + self.b[2][:,dim])
rando = np.random.rand(self.h[2].shape[0])
self.h[2][:,dim] = (rando <= q)/1
#NEG
#generate fantasy on h2:
z = np.dot(self.W[2][:,:,dim],self.h[2][:,dim])
p = expit(z + self.b[1][:,dim])
rando = np.random.rand(self.h[1].shape[0])
self.h_gen[1][:,dim] = (rando <= p)/1
#generate fantasy on h1:
z = np.dot(self.W[1][:,:,dim],self.h_gen[1][:,dim])
p = expit(z + self.b[0][:,dim])
rando = np.random.rand(self.h[0].shape[0])
self.h_gen[0][:,dim] = (rando <= p)/1
#fantasize data for error measure
z = np.dot(self.W[0][:,:,dim],self.h_gen[0][:,dim])
p1 = expit(z + self.a[:,dim])
rando = np.random.rand(self.x.shape[0])
self.x_gen[:,dim] = (rando <= p1)/1
#h3 again:
z = np.dot(np.transpose(self.W[2][:,:,dim]),self.h_gen[1][:,dim])
q1 = expit(z + self.b[2][:,dim])
rando = np.random.rand(self.h[2].shape[0])
self.h_gen[2][:,dim] = (rando <= q1)/1
#COST
#sum errors
self.err[dim] = (sum(self.x[:,dim]-self.x_gen[:,dim]))**2
self.total_err[dim] += sum(self.err[dim])
# {save examples code}
#CD-1 learning
self.pos_W[2][:,:,dim] = np.outer(self.h[1][:,dim],q)
self.neg_W[2][:,:,dim] = np.outer(self.h_gen[1][:,dim],q1)
self.W_inc[2][:,:,dim] = (momentum * self.W_inc[2][:,:,dim]) + alpha * (self.pos_W[2][:,:,dim] - self.neg_W[2][:,:,dim] - (self.W[2][:,:,dim]*weightcost))
self.b_inc[2][:,dim] = (momentum * self.b_inc[2][:,dim]) + alpha * (self.h[2][:,dim]-self.h_gen[2][:,dim])
self.W[2][:,:,dim] = self.W[2][:,:,dim] + self.W_inc[2][:,:,dim]
self.b[2][:,dim] = self.b[2][:,dim] + self.b_inc[2][:,dim]
#end epoch
i += 1
print "cost-channel " + str(dim) + ":" + str(self.total_err[dim]/data.Nums_u[typeflag][dim])
self.errHist[k,dim] = self.total_err[dim]
k += 1
elif (layer >= 4):
cur = layer - 1
#TRAIN LAYER 4+
while k < (iters):
self.total_err = np.zeros(self.num_channels)
self.err = np.zeros((N, self.num_channels))
print "epoch" + str(k) + "-"+str(layer)+"L:"
#loop through training set
for dim in range(self.num_channels):
i = 0
while i < data.Nums_u[typeflag][dim+cof]:
r = np.random.randint(0,data.Nums_u[typeflag][dim+cof])
self.x[:,dim] = C[dim,r,:]
#POS
#compute activation of h1:
z = np.dot(np.transpose(self.W[0][:,:,dim]),self.x[:,dim])
q = expit(z + self.b[0][:,dim])
rando = np.random.rand(self.h[0].shape[0])
self.h[0][:,dim] = (rando <= q)/1
for l in range(1,layer):
#compute activation of h2+:
z = np.dot(np.transpose(self.W[l][:,:,dim]),self.h[l-1][:,dim])
q = expit(z + self.b[l][:,dim])
rando = np.random.rand(self.h[l].shape[0])
self.h[l][:,dim] = (rando <= q)/1
self.h_gen[l][:,dim] = self.h[l][:,dim]
#NEG
for l in range(layer-1,0,-1):
#generate fantasy on hN-:
z = np.dot(self.W[l][:,:,dim],self.h_gen[l][:,dim])
p = expit(z + self.b[l-1][:,dim])
rando = np.random.rand(self.h[l-1].shape[0])
self.h_gen[l-1][:,dim] = (rando <= p)/1
#generate fantasy on h1:
z = np.dot(self.W[1][:,:,dim],self.h_gen[1][:,dim])
p = expit(z + self.b[0][:,dim])
rando = np.random.rand(self.h[0].shape[0])
self.h_gen[0][:,dim] = (rando <= p)/1
#fantasize data for error measure
z = np.dot(self.W[0][:,:,dim],self.h_gen[0][:,dim])
p1 = expit(z + self.a[:,dim])
rando = np.random.rand(self.x.shape[0])
self.x_gen[:,dim] = (rando <= p1)/1
#hN again:
z = np.dot(np.transpose(self.W[layer-1][:,:,dim]),self.h_gen[layer-2][:,dim])
q1 = expit(z + self.b[layer-1][:,dim])
rando = np.random.rand(self.h[layer-1].shape[0])
self.h_gen[layer-1][:,dim] = (rando <= q1)/1
#COST
#sum errors
self.err[r, dim] = (sum(self.x[:,dim]-self.x_gen[:,dim]))**2
self.total_err[dim] += sum(self.err[dim])
#CD-1 learning
self.pos_W[cur][:,:,dim] = np.outer(self.h[cur-1][:,dim],q)
self.neg_W[cur][:,:,dim] = np.outer(self.h_gen[cur-1][:,dim],q1)
self.W_inc[cur][:,:,dim] = (momentum * self.W_inc[cur][:,:,dim]) + alpha * (self.pos_W[cur][:,:,dim] - self.neg_W[cur][:,:,dim] - (self.W[cur][:,:,dim]*weightcost))
self.b_inc[cur][:,dim] = (momentum * self.b_inc[cur][:,dim]) + alpha * (self.h[cur][:,dim]-self.h_gen[cur][:,dim])
self.W[cur][:,:,dim] = self.W[cur][:,:,dim] + self.W_inc[cur][:,:,dim]
self.b[cur][:,dim] = self.b[cur][:,dim] + self.b_inc[cur][:,dim]
#end epoch
i += 1
print "cost-channel " + str(dim) + ":" + str(self.total_err[dim]/data.Nums_u[typeflag][dim])
self.errHist[k,dim] = self.total_err[dim]
k += 1
def autoencode(self, data, layer=1, datatype='phrases'):
Gibbs = 400
emptyPhrase = 0
if datatype == 'songs':
N = data.numSongs
C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])),dtype='int8') + 1
C[0,:data.whole_song.shape[0]] = data.whole_song
output = np.zeros((self.num_channels, N, 256, 4, 8)) + 255
if datatype == 'tables':
N = data.Nums[1][0]
C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])),dtype='int8') + 1
C[0,:data.Tb.shape[0]] = data.Tb
output = np.zeros((self.num_channels, N, 6, 16, 8)) + 255
elif datatype == 'phrases':
N = np.amax(data.Nums_u[0])
C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])),dtype='int8') + 1
C[0,:data.P1_u.shape[0]] = data.P1_u
C[1,:data.P2_u.shape[0]] = data.P2_u
C[2,:data.Wv_u.shape[0]] = data.Wv_u
C[3,:data.Ns_u.shape[0]] = data.Ns_u
output = np.zeros((self.num_channels, N, 4, 16, 8)) + 255
elif datatype == 'chains':
N = np.amax(data.Nums_u[2])
C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1
C[0,:data.PC_u.shape[0]] = data.PC_u
C[1,:data.P2C_u.shape[0]] = data.P2C_u
C[2,:data.WC_u.shape[0]] = data.WC_u
C[3,:data.NC_u.shape[0]] = data.NC_u
output = np.zeros((self.num_channels, N, 2, 16, 8)) + 255
elif datatype == 'instr_pulse':
N = data.Nums_u[3][0]
C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1
C[0,:data.PI.shape[0]] = data.PI
output_inc = np.array([np.array(np.zeros(2),dtype='int'),np.array([1,0,1,0,1,0,0,0],dtype='int'),np.array(np.zeros(8),dtype='int'),0,np.array(np.zeros(6),dtype='int'),np.array(np.zeros(8)+1,dtype='int'),0,0,np.array(np.zeros(2),dtype='int'),0,0,np.array(np.zeros(5),dtype='int'),np.array(np.zeros(2),dtype='int'),np.array(np.zeros(4),dtype='int'),np.array(np.zeros(2)+1,dtype='int')],dtype=object)
output = np.array([output_inc for x in range(N)])
elif datatype == 'instr_wave':
N = data.Nums_u[3][1]
C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1
C[0,:data.WI.shape[0]] = data.WI
output_inc = np.array([np.array([0,1],dtype='int'),np.array([0,0,1,0,0,0,0,0],dtype='int'),np.array(np.zeros(4),dtype='int'),np.array([0,0,1,1],dtype='int'),0,0,np.array(np.zeros(2),dtype='int'),0,0,np.array(np.zeros(5),dtype='int'),np.array(np.zeros(2)+1,dtype='int'),np.array(np.zeros(2),dtype='int'),np.array(np.zeros(4)+1,dtype='int'),np.array(np.zeros(4),dtype='int')],dtype=object)
output = np.array([output_inc for x in range(N)])
elif datatype == 'instr_kit':
N = data.Nums_u[3][2]
C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1
C[0,:data.KI.shape[0]] = data.KI
output_inc = np.array([np.array([1,0],dtype='int'),np.array([1,0,1,0,1,0,0,0],dtype='int'),0,0,np.array(np.zeros(6),dtype='int'),np.array(np.zeros(8),dtype='int'),0,0,0,1,np.array([np.array(np.zeros(2),dtype='int'),0]),0,np.array(np.zeros(5),dtype='int'),np.array(np.zeros(2)+1,dtype='int'),np.array(np.zeros(8),dtype='int'),0,np.array(np.zeros(6),dtype='int'),np.array(np.zeros(8),dtype='int'),np.array(np.zeros(8),dtype='int'), np.array(np.zeros(8),dtype='int'), np.array(np.zeros(8),dtype='int')],dtype=object)
output = np.array([output_inc for x in range(N)])
elif datatype == 'instr_noise':
N = data.Nums_u[3][3]
C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1
C[0,:data.NI.shape[0]] = data.NI
output_inc = np.array([np.array([1,1],dtype='int'),np.array([0,1,0,1,0,0,0,0],dtype='int'),np.array(np.zeros(8),dtype='int'),0,np.array(np.zeros(6),dtype='int'),np.array(np.zeros(8),dtype='int'),0,0,1,np.array(np.zeros(5),dtype='int'),np.array([1,1],dtype='int')],dtype=object)
output = np.array([output_inc for x in range(N)])
i = 0
reconstructions = np.array(np.zeros(C.shape),dtype='int')
hidden_reps = np.array(np.zeros((self.num_channels, N, self.h_gen[layer-1].shape[0]))+2,dtype='int')
while (i < N):
for dim in range(self.num_channels):
if datatype == 'chains':
if (sum(C[dim,i,:]) < 256):
self.x[:,dim] = C[dim,i,:]
elif (sum(C[dim,i,:]) == 256):
emptyPhrase = 1
if datatype == 'phrases':
if (sum(C[dim,i,:]) < 512):
self.x[:,dim] = C[dim,i,:]
elif (sum(C[dim,i,:]) == 512):
emptyPhrase = 1
elif datatype == 'tables':
if (sum(C[dim,i,:]) < 768):
self.x[:,dim] = C[dim,i,:]
elif (sum(C[dim,i,:]) == 768):
emptyPhrase = 1
elif datatype == 'instr_pulse':
pass
elif datatype == 'instr_wave':
pass
elif datatype == 'instr_kit':
pass
elif datatype == 'instr_noise':
pass
elif datatype == 'songs':
pass
if (emptyPhrase == 0):
#POS
#compute activation of h1:
for h in range(self.h[0].shape[0]):
sample = np.hstack(receptive_field(self.x[:,dim],1,self.horiz[0],self.vert[0], self.centers[0][h], self.vspans[0][h], self.hspans[0][h]))
z = np.dot(np.transpose(self.W[0][h]),sample)
self.q[0][h,dim] = expit(z + self.b[0][h,dim])
rando = np.random.rand(1)
self.h[0][h,dim] = (rando[0] <= self.q[0][h,dim])/1
if (layer >= 2):
for l in range(1,layer):
for h in range(self.h[l].shape[0]):
sample = np.hstack(receptive_field(self.h[l-1][:,dim],1,self.horiz[l-1],self.vert[l-1], self.centers[l-1][h], self.vspans[l-1][h], self.hspans[l-1][h]))
z = np.dot(np.transpose(self.W[0][h]),sample)
q = expit(z + self.b[l][h,dim])
rando = np.random.rand(1)
self.h[l][h,dim] = (rando[0] <= q)/1
self.h_gen[l][:,dim] = self.h[l][:,dim]
sleep = 0
#NEG
while sleep < Gibbs:
self.z = np.array([np.zeros((self.layer_sizes[lay], self.num_channels)) for lay in range(0, self.numLayers)])
if (layer == 1):
#generate fantasy
for h in range(self.h[0].shape[0]):
self.z[0][self.fields[0][h],dim] += np.dot(self.W[0][h],self.h_gen[0][h,dim])
p1 = expit(self.z[0][:,dim] + self.a[:,dim])
rando = np.random.rand(self.x.shape[0])
self.x_gen[:,dim] = (rando <= p1)/1
#print sleep
#h1 again:
for h in range(self.h[0].shape[0]):
sample = np.hstack(receptive_field(self.x_gen[:,dim],1,self.horiz[0],self.vert[0], self.centers[0][h], self.vspans[0][h], self.hspans[0][h]))
z = np.dot(np.transpose(self.W[0][h]),sample)
q1 = expit(z + self.b[0][h,dim])
rando = np.random.rand(1)
self.h_gen[0][h,dim] = (rando[0] <= q1)/1
elif (layer >= 2):
#generate fantasy on hN-:
for h in range(self.h[layer].shape[0]):
self.z[layer-1][self.fields[layer-1][h],dim] += np.dot(self.W[layer-1][h],self.h_gen[layer-1][h,dim])
p1 = expit(self.z[0][:,dim] + self.a[:,dim])
rando = np.random.rand(self.h[layer-2].shape[0])
self.h_gen[layer-2][:,dim] = (rando <= p1)/1
#hN again:
for h in range(self.h[layer].shape[0]):
sample = np.hstack(receptive_field(self.h_gen[layer-2][:,dim],1,self.horiz[layer-1],self.vert[layer-1], self.centers[layer-1][h], self.vspans[layer-1][h], self.hspans[layer-1][h]))
z = np.dot(np.transpose(self.W[layer-1][h]),sample)
q1 = expit(z + self.b[layer-1][h,dim])
rando = np.random.rand(1)
self.h_gen[layer-1][h,dim] = (rando[0] <= q1)/1
sleep += 1
for l in range(layer-1,0,-1):
self.z = np.array([np.zeros((self.layer_sizes[lay], self.num_channels)) for lay in range(0, self.numLayers)])
#generate fantasy on hN-:
for h in range(self.h[l].shape[0]):
self.z[l][self.fields[l][h],dim] += np.dot(self.W[l][h],self.h_gen[l][h,dim])
p1 = expit(self.z[0][:,dim] + self.a[:,dim])
rando = np.random.rand(self.h[l-1].shape[0])
self.h_gen[l-1][:,dim] = (rando <= p1)/1
#fantasize data
for h in range(self.h[0].shape[0]):
self.z[0][self.fields[0][h],dim] += np.dot(self.W[0][h],self.h[0][h,dim])
p1 = expit(self.z[0][:,dim] + self.a[:,dim])
rando = np.random.rand(self.x.shape[0])
self.x_gen[:,dim] = (rando <= p1)/1
#print reconstructions.shape
#print dim
#print i
#print self.x_gen.shape
reconstructions[dim,i,:] = self.x_gen[:,dim]
hidden_reps[dim,i,:] = self.h_gen[layer-1][:,dim]
print "example " + str(i) + " channel " + str(dim) + " encoded with error " + str((sum(self.x[:,dim]-self.x_gen[:,dim]))**2) + " using " + str(self.h_gen[layer-1].shape[0]) + " bits"
emptyPhrase = 0
i += 1
if datatype == 'phrases':
output[0] = np.reshape(reconstructions[0,:,:],(C[0].shape[0], 4, 16, 8))
output[1] = np.reshape(reconstructions[1,:,:],(C[1].shape[0], 4, 16, 8))
output[2] = np.reshape(reconstructions[2,:,:],(C[2].shape[0], 4, 16, 8))
output[3] = np.reshape(reconstructions[3,:,:],(C[3].shape[0], 4, 16, 8))
elif datatype == 'chains':
output[0] = np.reshape(reconstructions[0,:,:],(C[0].shape[0], 2, 16, 8))
output[1] = np.reshape(reconstructions[1,:,:],(C[1].shape[0], 2, 16, 8))
output[2] = np.reshape(reconstructions[2,:,:],(C[2].shape[0], 2, 16, 8))
output[3] = np.reshape(reconstructions[3,:,:],(C[3].shape[0], 2, 16, 8))
elif datatype == 'tables':
output[0] = np.reshape(reconstructions[0,:,:],(C[0].shape[0], 6, 16, 8))
output = output[0] #Note - this doesn't seem to work?!
elif datatype == 'instr_pulse':
for i in range(N):
output[i,0] = reconstructions[0][i,0:8]
output[i,1] = reconstructions[0][i,8:16]
output[i,2] = reconstructions[0][i,16]
output[i,3] = reconstructions[0][i,17:23]
output[i,4] = reconstructions[0][i,23:31]
output[i,5] = reconstructions[0][i,31]
output[i,6] = reconstructions[0][i,32]
output[i,7] = reconstructions[0][i,33:35]
output[i,8] = reconstructions[0][i,35]
output[i,9] = reconstructions[0][i,36]
output[i,10] = reconstructions[0][i,37:42]
output[i,11] = reconstructions[0][i,42:44]
output[i,12] = reconstructions[0][i,44:48]
output[i,13] = reconstructions[0][i,48:]
elif datatype == 'instr_wave':
for i in range(N):
output[i,0] = reconstructions[0][i,0:8]
output[i,1] = reconstructions[0][i,8:12]
output[i,2] = reconstructions[0][i,12:16]
output[i,3] = reconstructions[0][i,16]
output[i,4] = reconstructions[0][i,17]
output[i,5] = reconstructions[0][i,18:20]
output[i,6] = reconstructions[0][i,20]
output[i,7] = reconstructions[0][i,21]
output[i,8] = reconstructions[0][i,22:27]
output[i,9] = reconstructions[0][i,27:29]
output[i,10] = reconstructions[0][i,29:31]
output[i,11] = reconstructions[0][i,31:36]
output[i,12] = reconstructions[0][i,36:]
elif datatype == 'instr_kit':
for i in range(N):
output[i,0] = reconstructions[0][i,0:8]
output[i,1] = reconstructions[0][i,8]
output[i,2] = reconstructions[0][i,9]
output[i,3] = reconstructions[0][i,10:16]
output[i,4] = reconstructions[0][i,16:24]
output[i,5] = reconstructions[0][i,24]
output[i,6] = reconstructions[0][i,25]
output[i,7] = reconstructions[0][i,26]
output[i,8] = reconstructions[0][i,27]
output[i,9] = reconstructions[0][i,28:30]
output[i,10] = reconstructions[0][i,30]
output[i,11] = reconstructions[0][i,31]
output[i,12] = reconstructions[0][i,32:37]
output[i,13] = reconstructions[0][i,37:39]
output[i,14] = reconstructions[0][i,39:47]
output[i,15] = reconstructions[0][i,47]
output[i,16] = reconstructions[0][i,48:54]
output[i,17] = reconstructions[0][i,54:62]
output[i,18] = reconstructions[0][i,62:70]
output[i,19] = reconstructions[0][i,70:78]
output[i,20] = reconstructions[0][i,78:]
elif datatype == 'instr_noise':
for i in range(N):
output[i,1] = reconstructions[0][i,0:8]
output[i,2] = reconstructions[0][i,8:16]
output[i,3] = reconstructions[0][i,16]
output[i,4] = reconstructions[0][i,17:23]
output[i,5] = reconstructions[0][i,23:31]
output[i,6] = reconstructions[0][i,31]
output[i,7] = reconstructions[0][i,32]
output[i,8] = reconstructions[0][i,33]
output[i,9] = reconstructions[0][i,34:39]
output[i,10] = reconstructions[0][i,39:]
elif datatype == 'songs':
output[0] = np.reshape(reconstructions[0,:,:],(C[0].shape[0],256,4,8))
output = np.array(output[0][0], dtype='int')
return [[hidden_reps, reconstructions], output]
def get_neuron_features(model,
dataset,
batch_size=128,
top_n=100,
out_dir='./output',
mean=[0, 0, 0],
std=[1, 1, 1]):
"""
Generates neuron features for given model definition using given dataset.
:param model: Pytorch model definition.
:param dataset: Dataset used for generating NFs.
:param batch_size: Batch size used for predicting feature maps.
:top_n: Use top_n input patch activations for generating NF.
:out_dir: Directory where generated images are stored.
:mean: Dataset mean used for normalization in transform function.
:std: Dataset std used for normalization in transform function.
:return: returns nothing
"""
# make output directory of not exists
if not os.path.exists(out_dir):
os.makedirs(out_dir)
# set model in eval mode
if torch.cuda.is_available():
model = model.cuda()
# Set model in eval mode
model.eval()
# Dataset
dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False)
mean = np.asarray(mean)
std = np.asarray(std)
# input shape (c,w,h)
in_shape = next(iter(dataloader))[0].shape[1:]
# receptive field info for entire model
receptive_field_dict = receptive_field(model, in_shape)
# output layer info
output_layer_info = receptive_field_dict[str(
list(receptive_field_dict.keys())[-2])]
# check if fm has group convs
fm_groups = output_layer_info['output_shape'][2] if len(
output_layer_info['output_shape']) == 5 else 0
# number of filters in last fm
n_filters = output_layer_info['output_shape'][1]
# Create placeholder for input patches
rf = int(output_layer_info['r'])
if fm_groups > 0:
fm_im = np.zeros((top_n, n_filters, fm_groups, rf, rf, 3))
fm_w = -1e5 * np.ones((top_n, n_filters, fm_groups))
else:
fm_im = np.zeros((top_n, n_filters, rf, rf, 3))
fm_w = -1e5 * np.ones((top_n, n_filters))
# Calculate amount of padding needed for input visualization
# Get range for rf at position 0,0 in final feature map
rf_range = receptive_field_for_unit(
receptive_field_dict, str(list(receptive_field_dict.keys())[-2]),
(0, 0))
pad_y = int(rf - (rf_range[0][1] - rf_range[0][0]))
pad_x = int(rf - (rf_range[1][1] - rf_range[1][0]))
# Print summary
print('Group Convolutions: \t {}, {} elements'.format(
fm_groups > 0, fm_groups))
print('Number of filters: \t {}'.format(n_filters))
print('Receptive field size: \t {}'.format(rf))
print('RF range at (0,0): \t {}'.format(rf_range))
print('Input padding (x,y): \t {}, {}'.format(pad_x, pad_y))
print(
'=============================================================================='
)
# Iterate over all data samples to get input patch for highest neuron activation
# for each filter and transformation
with torch.no_grad():
for inputs, _ in tqdm(dataloader,
total=len(dataloader),
desc='Extracting input patches: '):
if torch.cuda.is_available():
model_inputs = inputs.cuda()
else:
model_inputs = inputs
# Predict feature map
fm = model(model_inputs)
# Convert inputs to numpy w,h,c for visualization
inputs = inputs.permute((0, 2, 3, 1)).numpy()
# Unnormalize
inputs *= std[None, None, None, :]
inputs += mean[None, None, None, :]
inputs = np.clip(inputs, 0, 1)
# Pad inputs for visualization to compensate for padding in layers
inputs = np.pad(inputs,
((0, 0), (pad_y, pad_y), (pad_x, pad_x), (0, 0)),
mode='constant')
# get batch shape
fm_shape = fm.shape
# if gconv: reshape groups into channels
if fm_groups > 0:
fm = fm.view((fm_shape[0], -1, fm_shape[3], fm_shape[4]))
# Get max values and locations of feature maps
# pool size = fm size = fm_shape[-1]
a, b = F.max_pool2d(fm, (fm_shape[-2], fm_shape[-1]),
return_indices=True)
# if gconv: reshape groups back to own dimension
if fm_groups > 0:
a = a.view((fm_shape[0], fm_shape[1], fm_shape[2]))
b = b.view((fm_shape[0], fm_shape[1], fm_shape[2]))
a = a.cpu().numpy()
b = b.cpu().numpy()
# coordinates of max activations
x = b % fm.shape[-1]
y = b // fm.shape[-1]
# store weight and input patches for each max position
for i in range(inputs.shape[0]):
for j in range(n_filters):
if fm_groups == 0:
# check if weight is higher than current lowest weight
if a[i, j] > np.min(fm_w[:, j]):
# replace lowest weight by current weight
m = np.argmin(fm_w[:, j])
fm_w[m, j] = a[i, j]
# store input patch
rf_range = receptive_field_for_unit(
receptive_field_dict,
str(list(receptive_field_dict.keys())[-2]),
(y[i, j], x[i, j]),
bound=False)
fm_im[m,
j, :, :, :] = inputs[i, rf_range[0][0] +
pad_y:rf_range[0][1] +
pad_y, rf_range[1][0] +
pad_x:rf_range[1][1] +
pad_x, :]
else:
# loop over extra dimension for gconv
for k in range(fm_groups):
# check if weight is higher than current lowest weight
if a[i, j, k] > np.min(fm_w[:, j, k]):
# replace lowest weight by current weight
m = np.argmin(fm_w[:, j, k])
# store weight
fm_w[m, j, k] = a[i, j, k]
# store input patch
rf_range = receptive_field_for_unit(
receptive_field_dict,
str(list(receptive_field_dict.keys())[-2]),
(y[i, j, k], x[i, j, k]),
bound=False)
fm_im[m, j, k, :, :, :] = inputs[
i, rf_range[0][0] + pad_y:rf_range[0][1] +
pad_y, rf_range[1][0] +
pad_x:rf_range[1][1] + pad_x, :]
# Calculate and save neuron feature for each filter and transformation
for i in tqdm(range(n_filters),
total=n_filters,
desc='Generating neuron features: '):
if fm_groups == 0:
w_sum = np.sum(fm_w[:, i])
if w_sum > 0:
# Sort patches in order of highest neuron activations
idx = np.argsort(fm_w[:, i])[::-1] # ::-1 for high to low sort
fm_w[:, i] = fm_w[idx, i]
fm_im[:, i, :, :, :] = fm_im[idx, i, :, :, :]
# Calculate neuron feature
fm_nfw = fm_w[:, i, None, None, None] / w_sum
nf = np.sum(fm_im[:, i, :, :, :] * fm_nfw, axis=0)
# Plot 19 highest activated patches
plt.figure(figsize=(40, 2))
for j in range(19):
plt.subplot(1, 20, j + 2)
plt.title('{:.3f}'.format(fm_w[j, i]))
plt.imshow(fm_im[j, i, :, :, :])
# Plot NF
plt.subplot(1, 20, 1)
plt.imshow(nf)
plt.title('NF')
_ = plt.setp(plt.gcf().get_axes(), xticks=[], yticks=[])
plt.savefig(os.path.join(out_dir, 'f_{:02d}.png'.format(i)),
bbox_inches='tight')
plt.savefig(os.path.join(out_dir, 'f_{:02d}.pdf'.format(i)),
bbox_inches='tight')
plt.close()
else:
plt.figure(figsize=(40, 6))
for k in range(fm_groups):
w_sum = np.sum(fm_w[:, i, k])
if w_sum > 0:
# Sort patches in order of highest neuron activations
idx = np.argsort(
fm_w[:, i, k])[::-1] # ::-1 for high to low sort
fm_w[:, i, k] = fm_w[idx, i, k]
fm_im[:, i, k, :, :, :] = fm_im[idx, i, k, :, :, :]
# Calculate neuron feature
fm_nfw = fm_w[:, i, k, None, None, None] / w_sum
nf = np.sum(fm_im[:, i, k, :, :, :] * fm_nfw, axis=0)
# Plot 19 highest activated patches
for j in range(19):
plt.subplot(fm_groups, 20, j + 2 + 20 * k)
plt.title('{:.3f}'.format(fm_w[j, i, k]))
plt.imshow(fm_im[j, i, k, :, :, :])
# Plot NF
plt.subplot(fm_groups, 20, 1 + 20 * k)
plt.imshow(nf)
plt.title('NF')
_ = plt.setp(plt.gcf().get_axes(), xticks=[], yticks=[])
plt.savefig(os.path.join(out_dir, 'f_{:02d}.png'.format(i)),
bbox_inches='tight')
plt.savefig(os.path.join(out_dir, 'f_{:02d}.pdf'.format(i)),
bbox_inches='tight')
plt.close()
print('Done!')
