def init_vars(self): self.gran[1:] = [1 for l in range(1,self.numLayers)] self.horiz[1:] = [max((self.horiz[0]/(self.layer_sizes[l-1]/self.layer_sizes[l]/4)),1) for l in range(1,self.numLayers)] self.vert[1:] = [max((self.vert[0]/(self.layer_sizes[l-1]/self.layer_sizes[l]/4)),1) for l in range(1,self.numLayers)] self.scale = np.array([self.layer_sizes[l-1]/self.layer_sizes[l] for l in range(self.numLayers-1,0,-1)]) #self.scale should = 14 for layer 1 #self.gran should = 7 self.indices = np.array([np.array(range(self.layer_sizes[l])) for l in range(self.numLayers)]) self.centers = np.array([np.array([max((self.indices[l][i]*self.scale[l-1])+(np.random.randint(2)-2)*self.gran[l-1],0) for i in range(self.layer_sizes[l])]) for l in range(self.numLayers-1,0,-1)]) self.hspans = [[(np.random.randint(3)+1)*self.gran[l-1] for i in range(self.layer_sizes[l])] for l in range(self.numLayers-1,0,-1)] self.vspans = [[(np.random.randint(8)+1) for i in range(self.layer_sizes[l])] for l in range(self.numLayers-1,0,-1)] #self.fields = np.array([[np.array(np.hstack(receptive_field(range(self.layer_sizes[l-1]),1,self.horiz[l-1],self.vert[l-1], self.centers[l-1][i], self.vspans[l-1][i], self.hspans[l-1][i])),dtype='int') for i in range(self.layer_sizes[l])] for l in range(self.numLayers-1,0,-1)]) if self.numLayers == 3: fields = [np.array(np.hstack(rf4(self.indices[0],1,self.horiz[0],self.vert[0], self.centers[0][i], self.vspans[0][i], self.hspans[0][i])),dtype='int') for i in range(self.layer_sizes[1])] self.fields = np.array([np.array([np.array(np.hstack(receptive_field(self.indices[l-1],1,self.horiz[l-1],self.vert[l-1], self.centers[l-1][i], self.vspans[l-1][i], self.hspans[l-1][i])),dtype='int') for i in range(self.layer_sizes[l])]) for l in range(self.numLayers-1,0,-1)]) #3 and 8 are pretty arbitrary constants here--should turn them into variables at some point self.x = np.array(np.zeros((self.layer_sizes[0], self.num_channels)),dtype='int') self.h = np.array([np.zeros((self.layer_sizes[i], self.num_channels)) for i in range(1, self.numLayers)],dtype='int') self.q = np.array([np.zeros((self.layer_sizes[i], self.num_channels)) for i in range(1, self.numLayers)]) self.q1 = self.q self.x_gen = np.array(np.zeros(self.x.shape),dtype='int') self.h_gen = np.array([np.zeros((self.layer_sizes[i], self.num_channels)) for i in range(1, self.numLayers)],dtype='int') self.W = np.array([np.array([np.random.rand(self.fields[l-1][h].shape[0])*0.1-0.05 for h in range(self.layer_sizes[l])]) for l in range(self.numLayers-1,0,-1)]) self.pos_W = np.array([np.array([np.zeros(self.W[l][i].shape) for i in range(self.layer_sizes[l+1])]) for l in range(self.numLayers - 1)]) self.neg_W = np.array([np.array([np.zeros(self.W[l][i].shape) for i in range(self.layer_sizes[l+1])]) for l in range(self.numLayers - 1)]) self.W_inc = np.array([np.array([np.zeros(self.W[l][i].shape) for i in range(self.layer_sizes[l+1])]) for l in range(self.numLayers - 1)]) self.a = np.zeros(self.x.shape) self.b = np.array([np.zeros(self.h[i].shape) for i in range(0, self.numLayers - 1)]) self.a_inc = np.zeros(self.a.shape) self.b_inc = np.array([np.zeros(self.b[i].shape) for i in range(0, self.numLayers - 1)]) self.errHist = np.zeros((1000, self.num_channels))
def train(self, data, layer=1, datatype='songs', iterations=400, rate=0.1, moment=0, reg=0.0002, noise=0.05): alpha = rate momentum = moment weightcost = reg iters = iterations log = np.fromfile('log.txt', dtype='int8', count=1) if datatype == 'songs': N = data.numSongs typeflag = 4 cof = 0 #Assumes dimensionality of data matches network architecture C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])),dtype='int8') + 1 C[0,:data.whole_song.shape[0]] = data.whole_song elif datatype == 'phrases': N = np.amax(data.Nums_u[0]) C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])),dtype='int8') + 1 C[0,:data.P1_u.shape[0]] = data.P1_u C[1,:data.P2_u.shape[0]] = data.P2_u C[2,:data.Wv_u.shape[0]] = data.Wv_u C[3,:data.Ns_u.shape[0]] = data.Ns_u output = np.zeros((self.num_channels, N, 4, 16, 8)) + 255 elif datatype == 'chains': N = np.amax(data.Nums_u[2]) typeflag = 2 cof = 0 C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1 C[0,:data.PC_u.shape[0]] = data.PC_u C[0,:data.P2C_u.shape[0]] = data.P2C_u C[0,:data.WC_u.shape[0]] = data.WC_u C[0,:data.NC_u.shape[0]] = data.NC_u elif datatype == 'tables': typeflag = 1 cof = 0 N = data.Nums[1][0] C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])),dtype='int8') + 1 C[0,:data.Tb.shape[0]] = data.Tb elif datatype == 'instr_pulse': typeflag = 3 cof = 0 N = data.Nums_u[3][0] C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1 C[0,:data.PI.shape[0]] = data.PI elif datatype == 'instr_wave': typeflag = 3 cof = 1 N = data.Nums_u[3][1] C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1 C[0,:data.WI.shape[0]] = data.WI elif datatype == 'instr_kit': typeflag = 3 cof = 2 N = data.Nums_u[3][2] C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1 C[0,:data.KI.shape[0]] = data.KI elif datatype == 'instr_noise': typeflag = 3 cof = 3 N = data.Nums_u[3][3] C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1 C[0,:data.NI.shape[0]] = data.NI k = 0 if (layer==1): #Main loop - TRAIN LAYER 1 while k < iters: self.total_err = np.zeros(self.num_channels) self.err = np.zeros((N, self.num_channels)) print "epoch" + str(k) + "-1L:" #loop through training set, choosing N examples randomly for dim in range(self.num_channels): i = 0 while i < data.Nums_u[typeflag][dim+cof]: #fetch set of examples r = np.random.randint(0,data.Nums_u[typeflag][dim+cof]) #r = i noiseMatrix = np.random.rand(self.x.shape[0]) noiseMatrix = (noiseMatrix >= noise)/1 self.x[:,dim] = np.multiply(noiseMatrix, C[dim,r,:]) self.z = np.array([np.zeros((self.layer_sizes[lay], self.num_channels)) for lay in range(0, self.numLayers)]) #POS #compute activation of h1: for h in range(self.h[0].shape[0]): #print "h: " + str(h) sample = np.hstack(receptive_field(self.x[:,dim],1,self.horiz[0],self.vert[0], self.centers[0][h], self.vspans[0][h], self.hspans[0][h])) #print "W: " + str(self.W[0][h]) #print "sample: " + str(sample) #print "dot: " + str(np.dot(np.transpose(self.W[0][h]),sample)) z = np.dot(np.transpose(self.W[0][h]),sample) #print "b: " + str(self.b[0][h,dim]) #print "q: " + str(self.q[0][h,dim]) #print "z: " + str(z) #print "expit: " + str(expit(z + self.b[0][h,dim])) self.q[0][h,dim] = expit(z + self.b[0][h,dim]) #print self.q[0][h,dim] rando = np.random.rand(1) self.h[0][h,dim] = (rando[0] <= self.q[0][h,dim])/1 #print self.h[0][h,dim] #NEG #generate fantasy for h in range(self.h[0].shape[0]): self.z[0][self.fields[0][h],dim] += np.dot(self.W[0][h],self.h[0][h,dim]) p1 = expit(self.z[0][:,dim] + self.a[:,dim]) rando = np.random.rand(self.x.shape[0]) self.x_gen[:,dim] = (rando <= p1)/1 #h1 again: for h in range(self.h[0].shape[0]): sample = np.hstack(receptive_field(self.x_gen[:,dim],1,self.horiz[0],self.vert[0], self.centers[0][h], self.vspans[0][h], self.hspans[0][h])) z = np.dot(np.transpose(self.W[0][h]),sample) self.q1[0][h,dim] = expit(z + self.b[0][h,dim]) rando = np.random.rand(1) self.h_gen[0][h,dim] = (rando[0] <= self.q1[0][h,dim])/1 #COST #sum errors self.err[r,dim] = (sum(self.x[:,dim]-self.x_gen[:,dim]))**2 self.total_err[dim] = sum(self.err[:,dim]) #CD-1 learning for h in range(self.h[0].shape[0]): #self.pos_W[0][h] = np.outer(self.x[:,dim][self.fields[0][h]],self.q[0][h,dim]) self.pos_W[0][h] = np.multiply(self.x[:,dim][self.fields[0][h]],self.q[0][h,dim]) self.neg_W[0][h] = np.multiply(self.x_gen[:,dim][self.fields[0][h]],self.q1[0][h,dim]) self.W_inc[0][h] = (momentum * self.W_inc[0][h]) + alpha * (self.pos_W[0][h] - self.neg_W[0][h] - (self.W[0][h] * weightcost)) self.b_inc[0][:,dim] = (momentum * self.b_inc[0][:,dim]) + alpha * (self.h[0][:,dim]-self.h_gen[0][:,dim]) self.a_inc[:,dim] = (momentum * self.a_inc[:,dim]) + alpha * (self.x[:,dim]-self.x_gen[:,dim]) self.W[0][:] = self.W[0][:] + self.W_inc[0][:] self.b[0][:,dim] = self.b[0][:,dim] + self.b_inc[0][:,dim] self.a[:,dim] = self.a[:,dim] + self.a_inc[:,dim] #end epoch i += 1 print "cost-channel " + str(dim) + ":" + str(self.total_err[dim]/data.Nums_u[typeflag][dim]) self.errHist[k,dim] = self.total_err[dim] k += 1 elif (layer == 2): #TRAIN LAYER 2 while k < (iters): self.total_err = np.zeros(self.num_channels) self.err = np.zeros((N, self.num_channels)) print "epoch" + str(k) + "-2L:" #loop through training set for dim in range(self.num_channels): i = 0 while i < data.Nums_u[typeflag][dim+cof]: r = np.random.randint(0,data.Nums_u[typeflag][dim+cof]) self.x[:,dim] = C[dim,r,:] self.z = np.array([np.zeros((self.layer_sizes[lay], self.num_channels)) for lay in range(0, self.numLayers)]) #POS #compute activation of h1: z = np.dot(np.transpose(self.W[0][:,:,dim]),self.x[:,dim]) q0 = expit(z + self.b[0][:,dim]) rando = np.random.rand(self.h[0].shape[0]) self.h[0][:,dim] = (rando <= q0)/1 for h in range(self.h[0].shape[0]): sample = np.hstack(receptive_field(self.x[:,dim],1,self.horiz[0],self.vert[0], self.centers[0][h], self.vspans[0][h], self.hspans[0][h])) z = np.dot(np.transpose(self.W[0][h]),sample) self.q[0][h,dim] = expit(z + self.b[0][h,dim]) rando = np.random.rand(1) self.h[0][h,dim] = (rando[0] <= self.q[0][h,dim])/1 #compute activation of h2: for h in range(self.h[1].shape[0]): sample = np.hstack(receptive_field(self.h[0][:,dim],1,self.horiz[1],self.vert[1], self.centers[1][h], self.vspans[1][h], self.hspans[1][h])) z = np.dot(np.transpose(self.W[1][h]),sample) self.q[1][h,dim] = expit(z + self.b[1][h,dim]) rando = np.random.rand(1) self.h[1][h,dim] = (rando[0] <= self.q[1][h,dim])/1 #NEG #generate fantasy on h1: for h in range(self.h[1].shape[0]): self.z[1][self.fields[1][h],dim] += np.dot(self.W[1][h],self.h[1][h,dim]) p1 = expit(self.z[1][:,dim] + self.a[:,dim]) rando = np.random.rand(self.x.shape[0]) self.h_gen[0][:,dim] = (rando <= p1)/1 #fantasize data for error measure for h in range(self.h[0].shape[0]): self.z[0][self.fields[0][h],dim] += np.dot(self.W[0][h],self.h_gen[0][h,dim]) p1 = expit(self.z[0][:,dim] + self.a[:,dim]) rando = np.random.rand(self.x.shape[0]) self.x_gen[:,dim] = (rando <= p1)/1 #h2 again: for h in range(self.h[1].shape[0]): sample = np.hstack(receptive_field(self.x_gen[:,dim],1,self.horiz[1],self.vert[1], self.centers[1][h], self.vspans[1][h], self.hspans[1][h])) z = np.dot(np.transpose(self.W[1][h]),sample) self.q1[1][h,dim] = expit(z + self.b[1][h,dim]) rando = np.random.rand(1) self.h_gen[1][h,dim] = (rando[0] <= self.q1[1][h,dim])/1 #COST #sum errors self.err[r,dim] = (sum(self.x[:,dim]-self.x_gen[:,dim]))**2 self.total_err[dim] += sum(self.err[:,dim]) # {save examples code} #CD-1 learning for h in range(self.h[1].shape[0]): self.pos_W[1][h] = np.multiply(self.x[:,dim][self.fields[1][h]],self.q[1][h,dim]) self.neg_W[1][h] = np.multiply(self.x_gen[:,dim][self.fields[1][h]],self.q1[1][h,dim]) self.W_inc[1][h] = (momentum * self.W_inc[1][h]) + alpha * (self.pos_W[1][h] - self.neg_W[1][h] - (self.W[1][h] * weightcost)) self.b_inc[1][:,dim] = (momentum * self.b_inc[1][:,dim]) + alpha * (self.h[1][:,dim]-self.h_gen[1][:,dim]) self.W[1][:,:,dim] = self.W[1][:,:,dim] + self.W_inc[1][:,:,dim] self.b[1][:,dim] = self.b[1][:,dim] + self.b_inc[1][:,dim] #end epoch i += 1 print "cost-channel " + str(dim) + ":" + str(self.total_err[dim]/data.Nums_u[typeflag][dim]) self.errHist[k,dim] = self.total_err[dim] k += 1 """I've only coded up to layer 2 for CRBM training.""" elif (layer == 3): #TRAIN LAYER 3 while k < (iters): self.total_err = np.zeros(self.num_channels) self.err = np.zeros((N, self.num_channels)) print "epoch" + str(k) + "-3L:" #loop through training set for dim in range(self.num_channels): i = 0 while i < data.Nums_u[typeflag][dim+cof]: r = np.random.randint(0,data.Nums_u[typeflag][dim+cof]) self.x[:,dim] = C[dim,r,:] #POS #compute activation of h1: z = np.dot(np.transpose(self.W[0][:,:,dim]),self.x[:,dim]) q = expit(z + self.b[0][:,dim]) rando = np.random.rand(self.h[0].shape[0]) self.h[0][:,dim] = (rando <= q)/1 #compute activation of h2: z = np.dot(np.transpose(self.W[1][:,:,dim]),self.h[0][:,dim]) q = expit(z + self.b[1][:,dim]) rando = np.random.rand(self.h[1].shape[0]) self.h[1][:,dim] = (rando <= q)/1 #compute activation of h3: z = np.dot(np.transpose(self.W[2][:,:,dim]),self.h[1][:,dim]) q = expit(z + self.b[2][:,dim]) rando = np.random.rand(self.h[2].shape[0]) self.h[2][:,dim] = (rando <= q)/1 #NEG #generate fantasy on h2: z = np.dot(self.W[2][:,:,dim],self.h[2][:,dim]) p = expit(z + self.b[1][:,dim]) rando = np.random.rand(self.h[1].shape[0]) self.h_gen[1][:,dim] = (rando <= p)/1 #generate fantasy on h1: z = np.dot(self.W[1][:,:,dim],self.h_gen[1][:,dim]) p = expit(z + self.b[0][:,dim]) rando = np.random.rand(self.h[0].shape[0]) self.h_gen[0][:,dim] = (rando <= p)/1 #fantasize data for error measure z = np.dot(self.W[0][:,:,dim],self.h_gen[0][:,dim]) p1 = expit(z + self.a[:,dim]) rando = np.random.rand(self.x.shape[0]) self.x_gen[:,dim] = (rando <= p1)/1 #h3 again: z = np.dot(np.transpose(self.W[2][:,:,dim]),self.h_gen[1][:,dim]) q1 = expit(z + self.b[2][:,dim]) rando = np.random.rand(self.h[2].shape[0]) self.h_gen[2][:,dim] = (rando <= q1)/1 #COST #sum errors self.err[dim] = (sum(self.x[:,dim]-self.x_gen[:,dim]))**2 self.total_err[dim] += sum(self.err[dim]) # {save examples code} #CD-1 learning self.pos_W[2][:,:,dim] = np.outer(self.h[1][:,dim],q) self.neg_W[2][:,:,dim] = np.outer(self.h_gen[1][:,dim],q1) self.W_inc[2][:,:,dim] = (momentum * self.W_inc[2][:,:,dim]) + alpha * (self.pos_W[2][:,:,dim] - self.neg_W[2][:,:,dim] - (self.W[2][:,:,dim]*weightcost)) self.b_inc[2][:,dim] = (momentum * self.b_inc[2][:,dim]) + alpha * (self.h[2][:,dim]-self.h_gen[2][:,dim]) self.W[2][:,:,dim] = self.W[2][:,:,dim] + self.W_inc[2][:,:,dim] self.b[2][:,dim] = self.b[2][:,dim] + self.b_inc[2][:,dim] #end epoch i += 1 print "cost-channel " + str(dim) + ":" + str(self.total_err[dim]/data.Nums_u[typeflag][dim]) self.errHist[k,dim] = self.total_err[dim] k += 1 elif (layer >= 4): cur = layer - 1 #TRAIN LAYER 4+ while k < (iters): self.total_err = np.zeros(self.num_channels) self.err = np.zeros((N, self.num_channels)) print "epoch" + str(k) + "-"+str(layer)+"L:" #loop through training set for dim in range(self.num_channels): i = 0 while i < data.Nums_u[typeflag][dim+cof]: r = np.random.randint(0,data.Nums_u[typeflag][dim+cof]) self.x[:,dim] = C[dim,r,:] #POS #compute activation of h1: z = np.dot(np.transpose(self.W[0][:,:,dim]),self.x[:,dim]) q = expit(z + self.b[0][:,dim]) rando = np.random.rand(self.h[0].shape[0]) self.h[0][:,dim] = (rando <= q)/1 for l in range(1,layer): #compute activation of h2+: z = np.dot(np.transpose(self.W[l][:,:,dim]),self.h[l-1][:,dim]) q = expit(z + self.b[l][:,dim]) rando = np.random.rand(self.h[l].shape[0]) self.h[l][:,dim] = (rando <= q)/1 self.h_gen[l][:,dim] = self.h[l][:,dim] #NEG for l in range(layer-1,0,-1): #generate fantasy on hN-: z = np.dot(self.W[l][:,:,dim],self.h_gen[l][:,dim]) p = expit(z + self.b[l-1][:,dim]) rando = np.random.rand(self.h[l-1].shape[0]) self.h_gen[l-1][:,dim] = (rando <= p)/1 #generate fantasy on h1: z = np.dot(self.W[1][:,:,dim],self.h_gen[1][:,dim]) p = expit(z + self.b[0][:,dim]) rando = np.random.rand(self.h[0].shape[0]) self.h_gen[0][:,dim] = (rando <= p)/1 #fantasize data for error measure z = np.dot(self.W[0][:,:,dim],self.h_gen[0][:,dim]) p1 = expit(z + self.a[:,dim]) rando = np.random.rand(self.x.shape[0]) self.x_gen[:,dim] = (rando <= p1)/1 #hN again: z = np.dot(np.transpose(self.W[layer-1][:,:,dim]),self.h_gen[layer-2][:,dim]) q1 = expit(z + self.b[layer-1][:,dim]) rando = np.random.rand(self.h[layer-1].shape[0]) self.h_gen[layer-1][:,dim] = (rando <= q1)/1 #COST #sum errors self.err[r, dim] = (sum(self.x[:,dim]-self.x_gen[:,dim]))**2 self.total_err[dim] += sum(self.err[dim]) #CD-1 learning self.pos_W[cur][:,:,dim] = np.outer(self.h[cur-1][:,dim],q) self.neg_W[cur][:,:,dim] = np.outer(self.h_gen[cur-1][:,dim],q1) self.W_inc[cur][:,:,dim] = (momentum * self.W_inc[cur][:,:,dim]) + alpha * (self.pos_W[cur][:,:,dim] - self.neg_W[cur][:,:,dim] - (self.W[cur][:,:,dim]*weightcost)) self.b_inc[cur][:,dim] = (momentum * self.b_inc[cur][:,dim]) + alpha * (self.h[cur][:,dim]-self.h_gen[cur][:,dim]) self.W[cur][:,:,dim] = self.W[cur][:,:,dim] + self.W_inc[cur][:,:,dim] self.b[cur][:,dim] = self.b[cur][:,dim] + self.b_inc[cur][:,dim] #end epoch i += 1 print "cost-channel " + str(dim) + ":" + str(self.total_err[dim]/data.Nums_u[typeflag][dim]) self.errHist[k,dim] = self.total_err[dim] k += 1
def autoencode(self, data, layer=1, datatype='phrases'): Gibbs = 400 emptyPhrase = 0 if datatype == 'songs': N = data.numSongs C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])),dtype='int8') + 1 C[0,:data.whole_song.shape[0]] = data.whole_song output = np.zeros((self.num_channels, N, 256, 4, 8)) + 255 if datatype == 'tables': N = data.Nums[1][0] C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])),dtype='int8') + 1 C[0,:data.Tb.shape[0]] = data.Tb output = np.zeros((self.num_channels, N, 6, 16, 8)) + 255 elif datatype == 'phrases': N = np.amax(data.Nums_u[0]) C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])),dtype='int8') + 1 C[0,:data.P1_u.shape[0]] = data.P1_u C[1,:data.P2_u.shape[0]] = data.P2_u C[2,:data.Wv_u.shape[0]] = data.Wv_u C[3,:data.Ns_u.shape[0]] = data.Ns_u output = np.zeros((self.num_channels, N, 4, 16, 8)) + 255 elif datatype == 'chains': N = np.amax(data.Nums_u[2]) C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1 C[0,:data.PC_u.shape[0]] = data.PC_u C[1,:data.P2C_u.shape[0]] = data.P2C_u C[2,:data.WC_u.shape[0]] = data.WC_u C[3,:data.NC_u.shape[0]] = data.NC_u output = np.zeros((self.num_channels, N, 2, 16, 8)) + 255 elif datatype == 'instr_pulse': N = data.Nums_u[3][0] C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1 C[0,:data.PI.shape[0]] = data.PI output_inc = np.array([np.array(np.zeros(2),dtype='int'),np.array([1,0,1,0,1,0,0,0],dtype='int'),np.array(np.zeros(8),dtype='int'),0,np.array(np.zeros(6),dtype='int'),np.array(np.zeros(8)+1,dtype='int'),0,0,np.array(np.zeros(2),dtype='int'),0,0,np.array(np.zeros(5),dtype='int'),np.array(np.zeros(2),dtype='int'),np.array(np.zeros(4),dtype='int'),np.array(np.zeros(2)+1,dtype='int')],dtype=object) output = np.array([output_inc for x in range(N)]) elif datatype == 'instr_wave': N = data.Nums_u[3][1] C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1 C[0,:data.WI.shape[0]] = data.WI output_inc = np.array([np.array([0,1],dtype='int'),np.array([0,0,1,0,0,0,0,0],dtype='int'),np.array(np.zeros(4),dtype='int'),np.array([0,0,1,1],dtype='int'),0,0,np.array(np.zeros(2),dtype='int'),0,0,np.array(np.zeros(5),dtype='int'),np.array(np.zeros(2)+1,dtype='int'),np.array(np.zeros(2),dtype='int'),np.array(np.zeros(4)+1,dtype='int'),np.array(np.zeros(4),dtype='int')],dtype=object) output = np.array([output_inc for x in range(N)]) elif datatype == 'instr_kit': N = data.Nums_u[3][2] C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1 C[0,:data.KI.shape[0]] = data.KI output_inc = np.array([np.array([1,0],dtype='int'),np.array([1,0,1,0,1,0,0,0],dtype='int'),0,0,np.array(np.zeros(6),dtype='int'),np.array(np.zeros(8),dtype='int'),0,0,0,1,np.array([np.array(np.zeros(2),dtype='int'),0]),0,np.array(np.zeros(5),dtype='int'),np.array(np.zeros(2)+1,dtype='int'),np.array(np.zeros(8),dtype='int'),0,np.array(np.zeros(6),dtype='int'),np.array(np.zeros(8),dtype='int'),np.array(np.zeros(8),dtype='int'), np.array(np.zeros(8),dtype='int'), np.array(np.zeros(8),dtype='int')],dtype=object) output = np.array([output_inc for x in range(N)]) elif datatype == 'instr_noise': N = data.Nums_u[3][3] C = np.array(np.zeros((self.num_channels, N, self.x.shape[0])), dtype='int8') + 1 C[0,:data.NI.shape[0]] = data.NI output_inc = np.array([np.array([1,1],dtype='int'),np.array([0,1,0,1,0,0,0,0],dtype='int'),np.array(np.zeros(8),dtype='int'),0,np.array(np.zeros(6),dtype='int'),np.array(np.zeros(8),dtype='int'),0,0,1,np.array(np.zeros(5),dtype='int'),np.array([1,1],dtype='int')],dtype=object) output = np.array([output_inc for x in range(N)]) i = 0 reconstructions = np.array(np.zeros(C.shape),dtype='int') hidden_reps = np.array(np.zeros((self.num_channels, N, self.h_gen[layer-1].shape[0]))+2,dtype='int') while (i < N): for dim in range(self.num_channels): if datatype == 'chains': if (sum(C[dim,i,:]) < 256): self.x[:,dim] = C[dim,i,:] elif (sum(C[dim,i,:]) == 256): emptyPhrase = 1 if datatype == 'phrases': if (sum(C[dim,i,:]) < 512): self.x[:,dim] = C[dim,i,:] elif (sum(C[dim,i,:]) == 512): emptyPhrase = 1 elif datatype == 'tables': if (sum(C[dim,i,:]) < 768): self.x[:,dim] = C[dim,i,:] elif (sum(C[dim,i,:]) == 768): emptyPhrase = 1 elif datatype == 'instr_pulse': pass elif datatype == 'instr_wave': pass elif datatype == 'instr_kit': pass elif datatype == 'instr_noise': pass elif datatype == 'songs': pass if (emptyPhrase == 0): #POS #compute activation of h1: for h in range(self.h[0].shape[0]): sample = np.hstack(receptive_field(self.x[:,dim],1,self.horiz[0],self.vert[0], self.centers[0][h], self.vspans[0][h], self.hspans[0][h])) z = np.dot(np.transpose(self.W[0][h]),sample) self.q[0][h,dim] = expit(z + self.b[0][h,dim]) rando = np.random.rand(1) self.h[0][h,dim] = (rando[0] <= self.q[0][h,dim])/1 if (layer >= 2): for l in range(1,layer): for h in range(self.h[l].shape[0]): sample = np.hstack(receptive_field(self.h[l-1][:,dim],1,self.horiz[l-1],self.vert[l-1], self.centers[l-1][h], self.vspans[l-1][h], self.hspans[l-1][h])) z = np.dot(np.transpose(self.W[0][h]),sample) q = expit(z + self.b[l][h,dim]) rando = np.random.rand(1) self.h[l][h,dim] = (rando[0] <= q)/1 self.h_gen[l][:,dim] = self.h[l][:,dim] sleep = 0 #NEG while sleep < Gibbs: self.z = np.array([np.zeros((self.layer_sizes[lay], self.num_channels)) for lay in range(0, self.numLayers)]) if (layer == 1): #generate fantasy for h in range(self.h[0].shape[0]): self.z[0][self.fields[0][h],dim] += np.dot(self.W[0][h],self.h_gen[0][h,dim]) p1 = expit(self.z[0][:,dim] + self.a[:,dim]) rando = np.random.rand(self.x.shape[0]) self.x_gen[:,dim] = (rando <= p1)/1 #print sleep #h1 again: for h in range(self.h[0].shape[0]): sample = np.hstack(receptive_field(self.x_gen[:,dim],1,self.horiz[0],self.vert[0], self.centers[0][h], self.vspans[0][h], self.hspans[0][h])) z = np.dot(np.transpose(self.W[0][h]),sample) q1 = expit(z + self.b[0][h,dim]) rando = np.random.rand(1) self.h_gen[0][h,dim] = (rando[0] <= q1)/1 elif (layer >= 2): #generate fantasy on hN-: for h in range(self.h[layer].shape[0]): self.z[layer-1][self.fields[layer-1][h],dim] += np.dot(self.W[layer-1][h],self.h_gen[layer-1][h,dim]) p1 = expit(self.z[0][:,dim] + self.a[:,dim]) rando = np.random.rand(self.h[layer-2].shape[0]) self.h_gen[layer-2][:,dim] = (rando <= p1)/1 #hN again: for h in range(self.h[layer].shape[0]): sample = np.hstack(receptive_field(self.h_gen[layer-2][:,dim],1,self.horiz[layer-1],self.vert[layer-1], self.centers[layer-1][h], self.vspans[layer-1][h], self.hspans[layer-1][h])) z = np.dot(np.transpose(self.W[layer-1][h]),sample) q1 = expit(z + self.b[layer-1][h,dim]) rando = np.random.rand(1) self.h_gen[layer-1][h,dim] = (rando[0] <= q1)/1 sleep += 1 for l in range(layer-1,0,-1): self.z = np.array([np.zeros((self.layer_sizes[lay], self.num_channels)) for lay in range(0, self.numLayers)]) #generate fantasy on hN-: for h in range(self.h[l].shape[0]): self.z[l][self.fields[l][h],dim] += np.dot(self.W[l][h],self.h_gen[l][h,dim]) p1 = expit(self.z[0][:,dim] + self.a[:,dim]) rando = np.random.rand(self.h[l-1].shape[0]) self.h_gen[l-1][:,dim] = (rando <= p1)/1 #fantasize data for h in range(self.h[0].shape[0]): self.z[0][self.fields[0][h],dim] += np.dot(self.W[0][h],self.h[0][h,dim]) p1 = expit(self.z[0][:,dim] + self.a[:,dim]) rando = np.random.rand(self.x.shape[0]) self.x_gen[:,dim] = (rando <= p1)/1 #print reconstructions.shape #print dim #print i #print self.x_gen.shape reconstructions[dim,i,:] = self.x_gen[:,dim] hidden_reps[dim,i,:] = self.h_gen[layer-1][:,dim] print "example " + str(i) + " channel " + str(dim) + " encoded with error " + str((sum(self.x[:,dim]-self.x_gen[:,dim]))**2) + " using " + str(self.h_gen[layer-1].shape[0]) + " bits" emptyPhrase = 0 i += 1 if datatype == 'phrases': output[0] = np.reshape(reconstructions[0,:,:],(C[0].shape[0], 4, 16, 8)) output[1] = np.reshape(reconstructions[1,:,:],(C[1].shape[0], 4, 16, 8)) output[2] = np.reshape(reconstructions[2,:,:],(C[2].shape[0], 4, 16, 8)) output[3] = np.reshape(reconstructions[3,:,:],(C[3].shape[0], 4, 16, 8)) elif datatype == 'chains': output[0] = np.reshape(reconstructions[0,:,:],(C[0].shape[0], 2, 16, 8)) output[1] = np.reshape(reconstructions[1,:,:],(C[1].shape[0], 2, 16, 8)) output[2] = np.reshape(reconstructions[2,:,:],(C[2].shape[0], 2, 16, 8)) output[3] = np.reshape(reconstructions[3,:,:],(C[3].shape[0], 2, 16, 8)) elif datatype == 'tables': output[0] = np.reshape(reconstructions[0,:,:],(C[0].shape[0], 6, 16, 8)) output = output[0] #Note - this doesn't seem to work?! elif datatype == 'instr_pulse': for i in range(N): output[i,0] = reconstructions[0][i,0:8] output[i,1] = reconstructions[0][i,8:16] output[i,2] = reconstructions[0][i,16] output[i,3] = reconstructions[0][i,17:23] output[i,4] = reconstructions[0][i,23:31] output[i,5] = reconstructions[0][i,31] output[i,6] = reconstructions[0][i,32] output[i,7] = reconstructions[0][i,33:35] output[i,8] = reconstructions[0][i,35] output[i,9] = reconstructions[0][i,36] output[i,10] = reconstructions[0][i,37:42] output[i,11] = reconstructions[0][i,42:44] output[i,12] = reconstructions[0][i,44:48] output[i,13] = reconstructions[0][i,48:] elif datatype == 'instr_wave': for i in range(N): output[i,0] = reconstructions[0][i,0:8] output[i,1] = reconstructions[0][i,8:12] output[i,2] = reconstructions[0][i,12:16] output[i,3] = reconstructions[0][i,16] output[i,4] = reconstructions[0][i,17] output[i,5] = reconstructions[0][i,18:20] output[i,6] = reconstructions[0][i,20] output[i,7] = reconstructions[0][i,21] output[i,8] = reconstructions[0][i,22:27] output[i,9] = reconstructions[0][i,27:29] output[i,10] = reconstructions[0][i,29:31] output[i,11] = reconstructions[0][i,31:36] output[i,12] = reconstructions[0][i,36:] elif datatype == 'instr_kit': for i in range(N): output[i,0] = reconstructions[0][i,0:8] output[i,1] = reconstructions[0][i,8] output[i,2] = reconstructions[0][i,9] output[i,3] = reconstructions[0][i,10:16] output[i,4] = reconstructions[0][i,16:24] output[i,5] = reconstructions[0][i,24] output[i,6] = reconstructions[0][i,25] output[i,7] = reconstructions[0][i,26] output[i,8] = reconstructions[0][i,27] output[i,9] = reconstructions[0][i,28:30] output[i,10] = reconstructions[0][i,30] output[i,11] = reconstructions[0][i,31] output[i,12] = reconstructions[0][i,32:37] output[i,13] = reconstructions[0][i,37:39] output[i,14] = reconstructions[0][i,39:47] output[i,15] = reconstructions[0][i,47] output[i,16] = reconstructions[0][i,48:54] output[i,17] = reconstructions[0][i,54:62] output[i,18] = reconstructions[0][i,62:70] output[i,19] = reconstructions[0][i,70:78] output[i,20] = reconstructions[0][i,78:] elif datatype == 'instr_noise': for i in range(N): output[i,1] = reconstructions[0][i,0:8] output[i,2] = reconstructions[0][i,8:16] output[i,3] = reconstructions[0][i,16] output[i,4] = reconstructions[0][i,17:23] output[i,5] = reconstructions[0][i,23:31] output[i,6] = reconstructions[0][i,31] output[i,7] = reconstructions[0][i,32] output[i,8] = reconstructions[0][i,33] output[i,9] = reconstructions[0][i,34:39] output[i,10] = reconstructions[0][i,39:] elif datatype == 'songs': output[0] = np.reshape(reconstructions[0,:,:],(C[0].shape[0],256,4,8)) output = np.array(output[0][0], dtype='int') return [[hidden_reps, reconstructions], output]
def get_neuron_features(model, dataset, batch_size=128, top_n=100, out_dir='./output', mean=[0, 0, 0], std=[1, 1, 1]): """ Generates neuron features for given model definition using given dataset. :param model: Pytorch model definition. :param dataset: Dataset used for generating NFs. :param batch_size: Batch size used for predicting feature maps. :top_n: Use top_n input patch activations for generating NF. :out_dir: Directory where generated images are stored. :mean: Dataset mean used for normalization in transform function. :std: Dataset std used for normalization in transform function. :return: returns nothing """ # make output directory of not exists if not os.path.exists(out_dir): os.makedirs(out_dir) # set model in eval mode if torch.cuda.is_available(): model = model.cuda() # Set model in eval mode model.eval() # Dataset dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False) mean = np.asarray(mean) std = np.asarray(std) # input shape (c,w,h) in_shape = next(iter(dataloader))[0].shape[1:] # receptive field info for entire model receptive_field_dict = receptive_field(model, in_shape) # output layer info output_layer_info = receptive_field_dict[str( list(receptive_field_dict.keys())[-2])] # check if fm has group convs fm_groups = output_layer_info['output_shape'][2] if len( output_layer_info['output_shape']) == 5 else 0 # number of filters in last fm n_filters = output_layer_info['output_shape'][1] # Create placeholder for input patches rf = int(output_layer_info['r']) if fm_groups > 0: fm_im = np.zeros((top_n, n_filters, fm_groups, rf, rf, 3)) fm_w = -1e5 * np.ones((top_n, n_filters, fm_groups)) else: fm_im = np.zeros((top_n, n_filters, rf, rf, 3)) fm_w = -1e5 * np.ones((top_n, n_filters)) # Calculate amount of padding needed for input visualization # Get range for rf at position 0,0 in final feature map rf_range = receptive_field_for_unit( receptive_field_dict, str(list(receptive_field_dict.keys())[-2]), (0, 0)) pad_y = int(rf - (rf_range[0][1] - rf_range[0][0])) pad_x = int(rf - (rf_range[1][1] - rf_range[1][0])) # Print summary print('Group Convolutions: \t {}, {} elements'.format( fm_groups > 0, fm_groups)) print('Number of filters: \t {}'.format(n_filters)) print('Receptive field size: \t {}'.format(rf)) print('RF range at (0,0): \t {}'.format(rf_range)) print('Input padding (x,y): \t {}, {}'.format(pad_x, pad_y)) print( '==============================================================================' ) # Iterate over all data samples to get input patch for highest neuron activation # for each filter and transformation with torch.no_grad(): for inputs, _ in tqdm(dataloader, total=len(dataloader), desc='Extracting input patches: '): if torch.cuda.is_available(): model_inputs = inputs.cuda() else: model_inputs = inputs # Predict feature map fm = model(model_inputs) # Convert inputs to numpy w,h,c for visualization inputs = inputs.permute((0, 2, 3, 1)).numpy() # Unnormalize inputs *= std[None, None, None, :] inputs += mean[None, None, None, :] inputs = np.clip(inputs, 0, 1) # Pad inputs for visualization to compensate for padding in layers inputs = np.pad(inputs, ((0, 0), (pad_y, pad_y), (pad_x, pad_x), (0, 0)), mode='constant') # get batch shape fm_shape = fm.shape # if gconv: reshape groups into channels if fm_groups > 0: fm = fm.view((fm_shape[0], -1, fm_shape[3], fm_shape[4])) # Get max values and locations of feature maps # pool size = fm size = fm_shape[-1] a, b = F.max_pool2d(fm, (fm_shape[-2], fm_shape[-1]), return_indices=True) # if gconv: reshape groups back to own dimension if fm_groups > 0: a = a.view((fm_shape[0], fm_shape[1], fm_shape[2])) b = b.view((fm_shape[0], fm_shape[1], fm_shape[2])) a = a.cpu().numpy() b = b.cpu().numpy() # coordinates of max activations x = b % fm.shape[-1] y = b // fm.shape[-1] # store weight and input patches for each max position for i in range(inputs.shape[0]): for j in range(n_filters): if fm_groups == 0: # check if weight is higher than current lowest weight if a[i, j] > np.min(fm_w[:, j]): # replace lowest weight by current weight m = np.argmin(fm_w[:, j]) fm_w[m, j] = a[i, j] # store input patch rf_range = receptive_field_for_unit( receptive_field_dict, str(list(receptive_field_dict.keys())[-2]), (y[i, j], x[i, j]), bound=False) fm_im[m, j, :, :, :] = inputs[i, rf_range[0][0] + pad_y:rf_range[0][1] + pad_y, rf_range[1][0] + pad_x:rf_range[1][1] + pad_x, :] else: # loop over extra dimension for gconv for k in range(fm_groups): # check if weight is higher than current lowest weight if a[i, j, k] > np.min(fm_w[:, j, k]): # replace lowest weight by current weight m = np.argmin(fm_w[:, j, k]) # store weight fm_w[m, j, k] = a[i, j, k] # store input patch rf_range = receptive_field_for_unit( receptive_field_dict, str(list(receptive_field_dict.keys())[-2]), (y[i, j, k], x[i, j, k]), bound=False) fm_im[m, j, k, :, :, :] = inputs[ i, rf_range[0][0] + pad_y:rf_range[0][1] + pad_y, rf_range[1][0] + pad_x:rf_range[1][1] + pad_x, :] # Calculate and save neuron feature for each filter and transformation for i in tqdm(range(n_filters), total=n_filters, desc='Generating neuron features: '): if fm_groups == 0: w_sum = np.sum(fm_w[:, i]) if w_sum > 0: # Sort patches in order of highest neuron activations idx = np.argsort(fm_w[:, i])[::-1] # ::-1 for high to low sort fm_w[:, i] = fm_w[idx, i] fm_im[:, i, :, :, :] = fm_im[idx, i, :, :, :] # Calculate neuron feature fm_nfw = fm_w[:, i, None, None, None] / w_sum nf = np.sum(fm_im[:, i, :, :, :] * fm_nfw, axis=0) # Plot 19 highest activated patches plt.figure(figsize=(40, 2)) for j in range(19): plt.subplot(1, 20, j + 2) plt.title('{:.3f}'.format(fm_w[j, i])) plt.imshow(fm_im[j, i, :, :, :]) # Plot NF plt.subplot(1, 20, 1) plt.imshow(nf) plt.title('NF') _ = plt.setp(plt.gcf().get_axes(), xticks=[], yticks=[]) plt.savefig(os.path.join(out_dir, 'f_{:02d}.png'.format(i)), bbox_inches='tight') plt.savefig(os.path.join(out_dir, 'f_{:02d}.pdf'.format(i)), bbox_inches='tight') plt.close() else: plt.figure(figsize=(40, 6)) for k in range(fm_groups): w_sum = np.sum(fm_w[:, i, k]) if w_sum > 0: # Sort patches in order of highest neuron activations idx = np.argsort( fm_w[:, i, k])[::-1] # ::-1 for high to low sort fm_w[:, i, k] = fm_w[idx, i, k] fm_im[:, i, k, :, :, :] = fm_im[idx, i, k, :, :, :] # Calculate neuron feature fm_nfw = fm_w[:, i, k, None, None, None] / w_sum nf = np.sum(fm_im[:, i, k, :, :, :] * fm_nfw, axis=0) # Plot 19 highest activated patches for j in range(19): plt.subplot(fm_groups, 20, j + 2 + 20 * k) plt.title('{:.3f}'.format(fm_w[j, i, k])) plt.imshow(fm_im[j, i, k, :, :, :]) # Plot NF plt.subplot(fm_groups, 20, 1 + 20 * k) plt.imshow(nf) plt.title('NF') _ = plt.setp(plt.gcf().get_axes(), xticks=[], yticks=[]) plt.savefig(os.path.join(out_dir, 'f_{:02d}.png'.format(i)), bbox_inches='tight') plt.savefig(os.path.join(out_dir, 'f_{:02d}.pdf'.format(i)), bbox_inches='tight') plt.close() print('Done!')