def test_momentum(): data = saved_data[21] assert len(data) == 8 x = data[0] y = data[1] solW = data[2:5] solb = data[5:] reset_prng() mlp = hw1.MLP(784, 10, [64, 32], [activation.Sigmoid(), activation.Sigmoid(), activation.Identity()], weight_init, bias_init, loss.SoftmaxCrossEntropy(), 0.008, momentum=0.856, num_bn_layers=0) num_test_updates = 5 for u in range(num_test_updates): mlp.zero_grads() mlp.forward(x) mlp.backward(y) mlp.step() mlp.eval() W = [x.W for x in mlp.linear_layers] b = [x.b for x in mlp.linear_layers] for i, (pred, gt) in enumerate(zip(W, solW)): closeness_test(pred, gt, "mlp.linear_layers[%d].W" % i) for i, (pred, gt) in enumerate(zip(b, solb)): closeness_test(pred, gt, "mlp.linear_layers[%d].b" % i)
def test_batch_norm_train(): data = saved_data[19] assert len(data) == 10 x = data[0] y = data[1] soldW = data[2:5] soldb = data[5:8] soldbeta = data[8] soldgamma = data[9] reset_prng() mlp = hw1.MLP(784, 10, [64, 32], [activation.Sigmoid(), activation.Sigmoid(), activation.Identity()], weight_init, bias_init, loss.SoftmaxCrossEntropy(), 0.008, momentum=0.0, num_bn_layers=1) mlp.forward(x) mlp.backward(y) dW = [x.dW for x in mlp.linear_layers] db = [x.db for x in mlp.linear_layers] for i, (pred, gt) in enumerate(zip(dW, soldW)): closeness_test(pred, gt, "mlp.dW[%d]" % i) for i, (pred, gt) in enumerate(zip(db, soldb)): closeness_test(pred, gt, "mlp.db[%d]" % i) closeness_test(mlp.bn_layers[0].dbeta, soldbeta, "mlp.bn_layers[0].dbeta") closeness_test(mlp.bn_layers[0].dgamma, soldgamma, "mlp.bn_layers[0].dgamma")
def test_batch_norm_inference(): num_examples = 1000 data = saved_data[20] assert len(data) == 15 x = data[0] y = data[1] soldbeta = data[2] soldgamma = data[3] xs = data[4] solground = data[5:] reset_prng() mlp = hw1.MLP(784, 10, [64, 32], [activation.Sigmoid(), activation.Sigmoid(), activation.Identity()], weight_init, bias_init, loss.SoftmaxCrossEntropy(), 0.008, momentum=0.0, num_bn_layers=1) batch_size = 100 mlp.train() for b in range(0, 1): mlp.zero_grads() mlp.forward(x[b:b + batch_size]) mlp.backward(y[b:b + batch_size]) mlp.step() closeness_test(mlp.bn_layers[0].dbeta, soldbeta, "mlp.bn_layers[0].dbeta") closeness_test(mlp.bn_layers[0].dgamma, soldgamma, "mlp.bn_layers[0].dgamma") for b in range(0, num_examples, batch_size): mlp.eval() student = mlp.forward(xs[b:b + batch_size]) ground = solground[b//batch_size] closeness_test(student, ground, "mlp.forward(x)")
def test_mystery_hidden_backward3(): data = saved_data[18] assert len(data) == 6 x = data[0] y = data[1] soldW = data[2:4] soldb = data[4:] reset_prng() mlp = hw1.MLP( 784, 10, [32], [activation.Sigmoid(), activation.Identity()], weight_init, bias_init, loss.SoftmaxCrossEntropy(), 0.008, momentum=0.0, num_bn_layers=0) mlp.forward(x) mlp.backward(y) dW = [x.dW for x in mlp.linear_layers] db = [x.db for x in mlp.linear_layers] for i, (pred, gt) in enumerate(zip(dW, soldW)): closeness_test(pred, gt, "mlp.linear_layers[%d].dW" % i) for i, (pred, gt) in enumerate(zip(db, soldb)): closeness_test(pred, gt, "mlp.linear_layers[%d].db" % i)
def test_linear_classifier_forward(): data = saved_data[2] x = data[0] gt = data[1] reset_prng() mlp = hw1.MLP(784, 10, [], [activation.Identity()], weight_init, bias_init, loss.SoftmaxCrossEntropy(), 0.008, momentum=0.0, num_bn_layers=0) pred = mlp.forward(x) closeness_test(pred, gt, "mlp.forward(x)")
def test_mystery_hidden_forward3(): data = saved_data[15] x = data[0] gt = data[1] reset_prng() mlp = hw1.MLP(784, 10, [32], [activation.Sigmoid(), activation.Identity()], weight_init, bias_init, loss.SoftmaxCrossEntropy(), 0.008, momentum=0.0, num_bn_layers=0) pred = mlp.forward(x) closeness_test(pred, gt, "mlp.forward(x)")
def test_linear_classifier_backward(): data = saved_data[3] x = data[0] y = data[1] soldW = data[2] soldb = data[3] reset_prng() mlp = hw1.MLP(784, 10, [], [activation.Identity()], weight_init, bias_init, loss.SoftmaxCrossEntropy(), 0.008, momentum=0.0, num_bn_layers=0) mlp.forward(x) mlp.backward(y) closeness_test(mlp.linear_layers[0].dW, soldW, "mlp.linear_layers[0].dW") closeness_test(mlp.linear_layers[0].db, soldb, "mlp.linear_layers[0].db")
def __real_step__(*args): rnn_input = list(args) static_inputs = filter(lambda x: isinstance(x, StaticInputV2), input) for static_input in static_inputs: mem_name = "__%s_memory__" % static_input.input.name mem = memory(name=mem_name, is_seq=static_input.is_seq, size=static_input.input.calculate_size, boot_layer=static_input.input) with mixed(name=mem_name, size=static_input.input.calculate_size, act=activation.Identity()) as mix: mix += identity_projection(input=mem) rnn_input.insert(input.index(static_input), mix) return step(*rnn_input)
def test_linear_classifier_step(): data = saved_data[4] x = data[0] y = data[1] solW = data[2] solb = data[3] reset_prng() mlp = hw1.MLP(784, 10, [], [activation.Identity()], weight_init, bias_init, loss.SoftmaxCrossEntropy(), 0.008, momentum=0.0, num_bn_layers=0) num_test_updates = 5 for u in range(num_test_updates): mlp.zero_grads() mlp.forward(x) mlp.backward(y) mlp.step() closeness_test(mlp.linear_layers[0].W, solW, "mlp.linear_layers[0].W") closeness_test(mlp.linear_layers[0].b, solb, "mlp.linear_layers[0].b")
def __init__(self, is_leaky_relu: bool = False, leakage_coeff: float = 0.2): self._relu = activation.ReLU( is_leaky=is_leaky_relu, leakage=leakage_coeff if is_leaky_relu else 0) self._sigmoid = activation.Sigmoid() self._identity = activation.Identity() self._network = [[ Perceptron(num_inputs, self._relu) for _ in range(hiddenlayer1_size) ], [ Perceptron(hiddenlayer1_size, self._relu) for _ in range(hiddenlayer2_size) ], [ Perceptron(hiddenlayer2_size, self._sigmoid) for _ in range(num_outputs) ]] self._mn_data = MNIST('./images') self._desired_changes = None self._layer_inputs = None
import cost_functions as cost from data_prep import DataPrep from NeuralNetwork import DenseLayer, NeuralNetwork from NN_keras import Keras # set the parameters n = 100 n_epochs = 300 n_batches = 100 eta = 0.5 lmbda = 0 neurons = [10, 10] n_outputs = 1 hidden_act = act.Sigmoid() output_act = act.Identity() # create data using franke function seed = 2034 np.random.seed(seed) x = np.sort(np.random.uniform(0, 1, n)) y = np.sort(np.random.uniform(0, 1, n)) x, y = np.meshgrid(x, y) z = np.ravel(f.FrankeFunction(x, y) + 0.1*np.random.randn(x.shape[0], x.shape[1])) z = z.reshape(-1, 1) # set up the design matrix data = DataPrep() X = data.design_matrix(x, y, degree=1)[:, 1:] # split data in train and test and scale it
def mixlearnNNbuff(chromaNorm='L1', constantQNorm=None, deltaTrain=2, nnStruct=[256, 150, 24], errorFunc='SSE', verbose=False, numDataPass=1): ''' Learns neural network weights with buffered feature input (batch training in segments). Use this function when thrashing to disk is a possibility PARAMETERS ---------- chromaNorm: L1, L2, Linf, or None, normalization of chroma constantQNorm: L1, L2, Linf, or None, normalization of constant q transform deltaTrain {int}: how many songs to train after. Buffering features prevents thrashing to disk. nnStruct {List}: neural network layer list (each element is number of neurons at the layer errorFunc {String}: 'KLDiv' or 'SSE' verbose {Boolean}: report progress to standard out RETURN ------ net: trained neural network ''' # initialize feature storage Xtrain = [] # Constant-Q transform Xtarget = [] # Bass and treble chromagram # Set up neural network # uses sigmoid activation function by default at each layer # output activation depends on the type of chromaNorm specified activations = [act.Sigmoid()] * (len(nnStruct) - 2) if chromaNorm == 'L1': # partitioned SoftMax output (L1 normalization for each chromagram) activations.append(act.SoftMax([12])) elif chromaNorm == 'L2': activations.append(act.Identity()) elif chromaNorm == 'Linf': activations.append(act.Sigmoid()) else: activations.append(act.Identity()) # Instantiate neural network # assumes full connectivity between layer neurons. net = nn.NeuralNet(nnStruct, actFunc=activations) # hire a trainer for the network trainer = nn.Trainer(net, errorFunc, 1) for iDataset in range(numDataPass): print "DATASET PASS %d" % (iDataset + 1) # get feature file pointers qtransFiles, chromaFiles = process_dir("data/burgoyne2011chords") # until all the feature files have been exhausted passInd = 0 while len(qtransFiles) > 0 and len(chromaFiles) > 0: # for each pair of feature files that remain i = 0 for qFile, cFile in izip(qtransFiles[:], chromaFiles[:]): # open Constant-Q file and restore reading offset qFilePtx = open(qFile["path"], 'r') qFilePtx.seek(qFile["offset"]) # read an observation qObs = qFilePtx.readline().strip() # we've exhausted the observations in this song if not qObs: print "DONE WITH SONG: %s" % qFile["path"] # remove from the processing queue del qtransFiles[i] del chromaFiles[i] continue # update offset qtransFiles[i]["offset"] = qFilePtx.tell() # close the file pointer qFilePtx.close() # open chroma file and restore reading offset cFilePtx = open(cFile["path"], 'r') cFilePtx.seek(cFile["offset"]) # read an observation cObs = cFilePtx.readline().strip() # update offset chromaFiles[i]["offset"] = cFilePtx.tell() # close the file pointer cFilePtx.close() i += 1 if passInd < 50: continue qObs = qObs.split(",") cObs = cObs.split(",") # check features are in sync by timestamp if float(cObs[0]) != float(qObs[0]): raise ValueError("Feature files out of sync") # get chromagrams chroma = np.asfarray(cObs[1:]) # avoid divide by zero (this is silence in audio) if np.sum(chroma) < 3.0: continue # perform feature normalization if chromaNorm == 'L1': if np.sum(chroma[0:12]) != 0: chroma[0:12] /= np.sum(np.abs(chroma[0:12])) if np.sum(chroma[12:24]) != 0: chroma[12:24] /= np.sum(np.abs(chroma[12:24])) elif chromaNorm == 'L2': if np.sum(chroma[0:12]) != 0: chroma[0:12] /= np.sum(chroma[0:12]**2) if np.sum(chroma[12:24]) != 0: chroma[12:24] /= np.sum(chroma[12:24]**2) elif chromaNorm == 'Linf': if np.sum(chroma[0:12]) != 0: chroma[0:12] /= np.max(np.abs(chroma[0:12])) if np.sum(chroma[12:24]) != 0: chroma[12:24] /= np.max(np.abs(chroma[12:24])) Xtarget.append(chroma) # get Constant-Q transform constantQ = np.asfarray(qObs[1:]) # perform feature normalization if constantQNorm is not None and np.sum(constantQ) != 0: if constantQNorm == 'L1': constantQ /= np.sum(np.abs(constantQ)) elif constantQNorm == 'L2': constantQ /= np.sum(constantQ**2) elif constantQNorm == 'Linf': constantQ /= np.max(np.abs(constantQ)) Xtrain.append(constantQ) # train on this pass if len(Xtrain) > 0: print "Xtrain: ", len(Xtrain), ", Xtarget: ", len(Xtarget) train = np.asarray(Xtrain) target = np.asarray(Xtarget) trainer.setData(train, target) trainNet(trainer, verbose) # clear feature buffers del Xtrain[:] del Xtarget[:] passInd += 1 print "pass: "******"Done training neural network." return net
def testNNFeature(getSong=1, chromaNorm='L1', constantQNorm='L1'): # initialize feature storage Xtrain = [] # Constant-Q transform Xtarget = [] # Bass and treble chromagram nnStruct = [256, 24] # Set up neural network # uses sigmoid activation function by default at each layer # output activation depends on the type of chromaNorm specified activations = [act.Sigmoid()] * (len(nnStruct) - 2) if chromaNorm == 'L1': # partitioned SoftMax output (L1 normalization for each chromagram) activations.append(act.SoftMax([12])) elif chromaNorm == 'L2': activations.append(act.Identity()) elif chromaNorm == 'Linf': activations.append(act.Sigmoid()) else: activations.append(act.Identity()) # Instantiate neural network # assumes full connectivity between layer neurons. net = nn.NeuralNet(nnStruct, actFunc=activations) # load in trained weights wstar = np.load("trainedweights/wstar_grad_KLDiv_[0]_0.75_100iter.npy") net.setWeights(wstar) # read constant-q transform preliminary features qFile = open('data/logfreqspec.csv', 'r') # read bass and treble chromagram features cFile = open('data/bothchroma.csv', 'r') songNum = 0 songPath = '' for cObs, qObs in izip(cFile, qFile): cObs = cObs.split(",") qObs = qObs.split(",") # check if we have moved to a new song if cObs[0]: # check features are in sync by audio file path if not qObs[0] or cObs[0] != qObs[0]: raise ValueError("Feature files out of sync") # run gathered features through the neural net and return the values if songNum > 0 and songNum == getSong: print "Processing song #%d, %s" % (songNum, songPath) train = np.asarray(Xtrain) target = np.asarray(Xtarget) output = net.calcOutput(train) return train, output, target songNum += 1 songPath = cObs[0] if songNum != getSong: continue # double check features are in sync by timestamp if float(cObs[1]) != float(qObs[1]): raise ValueError("Feature files out of sync") # get chromagrams chroma = np.asfarray(cObs[2:]) # avoid divide by zero (this is silence in audio) #if np.sum(chroma) == 0: # continue # perform feature normalization if chromaNorm == 'L1': if np.sum(chroma[0:12]) != 0: chroma[0:12] /= np.sum(np.abs(chroma[0:12])) if np.sum(chroma[12:24]) != 0: chroma[12:24] /= np.sum(np.abs(chroma[12:24])) elif chromaNorm == 'L2': if np.sum(chroma[0:12]) != 0: chroma[0:12] /= np.sum(chroma[0:12]**2) if np.sum(chroma[12:24]) != 0: chroma[12:24] /= np.sum(chroma[12:24]**2) elif chromaNorm == 'Linf': if np.sum(chroma[0:12]) != 0: chroma[0:12] /= np.max(np.abs(chroma[0:12])) if np.sum(chroma[12:24]) != 0: chroma[12:24] /= np.max(np.abs(chroma[12:24])) Xtarget.append(chroma) # get Constant-Q transform constantQ = np.asfarray(qObs[2:]) # perform feature normalization if constantQNorm is not None and np.sum(constantQ) != 0: if constantQNorm == 'L1': constantQ /= np.sum(np.abs(constantQ)) elif constantQNorm == 'L2': constantQ /= np.sum(constantQ**2) elif constantQNorm == 'Linf': constantQ /= np.max(np.abs(constantQ)) Xtrain.append(constantQ)
def learnNNbuff(chromaNorm = 'L1', constantQNorm = None, deltaTrain = 2, nnStruct = [256, 150, 24], errorFunc = 'SSE', verbose = False): ''' Learns neural network weights with buffered feature input (batch training in segments). Use this function when thrashing to disk is a possibility PARAMETERS ---------- chromaNorm: L1, L2, Linf, or None, normalization of chroma constantQNorm: L1, L2, Linf, or None, normalization of constant q transform deltaTrain {int}: how many songs to train after. Buffering features prevents thrashing to disk. nnStruct {List}: neural network layer list (each element is number of neurons at the layer errorFunc {String}: 'KLDiv' or 'SSE' verbose {Boolean}: report progress to standard out RETURN ------ net: trained neural network ''' # initialize feature storage Xtrain = [] # Constant-Q transform Xtarget = [] # Bass and treble chromagram # Set up neural network # uses sigmoid activation function by default at each layer # output activation depends on the type of chromaNorm specified activations = [act.Sigmoid()] * (len(nnStruct)-2) if chromaNorm == 'L1': # partitioned SoftMax output (L1 normalization for each chromagram) activations.append(act.SoftMax([12])) elif chromaNorm == 'L2': activations.append(act.Identity()) elif chromaNorm == 'Linf': activations.append(act.Sigmoid()) else: activations.append(act.Identity()) # Instantiate neural network # assumes full connectivity between layer neurons. net = nn.NeuralNet(nnStruct, actFunc=activations) # hire a trainer for the network trainer = nn.Trainer(net, errorFunc, 1) # read constant-q transform preliminary features qFile = open('data/logfreqspec.csv', 'r') # read bass and treble chromagram features cFile = open('data/bothchroma.csv', 'r') songNum = 0 eligibleTrain = False for cObs, qObs in izip(cFile, qFile): cObs = cObs.split(",") qObs = qObs.split(",") # check if we have moved to a new song if cObs[0]: # check features are in sync by audio file path if not qObs[0] or cObs[0] != qObs[0]: raise ValueError("Feature files out of sync") # train the neural net with buffered features from the previous songs if songNum > 0 and songNum % deltaTrain == 0: trainer.setData(np.asarray(Xtrain), np.asarray(Xtarget)) trainNet(trainer, verbose) # clear feature buffers del Xtrain[:] del Xtarget[:] songNum += 1 if verbose: print "Processing song: ", cObs[0] # double check features are in sync by timestamp if float(cObs[1]) != float(qObs[1]): raise ValueError("Feature files out of sync") # get chromagrams chroma = np.asfarray(cObs[2:]) # avoid divide by zero (this is silence in audio) if np.sum(chroma) == 0: continue # perform feature normalization if chromaNorm == 'L1': if np.sum(chroma[0:12]) != 0: chroma[0:12] /= np.sum(np.abs(chroma[0:12])) if np.sum(chroma[12:24]) != 0: chroma[12:24] /= np.sum(np.abs(chroma[12:24])) elif chromaNorm == 'L2': if np.sum(chroma[0:12]) != 0: chroma[0:12] /= np.sum(chroma[0:12] ** 2) if np.sum(chroma[12:24]) != 0: chroma[12:24] /= np.sum(chroma[12:24] ** 2) elif chromaNorm == 'Linf': if np.sum(chroma[0:12]) != 0: chroma[0:12] /= np.max(np.abs(chroma[0:12])) if np.sum(chroma[12:24]) != 0: chroma[12:24] /= np.max(np.abs(chroma[12:24])) Xtarget.append(chroma) # get Constant-Q transform constantQ = np.asfarray(qObs[2:]) # perform feature normalization if constantQNorm is not None and np.sum(constantQ) != 0: if constantQNorm == 'L1': constantQ /= np.sum(np.abs(constantQ)) elif constantQNorm == 'L2': constantQ /= np.sum(constantQ ** 2) elif constantQNorm == 'Linf': constantQ /= np.max(np.abs(constantQ)) Xtrain.append(constantQ) # train leftovers (< deltaTrain songs) if len(Xtrain) > 0: trainer.setData(np.asarray(Xtrain), np.asarray(Xtarget)) trainNet(trainer, verbose) if verbose: print "Done training neural network." qFile.close() cFile.close() return net
def learnNN(chromaNorm = 'L1', constantQNorm = 'Linf', deltaTrain = 2, nnStruct = [256, 50, 24], errorFunc = 'SSE', verbose = False): ''' Learns neural network weights with unbuffered feature input: store all features in main memory and do one giant batch train. PARAMETERS ---------- chromaNorm: L1, L2, Linf, or None, normalization of chroma constantQNorm: L1, L2, Linf, or None, normalization of constant q transform nnStruct {List}: neural network layer list (each element is number of neurons at the layer errorFunc {String}: 'KLDiv' or 'SSE' verbose {Boolean}: report progress to standard out RETURN ------ net: trained neural network ''' # Set up neural network # uses sigmoid activation function by default at each layer # output activation depends on the type of chromaNorm specified activations = [act.Sigmoid()] * (len(nnStruct)-2) if chromaNorm == 'L1': # partitioned SoftMax output (L1 normalization for each chromagram) activations.append(act.SoftMax([12])) elif chromaNorm == 'L2': activations.append(act.Identity()) elif chromaNorm == 'Linf': activations.append(act.Sigmoid()) else: activations.append(act.Identity()) # Instantiate neural network # assumes full connectivity between layer neurons. net = nn.NeuralNet(nnStruct, actFunc=activations) if verbose: print "Retrieving Features." # read constant-q transform preliminary features Xtrain = np.loadtxt('data/logfreqspec.csv', dtype=np.float, delimiter=',', usecols=range(1,257)) # read bass and treble chromagram features Xtarget = np.loadtxt('data/bothchroma.csv', dtype=np.float, delimiter=',', usecols=range(1,25)) if verbose: print "Normalizing Features." divInd = np.sum(Xtrain, axis=1) != 0 # perform feature normalization if constantQNorm == 'L1': Xtrain[divInd,:] /= np.sum(np.abs(Xtrain[divInd,:]), axis=1)[:,np.newaxis] elif constantQNorm == 'L2': Xtrain[divInd,:] /= np.sum(Xtrain[divInd,:] ** 2, axis=1)[:,np.newaxis] elif constantQNorm == 'Linf': Xtrain[divInd,:] /= np.max(np.abs(Xtrain[divInd,:]), axis=1)[:,np.newaxis] del divInd divIndTreble = np.sum(Xtarget[:,0:12], axis=1) != 0 divIndBass = np.sum(Xtarget[:,12:24], axis=1) != 0 # perform feature normalization if chromaNorm == 'L1': Xtarget[divIndTreble,0:12] /= np.sum(np.abs(Xtarget[divIndTreble,0:12]), axis=1)[:,np.newaxis] Xtarget[divIndBass,12:24] /= np.sum(np.abs(Xtarget[divIndBass,12:24]), axis=1)[:,np.newaxis] elif chromaNorm == 'L2': Xtarget[divIndTreble,0:12] /= np.sum(Xtarget[divIndTreble,0:12] ** 2, axis=1)[:,np.newaxis] Xtarget[divIndBass,12:24] /= np.sum(Xtarget[divIndBass,12:24] ** 2, axis=1)[:,np.newaxis] elif chromaNorm == 'Linf': Xtarget[divIndTreble,0:12] /= np.max(np.abs(Xtarget[divIndTreble,0:12]), axis=1)[:,np.newaxis] Xtarget[divIndBass,12:24] /= np.max(np.abs(Xtarget[divIndBass,12:24]), axis=1)[:,np.newaxis] del divIndTreble del divIndBass # batch train Neural Network trainNet(Xtrain, Xtarget, net, errorFunc, verbose) if verbose: print "All done!" return net