def test_momentum():
data = saved_data[21]
assert len(data) == 8
x = data[0]
y = data[1]
solW = data[2:5]
solb = data[5:]
reset_prng()
mlp = hw1.MLP(784, 10, [64, 32], [activation.Sigmoid(), activation.Sigmoid(), activation.Identity()], weight_init, bias_init, loss.SoftmaxCrossEntropy(), 0.008,
momentum=0.856, num_bn_layers=0)
num_test_updates = 5
for u in range(num_test_updates):
mlp.zero_grads()
mlp.forward(x)
mlp.backward(y)
mlp.step()
mlp.eval()
W = [x.W for x in mlp.linear_layers]
b = [x.b for x in mlp.linear_layers]
for i, (pred, gt) in enumerate(zip(W, solW)):
closeness_test(pred, gt, "mlp.linear_layers[%d].W" % i)
for i, (pred, gt) in enumerate(zip(b, solb)):
closeness_test(pred, gt, "mlp.linear_layers[%d].b" % i)
def test_batch_norm_train():
data = saved_data[19]
assert len(data) == 10
x = data[0]
y = data[1]
soldW = data[2:5]
soldb = data[5:8]
soldbeta = data[8]
soldgamma = data[9]
reset_prng()
mlp = hw1.MLP(784, 10, [64, 32], [activation.Sigmoid(), activation.Sigmoid(), activation.Identity()],
weight_init, bias_init, loss.SoftmaxCrossEntropy(), 0.008,
momentum=0.0, num_bn_layers=1)
mlp.forward(x)
mlp.backward(y)
dW = [x.dW for x in mlp.linear_layers]
db = [x.db for x in mlp.linear_layers]
for i, (pred, gt) in enumerate(zip(dW, soldW)):
closeness_test(pred, gt, "mlp.dW[%d]" % i)
for i, (pred, gt) in enumerate(zip(db, soldb)):
closeness_test(pred, gt, "mlp.db[%d]" % i)
closeness_test(mlp.bn_layers[0].dbeta, soldbeta, "mlp.bn_layers[0].dbeta")
closeness_test(mlp.bn_layers[0].dgamma, soldgamma, "mlp.bn_layers[0].dgamma")
def test_batch_norm_inference():
num_examples = 1000
data = saved_data[20]
assert len(data) == 15
x = data[0]
y = data[1]
soldbeta = data[2]
soldgamma = data[3]
xs = data[4]
solground = data[5:]
reset_prng()
mlp = hw1.MLP(784, 10, [64, 32], [activation.Sigmoid(), activation.Sigmoid(), activation.Identity()],
weight_init, bias_init, loss.SoftmaxCrossEntropy(), 0.008,
momentum=0.0, num_bn_layers=1)
batch_size = 100
mlp.train()
for b in range(0, 1):
mlp.zero_grads()
mlp.forward(x[b:b + batch_size])
mlp.backward(y[b:b + batch_size])
mlp.step()
closeness_test(mlp.bn_layers[0].dbeta, soldbeta, "mlp.bn_layers[0].dbeta")
closeness_test(mlp.bn_layers[0].dgamma, soldgamma, "mlp.bn_layers[0].dgamma")
for b in range(0, num_examples, batch_size):
mlp.eval()
student = mlp.forward(xs[b:b + batch_size])
ground = solground[b//batch_size]
closeness_test(student, ground, "mlp.forward(x)")
def test_mystery_hidden_backward3():
data = saved_data[18]
assert len(data) == 6
x = data[0]
y = data[1]
soldW = data[2:4]
soldb = data[4:]
reset_prng()
mlp = hw1.MLP(
784,
10, [32],
[activation.Sigmoid(), activation.Identity()],
weight_init,
bias_init,
loss.SoftmaxCrossEntropy(),
0.008,
momentum=0.0,
num_bn_layers=0)
mlp.forward(x)
mlp.backward(y)
dW = [x.dW for x in mlp.linear_layers]
db = [x.db for x in mlp.linear_layers]
for i, (pred, gt) in enumerate(zip(dW, soldW)):
closeness_test(pred, gt, "mlp.linear_layers[%d].dW" % i)
for i, (pred, gt) in enumerate(zip(db, soldb)):
closeness_test(pred, gt, "mlp.linear_layers[%d].db" % i)
def test_linear_classifier_forward():
data = saved_data[2]
x = data[0]
gt = data[1]
reset_prng()
mlp = hw1.MLP(784, 10, [], [activation.Identity()], weight_init, bias_init,
loss.SoftmaxCrossEntropy(), 0.008, momentum=0.0,
num_bn_layers=0)
pred = mlp.forward(x)
closeness_test(pred, gt, "mlp.forward(x)")
def test_mystery_hidden_forward3():
data = saved_data[15]
x = data[0]
gt = data[1]
reset_prng()
mlp = hw1.MLP(784, 10, [32], [activation.Sigmoid(), activation.Identity()],
weight_init, bias_init, loss.SoftmaxCrossEntropy(), 0.008,
momentum=0.0, num_bn_layers=0)
pred = mlp.forward(x)
closeness_test(pred, gt, "mlp.forward(x)")
def test_linear_classifier_backward():
data = saved_data[3]
x = data[0]
y = data[1]
soldW = data[2]
soldb = data[3]
reset_prng()
mlp = hw1.MLP(784, 10, [], [activation.Identity()], weight_init, bias_init,
loss.SoftmaxCrossEntropy(), 0.008, momentum=0.0,
num_bn_layers=0)
mlp.forward(x)
mlp.backward(y)
closeness_test(mlp.linear_layers[0].dW, soldW, "mlp.linear_layers[0].dW")
closeness_test(mlp.linear_layers[0].db, soldb, "mlp.linear_layers[0].db")
def __real_step__(*args):
rnn_input = list(args)
static_inputs = filter(lambda x: isinstance(x, StaticInputV2), input)
for static_input in static_inputs:
mem_name = "__%s_memory__" % static_input.input.name
mem = memory(name=mem_name,
is_seq=static_input.is_seq,
size=static_input.input.calculate_size,
boot_layer=static_input.input)
with mixed(name=mem_name,
size=static_input.input.calculate_size,
act=activation.Identity()) as mix:
mix += identity_projection(input=mem)
rnn_input.insert(input.index(static_input), mix)
return step(*rnn_input)
def test_linear_classifier_step():
data = saved_data[4]
x = data[0]
y = data[1]
solW = data[2]
solb = data[3]
reset_prng()
mlp = hw1.MLP(784, 10, [], [activation.Identity()], weight_init, bias_init,
loss.SoftmaxCrossEntropy(), 0.008, momentum=0.0,
num_bn_layers=0)
num_test_updates = 5
for u in range(num_test_updates):
mlp.zero_grads()
mlp.forward(x)
mlp.backward(y)
mlp.step()
closeness_test(mlp.linear_layers[0].W, solW, "mlp.linear_layers[0].W")
closeness_test(mlp.linear_layers[0].b, solb, "mlp.linear_layers[0].b")
def __init__(self,
is_leaky_relu: bool = False,
leakage_coeff: float = 0.2):
self._relu = activation.ReLU(
is_leaky=is_leaky_relu,
leakage=leakage_coeff if is_leaky_relu else 0)
self._sigmoid = activation.Sigmoid()
self._identity = activation.Identity()
self._network = [[
Perceptron(num_inputs, self._relu)
for _ in range(hiddenlayer1_size)
],
[
Perceptron(hiddenlayer1_size, self._relu)
for _ in range(hiddenlayer2_size)
],
[
Perceptron(hiddenlayer2_size, self._sigmoid)
for _ in range(num_outputs)
]]
self._mn_data = MNIST('./images')
self._desired_changes = None
self._layer_inputs = None
import cost_functions as cost from data_prep import DataPrep from NeuralNetwork import DenseLayer, NeuralNetwork from NN_keras import Keras # set the parameters n = 100 n_epochs = 300 n_batches = 100 eta = 0.5 lmbda = 0 neurons = [10, 10] n_outputs = 1 hidden_act = act.Sigmoid() output_act = act.Identity() # create data using franke function seed = 2034 np.random.seed(seed) x = np.sort(np.random.uniform(0, 1, n)) y = np.sort(np.random.uniform(0, 1, n)) x, y = np.meshgrid(x, y) z = np.ravel(f.FrankeFunction(x, y) + 0.1*np.random.randn(x.shape[0], x.shape[1])) z = z.reshape(-1, 1) # set up the design matrix data = DataPrep() X = data.design_matrix(x, y, degree=1)[:, 1:] # split data in train and test and scale it
def mixlearnNNbuff(chromaNorm='L1',
constantQNorm=None,
deltaTrain=2,
nnStruct=[256, 150, 24],
errorFunc='SSE',
verbose=False,
numDataPass=1):
'''
Learns neural network weights with buffered feature input (batch training in segments).
Use this function when thrashing to disk is a possibility
PARAMETERS
----------
chromaNorm: L1, L2, Linf, or None, normalization of chroma
constantQNorm: L1, L2, Linf, or None, normalization of constant q transform
deltaTrain {int}: how many songs to train after. Buffering features prevents thrashing to disk.
nnStruct {List}: neural network layer list (each element is number of neurons at the layer
errorFunc {String}: 'KLDiv' or 'SSE'
verbose {Boolean}: report progress to standard out
RETURN
------
net: trained neural network
'''
# initialize feature storage
Xtrain = [] # Constant-Q transform
Xtarget = [] # Bass and treble chromagram
# Set up neural network
# uses sigmoid activation function by default at each layer
# output activation depends on the type of chromaNorm specified
activations = [act.Sigmoid()] * (len(nnStruct) - 2)
if chromaNorm == 'L1':
# partitioned SoftMax output (L1 normalization for each chromagram)
activations.append(act.SoftMax([12]))
elif chromaNorm == 'L2':
activations.append(act.Identity())
elif chromaNorm == 'Linf':
activations.append(act.Sigmoid())
else:
activations.append(act.Identity())
# Instantiate neural network
# assumes full connectivity between layer neurons.
net = nn.NeuralNet(nnStruct, actFunc=activations)
# hire a trainer for the network
trainer = nn.Trainer(net, errorFunc, 1)
for iDataset in range(numDataPass):
print "DATASET PASS %d" % (iDataset + 1)
# get feature file pointers
qtransFiles, chromaFiles = process_dir("data/burgoyne2011chords")
# until all the feature files have been exhausted
passInd = 0
while len(qtransFiles) > 0 and len(chromaFiles) > 0:
# for each pair of feature files that remain
i = 0
for qFile, cFile in izip(qtransFiles[:], chromaFiles[:]):
# open Constant-Q file and restore reading offset
qFilePtx = open(qFile["path"], 'r')
qFilePtx.seek(qFile["offset"])
# read an observation
qObs = qFilePtx.readline().strip()
# we've exhausted the observations in this song
if not qObs:
print "DONE WITH SONG: %s" % qFile["path"]
# remove from the processing queue
del qtransFiles[i]
del chromaFiles[i]
continue
# update offset
qtransFiles[i]["offset"] = qFilePtx.tell()
# close the file pointer
qFilePtx.close()
# open chroma file and restore reading offset
cFilePtx = open(cFile["path"], 'r')
cFilePtx.seek(cFile["offset"])
# read an observation
cObs = cFilePtx.readline().strip()
# update offset
chromaFiles[i]["offset"] = cFilePtx.tell()
# close the file pointer
cFilePtx.close()
i += 1
if passInd < 50:
continue
qObs = qObs.split(",")
cObs = cObs.split(",")
# check features are in sync by timestamp
if float(cObs[0]) != float(qObs[0]):
raise ValueError("Feature files out of sync")
# get chromagrams
chroma = np.asfarray(cObs[1:])
# avoid divide by zero (this is silence in audio)
if np.sum(chroma) < 3.0:
continue
# perform feature normalization
if chromaNorm == 'L1':
if np.sum(chroma[0:12]) != 0:
chroma[0:12] /= np.sum(np.abs(chroma[0:12]))
if np.sum(chroma[12:24]) != 0:
chroma[12:24] /= np.sum(np.abs(chroma[12:24]))
elif chromaNorm == 'L2':
if np.sum(chroma[0:12]) != 0:
chroma[0:12] /= np.sum(chroma[0:12]**2)
if np.sum(chroma[12:24]) != 0:
chroma[12:24] /= np.sum(chroma[12:24]**2)
elif chromaNorm == 'Linf':
if np.sum(chroma[0:12]) != 0:
chroma[0:12] /= np.max(np.abs(chroma[0:12]))
if np.sum(chroma[12:24]) != 0:
chroma[12:24] /= np.max(np.abs(chroma[12:24]))
Xtarget.append(chroma)
# get Constant-Q transform
constantQ = np.asfarray(qObs[1:])
# perform feature normalization
if constantQNorm is not None and np.sum(constantQ) != 0:
if constantQNorm == 'L1':
constantQ /= np.sum(np.abs(constantQ))
elif constantQNorm == 'L2':
constantQ /= np.sum(constantQ**2)
elif constantQNorm == 'Linf':
constantQ /= np.max(np.abs(constantQ))
Xtrain.append(constantQ)
# train on this pass
if len(Xtrain) > 0:
print "Xtrain: ", len(Xtrain), ", Xtarget: ", len(Xtarget)
train = np.asarray(Xtrain)
target = np.asarray(Xtarget)
trainer.setData(train, target)
trainNet(trainer, verbose)
# clear feature buffers
del Xtrain[:]
del Xtarget[:]
passInd += 1
print "pass: "******"Done training neural network."
return net
def testNNFeature(getSong=1, chromaNorm='L1', constantQNorm='L1'):
# initialize feature storage
Xtrain = [] # Constant-Q transform
Xtarget = [] # Bass and treble chromagram
nnStruct = [256, 24]
# Set up neural network
# uses sigmoid activation function by default at each layer
# output activation depends on the type of chromaNorm specified
activations = [act.Sigmoid()] * (len(nnStruct) - 2)
if chromaNorm == 'L1':
# partitioned SoftMax output (L1 normalization for each chromagram)
activations.append(act.SoftMax([12]))
elif chromaNorm == 'L2':
activations.append(act.Identity())
elif chromaNorm == 'Linf':
activations.append(act.Sigmoid())
else:
activations.append(act.Identity())
# Instantiate neural network
# assumes full connectivity between layer neurons.
net = nn.NeuralNet(nnStruct, actFunc=activations)
# load in trained weights
wstar = np.load("trainedweights/wstar_grad_KLDiv_[0]_0.75_100iter.npy")
net.setWeights(wstar)
# read constant-q transform preliminary features
qFile = open('data/logfreqspec.csv', 'r')
# read bass and treble chromagram features
cFile = open('data/bothchroma.csv', 'r')
songNum = 0
songPath = ''
for cObs, qObs in izip(cFile, qFile):
cObs = cObs.split(",")
qObs = qObs.split(",")
# check if we have moved to a new song
if cObs[0]:
# check features are in sync by audio file path
if not qObs[0] or cObs[0] != qObs[0]:
raise ValueError("Feature files out of sync")
# run gathered features through the neural net and return the values
if songNum > 0 and songNum == getSong:
print "Processing song #%d, %s" % (songNum, songPath)
train = np.asarray(Xtrain)
target = np.asarray(Xtarget)
output = net.calcOutput(train)
return train, output, target
songNum += 1
songPath = cObs[0]
if songNum != getSong:
continue
# double check features are in sync by timestamp
if float(cObs[1]) != float(qObs[1]):
raise ValueError("Feature files out of sync")
# get chromagrams
chroma = np.asfarray(cObs[2:])
# avoid divide by zero (this is silence in audio)
#if np.sum(chroma) == 0:
# continue
# perform feature normalization
if chromaNorm == 'L1':
if np.sum(chroma[0:12]) != 0:
chroma[0:12] /= np.sum(np.abs(chroma[0:12]))
if np.sum(chroma[12:24]) != 0:
chroma[12:24] /= np.sum(np.abs(chroma[12:24]))
elif chromaNorm == 'L2':
if np.sum(chroma[0:12]) != 0:
chroma[0:12] /= np.sum(chroma[0:12]**2)
if np.sum(chroma[12:24]) != 0:
chroma[12:24] /= np.sum(chroma[12:24]**2)
elif chromaNorm == 'Linf':
if np.sum(chroma[0:12]) != 0:
chroma[0:12] /= np.max(np.abs(chroma[0:12]))
if np.sum(chroma[12:24]) != 0:
chroma[12:24] /= np.max(np.abs(chroma[12:24]))
Xtarget.append(chroma)
# get Constant-Q transform
constantQ = np.asfarray(qObs[2:])
# perform feature normalization
if constantQNorm is not None and np.sum(constantQ) != 0:
if constantQNorm == 'L1':
constantQ /= np.sum(np.abs(constantQ))
elif constantQNorm == 'L2':
constantQ /= np.sum(constantQ**2)
elif constantQNorm == 'Linf':
constantQ /= np.max(np.abs(constantQ))
Xtrain.append(constantQ)
def learnNNbuff(chromaNorm = 'L1', constantQNorm = None, deltaTrain = 2, nnStruct = [256, 150, 24], errorFunc = 'SSE', verbose = False):
'''
Learns neural network weights with buffered feature input (batch training in segments).
Use this function when thrashing to disk is a possibility
PARAMETERS
----------
chromaNorm: L1, L2, Linf, or None, normalization of chroma
constantQNorm: L1, L2, Linf, or None, normalization of constant q transform
deltaTrain {int}: how many songs to train after. Buffering features prevents thrashing to disk.
nnStruct {List}: neural network layer list (each element is number of neurons at the layer
errorFunc {String}: 'KLDiv' or 'SSE'
verbose {Boolean}: report progress to standard out
RETURN
------
net: trained neural network
'''
# initialize feature storage
Xtrain = [] # Constant-Q transform
Xtarget = [] # Bass and treble chromagram
# Set up neural network
# uses sigmoid activation function by default at each layer
# output activation depends on the type of chromaNorm specified
activations = [act.Sigmoid()] * (len(nnStruct)-2)
if chromaNorm == 'L1':
# partitioned SoftMax output (L1 normalization for each chromagram)
activations.append(act.SoftMax([12]))
elif chromaNorm == 'L2':
activations.append(act.Identity())
elif chromaNorm == 'Linf':
activations.append(act.Sigmoid())
else:
activations.append(act.Identity())
# Instantiate neural network
# assumes full connectivity between layer neurons.
net = nn.NeuralNet(nnStruct, actFunc=activations)
# hire a trainer for the network
trainer = nn.Trainer(net, errorFunc, 1)
# read constant-q transform preliminary features
qFile = open('data/logfreqspec.csv', 'r')
# read bass and treble chromagram features
cFile = open('data/bothchroma.csv', 'r')
songNum = 0
eligibleTrain = False
for cObs, qObs in izip(cFile, qFile):
cObs = cObs.split(",")
qObs = qObs.split(",")
# check if we have moved to a new song
if cObs[0]:
# check features are in sync by audio file path
if not qObs[0] or cObs[0] != qObs[0]:
raise ValueError("Feature files out of sync")
# train the neural net with buffered features from the previous songs
if songNum > 0 and songNum % deltaTrain == 0:
trainer.setData(np.asarray(Xtrain), np.asarray(Xtarget))
trainNet(trainer, verbose)
# clear feature buffers
del Xtrain[:]
del Xtarget[:]
songNum += 1
if verbose:
print "Processing song: ", cObs[0]
# double check features are in sync by timestamp
if float(cObs[1]) != float(qObs[1]):
raise ValueError("Feature files out of sync")
# get chromagrams
chroma = np.asfarray(cObs[2:])
# avoid divide by zero (this is silence in audio)
if np.sum(chroma) == 0:
continue
# perform feature normalization
if chromaNorm == 'L1':
if np.sum(chroma[0:12]) != 0:
chroma[0:12] /= np.sum(np.abs(chroma[0:12]))
if np.sum(chroma[12:24]) != 0:
chroma[12:24] /= np.sum(np.abs(chroma[12:24]))
elif chromaNorm == 'L2':
if np.sum(chroma[0:12]) != 0:
chroma[0:12] /= np.sum(chroma[0:12] ** 2)
if np.sum(chroma[12:24]) != 0:
chroma[12:24] /= np.sum(chroma[12:24] ** 2)
elif chromaNorm == 'Linf':
if np.sum(chroma[0:12]) != 0:
chroma[0:12] /= np.max(np.abs(chroma[0:12]))
if np.sum(chroma[12:24]) != 0:
chroma[12:24] /= np.max(np.abs(chroma[12:24]))
Xtarget.append(chroma)
# get Constant-Q transform
constantQ = np.asfarray(qObs[2:])
# perform feature normalization
if constantQNorm is not None and np.sum(constantQ) != 0:
if constantQNorm == 'L1':
constantQ /= np.sum(np.abs(constantQ))
elif constantQNorm == 'L2':
constantQ /= np.sum(constantQ ** 2)
elif constantQNorm == 'Linf':
constantQ /= np.max(np.abs(constantQ))
Xtrain.append(constantQ)
# train leftovers (< deltaTrain songs)
if len(Xtrain) > 0:
trainer.setData(np.asarray(Xtrain), np.asarray(Xtarget))
trainNet(trainer, verbose)
if verbose:
print "Done training neural network."
qFile.close()
cFile.close()
return net
def learnNN(chromaNorm = 'L1', constantQNorm = 'Linf', deltaTrain = 2, nnStruct = [256, 50, 24], errorFunc = 'SSE', verbose = False):
'''
Learns neural network weights with unbuffered feature input: store all features in main memory and do one giant batch train.
PARAMETERS
----------
chromaNorm: L1, L2, Linf, or None, normalization of chroma
constantQNorm: L1, L2, Linf, or None, normalization of constant q transform
nnStruct {List}: neural network layer list (each element is number of neurons at the layer
errorFunc {String}: 'KLDiv' or 'SSE'
verbose {Boolean}: report progress to standard out
RETURN
------
net: trained neural network
'''
# Set up neural network
# uses sigmoid activation function by default at each layer
# output activation depends on the type of chromaNorm specified
activations = [act.Sigmoid()] * (len(nnStruct)-2)
if chromaNorm == 'L1':
# partitioned SoftMax output (L1 normalization for each chromagram)
activations.append(act.SoftMax([12]))
elif chromaNorm == 'L2':
activations.append(act.Identity())
elif chromaNorm == 'Linf':
activations.append(act.Sigmoid())
else:
activations.append(act.Identity())
# Instantiate neural network
# assumes full connectivity between layer neurons.
net = nn.NeuralNet(nnStruct, actFunc=activations)
if verbose:
print "Retrieving Features."
# read constant-q transform preliminary features
Xtrain = np.loadtxt('data/logfreqspec.csv', dtype=np.float, delimiter=',', usecols=range(1,257))
# read bass and treble chromagram features
Xtarget = np.loadtxt('data/bothchroma.csv', dtype=np.float, delimiter=',', usecols=range(1,25))
if verbose:
print "Normalizing Features."
divInd = np.sum(Xtrain, axis=1) != 0
# perform feature normalization
if constantQNorm == 'L1':
Xtrain[divInd,:] /= np.sum(np.abs(Xtrain[divInd,:]), axis=1)[:,np.newaxis]
elif constantQNorm == 'L2':
Xtrain[divInd,:] /= np.sum(Xtrain[divInd,:] ** 2, axis=1)[:,np.newaxis]
elif constantQNorm == 'Linf':
Xtrain[divInd,:] /= np.max(np.abs(Xtrain[divInd,:]), axis=1)[:,np.newaxis]
del divInd
divIndTreble = np.sum(Xtarget[:,0:12], axis=1) != 0
divIndBass = np.sum(Xtarget[:,12:24], axis=1) != 0
# perform feature normalization
if chromaNorm == 'L1':
Xtarget[divIndTreble,0:12] /= np.sum(np.abs(Xtarget[divIndTreble,0:12]), axis=1)[:,np.newaxis]
Xtarget[divIndBass,12:24] /= np.sum(np.abs(Xtarget[divIndBass,12:24]), axis=1)[:,np.newaxis]
elif chromaNorm == 'L2':
Xtarget[divIndTreble,0:12] /= np.sum(Xtarget[divIndTreble,0:12] ** 2, axis=1)[:,np.newaxis]
Xtarget[divIndBass,12:24] /= np.sum(Xtarget[divIndBass,12:24] ** 2, axis=1)[:,np.newaxis]
elif chromaNorm == 'Linf':
Xtarget[divIndTreble,0:12] /= np.max(np.abs(Xtarget[divIndTreble,0:12]), axis=1)[:,np.newaxis]
Xtarget[divIndBass,12:24] /= np.max(np.abs(Xtarget[divIndBass,12:24]), axis=1)[:,np.newaxis]
del divIndTreble
del divIndBass
# batch train Neural Network
trainNet(Xtrain, Xtarget, net, errorFunc, verbose)
if verbose:
print "All done!"
return net
